|
111
|
1 // wb.cc -- Add write barriers as needed.
|
|
|
2
|
|
|
3 // Copyright 2017 The Go Authors. All rights reserved.
|
|
|
4 // Use of this source code is governed by a BSD-style
|
|
|
5 // license that can be found in the LICENSE file.
|
|
|
6
|
|
|
7 #include "go-system.h"
|
|
|
8
|
|
|
9 #include "go-c.h"
|
|
|
10 #include "go-diagnostics.h"
|
|
|
11 #include "operator.h"
|
|
|
12 #include "lex.h"
|
|
|
13 #include "types.h"
|
|
|
14 #include "expressions.h"
|
|
|
15 #include "statements.h"
|
|
|
16 #include "runtime.h"
|
|
|
17 #include "gogo.h"
|
|
|
18
|
|
|
19 // Mark variables whose addresses are taken. This has to be done
|
|
|
20 // before the write barrier pass and after the escape analysis pass.
|
|
|
21 // It would be nice to do this elsewhere but there isn't an obvious
|
|
|
22 // place.
|
|
|
23
|
|
|
24 class Mark_address_taken : public Traverse
|
|
|
25 {
|
|
|
26 public:
|
|
|
27 Mark_address_taken(Gogo* gogo)
|
|
|
28 : Traverse(traverse_expressions),
|
|
|
29 gogo_(gogo)
|
|
|
30 { }
|
|
|
31
|
|
|
32 int
|
|
|
33 expression(Expression**);
|
|
|
34
|
|
|
35 private:
|
|
|
36 Gogo* gogo_;
|
|
|
37 };
|
|
|
38
|
|
|
39 // Mark variable addresses taken.
|
|
|
40
|
|
|
41 int
|
|
|
42 Mark_address_taken::expression(Expression** pexpr)
|
|
|
43 {
|
|
|
44 Expression* expr = *pexpr;
|
|
|
45 Unary_expression* ue = expr->unary_expression();
|
|
|
46 if (ue != NULL)
|
|
|
47 ue->check_operand_address_taken(this->gogo_);
|
|
|
48 return TRAVERSE_CONTINUE;
|
|
|
49 }
|
|
|
50
|
|
|
51 // Add write barriers to the IR. This are required by the concurrent
|
|
|
52 // garbage collector. A write barrier is needed for any write of a
|
|
|
53 // pointer into memory controlled by the garbage collector. Write
|
|
|
54 // barriers are not required for writes to local variables that live
|
|
|
55 // on the stack. Write barriers are only required when the runtime
|
|
|
56 // enables them, which can be checked using a run time check on
|
|
|
57 // runtime.writeBarrier.enabled.
|
|
|
58 //
|
|
|
59 // Essentially, for each assignment A = B, where A is or contains a
|
|
|
60 // pointer, and where A is not, or at any rate may not be, a stack
|
|
|
61 // variable, we rewrite it into
|
|
|
62 // if runtime.writeBarrier.enabled {
|
|
|
63 // typedmemmove(typeof(A), &A, &B)
|
|
|
64 // } else {
|
|
|
65 // A = B
|
|
|
66 // }
|
|
|
67 //
|
|
|
68 // The test of runtime.writeBarrier.Enabled is implemented by treating
|
|
|
69 // the variable as a *uint32, and testing *runtime.writeBarrier != 0.
|
|
|
70 // This is compatible with the definition in the runtime package.
|
|
|
71 //
|
|
|
72 // For types that are pointer shared (pointers, maps, chans, funcs),
|
|
|
73 // we replaced the call to typedmemmove with writebarrierptr(&A, B).
|
|
|
74 // As far as the GC is concerned, all pointers are the same, so it
|
|
|
75 // doesn't need the type descriptor.
|
|
|
76 //
|
|
|
77 // There are possible optimizations that are not implemented.
|
|
|
78 //
|
|
|
79 // runtime.writeBarrier can only change when the goroutine is
|
|
|
80 // preempted, which in practice means when a call is made into the
|
|
|
81 // runtime package, so we could optimize by only testing it once
|
|
|
82 // between function calls.
|
|
|
83 //
|
|
|
84 // A slice could be handled with a call to writebarrierptr plus two
|
|
|
85 // integer moves.
|
|
|
86
|
|
|
87 // Traverse the IR adding write barriers.
|
|
|
88
|
|
|
89 class Write_barriers : public Traverse
|
|
|
90 {
|
|
|
91 public:
|
|
|
92 Write_barriers(Gogo* gogo)
|
|
|
93 : Traverse(traverse_functions | traverse_variables | traverse_statements),
|
|
|
94 gogo_(gogo), function_(NULL)
|
|
|
95 { }
|
|
|
96
|
|
|
97 int
|
|
|
98 function(Named_object*);
|
|
|
99
|
|
|
100 int
|
|
|
101 variable(Named_object*);
|
|
|
102
|
|
|
103 int
|
|
|
104 statement(Block*, size_t* pindex, Statement*);
|
|
|
105
|
|
|
106 private:
|
|
|
107 // General IR.
|
|
|
108 Gogo* gogo_;
|
|
|
109 // Current function.
|
|
|
110 Function* function_;
|
|
|
111 };
|
|
|
112
|
|
|
113 // Traverse a function. Just record it for later.
|
|
|
114
|
|
|
115 int
|
|
|
116 Write_barriers::function(Named_object* no)
|
|
|
117 {
|
|
|
118 go_assert(this->function_ == NULL);
|
|
|
119 this->function_ = no->func_value();
|
|
|
120 int t = this->function_->traverse(this);
|
|
|
121 this->function_ = NULL;
|
|
|
122
|
|
|
123 if (t == TRAVERSE_EXIT)
|
|
|
124 return t;
|
|
|
125 return TRAVERSE_SKIP_COMPONENTS;
|
|
|
126 }
|
|
|
127
|
|
|
128 // Insert write barriers for a global variable: ensure that variable
|
|
|
129 // initialization is handled correctly. This is rarely needed, since
|
|
|
130 // we currently don't enable background GC until after all global
|
|
|
131 // variables are initialized. But we do need this if an init function
|
|
|
132 // calls runtime.GC.
|
|
|
133
|
|
|
134 int
|
|
|
135 Write_barriers::variable(Named_object* no)
|
|
|
136 {
|
|
|
137 // We handle local variables in the variable declaration statement.
|
|
|
138 // We only have to handle global variables here.
|
|
|
139 if (!no->is_variable())
|
|
|
140 return TRAVERSE_CONTINUE;
|
|
|
141 Variable* var = no->var_value();
|
|
|
142 if (!var->is_global())
|
|
|
143 return TRAVERSE_CONTINUE;
|
|
|
144
|
|
|
145 // Nothing to do if there is no initializer.
|
|
|
146 Expression* init = var->init();
|
|
|
147 if (init == NULL)
|
|
|
148 return TRAVERSE_CONTINUE;
|
|
|
149
|
|
|
150 // Nothing to do for variables that do not contain any pointers.
|
|
|
151 if (!var->type()->has_pointer())
|
|
|
152 return TRAVERSE_CONTINUE;
|
|
|
153
|
|
|
154 // Nothing to do if the initializer is static.
|
|
|
155 init = Expression::make_cast(var->type(), init, var->location());
|
|
|
156 if (!var->has_pre_init() && init->is_static_initializer())
|
|
|
157 return TRAVERSE_CONTINUE;
|
|
|
158
|
|
|
159 // Nothing to do for a type that can not be in the heap, or a
|
|
|
160 // pointer to a type that can not be in the heap.
|
|
|
161 if (!var->type()->in_heap())
|
|
|
162 return TRAVERSE_CONTINUE;
|
|
|
163 if (var->type()->points_to() != NULL && !var->type()->points_to()->in_heap())
|
|
|
164 return TRAVERSE_CONTINUE;
|
|
|
165
|
|
|
166 // Otherwise change the initializer into a pre_init assignment
|
|
|
167 // statement with a write barrier.
|
|
|
168
|
|
|
169 // We can't check for a dependency of the variable on itself after
|
|
|
170 // we make this change, because the preinit statement will always
|
|
|
171 // depend on the variable (since it assigns to it). So check for a
|
|
|
172 // self-dependency now.
|
|
|
173 this->gogo_->check_self_dep(no);
|
|
|
174
|
|
|
175 // Replace the initializer.
|
|
|
176 Location loc = init->location();
|
|
|
177 Expression* ref = Expression::make_var_reference(no, loc);
|
|
|
178 ref->var_expression()->set_in_lvalue_pos();
|
|
|
179
|
|
|
180 Statement_inserter inserter(this->gogo_, var);
|
|
|
181 Statement* s = this->gogo_->assign_with_write_barrier(NULL, NULL, &inserter,
|
|
|
182 ref, init, loc);
|
|
|
183
|
|
|
184 var->add_preinit_statement(this->gogo_, s);
|
|
|
185 var->clear_init();
|
|
|
186
|
|
|
187 return TRAVERSE_CONTINUE;
|
|
|
188 }
|
|
|
189
|
|
|
190 // Insert write barriers for statements.
|
|
|
191
|
|
|
192 int
|
|
|
193 Write_barriers::statement(Block* block, size_t* pindex, Statement* s)
|
|
|
194 {
|
|
|
195 switch (s->classification())
|
|
|
196 {
|
|
|
197 default:
|
|
|
198 break;
|
|
|
199
|
|
|
200 case Statement::STATEMENT_VARIABLE_DECLARATION:
|
|
|
201 {
|
|
|
202 Variable_declaration_statement* vds =
|
|
|
203 s->variable_declaration_statement();
|
|
|
204 Named_object* no = vds->var();
|
|
|
205 Variable* var = no->var_value();
|
|
|
206
|
|
|
207 // We may need to emit a write barrier for the initialization
|
|
|
208 // of the variable.
|
|
|
209
|
|
|
210 // Nothing to do for a variable with no initializer.
|
|
|
211 Expression* init = var->init();
|
|
|
212 if (init == NULL)
|
|
|
213 break;
|
|
|
214
|
|
|
215 // Nothing to do if the variable is not in the heap. Only
|
|
|
216 // local variables get declaration statements, and local
|
|
|
217 // variables on the stack do not require write barriers.
|
|
|
218 if (!var->is_in_heap())
|
|
|
219 break;
|
|
|
220
|
|
|
221 // Nothing to do if the variable does not contain any pointers.
|
|
|
222 if (!var->type()->has_pointer())
|
|
|
223 break;
|
|
|
224
|
|
|
225 // Nothing to do for a type that can not be in the heap, or a
|
|
|
226 // pointer to a type that can not be in the heap.
|
|
|
227 if (!var->type()->in_heap())
|
|
|
228 break;
|
|
|
229 if (var->type()->points_to() != NULL
|
|
|
230 && !var->type()->points_to()->in_heap())
|
|
|
231 break;
|
|
|
232
|
|
|
233 // Otherwise initialize the variable with a write barrier.
|
|
|
234
|
|
|
235 Function* function = this->function_;
|
|
|
236 Location loc = init->location();
|
|
|
237 Statement_inserter inserter(block, pindex);
|
|
|
238
|
|
|
239 // Insert the variable declaration statement with no
|
|
|
240 // initializer, so that the variable exists.
|
|
|
241 var->clear_init();
|
|
|
242 inserter.insert(s);
|
|
|
243
|
|
|
244 // Create a statement that initializes the variable with a
|
|
|
245 // write barrier.
|
|
|
246 Expression* ref = Expression::make_var_reference(no, loc);
|
|
|
247 Statement* assign = this->gogo_->assign_with_write_barrier(function,
|
|
|
248 block,
|
|
|
249 &inserter,
|
|
|
250 ref, init,
|
|
|
251 loc);
|
|
|
252
|
|
|
253 // Replace the old variable declaration statement with the new
|
|
|
254 // initialization.
|
|
|
255 block->replace_statement(*pindex, assign);
|
|
|
256 }
|
|
|
257 break;
|
|
|
258
|
|
|
259 case Statement::STATEMENT_ASSIGNMENT:
|
|
|
260 {
|
|
|
261 Assignment_statement* as = s->assignment_statement();
|
|
|
262 Expression* lhs = as->lhs();
|
|
|
263 Expression* rhs = as->rhs();
|
|
|
264
|
|
|
265 // We may need to emit a write barrier for the assignment.
|
|
|
266
|
|
|
267 if (!this->gogo_->assign_needs_write_barrier(lhs))
|
|
|
268 break;
|
|
|
269
|
|
|
270 // Change the assignment to use a write barrier.
|
|
|
271 Function* function = this->function_;
|
|
|
272 Location loc = as->location();
|
|
|
273 Statement_inserter inserter = Statement_inserter(block, pindex);
|
|
|
274 Statement* assign = this->gogo_->assign_with_write_barrier(function,
|
|
|
275 block,
|
|
|
276 &inserter,
|
|
|
277 lhs, rhs,
|
|
|
278 loc);
|
|
|
279 block->replace_statement(*pindex, assign);
|
|
|
280 }
|
|
|
281 break;
|
|
|
282 }
|
|
|
283
|
|
|
284 return TRAVERSE_CONTINUE;
|
|
|
285 }
|
|
|
286
|
|
|
287 // The write barrier pass.
|
|
|
288
|
|
|
289 void
|
|
|
290 Gogo::add_write_barriers()
|
|
|
291 {
|
|
|
292 Mark_address_taken mat(this);
|
|
|
293 this->traverse(&mat);
|
|
|
294
|
|
|
295 Write_barriers wb(this);
|
|
|
296 this->traverse(&wb);
|
|
|
297 }
|
|
|
298
|
|
|
299 // Return the runtime.writeBarrier variable.
|
|
|
300
|
|
|
301 Named_object*
|
|
|
302 Gogo::write_barrier_variable()
|
|
|
303 {
|
|
|
304 static Named_object* write_barrier_var;
|
|
|
305 if (write_barrier_var == NULL)
|
|
|
306 {
|
|
|
307 Location bloc = Linemap::predeclared_location();
|
|
|
308
|
|
|
309 // We pretend that writeBarrier is a uint32, so that we do a
|
|
|
310 // 32-bit load. That is what the gc toolchain does.
|
|
|
311 Type* uint32_type = Type::lookup_integer_type("uint32");
|
|
|
312 Variable* var = new Variable(uint32_type, NULL, true, false, false,
|
|
|
313 bloc);
|
|
|
314
|
|
|
315 bool add_to_globals;
|
|
|
316 Package* package = this->add_imported_package("runtime", "_", false,
|
|
|
317 "runtime", "runtime",
|
|
|
318 bloc, &add_to_globals);
|
|
|
319 write_barrier_var = Named_object::make_variable("writeBarrier",
|
|
|
320 package, var);
|
|
|
321 }
|
|
|
322
|
|
|
323 return write_barrier_var;
|
|
|
324 }
|
|
|
325
|
|
|
326 // Return whether an assignment that sets LHS needs a write barrier.
|
|
|
327
|
|
|
328 bool
|
|
|
329 Gogo::assign_needs_write_barrier(Expression* lhs)
|
|
|
330 {
|
|
|
331 // Nothing to do if the variable does not contain any pointers.
|
|
|
332 if (!lhs->type()->has_pointer())
|
|
|
333 return false;
|
|
|
334
|
|
|
335 // Nothing to do for an assignment to a temporary.
|
|
|
336 if (lhs->temporary_reference_expression() != NULL)
|
|
|
337 return false;
|
|
|
338
|
|
|
339 // Nothing to do for an assignment to a sink.
|
|
|
340 if (lhs->is_sink_expression())
|
|
|
341 return false;
|
|
|
342
|
|
|
343 // Nothing to do for an assignment to a local variable that is not
|
|
|
344 // on the heap.
|
|
|
345 Var_expression* ve = lhs->var_expression();
|
|
|
346 if (ve != NULL)
|
|
|
347 {
|
|
|
348 Named_object* no = ve->named_object();
|
|
|
349 if (no->is_variable())
|
|
|
350 {
|
|
|
351 Variable* var = no->var_value();
|
|
|
352 if (!var->is_global() && !var->is_in_heap())
|
|
|
353 return false;
|
|
|
354 }
|
|
|
355 else if (no->is_result_variable())
|
|
|
356 {
|
|
|
357 Result_variable* rvar = no->result_var_value();
|
|
|
358 if (!rvar->is_in_heap())
|
|
|
359 return false;
|
|
|
360 }
|
|
|
361 }
|
|
|
362
|
|
|
363 // Nothing to do for a type that can not be in the heap, or a
|
|
|
364 // pointer to a type that can not be in the heap.
|
|
|
365 if (!lhs->type()->in_heap())
|
|
|
366 return false;
|
|
|
367 if (lhs->type()->points_to() != NULL && !lhs->type()->points_to()->in_heap())
|
|
|
368 return false;
|
|
|
369
|
|
|
370 // Write barrier needed in other cases.
|
|
|
371 return true;
|
|
|
372 }
|
|
|
373
|
|
|
374 // Return a statement that sets LHS to RHS using a write barrier.
|
|
|
375 // ENCLOSING is the enclosing block.
|
|
|
376
|
|
|
377 Statement*
|
|
|
378 Gogo::assign_with_write_barrier(Function* function, Block* enclosing,
|
|
|
379 Statement_inserter* inserter, Expression* lhs,
|
|
|
380 Expression* rhs, Location loc)
|
|
|
381 {
|
|
|
382 if (function != NULL
|
|
|
383 && ((function->pragmas() & GOPRAGMA_NOWRITEBARRIER) != 0
|
|
|
384 || (function->pragmas() & GOPRAGMA_NOWRITEBARRIERREC) != 0))
|
|
|
385 go_error_at(loc, "write barrier prohibited");
|
|
|
386
|
|
|
387 Type* type = lhs->type();
|
|
|
388 go_assert(type->has_pointer());
|
|
|
389
|
|
|
390 Expression* addr;
|
|
|
391 if (lhs->unary_expression() != NULL
|
|
|
392 && lhs->unary_expression()->op() == OPERATOR_MULT)
|
|
|
393 addr = lhs->unary_expression()->operand();
|
|
|
394 else
|
|
|
395 {
|
|
|
396 addr = Expression::make_unary(OPERATOR_AND, lhs, loc);
|
|
|
397 addr->unary_expression()->set_does_not_escape();
|
|
|
398 }
|
|
|
399 Temporary_statement* lhs_temp = Statement::make_temporary(NULL, addr, loc);
|
|
|
400 inserter->insert(lhs_temp);
|
|
|
401 lhs = Expression::make_temporary_reference(lhs_temp, loc);
|
|
|
402
|
|
|
403 if (!Type::are_identical(type, rhs->type(), false, NULL)
|
|
|
404 && rhs->type()->interface_type() != NULL
|
|
|
405 && !rhs->is_variable())
|
|
|
406 {
|
|
|
407 // May need a temporary for interface conversion.
|
|
|
408 Temporary_statement* temp = Statement::make_temporary(NULL, rhs, loc);
|
|
|
409 inserter->insert(temp);
|
|
|
410 rhs = Expression::make_temporary_reference(temp, loc);
|
|
|
411 }
|
|
|
412 rhs = Expression::convert_for_assignment(this, type, rhs, loc);
|
|
|
413 Temporary_statement* rhs_temp = NULL;
|
|
|
414 if (!rhs->is_variable() && !rhs->is_constant())
|
|
|
415 {
|
|
|
416 rhs_temp = Statement::make_temporary(NULL, rhs, loc);
|
|
|
417 inserter->insert(rhs_temp);
|
|
|
418 rhs = Expression::make_temporary_reference(rhs_temp, loc);
|
|
|
419 }
|
|
|
420
|
|
|
421 Expression* indir = Expression::make_unary(OPERATOR_MULT, lhs, loc);
|
|
|
422 Statement* assign = Statement::make_assignment(indir, rhs, loc);
|
|
|
423
|
|
|
424 lhs = Expression::make_temporary_reference(lhs_temp, loc);
|
|
|
425 if (rhs_temp != NULL)
|
|
|
426 rhs = Expression::make_temporary_reference(rhs_temp, loc);
|
|
|
427
|
|
|
428 Type* unsafe_ptr_type = Type::make_pointer_type(Type::make_void_type());
|
|
|
429 lhs = Expression::make_unsafe_cast(unsafe_ptr_type, lhs, loc);
|
|
|
430
|
|
|
431 Expression* call;
|
|
|
432 switch (type->base()->classification())
|
|
|
433 {
|
|
|
434 default:
|
|
|
435 go_unreachable();
|
|
|
436
|
|
|
437 case Type::TYPE_ERROR:
|
|
|
438 return assign;
|
|
|
439
|
|
|
440 case Type::TYPE_POINTER:
|
|
|
441 case Type::TYPE_FUNCTION:
|
|
|
442 case Type::TYPE_MAP:
|
|
|
443 case Type::TYPE_CHANNEL:
|
|
|
444 // These types are all represented by a single pointer.
|
|
|
445 call = Runtime::make_call(Runtime::WRITEBARRIERPTR, loc, 2, lhs, rhs);
|
|
|
446 break;
|
|
|
447
|
|
|
448 case Type::TYPE_STRING:
|
|
|
449 case Type::TYPE_STRUCT:
|
|
|
450 case Type::TYPE_ARRAY:
|
|
|
451 case Type::TYPE_INTERFACE:
|
|
|
452 {
|
|
|
453 rhs = Expression::make_unary(OPERATOR_AND, rhs, loc);
|
|
|
454 rhs->unary_expression()->set_does_not_escape();
|
|
|
455 call = Runtime::make_call(Runtime::TYPEDMEMMOVE, loc, 3,
|
|
|
456 Expression::make_type_descriptor(type, loc),
|
|
|
457 lhs, rhs);
|
|
|
458 }
|
|
|
459 break;
|
|
|
460 }
|
|
|
461
|
|
|
462 return this->check_write_barrier(enclosing, assign,
|
|
|
463 Statement::make_statement(call, false));
|
|
|
464 }
|
|
|
465
|
|
|
466 // Return a statement that tests whether write barriers are enabled
|
|
|
467 // and executes either the efficient code or the write barrier
|
|
|
468 // function call, depending.
|
|
|
469
|
|
|
470 Statement*
|
|
|
471 Gogo::check_write_barrier(Block* enclosing, Statement* without,
|
|
|
472 Statement* with)
|
|
|
473 {
|
|
|
474 Location loc = without->location();
|
|
|
475 Named_object* wb = this->write_barrier_variable();
|
|
|
476 Expression* ref = Expression::make_var_reference(wb, loc);
|
|
|
477 Expression* zero = Expression::make_integer_ul(0, ref->type(), loc);
|
|
|
478 Expression* cond = Expression::make_binary(OPERATOR_EQEQ, ref, zero, loc);
|
|
|
479
|
|
|
480 Block* then_block = new Block(enclosing, loc);
|
|
|
481 then_block->add_statement(without);
|
|
|
482
|
|
|
483 Block* else_block = new Block(enclosing, loc);
|
|
|
484 else_block->add_statement(with);
|
|
|
485
|
|
|
486 return Statement::make_if_statement(cond, then_block, else_block, loc);
|
|
|
487 }
|
