111
|
1 /* Internal functions.
|
145
|
2 Copyright (C) 2011-2020 Free Software Foundation, Inc.
|
111
|
3
|
|
4 This file is part of GCC.
|
|
5
|
|
6 GCC is free software; you can redistribute it and/or modify it under
|
|
7 the terms of the GNU General Public License as published by the Free
|
|
8 Software Foundation; either version 3, or (at your option) any later
|
|
9 version.
|
|
10
|
|
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
14 for more details.
|
|
15
|
|
16 You should have received a copy of the GNU General Public License
|
|
17 along with GCC; see the file COPYING3. If not see
|
|
18 <http://www.gnu.org/licenses/>. */
|
|
19
|
|
20 #include "config.h"
|
|
21 #include "system.h"
|
|
22 #include "coretypes.h"
|
|
23 #include "backend.h"
|
|
24 #include "target.h"
|
|
25 #include "rtl.h"
|
|
26 #include "tree.h"
|
|
27 #include "gimple.h"
|
|
28 #include "predict.h"
|
|
29 #include "stringpool.h"
|
|
30 #include "tree-vrp.h"
|
|
31 #include "tree-ssanames.h"
|
|
32 #include "expmed.h"
|
|
33 #include "memmodel.h"
|
|
34 #include "optabs.h"
|
|
35 #include "emit-rtl.h"
|
|
36 #include "diagnostic-core.h"
|
|
37 #include "fold-const.h"
|
|
38 #include "internal-fn.h"
|
|
39 #include "stor-layout.h"
|
|
40 #include "dojump.h"
|
|
41 #include "expr.h"
|
|
42 #include "stringpool.h"
|
|
43 #include "attribs.h"
|
|
44 #include "asan.h"
|
|
45 #include "ubsan.h"
|
|
46 #include "recog.h"
|
|
47 #include "builtins.h"
|
|
48 #include "optabs-tree.h"
|
131
|
49 #include "gimple-ssa.h"
|
|
50 #include "tree-phinodes.h"
|
|
51 #include "ssa-iterators.h"
|
111
|
52
|
|
53 /* The names of each internal function, indexed by function number. */
|
|
54 const char *const internal_fn_name_array[] = {
|
|
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
|
|
56 #include "internal-fn.def"
|
|
57 "<invalid-fn>"
|
|
58 };
|
|
59
|
|
60 /* The ECF_* flags of each internal function, indexed by function number. */
|
|
61 const int internal_fn_flags_array[] = {
|
|
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
|
|
63 #include "internal-fn.def"
|
|
64 0
|
|
65 };
|
|
66
|
131
|
67 /* Return the internal function called NAME, or IFN_LAST if there's
|
|
68 no such function. */
|
|
69
|
|
70 internal_fn
|
|
71 lookup_internal_fn (const char *name)
|
|
72 {
|
|
73 typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
|
|
74 static name_to_fn_map_type *name_to_fn_map;
|
|
75
|
|
76 if (!name_to_fn_map)
|
|
77 {
|
|
78 name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
|
|
79 for (unsigned int i = 0; i < IFN_LAST; ++i)
|
|
80 name_to_fn_map->put (internal_fn_name (internal_fn (i)),
|
|
81 internal_fn (i));
|
|
82 }
|
|
83 internal_fn *entry = name_to_fn_map->get (name);
|
|
84 return entry ? *entry : IFN_LAST;
|
|
85 }
|
|
86
|
111
|
87 /* Fnspec of each internal function, indexed by function number. */
|
|
88 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
|
|
89
|
|
90 void
|
|
91 init_internal_fns ()
|
|
92 {
|
|
93 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
94 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
|
|
95 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
|
|
96 #include "internal-fn.def"
|
|
97 internal_fn_fnspec_array[IFN_LAST] = 0;
|
|
98 }
|
|
99
|
|
100 /* Create static initializers for the information returned by
|
|
101 direct_internal_fn. */
|
|
102 #define not_direct { -2, -2, false }
|
|
103 #define mask_load_direct { -1, 2, false }
|
|
104 #define load_lanes_direct { -1, -1, false }
|
131
|
105 #define mask_load_lanes_direct { -1, -1, false }
|
145
|
106 #define gather_load_direct { 3, 1, false }
|
111
|
107 #define mask_store_direct { 3, 2, false }
|
|
108 #define store_lanes_direct { 0, 0, false }
|
131
|
109 #define mask_store_lanes_direct { 0, 0, false }
|
145
|
110 #define scatter_store_direct { 3, 1, false }
|
111
|
111 #define unary_direct { 0, 0, true }
|
|
112 #define binary_direct { 0, 0, true }
|
131
|
113 #define ternary_direct { 0, 0, true }
|
|
114 #define cond_unary_direct { 1, 1, true }
|
|
115 #define cond_binary_direct { 1, 1, true }
|
|
116 #define cond_ternary_direct { 1, 1, true }
|
|
117 #define while_direct { 0, 2, false }
|
|
118 #define fold_extract_direct { 2, 2, false }
|
|
119 #define fold_left_direct { 1, 1, false }
|
145
|
120 #define mask_fold_left_direct { 1, 1, false }
|
|
121 #define check_ptrs_direct { 0, 0, false }
|
111
|
122
|
|
123 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
|
|
124 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
|
|
125 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
|
131
|
126 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
127 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
|
111
|
128 #include "internal-fn.def"
|
|
129 not_direct
|
|
130 };
|
|
131
|
|
132 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
|
|
133 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
|
|
134
|
|
135 static enum insn_code
|
|
136 get_multi_vector_move (tree array_type, convert_optab optab)
|
|
137 {
|
|
138 machine_mode imode;
|
|
139 machine_mode vmode;
|
|
140
|
|
141 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
|
|
142 imode = TYPE_MODE (array_type);
|
|
143 vmode = TYPE_MODE (TREE_TYPE (array_type));
|
|
144
|
|
145 return convert_optab_handler (optab, imode, vmode);
|
|
146 }
|
|
147
|
|
148 /* Expand LOAD_LANES call STMT using optab OPTAB. */
|
|
149
|
|
150 static void
|
|
151 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
152 {
|
145
|
153 class expand_operand ops[2];
|
111
|
154 tree type, lhs, rhs;
|
|
155 rtx target, mem;
|
|
156
|
|
157 lhs = gimple_call_lhs (stmt);
|
|
158 rhs = gimple_call_arg (stmt, 0);
|
|
159 type = TREE_TYPE (lhs);
|
|
160
|
|
161 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
162 mem = expand_normal (rhs);
|
|
163
|
|
164 gcc_assert (MEM_P (mem));
|
|
165 PUT_MODE (mem, TYPE_MODE (type));
|
|
166
|
|
167 create_output_operand (&ops[0], target, TYPE_MODE (type));
|
|
168 create_fixed_operand (&ops[1], mem);
|
|
169 expand_insn (get_multi_vector_move (type, optab), 2, ops);
|
|
170 }
|
|
171
|
|
172 /* Expand STORE_LANES call STMT using optab OPTAB. */
|
|
173
|
|
174 static void
|
|
175 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
176 {
|
145
|
177 class expand_operand ops[2];
|
111
|
178 tree type, lhs, rhs;
|
|
179 rtx target, reg;
|
|
180
|
|
181 lhs = gimple_call_lhs (stmt);
|
|
182 rhs = gimple_call_arg (stmt, 0);
|
|
183 type = TREE_TYPE (rhs);
|
|
184
|
|
185 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
186 reg = expand_normal (rhs);
|
|
187
|
|
188 gcc_assert (MEM_P (target));
|
|
189 PUT_MODE (target, TYPE_MODE (type));
|
|
190
|
|
191 create_fixed_operand (&ops[0], target);
|
|
192 create_input_operand (&ops[1], reg, TYPE_MODE (type));
|
|
193 expand_insn (get_multi_vector_move (type, optab), 2, ops);
|
|
194 }
|
|
195
|
|
196 static void
|
|
197 expand_ANNOTATE (internal_fn, gcall *)
|
|
198 {
|
|
199 gcc_unreachable ();
|
|
200 }
|
|
201
|
|
202 /* This should get expanded in omp_device_lower pass. */
|
|
203
|
|
204 static void
|
|
205 expand_GOMP_USE_SIMT (internal_fn, gcall *)
|
|
206 {
|
|
207 gcc_unreachable ();
|
|
208 }
|
|
209
|
|
210 /* This should get expanded in omp_device_lower pass. */
|
|
211
|
|
212 static void
|
|
213 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
|
|
214 {
|
|
215 gcc_unreachable ();
|
|
216 }
|
|
217
|
|
218 /* Allocate per-lane storage and begin non-uniform execution region. */
|
|
219
|
|
220 static void
|
|
221 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
|
|
222 {
|
|
223 rtx target;
|
|
224 tree lhs = gimple_call_lhs (stmt);
|
|
225 if (lhs)
|
|
226 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
227 else
|
|
228 target = gen_reg_rtx (Pmode);
|
|
229 rtx size = expand_normal (gimple_call_arg (stmt, 0));
|
|
230 rtx align = expand_normal (gimple_call_arg (stmt, 1));
|
145
|
231 class expand_operand ops[3];
|
111
|
232 create_output_operand (&ops[0], target, Pmode);
|
|
233 create_input_operand (&ops[1], size, Pmode);
|
|
234 create_input_operand (&ops[2], align, Pmode);
|
|
235 gcc_assert (targetm.have_omp_simt_enter ());
|
|
236 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
|
|
237 }
|
|
238
|
|
239 /* Deallocate per-lane storage and leave non-uniform execution region. */
|
|
240
|
|
241 static void
|
|
242 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
|
|
243 {
|
|
244 gcc_checking_assert (!gimple_call_lhs (stmt));
|
|
245 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
|
145
|
246 class expand_operand ops[1];
|
111
|
247 create_input_operand (&ops[0], arg, Pmode);
|
|
248 gcc_assert (targetm.have_omp_simt_exit ());
|
|
249 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
|
|
250 }
|
|
251
|
|
252 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
|
|
253 without SIMT execution this should be expanded in omp_device_lower pass. */
|
|
254
|
|
255 static void
|
|
256 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
|
|
257 {
|
|
258 tree lhs = gimple_call_lhs (stmt);
|
|
259 if (!lhs)
|
|
260 return;
|
|
261
|
|
262 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
263 gcc_assert (targetm.have_omp_simt_lane ());
|
|
264 emit_insn (targetm.gen_omp_simt_lane (target));
|
|
265 }
|
|
266
|
|
267 /* This should get expanded in omp_device_lower pass. */
|
|
268
|
|
269 static void
|
|
270 expand_GOMP_SIMT_VF (internal_fn, gcall *)
|
|
271 {
|
|
272 gcc_unreachable ();
|
|
273 }
|
|
274
|
|
275 /* Lane index of the first SIMT lane that supplies a non-zero argument.
|
|
276 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
|
|
277 lane that executed the last iteration for handling OpenMP lastprivate. */
|
|
278
|
|
279 static void
|
|
280 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
|
|
281 {
|
|
282 tree lhs = gimple_call_lhs (stmt);
|
|
283 if (!lhs)
|
|
284 return;
|
|
285
|
|
286 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
287 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
|
|
288 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
145
|
289 class expand_operand ops[2];
|
111
|
290 create_output_operand (&ops[0], target, mode);
|
|
291 create_input_operand (&ops[1], cond, mode);
|
|
292 gcc_assert (targetm.have_omp_simt_last_lane ());
|
|
293 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
|
|
294 }
|
|
295
|
|
296 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
|
|
297
|
|
298 static void
|
|
299 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
|
|
300 {
|
|
301 tree lhs = gimple_call_lhs (stmt);
|
|
302 if (!lhs)
|
|
303 return;
|
|
304
|
|
305 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
306 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
|
|
307 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
145
|
308 class expand_operand ops[2];
|
111
|
309 create_output_operand (&ops[0], target, mode);
|
|
310 create_input_operand (&ops[1], ctr, mode);
|
|
311 gcc_assert (targetm.have_omp_simt_ordered ());
|
|
312 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
|
|
313 }
|
|
314
|
|
315 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
|
|
316 any lane supplies a non-zero argument. */
|
|
317
|
|
318 static void
|
|
319 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
|
|
320 {
|
|
321 tree lhs = gimple_call_lhs (stmt);
|
|
322 if (!lhs)
|
|
323 return;
|
|
324
|
|
325 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
326 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
|
|
327 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
145
|
328 class expand_operand ops[2];
|
111
|
329 create_output_operand (&ops[0], target, mode);
|
|
330 create_input_operand (&ops[1], cond, mode);
|
|
331 gcc_assert (targetm.have_omp_simt_vote_any ());
|
|
332 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
|
|
333 }
|
|
334
|
|
335 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
|
|
336 is destination lane index XOR given offset. */
|
|
337
|
|
338 static void
|
|
339 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
|
|
340 {
|
|
341 tree lhs = gimple_call_lhs (stmt);
|
|
342 if (!lhs)
|
|
343 return;
|
|
344
|
|
345 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
346 rtx src = expand_normal (gimple_call_arg (stmt, 0));
|
|
347 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
|
|
348 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
145
|
349 class expand_operand ops[3];
|
111
|
350 create_output_operand (&ops[0], target, mode);
|
|
351 create_input_operand (&ops[1], src, mode);
|
|
352 create_input_operand (&ops[2], idx, SImode);
|
|
353 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
|
|
354 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
|
|
355 }
|
|
356
|
|
357 /* Exchange between SIMT lanes according to given source lane index. */
|
|
358
|
|
359 static void
|
|
360 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
|
|
361 {
|
|
362 tree lhs = gimple_call_lhs (stmt);
|
|
363 if (!lhs)
|
|
364 return;
|
|
365
|
|
366 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
367 rtx src = expand_normal (gimple_call_arg (stmt, 0));
|
|
368 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
|
|
369 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
145
|
370 class expand_operand ops[3];
|
111
|
371 create_output_operand (&ops[0], target, mode);
|
|
372 create_input_operand (&ops[1], src, mode);
|
|
373 create_input_operand (&ops[2], idx, SImode);
|
|
374 gcc_assert (targetm.have_omp_simt_xchg_idx ());
|
|
375 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
|
|
376 }
|
|
377
|
|
378 /* This should get expanded in adjust_simduid_builtins. */
|
|
379
|
|
380 static void
|
|
381 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
|
|
382 {
|
|
383 gcc_unreachable ();
|
|
384 }
|
|
385
|
|
386 /* This should get expanded in adjust_simduid_builtins. */
|
|
387
|
|
388 static void
|
|
389 expand_GOMP_SIMD_VF (internal_fn, gcall *)
|
|
390 {
|
|
391 gcc_unreachable ();
|
|
392 }
|
|
393
|
|
394 /* This should get expanded in adjust_simduid_builtins. */
|
|
395
|
|
396 static void
|
|
397 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
|
|
398 {
|
|
399 gcc_unreachable ();
|
|
400 }
|
|
401
|
|
402 /* This should get expanded in adjust_simduid_builtins. */
|
|
403
|
|
404 static void
|
|
405 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
|
|
406 {
|
|
407 gcc_unreachable ();
|
|
408 }
|
|
409
|
|
410 /* This should get expanded in adjust_simduid_builtins. */
|
|
411
|
|
412 static void
|
|
413 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
|
|
414 {
|
|
415 gcc_unreachable ();
|
|
416 }
|
|
417
|
|
418 /* This should get expanded in the sanopt pass. */
|
|
419
|
|
420 static void
|
|
421 expand_UBSAN_NULL (internal_fn, gcall *)
|
|
422 {
|
|
423 gcc_unreachable ();
|
|
424 }
|
|
425
|
|
426 /* This should get expanded in the sanopt pass. */
|
|
427
|
|
428 static void
|
|
429 expand_UBSAN_BOUNDS (internal_fn, gcall *)
|
|
430 {
|
|
431 gcc_unreachable ();
|
|
432 }
|
|
433
|
|
434 /* This should get expanded in the sanopt pass. */
|
|
435
|
|
436 static void
|
|
437 expand_UBSAN_VPTR (internal_fn, gcall *)
|
|
438 {
|
|
439 gcc_unreachable ();
|
|
440 }
|
|
441
|
|
442 /* This should get expanded in the sanopt pass. */
|
|
443
|
|
444 static void
|
|
445 expand_UBSAN_PTR (internal_fn, gcall *)
|
|
446 {
|
|
447 gcc_unreachable ();
|
|
448 }
|
|
449
|
|
450 /* This should get expanded in the sanopt pass. */
|
|
451
|
|
452 static void
|
|
453 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
|
|
454 {
|
|
455 gcc_unreachable ();
|
|
456 }
|
|
457
|
|
458 /* This should get expanded in the sanopt pass. */
|
|
459
|
|
460 static void
|
|
461 expand_ASAN_CHECK (internal_fn, gcall *)
|
|
462 {
|
|
463 gcc_unreachable ();
|
|
464 }
|
|
465
|
|
466 /* This should get expanded in the sanopt pass. */
|
|
467
|
|
468 static void
|
|
469 expand_ASAN_MARK (internal_fn, gcall *)
|
|
470 {
|
|
471 gcc_unreachable ();
|
|
472 }
|
|
473
|
|
474 /* This should get expanded in the sanopt pass. */
|
|
475
|
|
476 static void
|
|
477 expand_ASAN_POISON (internal_fn, gcall *)
|
|
478 {
|
|
479 gcc_unreachable ();
|
|
480 }
|
|
481
|
|
482 /* This should get expanded in the sanopt pass. */
|
|
483
|
|
484 static void
|
|
485 expand_ASAN_POISON_USE (internal_fn, gcall *)
|
|
486 {
|
|
487 gcc_unreachable ();
|
|
488 }
|
|
489
|
|
490 /* This should get expanded in the tsan pass. */
|
|
491
|
|
492 static void
|
|
493 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
|
|
494 {
|
|
495 gcc_unreachable ();
|
|
496 }
|
|
497
|
|
498 /* This should get expanded in the lower pass. */
|
|
499
|
|
500 static void
|
|
501 expand_FALLTHROUGH (internal_fn, gcall *call)
|
|
502 {
|
|
503 error_at (gimple_location (call),
|
|
504 "invalid use of attribute %<fallthrough%>");
|
|
505 }
|
|
506
|
|
507 /* Return minimum precision needed to represent all values
|
|
508 of ARG in SIGNed integral type. */
|
|
509
|
|
510 static int
|
|
511 get_min_precision (tree arg, signop sign)
|
|
512 {
|
|
513 int prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
514 int cnt = 0;
|
|
515 signop orig_sign = sign;
|
|
516 if (TREE_CODE (arg) == INTEGER_CST)
|
|
517 {
|
|
518 int p;
|
|
519 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
|
|
520 {
|
|
521 widest_int w = wi::to_widest (arg);
|
|
522 w = wi::ext (w, prec, sign);
|
|
523 p = wi::min_precision (w, sign);
|
|
524 }
|
|
525 else
|
|
526 p = wi::min_precision (wi::to_wide (arg), sign);
|
|
527 return MIN (p, prec);
|
|
528 }
|
|
529 while (CONVERT_EXPR_P (arg)
|
|
530 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
|
|
531 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
|
|
532 {
|
|
533 arg = TREE_OPERAND (arg, 0);
|
|
534 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
|
|
535 {
|
|
536 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
|
|
537 sign = UNSIGNED;
|
|
538 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
|
|
539 return prec + (orig_sign != sign);
|
|
540 prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
541 }
|
|
542 if (++cnt > 30)
|
|
543 return prec + (orig_sign != sign);
|
|
544 }
|
|
545 if (TREE_CODE (arg) != SSA_NAME)
|
|
546 return prec + (orig_sign != sign);
|
|
547 wide_int arg_min, arg_max;
|
|
548 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
|
|
549 {
|
|
550 gimple *g = SSA_NAME_DEF_STMT (arg);
|
|
551 if (is_gimple_assign (g)
|
|
552 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
|
|
553 {
|
|
554 tree t = gimple_assign_rhs1 (g);
|
|
555 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
|
|
556 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
|
|
557 {
|
|
558 arg = t;
|
|
559 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
|
|
560 {
|
|
561 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
|
|
562 sign = UNSIGNED;
|
|
563 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
|
|
564 return prec + (orig_sign != sign);
|
|
565 prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
566 }
|
|
567 if (++cnt > 30)
|
|
568 return prec + (orig_sign != sign);
|
|
569 continue;
|
|
570 }
|
|
571 }
|
|
572 return prec + (orig_sign != sign);
|
|
573 }
|
|
574 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
|
|
575 {
|
|
576 int p1 = wi::min_precision (arg_min, sign);
|
|
577 int p2 = wi::min_precision (arg_max, sign);
|
|
578 p1 = MAX (p1, p2);
|
|
579 prec = MIN (prec, p1);
|
|
580 }
|
|
581 else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
|
|
582 {
|
|
583 int p = wi::min_precision (arg_max, UNSIGNED);
|
|
584 prec = MIN (prec, p);
|
|
585 }
|
|
586 return prec + (orig_sign != sign);
|
|
587 }
|
|
588
|
|
589 /* Helper for expand_*_overflow. Set the __imag__ part to true
|
|
590 (1 except for signed:1 type, in which case store -1). */
|
|
591
|
|
592 static void
|
|
593 expand_arith_set_overflow (tree lhs, rtx target)
|
|
594 {
|
|
595 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
|
|
596 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
|
|
597 write_complex_part (target, constm1_rtx, true);
|
|
598 else
|
|
599 write_complex_part (target, const1_rtx, true);
|
|
600 }
|
|
601
|
|
602 /* Helper for expand_*_overflow. Store RES into the __real__ part
|
|
603 of TARGET. If RES has larger MODE than __real__ part of TARGET,
|
|
604 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
|
|
605 if LHS has smaller precision than its mode. */
|
|
606
|
|
607 static void
|
|
608 expand_arith_overflow_result_store (tree lhs, rtx target,
|
|
609 scalar_int_mode mode, rtx res)
|
|
610 {
|
|
611 scalar_int_mode tgtmode
|
|
612 = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
|
|
613 rtx lres = res;
|
|
614 if (tgtmode != mode)
|
|
615 {
|
|
616 rtx_code_label *done_label = gen_label_rtx ();
|
|
617 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
|
|
618 lres = convert_modes (tgtmode, mode, res, uns);
|
|
619 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
|
|
620 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
|
|
621 EQ, true, mode, NULL_RTX, NULL, done_label,
|
|
622 profile_probability::very_likely ());
|
|
623 expand_arith_set_overflow (lhs, target);
|
|
624 emit_label (done_label);
|
|
625 }
|
|
626 int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
|
|
627 int tgtprec = GET_MODE_PRECISION (tgtmode);
|
|
628 if (prec < tgtprec)
|
|
629 {
|
|
630 rtx_code_label *done_label = gen_label_rtx ();
|
|
631 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
|
|
632 res = lres;
|
|
633 if (uns)
|
|
634 {
|
|
635 rtx mask
|
|
636 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
|
|
637 tgtmode);
|
|
638 lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
|
|
639 true, OPTAB_LIB_WIDEN);
|
|
640 }
|
|
641 else
|
|
642 {
|
|
643 lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
|
|
644 NULL_RTX, 1);
|
|
645 lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
|
|
646 NULL_RTX, 0);
|
|
647 }
|
|
648 do_compare_rtx_and_jump (res, lres,
|
|
649 EQ, true, tgtmode, NULL_RTX, NULL, done_label,
|
|
650 profile_probability::very_likely ());
|
|
651 expand_arith_set_overflow (lhs, target);
|
|
652 emit_label (done_label);
|
|
653 }
|
|
654 write_complex_part (target, lres, false);
|
|
655 }
|
|
656
|
|
657 /* Helper for expand_*_overflow. Store RES into TARGET. */
|
|
658
|
|
659 static void
|
|
660 expand_ubsan_result_store (rtx target, rtx res)
|
|
661 {
|
|
662 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
|
|
663 /* If this is a scalar in a register that is stored in a wider mode
|
|
664 than the declared mode, compute the result into its declared mode
|
|
665 and then convert to the wider mode. Our value is the computed
|
|
666 expression. */
|
|
667 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
|
|
668 else
|
|
669 emit_move_insn (target, res);
|
|
670 }
|
|
671
|
|
672 /* Add sub/add overflow checking to the statement STMT.
|
|
673 CODE says whether the operation is +, or -. */
|
|
674
|
|
675 static void
|
|
676 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
|
|
677 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
|
|
678 bool uns1_p, bool is_ubsan, tree *datap)
|
|
679 {
|
|
680 rtx res, target = NULL_RTX;
|
|
681 tree fn;
|
|
682 rtx_code_label *done_label = gen_label_rtx ();
|
|
683 rtx_code_label *do_error = gen_label_rtx ();
|
|
684 do_pending_stack_adjust ();
|
|
685 rtx op0 = expand_normal (arg0);
|
|
686 rtx op1 = expand_normal (arg1);
|
|
687 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
|
|
688 int prec = GET_MODE_PRECISION (mode);
|
|
689 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
|
|
690 bool do_xor = false;
|
|
691
|
|
692 if (is_ubsan)
|
|
693 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
|
|
694
|
|
695 if (lhs)
|
|
696 {
|
|
697 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
698 if (!is_ubsan)
|
|
699 write_complex_part (target, const0_rtx, true);
|
|
700 }
|
|
701
|
|
702 /* We assume both operands and result have the same precision
|
|
703 here (GET_MODE_BITSIZE (mode)), S stands for signed type
|
|
704 with that precision, U for unsigned type with that precision,
|
|
705 sgn for unsigned most significant bit in that precision.
|
|
706 s1 is signed first operand, u1 is unsigned first operand,
|
|
707 s2 is signed second operand, u2 is unsigned second operand,
|
|
708 sr is signed result, ur is unsigned result and the following
|
|
709 rules say how to compute result (which is always result of
|
|
710 the operands as if both were unsigned, cast to the right
|
|
711 signedness) and how to compute whether operation overflowed.
|
|
712
|
|
713 s1 + s2 -> sr
|
|
714 res = (S) ((U) s1 + (U) s2)
|
|
715 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
|
|
716 s1 - s2 -> sr
|
|
717 res = (S) ((U) s1 - (U) s2)
|
|
718 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
|
|
719 u1 + u2 -> ur
|
|
720 res = u1 + u2
|
|
721 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
|
|
722 u1 - u2 -> ur
|
|
723 res = u1 - u2
|
|
724 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
|
|
725 s1 + u2 -> sr
|
|
726 res = (S) ((U) s1 + u2)
|
|
727 ovf = ((U) res ^ sgn) < u2
|
|
728 s1 + u2 -> ur
|
|
729 t1 = (S) (u2 ^ sgn)
|
|
730 t2 = s1 + t1
|
|
731 res = (U) t2 ^ sgn
|
|
732 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
|
|
733 s1 - u2 -> sr
|
|
734 res = (S) ((U) s1 - u2)
|
|
735 ovf = u2 > ((U) s1 ^ sgn)
|
|
736 s1 - u2 -> ur
|
|
737 res = (U) s1 - u2
|
|
738 ovf = s1 < 0 || u2 > (U) s1
|
|
739 u1 - s2 -> sr
|
|
740 res = u1 - (U) s2
|
|
741 ovf = u1 >= ((U) s2 ^ sgn)
|
|
742 u1 - s2 -> ur
|
|
743 t1 = u1 ^ sgn
|
|
744 t2 = t1 - (U) s2
|
|
745 res = t2 ^ sgn
|
|
746 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
|
|
747 s1 + s2 -> ur
|
|
748 res = (U) s1 + (U) s2
|
|
749 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
|
|
750 u1 + u2 -> sr
|
|
751 res = (S) (u1 + u2)
|
|
752 ovf = (U) res < u2 || res < 0
|
|
753 u1 - u2 -> sr
|
|
754 res = (S) (u1 - u2)
|
|
755 ovf = u1 >= u2 ? res < 0 : res >= 0
|
|
756 s1 - s2 -> ur
|
|
757 res = (U) s1 - (U) s2
|
|
758 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
|
|
759
|
|
760 if (code == PLUS_EXPR && uns0_p && !uns1_p)
|
|
761 {
|
|
762 /* PLUS_EXPR is commutative, if operand signedness differs,
|
|
763 canonicalize to the first operand being signed and second
|
|
764 unsigned to simplify following code. */
|
|
765 std::swap (op0, op1);
|
|
766 std::swap (arg0, arg1);
|
|
767 uns0_p = false;
|
|
768 uns1_p = true;
|
|
769 }
|
|
770
|
|
771 /* u1 +- u2 -> ur */
|
|
772 if (uns0_p && uns1_p && unsr_p)
|
|
773 {
|
|
774 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
|
|
775 : usubv4_optab, mode);
|
|
776 if (icode != CODE_FOR_nothing)
|
|
777 {
|
145
|
778 class expand_operand ops[4];
|
111
|
779 rtx_insn *last = get_last_insn ();
|
|
780
|
|
781 res = gen_reg_rtx (mode);
|
|
782 create_output_operand (&ops[0], res, mode);
|
|
783 create_input_operand (&ops[1], op0, mode);
|
|
784 create_input_operand (&ops[2], op1, mode);
|
|
785 create_fixed_operand (&ops[3], do_error);
|
|
786 if (maybe_expand_insn (icode, 4, ops))
|
|
787 {
|
|
788 last = get_last_insn ();
|
|
789 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
790 && JUMP_P (last)
|
|
791 && any_condjump_p (last)
|
|
792 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
793 add_reg_br_prob_note (last,
|
|
794 profile_probability::very_unlikely ());
|
|
795 emit_jump (done_label);
|
|
796 goto do_error_label;
|
|
797 }
|
|
798
|
|
799 delete_insns_since (last);
|
|
800 }
|
|
801
|
|
802 /* Compute the operation. On RTL level, the addition is always
|
|
803 unsigned. */
|
|
804 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
805 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
806 rtx tem = op0;
|
|
807 /* For PLUS_EXPR, the operation is commutative, so we can pick
|
|
808 operand to compare against. For prec <= BITS_PER_WORD, I think
|
|
809 preferring REG operand is better over CONST_INT, because
|
|
810 the CONST_INT might enlarge the instruction or CSE would need
|
|
811 to figure out we'd already loaded it into a register before.
|
|
812 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
|
|
813 as then the multi-word comparison can be perhaps simplified. */
|
|
814 if (code == PLUS_EXPR
|
|
815 && (prec <= BITS_PER_WORD
|
|
816 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
|
|
817 : CONST_SCALAR_INT_P (op1)))
|
|
818 tem = op1;
|
|
819 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
|
|
820 true, mode, NULL_RTX, NULL, done_label,
|
|
821 profile_probability::very_likely ());
|
|
822 goto do_error_label;
|
|
823 }
|
|
824
|
|
825 /* s1 +- u2 -> sr */
|
|
826 if (!uns0_p && uns1_p && !unsr_p)
|
|
827 {
|
|
828 /* Compute the operation. On RTL level, the addition is always
|
|
829 unsigned. */
|
|
830 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
831 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
832 rtx tem = expand_binop (mode, add_optab,
|
|
833 code == PLUS_EXPR ? res : op0, sgn,
|
|
834 NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
835 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
|
|
836 done_label, profile_probability::very_likely ());
|
|
837 goto do_error_label;
|
|
838 }
|
|
839
|
|
840 /* s1 + u2 -> ur */
|
|
841 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
|
|
842 {
|
|
843 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
|
|
844 OPTAB_LIB_WIDEN);
|
|
845 /* As we've changed op1, we have to avoid using the value range
|
|
846 for the original argument. */
|
|
847 arg1 = error_mark_node;
|
|
848 do_xor = true;
|
|
849 goto do_signed;
|
|
850 }
|
|
851
|
|
852 /* u1 - s2 -> ur */
|
|
853 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
|
|
854 {
|
|
855 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
|
|
856 OPTAB_LIB_WIDEN);
|
|
857 /* As we've changed op0, we have to avoid using the value range
|
|
858 for the original argument. */
|
|
859 arg0 = error_mark_node;
|
|
860 do_xor = true;
|
|
861 goto do_signed;
|
|
862 }
|
|
863
|
|
864 /* s1 - u2 -> ur */
|
|
865 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
|
|
866 {
|
|
867 /* Compute the operation. On RTL level, the addition is always
|
|
868 unsigned. */
|
|
869 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
870 OPTAB_LIB_WIDEN);
|
|
871 int pos_neg = get_range_pos_neg (arg0);
|
|
872 if (pos_neg == 2)
|
|
873 /* If ARG0 is known to be always negative, this is always overflow. */
|
|
874 emit_jump (do_error);
|
|
875 else if (pos_neg == 3)
|
|
876 /* If ARG0 is not known to be always positive, check at runtime. */
|
|
877 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
|
|
878 NULL, do_error, profile_probability::very_unlikely ());
|
|
879 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
|
|
880 done_label, profile_probability::very_likely ());
|
|
881 goto do_error_label;
|
|
882 }
|
|
883
|
|
884 /* u1 - s2 -> sr */
|
|
885 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
|
|
886 {
|
|
887 /* Compute the operation. On RTL level, the addition is always
|
|
888 unsigned. */
|
|
889 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
890 OPTAB_LIB_WIDEN);
|
|
891 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
|
|
892 OPTAB_LIB_WIDEN);
|
|
893 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
|
|
894 done_label, profile_probability::very_likely ());
|
|
895 goto do_error_label;
|
|
896 }
|
|
897
|
|
898 /* u1 + u2 -> sr */
|
|
899 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
|
|
900 {
|
|
901 /* Compute the operation. On RTL level, the addition is always
|
|
902 unsigned. */
|
|
903 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
|
|
904 OPTAB_LIB_WIDEN);
|
|
905 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
|
|
906 NULL, do_error, profile_probability::very_unlikely ());
|
|
907 rtx tem = op1;
|
|
908 /* The operation is commutative, so we can pick operand to compare
|
|
909 against. For prec <= BITS_PER_WORD, I think preferring REG operand
|
|
910 is better over CONST_INT, because the CONST_INT might enlarge the
|
|
911 instruction or CSE would need to figure out we'd already loaded it
|
|
912 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
|
|
913 might be more beneficial, as then the multi-word comparison can be
|
|
914 perhaps simplified. */
|
|
915 if (prec <= BITS_PER_WORD
|
|
916 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
|
|
917 : CONST_SCALAR_INT_P (op0))
|
|
918 tem = op0;
|
|
919 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
|
|
920 done_label, profile_probability::very_likely ());
|
|
921 goto do_error_label;
|
|
922 }
|
|
923
|
|
924 /* s1 +- s2 -> ur */
|
|
925 if (!uns0_p && !uns1_p && unsr_p)
|
|
926 {
|
|
927 /* Compute the operation. On RTL level, the addition is always
|
|
928 unsigned. */
|
|
929 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
930 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
931 int pos_neg = get_range_pos_neg (arg1);
|
|
932 if (code == PLUS_EXPR)
|
|
933 {
|
|
934 int pos_neg0 = get_range_pos_neg (arg0);
|
|
935 if (pos_neg0 != 3 && pos_neg == 3)
|
|
936 {
|
|
937 std::swap (op0, op1);
|
|
938 pos_neg = pos_neg0;
|
|
939 }
|
|
940 }
|
|
941 rtx tem;
|
|
942 if (pos_neg != 3)
|
|
943 {
|
|
944 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
|
|
945 ? and_optab : ior_optab,
|
|
946 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
947 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
|
|
948 NULL, done_label, profile_probability::very_likely ());
|
|
949 }
|
|
950 else
|
|
951 {
|
|
952 rtx_code_label *do_ior_label = gen_label_rtx ();
|
|
953 do_compare_rtx_and_jump (op1, const0_rtx,
|
|
954 code == MINUS_EXPR ? GE : LT, false, mode,
|
|
955 NULL_RTX, NULL, do_ior_label,
|
|
956 profile_probability::even ());
|
|
957 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
|
|
958 OPTAB_LIB_WIDEN);
|
|
959 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
960 NULL, done_label, profile_probability::very_likely ());
|
|
961 emit_jump (do_error);
|
|
962 emit_label (do_ior_label);
|
|
963 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
|
|
964 OPTAB_LIB_WIDEN);
|
|
965 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
966 NULL, done_label, profile_probability::very_likely ());
|
|
967 }
|
|
968 goto do_error_label;
|
|
969 }
|
|
970
|
|
971 /* u1 - u2 -> sr */
|
|
972 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
|
|
973 {
|
|
974 /* Compute the operation. On RTL level, the addition is always
|
|
975 unsigned. */
|
|
976 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
977 OPTAB_LIB_WIDEN);
|
|
978 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
|
|
979 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
|
|
980 op0_geu_op1, profile_probability::even ());
|
|
981 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
|
|
982 NULL, done_label, profile_probability::very_likely ());
|
|
983 emit_jump (do_error);
|
|
984 emit_label (op0_geu_op1);
|
|
985 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
|
|
986 NULL, done_label, profile_probability::very_likely ());
|
|
987 goto do_error_label;
|
|
988 }
|
|
989
|
|
990 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
|
|
991
|
|
992 /* s1 +- s2 -> sr */
|
|
993 do_signed:
|
|
994 {
|
|
995 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
|
|
996 : subv4_optab, mode);
|
|
997 if (icode != CODE_FOR_nothing)
|
|
998 {
|
145
|
999 class expand_operand ops[4];
|
111
|
1000 rtx_insn *last = get_last_insn ();
|
|
1001
|
|
1002 res = gen_reg_rtx (mode);
|
|
1003 create_output_operand (&ops[0], res, mode);
|
|
1004 create_input_operand (&ops[1], op0, mode);
|
|
1005 create_input_operand (&ops[2], op1, mode);
|
|
1006 create_fixed_operand (&ops[3], do_error);
|
|
1007 if (maybe_expand_insn (icode, 4, ops))
|
|
1008 {
|
|
1009 last = get_last_insn ();
|
|
1010 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
1011 && JUMP_P (last)
|
|
1012 && any_condjump_p (last)
|
|
1013 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
1014 add_reg_br_prob_note (last,
|
|
1015 profile_probability::very_unlikely ());
|
|
1016 emit_jump (done_label);
|
|
1017 goto do_error_label;
|
|
1018 }
|
|
1019
|
|
1020 delete_insns_since (last);
|
|
1021 }
|
|
1022
|
|
1023 /* Compute the operation. On RTL level, the addition is always
|
|
1024 unsigned. */
|
|
1025 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
1026 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
1027
|
|
1028 /* If we can prove that one of the arguments (for MINUS_EXPR only
|
|
1029 the second operand, as subtraction is not commutative) is always
|
|
1030 non-negative or always negative, we can do just one comparison
|
|
1031 and conditional jump. */
|
|
1032 int pos_neg = get_range_pos_neg (arg1);
|
|
1033 if (code == PLUS_EXPR)
|
|
1034 {
|
|
1035 int pos_neg0 = get_range_pos_neg (arg0);
|
|
1036 if (pos_neg0 != 3 && pos_neg == 3)
|
|
1037 {
|
|
1038 std::swap (op0, op1);
|
|
1039 pos_neg = pos_neg0;
|
|
1040 }
|
|
1041 }
|
|
1042
|
|
1043 /* Addition overflows if and only if the two operands have the same sign,
|
|
1044 and the result has the opposite sign. Subtraction overflows if and
|
|
1045 only if the two operands have opposite sign, and the subtrahend has
|
|
1046 the same sign as the result. Here 0 is counted as positive. */
|
|
1047 if (pos_neg == 3)
|
|
1048 {
|
|
1049 /* Compute op0 ^ op1 (operands have opposite sign). */
|
|
1050 rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
|
|
1051 OPTAB_LIB_WIDEN);
|
|
1052
|
|
1053 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
|
|
1054 rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
|
|
1055 OPTAB_LIB_WIDEN);
|
|
1056
|
|
1057 rtx tem;
|
|
1058 if (code == PLUS_EXPR)
|
|
1059 {
|
|
1060 /* Compute (res ^ op1) & ~(op0 ^ op1). */
|
|
1061 tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
|
|
1062 tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
|
|
1063 OPTAB_LIB_WIDEN);
|
|
1064 }
|
|
1065 else
|
|
1066 {
|
|
1067 /* Compute (op0 ^ op1) & ~(res ^ op1). */
|
|
1068 tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
|
|
1069 tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
|
|
1070 OPTAB_LIB_WIDEN);
|
|
1071 }
|
|
1072
|
|
1073 /* No overflow if the result has bit sign cleared. */
|
|
1074 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
1075 NULL, done_label, profile_probability::very_likely ());
|
|
1076 }
|
|
1077
|
|
1078 /* Compare the result of the operation with the first operand.
|
|
1079 No overflow for addition if second operand is positive and result
|
|
1080 is larger or second operand is negative and result is smaller.
|
|
1081 Likewise for subtraction with sign of second operand flipped. */
|
|
1082 else
|
|
1083 do_compare_rtx_and_jump (res, op0,
|
|
1084 (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
|
|
1085 false, mode, NULL_RTX, NULL, done_label,
|
|
1086 profile_probability::very_likely ());
|
|
1087 }
|
|
1088
|
|
1089 do_error_label:
|
|
1090 emit_label (do_error);
|
|
1091 if (is_ubsan)
|
|
1092 {
|
|
1093 /* Expand the ubsan builtin call. */
|
|
1094 push_temp_slots ();
|
|
1095 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
|
|
1096 arg0, arg1, datap);
|
|
1097 expand_normal (fn);
|
|
1098 pop_temp_slots ();
|
|
1099 do_pending_stack_adjust ();
|
|
1100 }
|
|
1101 else if (lhs)
|
|
1102 expand_arith_set_overflow (lhs, target);
|
|
1103
|
|
1104 /* We're done. */
|
|
1105 emit_label (done_label);
|
|
1106
|
|
1107 if (lhs)
|
|
1108 {
|
|
1109 if (is_ubsan)
|
|
1110 expand_ubsan_result_store (target, res);
|
|
1111 else
|
|
1112 {
|
|
1113 if (do_xor)
|
|
1114 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
|
|
1115 OPTAB_LIB_WIDEN);
|
|
1116
|
|
1117 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
1118 }
|
|
1119 }
|
|
1120 }
|
|
1121
|
|
1122 /* Add negate overflow checking to the statement STMT. */
|
|
1123
|
|
1124 static void
|
|
1125 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
|
|
1126 tree *datap)
|
|
1127 {
|
|
1128 rtx res, op1;
|
|
1129 tree fn;
|
|
1130 rtx_code_label *done_label, *do_error;
|
|
1131 rtx target = NULL_RTX;
|
|
1132
|
|
1133 done_label = gen_label_rtx ();
|
|
1134 do_error = gen_label_rtx ();
|
|
1135
|
|
1136 do_pending_stack_adjust ();
|
|
1137 op1 = expand_normal (arg1);
|
|
1138
|
|
1139 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
|
|
1140 if (lhs)
|
|
1141 {
|
|
1142 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
1143 if (!is_ubsan)
|
|
1144 write_complex_part (target, const0_rtx, true);
|
|
1145 }
|
|
1146
|
|
1147 enum insn_code icode = optab_handler (negv3_optab, mode);
|
|
1148 if (icode != CODE_FOR_nothing)
|
|
1149 {
|
145
|
1150 class expand_operand ops[3];
|
111
|
1151 rtx_insn *last = get_last_insn ();
|
|
1152
|
|
1153 res = gen_reg_rtx (mode);
|
|
1154 create_output_operand (&ops[0], res, mode);
|
|
1155 create_input_operand (&ops[1], op1, mode);
|
|
1156 create_fixed_operand (&ops[2], do_error);
|
|
1157 if (maybe_expand_insn (icode, 3, ops))
|
|
1158 {
|
|
1159 last = get_last_insn ();
|
|
1160 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
1161 && JUMP_P (last)
|
|
1162 && any_condjump_p (last)
|
|
1163 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
1164 add_reg_br_prob_note (last,
|
|
1165 profile_probability::very_unlikely ());
|
|
1166 emit_jump (done_label);
|
|
1167 }
|
|
1168 else
|
|
1169 {
|
|
1170 delete_insns_since (last);
|
|
1171 icode = CODE_FOR_nothing;
|
|
1172 }
|
|
1173 }
|
|
1174
|
|
1175 if (icode == CODE_FOR_nothing)
|
|
1176 {
|
|
1177 /* Compute the operation. On RTL level, the addition is always
|
|
1178 unsigned. */
|
|
1179 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
|
|
1180
|
|
1181 /* Compare the operand with the most negative value. */
|
|
1182 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
|
|
1183 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
|
|
1184 done_label, profile_probability::very_likely ());
|
|
1185 }
|
|
1186
|
|
1187 emit_label (do_error);
|
|
1188 if (is_ubsan)
|
|
1189 {
|
|
1190 /* Expand the ubsan builtin call. */
|
|
1191 push_temp_slots ();
|
|
1192 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
|
|
1193 arg1, NULL_TREE, datap);
|
|
1194 expand_normal (fn);
|
|
1195 pop_temp_slots ();
|
|
1196 do_pending_stack_adjust ();
|
|
1197 }
|
|
1198 else if (lhs)
|
|
1199 expand_arith_set_overflow (lhs, target);
|
|
1200
|
|
1201 /* We're done. */
|
|
1202 emit_label (done_label);
|
|
1203
|
|
1204 if (lhs)
|
|
1205 {
|
|
1206 if (is_ubsan)
|
|
1207 expand_ubsan_result_store (target, res);
|
|
1208 else
|
|
1209 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
1210 }
|
|
1211 }
|
|
1212
|
131
|
1213 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
|
|
1214 mode MODE can be expanded without using a libcall. */
|
|
1215
|
|
1216 static bool
|
|
1217 can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
|
|
1218 rtx op0, rtx op1, bool uns)
|
|
1219 {
|
|
1220 if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
|
|
1221 != CODE_FOR_nothing)
|
|
1222 return true;
|
|
1223
|
|
1224 if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
|
|
1225 != CODE_FOR_nothing)
|
|
1226 return true;
|
|
1227
|
|
1228 rtx_insn *last = get_last_insn ();
|
|
1229 if (CONSTANT_P (op0))
|
|
1230 op0 = convert_modes (wmode, mode, op0, uns);
|
|
1231 else
|
|
1232 op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
|
|
1233 if (CONSTANT_P (op1))
|
|
1234 op1 = convert_modes (wmode, mode, op1, uns);
|
|
1235 else
|
|
1236 op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
|
|
1237 rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
|
|
1238 delete_insns_since (last);
|
|
1239 return ret != NULL_RTX;
|
|
1240 }
|
|
1241
|
111
|
1242 /* Add mul overflow checking to the statement STMT. */
|
|
1243
|
|
1244 static void
|
|
1245 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
|
|
1246 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
|
|
1247 tree *datap)
|
|
1248 {
|
|
1249 rtx res, op0, op1;
|
|
1250 tree fn, type;
|
|
1251 rtx_code_label *done_label, *do_error;
|
|
1252 rtx target = NULL_RTX;
|
|
1253 signop sign;
|
|
1254 enum insn_code icode;
|
|
1255
|
|
1256 done_label = gen_label_rtx ();
|
|
1257 do_error = gen_label_rtx ();
|
|
1258
|
|
1259 do_pending_stack_adjust ();
|
|
1260 op0 = expand_normal (arg0);
|
|
1261 op1 = expand_normal (arg1);
|
|
1262
|
|
1263 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
|
|
1264 bool uns = unsr_p;
|
|
1265 if (lhs)
|
|
1266 {
|
|
1267 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
1268 if (!is_ubsan)
|
|
1269 write_complex_part (target, const0_rtx, true);
|
|
1270 }
|
|
1271
|
|
1272 if (is_ubsan)
|
|
1273 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
|
|
1274
|
|
1275 /* We assume both operands and result have the same precision
|
|
1276 here (GET_MODE_BITSIZE (mode)), S stands for signed type
|
|
1277 with that precision, U for unsigned type with that precision,
|
|
1278 sgn for unsigned most significant bit in that precision.
|
|
1279 s1 is signed first operand, u1 is unsigned first operand,
|
|
1280 s2 is signed second operand, u2 is unsigned second operand,
|
|
1281 sr is signed result, ur is unsigned result and the following
|
|
1282 rules say how to compute result (which is always result of
|
|
1283 the operands as if both were unsigned, cast to the right
|
|
1284 signedness) and how to compute whether operation overflowed.
|
|
1285 main_ovf (false) stands for jump on signed multiplication
|
|
1286 overflow or the main algorithm with uns == false.
|
|
1287 main_ovf (true) stands for jump on unsigned multiplication
|
|
1288 overflow or the main algorithm with uns == true.
|
|
1289
|
|
1290 s1 * s2 -> sr
|
|
1291 res = (S) ((U) s1 * (U) s2)
|
|
1292 ovf = main_ovf (false)
|
|
1293 u1 * u2 -> ur
|
|
1294 res = u1 * u2
|
|
1295 ovf = main_ovf (true)
|
|
1296 s1 * u2 -> ur
|
|
1297 res = (U) s1 * u2
|
|
1298 ovf = (s1 < 0 && u2) || main_ovf (true)
|
|
1299 u1 * u2 -> sr
|
|
1300 res = (S) (u1 * u2)
|
|
1301 ovf = res < 0 || main_ovf (true)
|
|
1302 s1 * u2 -> sr
|
|
1303 res = (S) ((U) s1 * u2)
|
|
1304 ovf = (S) u2 >= 0 ? main_ovf (false)
|
|
1305 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
|
|
1306 s1 * s2 -> ur
|
|
1307 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
|
|
1308 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
|
|
1309 res = t1 * t2
|
|
1310 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
|
|
1311
|
|
1312 if (uns0_p && !uns1_p)
|
|
1313 {
|
|
1314 /* Multiplication is commutative, if operand signedness differs,
|
|
1315 canonicalize to the first operand being signed and second
|
|
1316 unsigned to simplify following code. */
|
|
1317 std::swap (op0, op1);
|
|
1318 std::swap (arg0, arg1);
|
|
1319 uns0_p = false;
|
|
1320 uns1_p = true;
|
|
1321 }
|
|
1322
|
|
1323 int pos_neg0 = get_range_pos_neg (arg0);
|
|
1324 int pos_neg1 = get_range_pos_neg (arg1);
|
|
1325
|
|
1326 /* s1 * u2 -> ur */
|
|
1327 if (!uns0_p && uns1_p && unsr_p)
|
|
1328 {
|
|
1329 switch (pos_neg0)
|
|
1330 {
|
|
1331 case 1:
|
|
1332 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
|
|
1333 goto do_main;
|
|
1334 case 2:
|
|
1335 /* If s1 is negative, avoid the main code, just multiply and
|
|
1336 signal overflow if op1 is not 0. */
|
|
1337 struct separate_ops ops;
|
|
1338 ops.code = MULT_EXPR;
|
|
1339 ops.type = TREE_TYPE (arg1);
|
|
1340 ops.op0 = make_tree (ops.type, op0);
|
|
1341 ops.op1 = make_tree (ops.type, op1);
|
|
1342 ops.op2 = NULL_TREE;
|
|
1343 ops.location = loc;
|
|
1344 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1345 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
1346 NULL, done_label, profile_probability::very_likely ());
|
|
1347 goto do_error_label;
|
|
1348 case 3:
|
|
1349 rtx_code_label *do_main_label;
|
|
1350 do_main_label = gen_label_rtx ();
|
|
1351 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
|
|
1352 NULL, do_main_label, profile_probability::very_likely ());
|
|
1353 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
1354 NULL, do_main_label, profile_probability::very_likely ());
|
|
1355 expand_arith_set_overflow (lhs, target);
|
|
1356 emit_label (do_main_label);
|
|
1357 goto do_main;
|
|
1358 default:
|
|
1359 gcc_unreachable ();
|
|
1360 }
|
|
1361 }
|
|
1362
|
|
1363 /* u1 * u2 -> sr */
|
|
1364 if (uns0_p && uns1_p && !unsr_p)
|
|
1365 {
|
|
1366 uns = true;
|
|
1367 /* Rest of handling of this case after res is computed. */
|
|
1368 goto do_main;
|
|
1369 }
|
|
1370
|
|
1371 /* s1 * u2 -> sr */
|
|
1372 if (!uns0_p && uns1_p && !unsr_p)
|
|
1373 {
|
|
1374 switch (pos_neg1)
|
|
1375 {
|
|
1376 case 1:
|
|
1377 goto do_main;
|
|
1378 case 2:
|
|
1379 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
|
|
1380 avoid the main code, just multiply and signal overflow
|
|
1381 unless 0 * u2 or -1 * ((U) Smin). */
|
|
1382 struct separate_ops ops;
|
|
1383 ops.code = MULT_EXPR;
|
|
1384 ops.type = TREE_TYPE (arg1);
|
|
1385 ops.op0 = make_tree (ops.type, op0);
|
|
1386 ops.op1 = make_tree (ops.type, op1);
|
|
1387 ops.op2 = NULL_TREE;
|
|
1388 ops.location = loc;
|
|
1389 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1390 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
1391 NULL, done_label, profile_probability::very_likely ());
|
|
1392 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
|
|
1393 NULL, do_error, profile_probability::very_unlikely ());
|
|
1394 int prec;
|
|
1395 prec = GET_MODE_PRECISION (mode);
|
|
1396 rtx sgn;
|
|
1397 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
|
|
1398 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
|
|
1399 NULL, done_label, profile_probability::very_likely ());
|
|
1400 goto do_error_label;
|
|
1401 case 3:
|
|
1402 /* Rest of handling of this case after res is computed. */
|
|
1403 goto do_main;
|
|
1404 default:
|
|
1405 gcc_unreachable ();
|
|
1406 }
|
|
1407 }
|
|
1408
|
|
1409 /* s1 * s2 -> ur */
|
|
1410 if (!uns0_p && !uns1_p && unsr_p)
|
|
1411 {
|
145
|
1412 rtx tem;
|
111
|
1413 switch (pos_neg0 | pos_neg1)
|
|
1414 {
|
|
1415 case 1: /* Both operands known to be non-negative. */
|
|
1416 goto do_main;
|
|
1417 case 2: /* Both operands known to be negative. */
|
|
1418 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
|
|
1419 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
|
|
1420 /* Avoid looking at arg0/arg1 ranges, as we've changed
|
|
1421 the arguments. */
|
|
1422 arg0 = error_mark_node;
|
|
1423 arg1 = error_mark_node;
|
|
1424 goto do_main;
|
|
1425 case 3:
|
|
1426 if ((pos_neg0 ^ pos_neg1) == 3)
|
|
1427 {
|
|
1428 /* If one operand is known to be negative and the other
|
|
1429 non-negative, this overflows always, unless the non-negative
|
|
1430 one is 0. Just do normal multiply and set overflow
|
|
1431 unless one of the operands is 0. */
|
|
1432 struct separate_ops ops;
|
|
1433 ops.code = MULT_EXPR;
|
|
1434 ops.type
|
|
1435 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
|
|
1436 1);
|
|
1437 ops.op0 = make_tree (ops.type, op0);
|
|
1438 ops.op1 = make_tree (ops.type, op1);
|
|
1439 ops.op2 = NULL_TREE;
|
|
1440 ops.location = loc;
|
|
1441 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
145
|
1442 do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
|
|
1443 true, mode, NULL_RTX, NULL, done_label,
|
111
|
1444 profile_probability::very_likely ());
|
|
1445 goto do_error_label;
|
|
1446 }
|
|
1447 /* The general case, do all the needed comparisons at runtime. */
|
|
1448 rtx_code_label *do_main_label, *after_negate_label;
|
|
1449 rtx rop0, rop1;
|
|
1450 rop0 = gen_reg_rtx (mode);
|
|
1451 rop1 = gen_reg_rtx (mode);
|
|
1452 emit_move_insn (rop0, op0);
|
|
1453 emit_move_insn (rop1, op1);
|
|
1454 op0 = rop0;
|
|
1455 op1 = rop1;
|
|
1456 do_main_label = gen_label_rtx ();
|
|
1457 after_negate_label = gen_label_rtx ();
|
|
1458 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
|
|
1459 OPTAB_LIB_WIDEN);
|
|
1460 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
1461 NULL, after_negate_label, profile_probability::very_likely ());
|
|
1462 /* Both arguments negative here, negate them and continue with
|
|
1463 normal unsigned overflow checking multiplication. */
|
|
1464 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
|
|
1465 NULL_RTX, false));
|
|
1466 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
|
|
1467 NULL_RTX, false));
|
|
1468 /* Avoid looking at arg0/arg1 ranges, as we might have changed
|
|
1469 the arguments. */
|
|
1470 arg0 = error_mark_node;
|
|
1471 arg1 = error_mark_node;
|
|
1472 emit_jump (do_main_label);
|
|
1473 emit_label (after_negate_label);
|
145
|
1474 tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
|
|
1475 OPTAB_LIB_WIDEN);
|
|
1476 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
1477 NULL, do_main_label,
|
|
1478 profile_probability::very_likely ());
|
111
|
1479 /* One argument is negative here, the other positive. This
|
|
1480 overflows always, unless one of the arguments is 0. But
|
|
1481 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
|
|
1482 is, thus we can keep do_main code oring in overflow as is. */
|
145
|
1483 if (pos_neg0 != 2)
|
|
1484 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
1485 NULL, do_main_label,
|
|
1486 profile_probability::very_unlikely ());
|
|
1487 if (pos_neg1 != 2)
|
|
1488 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
1489 NULL, do_main_label,
|
|
1490 profile_probability::very_unlikely ());
|
111
|
1491 expand_arith_set_overflow (lhs, target);
|
|
1492 emit_label (do_main_label);
|
|
1493 goto do_main;
|
|
1494 default:
|
|
1495 gcc_unreachable ();
|
|
1496 }
|
|
1497 }
|
|
1498
|
|
1499 do_main:
|
|
1500 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
|
|
1501 sign = uns ? UNSIGNED : SIGNED;
|
|
1502 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
|
131
|
1503 if (uns
|
|
1504 && (integer_pow2p (arg0) || integer_pow2p (arg1))
|
|
1505 && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
|
|
1506 {
|
|
1507 /* Optimize unsigned multiplication by power of 2 constant
|
|
1508 using 2 shifts, one for result, one to extract the shifted
|
|
1509 out bits to see if they are all zero.
|
|
1510 Don't do this if optimizing for size and we have umulv4_optab,
|
|
1511 in that case assume multiplication will be shorter.
|
|
1512 This is heuristics based on the single target that provides
|
|
1513 umulv4 right now (i?86/x86_64), if further targets add it, this
|
|
1514 might need to be revisited.
|
|
1515 Cases where both operands are constant should be folded already
|
|
1516 during GIMPLE, and cases where one operand is constant but not
|
|
1517 power of 2 are questionable, either the WIDEN_MULT_EXPR case
|
|
1518 below can be done without multiplication, just by shifts and adds,
|
|
1519 or we'd need to divide the result (and hope it actually doesn't
|
|
1520 really divide nor multiply) and compare the result of the division
|
|
1521 with the original operand. */
|
|
1522 rtx opn0 = op0;
|
|
1523 rtx opn1 = op1;
|
|
1524 tree argn0 = arg0;
|
|
1525 tree argn1 = arg1;
|
|
1526 if (integer_pow2p (arg0))
|
|
1527 {
|
|
1528 std::swap (opn0, opn1);
|
|
1529 std::swap (argn0, argn1);
|
|
1530 }
|
|
1531 int cnt = tree_log2 (argn1);
|
|
1532 if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
|
|
1533 {
|
|
1534 rtx upper = const0_rtx;
|
|
1535 res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
|
|
1536 if (cnt != 0)
|
|
1537 upper = expand_shift (RSHIFT_EXPR, mode, opn0,
|
|
1538 GET_MODE_PRECISION (mode) - cnt,
|
|
1539 NULL_RTX, uns);
|
|
1540 do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
|
|
1541 NULL_RTX, NULL, done_label,
|
|
1542 profile_probability::very_likely ());
|
|
1543 goto do_error_label;
|
|
1544 }
|
|
1545 }
|
111
|
1546 if (icode != CODE_FOR_nothing)
|
|
1547 {
|
145
|
1548 class expand_operand ops[4];
|
111
|
1549 rtx_insn *last = get_last_insn ();
|
|
1550
|
|
1551 res = gen_reg_rtx (mode);
|
|
1552 create_output_operand (&ops[0], res, mode);
|
|
1553 create_input_operand (&ops[1], op0, mode);
|
|
1554 create_input_operand (&ops[2], op1, mode);
|
|
1555 create_fixed_operand (&ops[3], do_error);
|
|
1556 if (maybe_expand_insn (icode, 4, ops))
|
|
1557 {
|
|
1558 last = get_last_insn ();
|
|
1559 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
1560 && JUMP_P (last)
|
|
1561 && any_condjump_p (last)
|
|
1562 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
1563 add_reg_br_prob_note (last,
|
|
1564 profile_probability::very_unlikely ());
|
|
1565 emit_jump (done_label);
|
|
1566 }
|
|
1567 else
|
|
1568 {
|
|
1569 delete_insns_since (last);
|
|
1570 icode = CODE_FOR_nothing;
|
|
1571 }
|
|
1572 }
|
|
1573
|
|
1574 if (icode == CODE_FOR_nothing)
|
|
1575 {
|
|
1576 struct separate_ops ops;
|
|
1577 int prec = GET_MODE_PRECISION (mode);
|
|
1578 scalar_int_mode hmode, wmode;
|
|
1579 ops.op0 = make_tree (type, op0);
|
|
1580 ops.op1 = make_tree (type, op1);
|
|
1581 ops.op2 = NULL_TREE;
|
|
1582 ops.location = loc;
|
131
|
1583
|
|
1584 /* Optimize unsigned overflow check where we don't use the
|
|
1585 multiplication result, just whether overflow happened.
|
|
1586 If we can do MULT_HIGHPART_EXPR, that followed by
|
|
1587 comparison of the result against zero is cheapest.
|
|
1588 We'll still compute res, but it should be DCEd later. */
|
|
1589 use_operand_p use;
|
|
1590 gimple *use_stmt;
|
|
1591 if (!is_ubsan
|
|
1592 && lhs
|
|
1593 && uns
|
|
1594 && !(uns0_p && uns1_p && !unsr_p)
|
|
1595 && can_mult_highpart_p (mode, uns) == 1
|
|
1596 && single_imm_use (lhs, &use, &use_stmt)
|
|
1597 && is_gimple_assign (use_stmt)
|
|
1598 && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
|
|
1599 goto highpart;
|
|
1600
|
111
|
1601 if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
|
131
|
1602 && targetm.scalar_mode_supported_p (wmode)
|
|
1603 && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
|
111
|
1604 {
|
131
|
1605 twoxwider:
|
111
|
1606 ops.code = WIDEN_MULT_EXPR;
|
|
1607 ops.type
|
|
1608 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
|
|
1609
|
|
1610 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
|
|
1611 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
|
|
1612 NULL_RTX, uns);
|
|
1613 hipart = convert_modes (mode, wmode, hipart, uns);
|
|
1614 res = convert_modes (mode, wmode, res, uns);
|
|
1615 if (uns)
|
|
1616 /* For the unsigned multiplication, there was overflow if
|
|
1617 HIPART is non-zero. */
|
|
1618 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
|
|
1619 NULL_RTX, NULL, done_label,
|
|
1620 profile_probability::very_likely ());
|
|
1621 else
|
|
1622 {
|
|
1623 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
|
|
1624 NULL_RTX, 0);
|
|
1625 /* RES is low half of the double width result, HIPART
|
|
1626 the high half. There was overflow if
|
|
1627 HIPART is different from RES < 0 ? -1 : 0. */
|
|
1628 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
|
|
1629 NULL_RTX, NULL, done_label,
|
|
1630 profile_probability::very_likely ());
|
|
1631 }
|
|
1632 }
|
131
|
1633 else if (can_mult_highpart_p (mode, uns) == 1)
|
|
1634 {
|
|
1635 highpart:
|
|
1636 ops.code = MULT_HIGHPART_EXPR;
|
|
1637 ops.type = type;
|
|
1638
|
|
1639 rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
|
|
1640 EXPAND_NORMAL);
|
|
1641 ops.code = MULT_EXPR;
|
|
1642 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1643 if (uns)
|
|
1644 /* For the unsigned multiplication, there was overflow if
|
|
1645 HIPART is non-zero. */
|
|
1646 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
|
|
1647 NULL_RTX, NULL, done_label,
|
|
1648 profile_probability::very_likely ());
|
|
1649 else
|
|
1650 {
|
|
1651 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
|
|
1652 NULL_RTX, 0);
|
|
1653 /* RES is low half of the double width result, HIPART
|
|
1654 the high half. There was overflow if
|
|
1655 HIPART is different from RES < 0 ? -1 : 0. */
|
|
1656 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
|
|
1657 NULL_RTX, NULL, done_label,
|
|
1658 profile_probability::very_likely ());
|
|
1659 }
|
|
1660
|
|
1661 }
|
111
|
1662 else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
|
|
1663 && 2 * GET_MODE_PRECISION (hmode) == prec)
|
|
1664 {
|
|
1665 rtx_code_label *large_op0 = gen_label_rtx ();
|
|
1666 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
|
|
1667 rtx_code_label *one_small_one_large = gen_label_rtx ();
|
|
1668 rtx_code_label *both_ops_large = gen_label_rtx ();
|
|
1669 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
|
|
1670 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
|
|
1671 rtx_code_label *do_overflow = gen_label_rtx ();
|
|
1672 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
|
|
1673
|
|
1674 unsigned int hprec = GET_MODE_PRECISION (hmode);
|
|
1675 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
|
|
1676 NULL_RTX, uns);
|
|
1677 hipart0 = convert_modes (hmode, mode, hipart0, uns);
|
|
1678 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
|
|
1679 rtx signbit0 = const0_rtx;
|
|
1680 if (!uns)
|
|
1681 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
|
|
1682 NULL_RTX, 0);
|
|
1683 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
|
|
1684 NULL_RTX, uns);
|
|
1685 hipart1 = convert_modes (hmode, mode, hipart1, uns);
|
|
1686 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
|
|
1687 rtx signbit1 = const0_rtx;
|
|
1688 if (!uns)
|
|
1689 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
|
|
1690 NULL_RTX, 0);
|
|
1691
|
|
1692 res = gen_reg_rtx (mode);
|
|
1693
|
|
1694 /* True if op0 resp. op1 are known to be in the range of
|
|
1695 halfstype. */
|
|
1696 bool op0_small_p = false;
|
|
1697 bool op1_small_p = false;
|
|
1698 /* True if op0 resp. op1 are known to have all zeros or all ones
|
|
1699 in the upper half of bits, but are not known to be
|
|
1700 op{0,1}_small_p. */
|
|
1701 bool op0_medium_p = false;
|
|
1702 bool op1_medium_p = false;
|
|
1703 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
|
|
1704 nonnegative, 1 if unknown. */
|
|
1705 int op0_sign = 1;
|
|
1706 int op1_sign = 1;
|
|
1707
|
|
1708 if (pos_neg0 == 1)
|
|
1709 op0_sign = 0;
|
|
1710 else if (pos_neg0 == 2)
|
|
1711 op0_sign = -1;
|
|
1712 if (pos_neg1 == 1)
|
|
1713 op1_sign = 0;
|
|
1714 else if (pos_neg1 == 2)
|
|
1715 op1_sign = -1;
|
|
1716
|
|
1717 unsigned int mprec0 = prec;
|
|
1718 if (arg0 != error_mark_node)
|
|
1719 mprec0 = get_min_precision (arg0, sign);
|
|
1720 if (mprec0 <= hprec)
|
|
1721 op0_small_p = true;
|
|
1722 else if (!uns && mprec0 <= hprec + 1)
|
|
1723 op0_medium_p = true;
|
|
1724 unsigned int mprec1 = prec;
|
|
1725 if (arg1 != error_mark_node)
|
|
1726 mprec1 = get_min_precision (arg1, sign);
|
|
1727 if (mprec1 <= hprec)
|
|
1728 op1_small_p = true;
|
|
1729 else if (!uns && mprec1 <= hprec + 1)
|
|
1730 op1_medium_p = true;
|
|
1731
|
|
1732 int smaller_sign = 1;
|
|
1733 int larger_sign = 1;
|
|
1734 if (op0_small_p)
|
|
1735 {
|
|
1736 smaller_sign = op0_sign;
|
|
1737 larger_sign = op1_sign;
|
|
1738 }
|
|
1739 else if (op1_small_p)
|
|
1740 {
|
|
1741 smaller_sign = op1_sign;
|
|
1742 larger_sign = op0_sign;
|
|
1743 }
|
|
1744 else if (op0_sign == op1_sign)
|
|
1745 {
|
|
1746 smaller_sign = op0_sign;
|
|
1747 larger_sign = op0_sign;
|
|
1748 }
|
|
1749
|
|
1750 if (!op0_small_p)
|
|
1751 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
|
|
1752 NULL_RTX, NULL, large_op0,
|
|
1753 profile_probability::unlikely ());
|
|
1754
|
|
1755 if (!op1_small_p)
|
|
1756 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
|
|
1757 NULL_RTX, NULL, small_op0_large_op1,
|
|
1758 profile_probability::unlikely ());
|
|
1759
|
|
1760 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
|
|
1761 hmode to mode, the multiplication will never overflow. We can
|
|
1762 do just one hmode x hmode => mode widening multiplication. */
|
|
1763 tree halfstype = build_nonstandard_integer_type (hprec, uns);
|
145
|
1764 ops.op0 = make_tree (halfstype, lopart0);
|
|
1765 ops.op1 = make_tree (halfstype, lopart1);
|
111
|
1766 ops.code = WIDEN_MULT_EXPR;
|
|
1767 ops.type = type;
|
|
1768 rtx thisres
|
|
1769 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1770 emit_move_insn (res, thisres);
|
|
1771 emit_jump (done_label);
|
|
1772
|
|
1773 emit_label (small_op0_large_op1);
|
|
1774
|
|
1775 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
|
|
1776 but op1 is not, just swap the arguments and handle it as op1
|
|
1777 sign/zero extended, op0 not. */
|
|
1778 rtx larger = gen_reg_rtx (mode);
|
|
1779 rtx hipart = gen_reg_rtx (hmode);
|
|
1780 rtx lopart = gen_reg_rtx (hmode);
|
|
1781 emit_move_insn (larger, op1);
|
|
1782 emit_move_insn (hipart, hipart1);
|
|
1783 emit_move_insn (lopart, lopart0);
|
|
1784 emit_jump (one_small_one_large);
|
|
1785
|
|
1786 emit_label (large_op0);
|
|
1787
|
|
1788 if (!op1_small_p)
|
|
1789 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
|
|
1790 NULL_RTX, NULL, both_ops_large,
|
|
1791 profile_probability::unlikely ());
|
|
1792
|
|
1793 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
|
|
1794 but op0 is not, prepare larger, hipart and lopart pseudos and
|
|
1795 handle it together with small_op0_large_op1. */
|
|
1796 emit_move_insn (larger, op0);
|
|
1797 emit_move_insn (hipart, hipart0);
|
|
1798 emit_move_insn (lopart, lopart1);
|
|
1799
|
|
1800 emit_label (one_small_one_large);
|
|
1801
|
|
1802 /* lopart is the low part of the operand that is sign extended
|
|
1803 to mode, larger is the other operand, hipart is the
|
|
1804 high part of larger and lopart0 and lopart1 are the low parts
|
|
1805 of both operands.
|
|
1806 We perform lopart0 * lopart1 and lopart * hipart widening
|
|
1807 multiplications. */
|
|
1808 tree halfutype = build_nonstandard_integer_type (hprec, 1);
|
|
1809 ops.op0 = make_tree (halfutype, lopart0);
|
|
1810 ops.op1 = make_tree (halfutype, lopart1);
|
|
1811 rtx lo0xlo1
|
|
1812 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1813
|
|
1814 ops.op0 = make_tree (halfutype, lopart);
|
|
1815 ops.op1 = make_tree (halfutype, hipart);
|
|
1816 rtx loxhi = gen_reg_rtx (mode);
|
|
1817 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1818 emit_move_insn (loxhi, tem);
|
|
1819
|
|
1820 if (!uns)
|
|
1821 {
|
|
1822 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
|
|
1823 if (larger_sign == 0)
|
|
1824 emit_jump (after_hipart_neg);
|
|
1825 else if (larger_sign != -1)
|
|
1826 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
|
|
1827 NULL_RTX, NULL, after_hipart_neg,
|
|
1828 profile_probability::even ());
|
|
1829
|
|
1830 tem = convert_modes (mode, hmode, lopart, 1);
|
|
1831 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
|
|
1832 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
|
131
|
1833 1, OPTAB_WIDEN);
|
111
|
1834 emit_move_insn (loxhi, tem);
|
|
1835
|
|
1836 emit_label (after_hipart_neg);
|
|
1837
|
|
1838 /* if (lopart < 0) loxhi -= larger; */
|
|
1839 if (smaller_sign == 0)
|
|
1840 emit_jump (after_lopart_neg);
|
|
1841 else if (smaller_sign != -1)
|
|
1842 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
|
|
1843 NULL_RTX, NULL, after_lopart_neg,
|
|
1844 profile_probability::even ());
|
|
1845
|
|
1846 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
|
131
|
1847 1, OPTAB_WIDEN);
|
111
|
1848 emit_move_insn (loxhi, tem);
|
|
1849
|
|
1850 emit_label (after_lopart_neg);
|
|
1851 }
|
|
1852
|
|
1853 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
|
|
1854 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
|
|
1855 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
|
131
|
1856 1, OPTAB_WIDEN);
|
111
|
1857 emit_move_insn (loxhi, tem);
|
|
1858
|
|
1859 /* if (loxhi >> (bitsize / 2)
|
|
1860 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
|
|
1861 if (loxhi >> (bitsize / 2) == 0 (if uns). */
|
|
1862 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
|
|
1863 NULL_RTX, 0);
|
|
1864 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
|
|
1865 rtx signbitloxhi = const0_rtx;
|
|
1866 if (!uns)
|
|
1867 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
|
|
1868 convert_modes (hmode, mode,
|
|
1869 loxhi, 0),
|
|
1870 hprec - 1, NULL_RTX, 0);
|
|
1871
|
|
1872 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
|
|
1873 NULL_RTX, NULL, do_overflow,
|
|
1874 profile_probability::very_unlikely ());
|
|
1875
|
|
1876 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
|
|
1877 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
|
|
1878 NULL_RTX, 1);
|
|
1879 tem = convert_modes (mode, hmode,
|
|
1880 convert_modes (hmode, mode, lo0xlo1, 1), 1);
|
|
1881
|
|
1882 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
|
131
|
1883 1, OPTAB_WIDEN);
|
111
|
1884 if (tem != res)
|
|
1885 emit_move_insn (res, tem);
|
|
1886 emit_jump (done_label);
|
|
1887
|
|
1888 emit_label (both_ops_large);
|
|
1889
|
|
1890 /* If both operands are large (not sign (!uns) or zero (uns)
|
|
1891 extended from hmode), then perform the full multiplication
|
|
1892 which will be the result of the operation.
|
|
1893 The only cases which don't overflow are for signed multiplication
|
|
1894 some cases where both hipart0 and highpart1 are 0 or -1.
|
|
1895 For unsigned multiplication when high parts are both non-zero
|
|
1896 this overflows always. */
|
|
1897 ops.code = MULT_EXPR;
|
|
1898 ops.op0 = make_tree (type, op0);
|
|
1899 ops.op1 = make_tree (type, op1);
|
|
1900 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1901 emit_move_insn (res, tem);
|
|
1902
|
|
1903 if (!uns)
|
|
1904 {
|
|
1905 if (!op0_medium_p)
|
|
1906 {
|
|
1907 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
|
131
|
1908 NULL_RTX, 1, OPTAB_WIDEN);
|
111
|
1909 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
|
|
1910 NULL_RTX, NULL, do_error,
|
|
1911 profile_probability::very_unlikely ());
|
|
1912 }
|
|
1913
|
|
1914 if (!op1_medium_p)
|
|
1915 {
|
|
1916 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
|
131
|
1917 NULL_RTX, 1, OPTAB_WIDEN);
|
111
|
1918 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
|
|
1919 NULL_RTX, NULL, do_error,
|
|
1920 profile_probability::very_unlikely ());
|
|
1921 }
|
|
1922
|
|
1923 /* At this point hipart{0,1} are both in [-1, 0]. If they are
|
|
1924 the same, overflow happened if res is non-positive, if they
|
|
1925 are different, overflow happened if res is positive. */
|
|
1926 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
|
|
1927 emit_jump (hipart_different);
|
|
1928 else if (op0_sign == 1 || op1_sign == 1)
|
|
1929 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
|
|
1930 NULL_RTX, NULL, hipart_different,
|
|
1931 profile_probability::even ());
|
|
1932
|
|
1933 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
|
|
1934 NULL_RTX, NULL, do_error,
|
|
1935 profile_probability::very_unlikely ());
|
|
1936 emit_jump (done_label);
|
|
1937
|
|
1938 emit_label (hipart_different);
|
|
1939
|
|
1940 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
|
|
1941 NULL_RTX, NULL, do_error,
|
|
1942 profile_probability::very_unlikely ());
|
|
1943 emit_jump (done_label);
|
|
1944 }
|
|
1945
|
|
1946 emit_label (do_overflow);
|
|
1947
|
|
1948 /* Overflow, do full multiplication and fallthru into do_error. */
|
|
1949 ops.op0 = make_tree (type, op0);
|
|
1950 ops.op1 = make_tree (type, op1);
|
|
1951 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1952 emit_move_insn (res, tem);
|
|
1953 }
|
131
|
1954 else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
|
|
1955 && targetm.scalar_mode_supported_p (wmode))
|
|
1956 /* Even emitting a libcall is better than not detecting overflow
|
|
1957 at all. */
|
|
1958 goto twoxwider;
|
111
|
1959 else
|
|
1960 {
|
|
1961 gcc_assert (!is_ubsan);
|
|
1962 ops.code = MULT_EXPR;
|
|
1963 ops.type = type;
|
|
1964 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
1965 emit_jump (done_label);
|
|
1966 }
|
|
1967 }
|
|
1968
|
|
1969 do_error_label:
|
|
1970 emit_label (do_error);
|
|
1971 if (is_ubsan)
|
|
1972 {
|
|
1973 /* Expand the ubsan builtin call. */
|
|
1974 push_temp_slots ();
|
|
1975 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
|
|
1976 arg0, arg1, datap);
|
|
1977 expand_normal (fn);
|
|
1978 pop_temp_slots ();
|
|
1979 do_pending_stack_adjust ();
|
|
1980 }
|
|
1981 else if (lhs)
|
|
1982 expand_arith_set_overflow (lhs, target);
|
|
1983
|
|
1984 /* We're done. */
|
|
1985 emit_label (done_label);
|
|
1986
|
|
1987 /* u1 * u2 -> sr */
|
|
1988 if (uns0_p && uns1_p && !unsr_p)
|
|
1989 {
|
|
1990 rtx_code_label *all_done_label = gen_label_rtx ();
|
|
1991 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
|
|
1992 NULL, all_done_label, profile_probability::very_likely ());
|
|
1993 expand_arith_set_overflow (lhs, target);
|
|
1994 emit_label (all_done_label);
|
|
1995 }
|
|
1996
|
|
1997 /* s1 * u2 -> sr */
|
|
1998 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
|
|
1999 {
|
|
2000 rtx_code_label *all_done_label = gen_label_rtx ();
|
|
2001 rtx_code_label *set_noovf = gen_label_rtx ();
|
|
2002 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
|
|
2003 NULL, all_done_label, profile_probability::very_likely ());
|
|
2004 expand_arith_set_overflow (lhs, target);
|
|
2005 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
2006 NULL, set_noovf, profile_probability::very_likely ());
|
|
2007 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
|
|
2008 NULL, all_done_label, profile_probability::very_unlikely ());
|
|
2009 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
|
|
2010 all_done_label, profile_probability::very_unlikely ());
|
|
2011 emit_label (set_noovf);
|
|
2012 write_complex_part (target, const0_rtx, true);
|
|
2013 emit_label (all_done_label);
|
|
2014 }
|
|
2015
|
|
2016 if (lhs)
|
|
2017 {
|
|
2018 if (is_ubsan)
|
|
2019 expand_ubsan_result_store (target, res);
|
|
2020 else
|
|
2021 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
2022 }
|
|
2023 }
|
|
2024
|
|
2025 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
|
|
2026
|
|
2027 static void
|
|
2028 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
|
|
2029 tree arg0, tree arg1)
|
|
2030 {
|
131
|
2031 poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
|
111
|
2032 rtx_code_label *loop_lab = NULL;
|
|
2033 rtx cntvar = NULL_RTX;
|
|
2034 tree cntv = NULL_TREE;
|
|
2035 tree eltype = TREE_TYPE (TREE_TYPE (arg0));
|
|
2036 tree sz = TYPE_SIZE (eltype);
|
|
2037 tree data = NULL_TREE;
|
|
2038 tree resv = NULL_TREE;
|
|
2039 rtx lhsr = NULL_RTX;
|
|
2040 rtx resvr = NULL_RTX;
|
131
|
2041 unsigned HOST_WIDE_INT const_cnt = 0;
|
|
2042 bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
|
111
|
2043
|
|
2044 if (lhs)
|
|
2045 {
|
|
2046 optab op;
|
|
2047 lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2048 if (!VECTOR_MODE_P (GET_MODE (lhsr))
|
|
2049 || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
|
|
2050 optab_default)) == unknown_optab
|
|
2051 || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
|
|
2052 == CODE_FOR_nothing))
|
|
2053 {
|
|
2054 if (MEM_P (lhsr))
|
|
2055 resv = make_tree (TREE_TYPE (lhs), lhsr);
|
|
2056 else
|
|
2057 {
|
|
2058 resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
|
|
2059 resv = make_tree (TREE_TYPE (lhs), resvr);
|
|
2060 }
|
|
2061 }
|
|
2062 }
|
131
|
2063 if (use_loop_p)
|
111
|
2064 {
|
|
2065 do_pending_stack_adjust ();
|
|
2066 loop_lab = gen_label_rtx ();
|
|
2067 cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
|
|
2068 cntv = make_tree (sizetype, cntvar);
|
|
2069 emit_move_insn (cntvar, const0_rtx);
|
|
2070 emit_label (loop_lab);
|
|
2071 }
|
|
2072 if (TREE_CODE (arg0) != VECTOR_CST)
|
|
2073 {
|
|
2074 rtx arg0r = expand_normal (arg0);
|
|
2075 arg0 = make_tree (TREE_TYPE (arg0), arg0r);
|
|
2076 }
|
|
2077 if (TREE_CODE (arg1) != VECTOR_CST)
|
|
2078 {
|
|
2079 rtx arg1r = expand_normal (arg1);
|
|
2080 arg1 = make_tree (TREE_TYPE (arg1), arg1r);
|
|
2081 }
|
131
|
2082 for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
|
111
|
2083 {
|
|
2084 tree op0, op1, res = NULL_TREE;
|
131
|
2085 if (use_loop_p)
|
111
|
2086 {
|
|
2087 tree atype = build_array_type_nelts (eltype, cnt);
|
|
2088 op0 = uniform_vector_p (arg0);
|
|
2089 if (op0 == NULL_TREE)
|
|
2090 {
|
|
2091 op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
|
|
2092 op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
|
|
2093 NULL_TREE, NULL_TREE);
|
|
2094 }
|
|
2095 op1 = uniform_vector_p (arg1);
|
|
2096 if (op1 == NULL_TREE)
|
|
2097 {
|
|
2098 op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
|
|
2099 op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
|
|
2100 NULL_TREE, NULL_TREE);
|
|
2101 }
|
|
2102 if (resv)
|
|
2103 {
|
|
2104 res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
|
|
2105 res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
|
|
2106 NULL_TREE, NULL_TREE);
|
|
2107 }
|
|
2108 }
|
|
2109 else
|
|
2110 {
|
|
2111 tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
|
|
2112 op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
|
|
2113 op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
|
|
2114 if (resv)
|
|
2115 res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
|
|
2116 bitpos);
|
|
2117 }
|
|
2118 switch (code)
|
|
2119 {
|
|
2120 case PLUS_EXPR:
|
|
2121 expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
|
|
2122 false, false, false, true, &data);
|
|
2123 break;
|
|
2124 case MINUS_EXPR:
|
131
|
2125 if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
|
111
|
2126 expand_neg_overflow (loc, res, op1, true, &data);
|
|
2127 else
|
|
2128 expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
|
|
2129 false, false, false, true, &data);
|
|
2130 break;
|
|
2131 case MULT_EXPR:
|
|
2132 expand_mul_overflow (loc, res, op0, op1, false, false, false,
|
|
2133 true, &data);
|
|
2134 break;
|
|
2135 default:
|
|
2136 gcc_unreachable ();
|
|
2137 }
|
|
2138 }
|
131
|
2139 if (use_loop_p)
|
111
|
2140 {
|
|
2141 struct separate_ops ops;
|
|
2142 ops.code = PLUS_EXPR;
|
|
2143 ops.type = TREE_TYPE (cntv);
|
|
2144 ops.op0 = cntv;
|
|
2145 ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
|
|
2146 ops.op2 = NULL_TREE;
|
|
2147 ops.location = loc;
|
|
2148 rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
|
|
2149 EXPAND_NORMAL);
|
|
2150 if (ret != cntvar)
|
|
2151 emit_move_insn (cntvar, ret);
|
131
|
2152 rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
|
|
2153 do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
|
111
|
2154 TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
|
|
2155 profile_probability::very_likely ());
|
|
2156 }
|
|
2157 if (lhs && resv == NULL_TREE)
|
|
2158 {
|
|
2159 struct separate_ops ops;
|
|
2160 ops.code = code;
|
|
2161 ops.type = TREE_TYPE (arg0);
|
|
2162 ops.op0 = arg0;
|
|
2163 ops.op1 = arg1;
|
|
2164 ops.op2 = NULL_TREE;
|
|
2165 ops.location = loc;
|
|
2166 rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
|
|
2167 EXPAND_NORMAL);
|
|
2168 if (ret != lhsr)
|
|
2169 emit_move_insn (lhsr, ret);
|
|
2170 }
|
|
2171 else if (resvr)
|
|
2172 emit_move_insn (lhsr, resvr);
|
|
2173 }
|
|
2174
|
|
2175 /* Expand UBSAN_CHECK_ADD call STMT. */
|
|
2176
|
|
2177 static void
|
|
2178 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
|
|
2179 {
|
|
2180 location_t loc = gimple_location (stmt);
|
|
2181 tree lhs = gimple_call_lhs (stmt);
|
|
2182 tree arg0 = gimple_call_arg (stmt, 0);
|
|
2183 tree arg1 = gimple_call_arg (stmt, 1);
|
|
2184 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
2185 expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
|
|
2186 else
|
|
2187 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
|
|
2188 false, false, false, true, NULL);
|
|
2189 }
|
|
2190
|
|
2191 /* Expand UBSAN_CHECK_SUB call STMT. */
|
|
2192
|
|
2193 static void
|
|
2194 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
|
|
2195 {
|
|
2196 location_t loc = gimple_location (stmt);
|
|
2197 tree lhs = gimple_call_lhs (stmt);
|
|
2198 tree arg0 = gimple_call_arg (stmt, 0);
|
|
2199 tree arg1 = gimple_call_arg (stmt, 1);
|
|
2200 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
2201 expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
|
|
2202 else if (integer_zerop (arg0))
|
|
2203 expand_neg_overflow (loc, lhs, arg1, true, NULL);
|
|
2204 else
|
|
2205 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
|
|
2206 false, false, false, true, NULL);
|
|
2207 }
|
|
2208
|
|
2209 /* Expand UBSAN_CHECK_MUL call STMT. */
|
|
2210
|
|
2211 static void
|
|
2212 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
|
|
2213 {
|
|
2214 location_t loc = gimple_location (stmt);
|
|
2215 tree lhs = gimple_call_lhs (stmt);
|
|
2216 tree arg0 = gimple_call_arg (stmt, 0);
|
|
2217 tree arg1 = gimple_call_arg (stmt, 1);
|
|
2218 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
2219 expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
|
|
2220 else
|
|
2221 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
|
|
2222 NULL);
|
|
2223 }
|
|
2224
|
|
2225 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
|
|
2226
|
|
2227 static void
|
|
2228 expand_arith_overflow (enum tree_code code, gimple *stmt)
|
|
2229 {
|
|
2230 tree lhs = gimple_call_lhs (stmt);
|
|
2231 if (lhs == NULL_TREE)
|
|
2232 return;
|
|
2233 tree arg0 = gimple_call_arg (stmt, 0);
|
|
2234 tree arg1 = gimple_call_arg (stmt, 1);
|
|
2235 tree type = TREE_TYPE (TREE_TYPE (lhs));
|
|
2236 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
|
|
2237 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
|
|
2238 int unsr_p = TYPE_UNSIGNED (type);
|
|
2239 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
|
|
2240 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
|
|
2241 int precres = TYPE_PRECISION (type);
|
|
2242 location_t loc = gimple_location (stmt);
|
|
2243 if (!uns0_p && get_range_pos_neg (arg0) == 1)
|
|
2244 uns0_p = true;
|
|
2245 if (!uns1_p && get_range_pos_neg (arg1) == 1)
|
|
2246 uns1_p = true;
|
|
2247 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
|
|
2248 prec0 = MIN (prec0, pr);
|
|
2249 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
|
|
2250 prec1 = MIN (prec1, pr);
|
|
2251
|
|
2252 /* If uns0_p && uns1_p, precop is minimum needed precision
|
|
2253 of unsigned type to hold the exact result, otherwise
|
|
2254 precop is minimum needed precision of signed type to
|
|
2255 hold the exact result. */
|
|
2256 int precop;
|
|
2257 if (code == MULT_EXPR)
|
|
2258 precop = prec0 + prec1 + (uns0_p != uns1_p);
|
|
2259 else
|
|
2260 {
|
|
2261 if (uns0_p == uns1_p)
|
|
2262 precop = MAX (prec0, prec1) + 1;
|
|
2263 else if (uns0_p)
|
|
2264 precop = MAX (prec0 + 1, prec1) + 1;
|
|
2265 else
|
|
2266 precop = MAX (prec0, prec1 + 1) + 1;
|
|
2267 }
|
|
2268 int orig_precres = precres;
|
|
2269
|
|
2270 do
|
|
2271 {
|
|
2272 if ((uns0_p && uns1_p)
|
|
2273 ? ((precop + !unsr_p) <= precres
|
|
2274 /* u1 - u2 -> ur can overflow, no matter what precision
|
|
2275 the result has. */
|
|
2276 && (code != MINUS_EXPR || !unsr_p))
|
|
2277 : (!unsr_p && precop <= precres))
|
|
2278 {
|
|
2279 /* The infinity precision result will always fit into result. */
|
|
2280 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2281 write_complex_part (target, const0_rtx, true);
|
|
2282 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
|
|
2283 struct separate_ops ops;
|
|
2284 ops.code = code;
|
|
2285 ops.type = type;
|
|
2286 ops.op0 = fold_convert_loc (loc, type, arg0);
|
|
2287 ops.op1 = fold_convert_loc (loc, type, arg1);
|
|
2288 ops.op2 = NULL_TREE;
|
|
2289 ops.location = loc;
|
|
2290 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
2291 expand_arith_overflow_result_store (lhs, target, mode, tem);
|
|
2292 return;
|
|
2293 }
|
|
2294
|
|
2295 /* For operations with low precision, if target doesn't have them, start
|
|
2296 with precres widening right away, otherwise do it only if the most
|
|
2297 simple cases can't be used. */
|
|
2298 const int min_precision = targetm.min_arithmetic_precision ();
|
|
2299 if (orig_precres == precres && precres < min_precision)
|
|
2300 ;
|
|
2301 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
|
|
2302 && prec1 <= precres)
|
|
2303 || ((!uns0_p || !uns1_p) && !unsr_p
|
|
2304 && prec0 + uns0_p <= precres
|
|
2305 && prec1 + uns1_p <= precres))
|
|
2306 {
|
|
2307 arg0 = fold_convert_loc (loc, type, arg0);
|
|
2308 arg1 = fold_convert_loc (loc, type, arg1);
|
|
2309 switch (code)
|
|
2310 {
|
|
2311 case MINUS_EXPR:
|
|
2312 if (integer_zerop (arg0) && !unsr_p)
|
|
2313 {
|
|
2314 expand_neg_overflow (loc, lhs, arg1, false, NULL);
|
|
2315 return;
|
|
2316 }
|
|
2317 /* FALLTHRU */
|
|
2318 case PLUS_EXPR:
|
|
2319 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
|
|
2320 unsr_p, unsr_p, false, NULL);
|
|
2321 return;
|
|
2322 case MULT_EXPR:
|
|
2323 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
|
|
2324 unsr_p, unsr_p, false, NULL);
|
|
2325 return;
|
|
2326 default:
|
|
2327 gcc_unreachable ();
|
|
2328 }
|
|
2329 }
|
|
2330
|
|
2331 /* For sub-word operations, retry with a wider type first. */
|
|
2332 if (orig_precres == precres && precop <= BITS_PER_WORD)
|
|
2333 {
|
|
2334 int p = MAX (min_precision, precop);
|
|
2335 scalar_int_mode m = smallest_int_mode_for_size (p);
|
|
2336 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
|
|
2337 uns0_p && uns1_p
|
|
2338 && unsr_p);
|
|
2339 p = TYPE_PRECISION (optype);
|
|
2340 if (p > precres)
|
|
2341 {
|
|
2342 precres = p;
|
|
2343 unsr_p = TYPE_UNSIGNED (optype);
|
|
2344 type = optype;
|
|
2345 continue;
|
|
2346 }
|
|
2347 }
|
|
2348
|
|
2349 if (prec0 <= precres && prec1 <= precres)
|
|
2350 {
|
|
2351 tree types[2];
|
|
2352 if (unsr_p)
|
|
2353 {
|
|
2354 types[0] = build_nonstandard_integer_type (precres, 0);
|
|
2355 types[1] = type;
|
|
2356 }
|
|
2357 else
|
|
2358 {
|
|
2359 types[0] = type;
|
|
2360 types[1] = build_nonstandard_integer_type (precres, 1);
|
|
2361 }
|
|
2362 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
|
|
2363 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
|
|
2364 if (code != MULT_EXPR)
|
|
2365 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
|
|
2366 uns0_p, uns1_p, false, NULL);
|
|
2367 else
|
|
2368 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
|
|
2369 uns0_p, uns1_p, false, NULL);
|
|
2370 return;
|
|
2371 }
|
|
2372
|
|
2373 /* Retry with a wider type. */
|
|
2374 if (orig_precres == precres)
|
|
2375 {
|
|
2376 int p = MAX (prec0, prec1);
|
|
2377 scalar_int_mode m = smallest_int_mode_for_size (p);
|
|
2378 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
|
|
2379 uns0_p && uns1_p
|
|
2380 && unsr_p);
|
|
2381 p = TYPE_PRECISION (optype);
|
|
2382 if (p > precres)
|
|
2383 {
|
|
2384 precres = p;
|
|
2385 unsr_p = TYPE_UNSIGNED (optype);
|
|
2386 type = optype;
|
|
2387 continue;
|
|
2388 }
|
|
2389 }
|
|
2390
|
|
2391 gcc_unreachable ();
|
|
2392 }
|
|
2393 while (1);
|
|
2394 }
|
|
2395
|
|
2396 /* Expand ADD_OVERFLOW STMT. */
|
|
2397
|
|
2398 static void
|
|
2399 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
|
|
2400 {
|
|
2401 expand_arith_overflow (PLUS_EXPR, stmt);
|
|
2402 }
|
|
2403
|
|
2404 /* Expand SUB_OVERFLOW STMT. */
|
|
2405
|
|
2406 static void
|
|
2407 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
|
|
2408 {
|
|
2409 expand_arith_overflow (MINUS_EXPR, stmt);
|
|
2410 }
|
|
2411
|
|
2412 /* Expand MUL_OVERFLOW STMT. */
|
|
2413
|
|
2414 static void
|
|
2415 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
|
|
2416 {
|
|
2417 expand_arith_overflow (MULT_EXPR, stmt);
|
|
2418 }
|
|
2419
|
|
2420 /* This should get folded in tree-vectorizer.c. */
|
|
2421
|
|
2422 static void
|
|
2423 expand_LOOP_VECTORIZED (internal_fn, gcall *)
|
|
2424 {
|
|
2425 gcc_unreachable ();
|
|
2426 }
|
|
2427
|
|
2428 /* This should get folded in tree-vectorizer.c. */
|
|
2429
|
|
2430 static void
|
|
2431 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
|
|
2432 {
|
|
2433 gcc_unreachable ();
|
|
2434 }
|
|
2435
|
131
|
2436 /* Return a memory reference of type TYPE for argument INDEX of STMT.
|
|
2437 Use argument INDEX + 1 to derive the second (TBAA) operand. */
|
|
2438
|
|
2439 static tree
|
|
2440 expand_call_mem_ref (tree type, gcall *stmt, int index)
|
|
2441 {
|
|
2442 tree addr = gimple_call_arg (stmt, index);
|
|
2443 tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
|
|
2444 unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
|
|
2445 if (TYPE_ALIGN (type) != align)
|
|
2446 type = build_aligned_type (type, align);
|
|
2447
|
|
2448 tree tmp = addr;
|
|
2449 if (TREE_CODE (tmp) == SSA_NAME)
|
|
2450 {
|
|
2451 gimple *def = SSA_NAME_DEF_STMT (tmp);
|
|
2452 if (gimple_assign_single_p (def))
|
|
2453 tmp = gimple_assign_rhs1 (def);
|
|
2454 }
|
|
2455
|
|
2456 if (TREE_CODE (tmp) == ADDR_EXPR)
|
|
2457 {
|
|
2458 tree mem = TREE_OPERAND (tmp, 0);
|
|
2459 if (TREE_CODE (mem) == TARGET_MEM_REF
|
|
2460 && types_compatible_p (TREE_TYPE (mem), type))
|
|
2461 {
|
|
2462 tree offset = TMR_OFFSET (mem);
|
|
2463 if (type != TREE_TYPE (mem)
|
|
2464 || alias_ptr_type != TREE_TYPE (offset)
|
|
2465 || !integer_zerop (offset))
|
|
2466 {
|
|
2467 mem = copy_node (mem);
|
|
2468 TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
|
|
2469 wi::to_poly_wide (offset));
|
|
2470 TREE_TYPE (mem) = type;
|
|
2471 }
|
|
2472 return mem;
|
|
2473 }
|
|
2474 }
|
|
2475
|
|
2476 return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
|
|
2477 }
|
|
2478
|
|
2479 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
|
111
|
2480
|
|
2481 static void
|
|
2482 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
2483 {
|
145
|
2484 class expand_operand ops[3];
|
131
|
2485 tree type, lhs, rhs, maskt;
|
111
|
2486 rtx mem, target, mask;
|
131
|
2487 insn_code icode;
|
111
|
2488
|
|
2489 maskt = gimple_call_arg (stmt, 2);
|
|
2490 lhs = gimple_call_lhs (stmt);
|
|
2491 if (lhs == NULL_TREE)
|
|
2492 return;
|
|
2493 type = TREE_TYPE (lhs);
|
131
|
2494 rhs = expand_call_mem_ref (type, stmt, 0);
|
|
2495
|
|
2496 if (optab == vec_mask_load_lanes_optab)
|
|
2497 icode = get_multi_vector_move (type, optab);
|
|
2498 else
|
|
2499 icode = convert_optab_handler (optab, TYPE_MODE (type),
|
|
2500 TYPE_MODE (TREE_TYPE (maskt)));
|
111
|
2501
|
|
2502 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2503 gcc_assert (MEM_P (mem));
|
|
2504 mask = expand_normal (maskt);
|
|
2505 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2506 create_output_operand (&ops[0], target, TYPE_MODE (type));
|
|
2507 create_fixed_operand (&ops[1], mem);
|
|
2508 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
|
131
|
2509 expand_insn (icode, 3, ops);
|
111
|
2510 }
|
|
2511
|
131
|
2512 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
|
|
2513
|
|
2514 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
|
111
|
2515
|
|
2516 static void
|
|
2517 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
2518 {
|
145
|
2519 class expand_operand ops[3];
|
131
|
2520 tree type, lhs, rhs, maskt;
|
111
|
2521 rtx mem, reg, mask;
|
131
|
2522 insn_code icode;
|
111
|
2523
|
|
2524 maskt = gimple_call_arg (stmt, 2);
|
|
2525 rhs = gimple_call_arg (stmt, 3);
|
|
2526 type = TREE_TYPE (rhs);
|
131
|
2527 lhs = expand_call_mem_ref (type, stmt, 0);
|
|
2528
|
|
2529 if (optab == vec_mask_store_lanes_optab)
|
|
2530 icode = get_multi_vector_move (type, optab);
|
|
2531 else
|
|
2532 icode = convert_optab_handler (optab, TYPE_MODE (type),
|
|
2533 TYPE_MODE (TREE_TYPE (maskt)));
|
111
|
2534
|
|
2535 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2536 gcc_assert (MEM_P (mem));
|
|
2537 mask = expand_normal (maskt);
|
|
2538 reg = expand_normal (rhs);
|
|
2539 create_fixed_operand (&ops[0], mem);
|
|
2540 create_input_operand (&ops[1], reg, TYPE_MODE (type));
|
|
2541 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
|
131
|
2542 expand_insn (icode, 3, ops);
|
111
|
2543 }
|
|
2544
|
131
|
2545 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
|
|
2546
|
111
|
2547 static void
|
|
2548 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
|
|
2549 {
|
|
2550 }
|
|
2551
|
|
2552 static void
|
|
2553 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
|
|
2554 {
|
|
2555 /* When guessing was done, the hints should be already stripped away. */
|
|
2556 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
|
|
2557
|
|
2558 rtx target;
|
|
2559 tree lhs = gimple_call_lhs (stmt);
|
|
2560 if (lhs)
|
|
2561 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2562 else
|
|
2563 target = const0_rtx;
|
|
2564 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
|
|
2565 if (lhs && val != target)
|
|
2566 emit_move_insn (target, val);
|
|
2567 }
|
|
2568
|
|
2569 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
|
|
2570 should never be called. */
|
|
2571
|
|
2572 static void
|
|
2573 expand_VA_ARG (internal_fn, gcall *)
|
|
2574 {
|
|
2575 gcc_unreachable ();
|
|
2576 }
|
|
2577
|
145
|
2578 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
|
|
2579 dummy function should never be called. */
|
|
2580
|
|
2581 static void
|
|
2582 expand_VEC_CONVERT (internal_fn, gcall *)
|
|
2583 {
|
|
2584 gcc_unreachable ();
|
|
2585 }
|
|
2586
|
111
|
2587 /* Expand the IFN_UNIQUE function according to its first argument. */
|
|
2588
|
|
2589 static void
|
|
2590 expand_UNIQUE (internal_fn, gcall *stmt)
|
|
2591 {
|
|
2592 rtx pattern = NULL_RTX;
|
|
2593 enum ifn_unique_kind kind
|
|
2594 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
|
|
2595
|
|
2596 switch (kind)
|
|
2597 {
|
|
2598 default:
|
|
2599 gcc_unreachable ();
|
|
2600
|
|
2601 case IFN_UNIQUE_UNSPEC:
|
|
2602 if (targetm.have_unique ())
|
|
2603 pattern = targetm.gen_unique ();
|
|
2604 break;
|
|
2605
|
|
2606 case IFN_UNIQUE_OACC_FORK:
|
|
2607 case IFN_UNIQUE_OACC_JOIN:
|
|
2608 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
|
|
2609 {
|
|
2610 tree lhs = gimple_call_lhs (stmt);
|
|
2611 rtx target = const0_rtx;
|
|
2612
|
|
2613 if (lhs)
|
|
2614 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2615
|
|
2616 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
|
|
2617 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
|
|
2618
|
|
2619 if (kind == IFN_UNIQUE_OACC_FORK)
|
|
2620 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
|
|
2621 else
|
|
2622 pattern = targetm.gen_oacc_join (target, data_dep, axis);
|
|
2623 }
|
|
2624 else
|
|
2625 gcc_unreachable ();
|
|
2626 break;
|
|
2627 }
|
|
2628
|
|
2629 if (pattern)
|
|
2630 emit_insn (pattern);
|
|
2631 }
|
|
2632
|
|
2633 /* The size of an OpenACC compute dimension. */
|
|
2634
|
|
2635 static void
|
|
2636 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
|
|
2637 {
|
|
2638 tree lhs = gimple_call_lhs (stmt);
|
|
2639
|
|
2640 if (!lhs)
|
|
2641 return;
|
|
2642
|
|
2643 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2644 if (targetm.have_oacc_dim_size ())
|
|
2645 {
|
|
2646 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
|
|
2647 VOIDmode, EXPAND_NORMAL);
|
|
2648 emit_insn (targetm.gen_oacc_dim_size (target, dim));
|
|
2649 }
|
|
2650 else
|
|
2651 emit_move_insn (target, GEN_INT (1));
|
|
2652 }
|
|
2653
|
|
2654 /* The position of an OpenACC execution engine along one compute axis. */
|
|
2655
|
|
2656 static void
|
|
2657 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
|
|
2658 {
|
|
2659 tree lhs = gimple_call_lhs (stmt);
|
|
2660
|
|
2661 if (!lhs)
|
|
2662 return;
|
|
2663
|
|
2664 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2665 if (targetm.have_oacc_dim_pos ())
|
|
2666 {
|
|
2667 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
|
|
2668 VOIDmode, EXPAND_NORMAL);
|
|
2669 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
|
|
2670 }
|
|
2671 else
|
|
2672 emit_move_insn (target, const0_rtx);
|
|
2673 }
|
|
2674
|
|
2675 /* This is expanded by oacc_device_lower pass. */
|
|
2676
|
|
2677 static void
|
|
2678 expand_GOACC_LOOP (internal_fn, gcall *)
|
|
2679 {
|
|
2680 gcc_unreachable ();
|
|
2681 }
|
|
2682
|
|
2683 /* This is expanded by oacc_device_lower pass. */
|
|
2684
|
|
2685 static void
|
|
2686 expand_GOACC_REDUCTION (internal_fn, gcall *)
|
|
2687 {
|
|
2688 gcc_unreachable ();
|
|
2689 }
|
|
2690
|
|
2691 /* This is expanded by oacc_device_lower pass. */
|
|
2692
|
|
2693 static void
|
|
2694 expand_GOACC_TILE (internal_fn, gcall *)
|
|
2695 {
|
|
2696 gcc_unreachable ();
|
|
2697 }
|
|
2698
|
|
2699 /* Set errno to EDOM. */
|
|
2700
|
|
2701 static void
|
|
2702 expand_SET_EDOM (internal_fn, gcall *)
|
|
2703 {
|
|
2704 #ifdef TARGET_EDOM
|
|
2705 #ifdef GEN_ERRNO_RTX
|
|
2706 rtx errno_rtx = GEN_ERRNO_RTX;
|
|
2707 #else
|
|
2708 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
|
|
2709 #endif
|
|
2710 emit_move_insn (errno_rtx,
|
|
2711 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
|
|
2712 #else
|
|
2713 gcc_unreachable ();
|
|
2714 #endif
|
|
2715 }
|
|
2716
|
|
2717 /* Expand atomic bit test and set. */
|
|
2718
|
|
2719 static void
|
|
2720 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
|
|
2721 {
|
|
2722 expand_ifn_atomic_bit_test_and (call);
|
|
2723 }
|
|
2724
|
|
2725 /* Expand atomic bit test and complement. */
|
|
2726
|
|
2727 static void
|
|
2728 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
|
|
2729 {
|
|
2730 expand_ifn_atomic_bit_test_and (call);
|
|
2731 }
|
|
2732
|
|
2733 /* Expand atomic bit test and reset. */
|
|
2734
|
|
2735 static void
|
|
2736 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
|
|
2737 {
|
|
2738 expand_ifn_atomic_bit_test_and (call);
|
|
2739 }
|
|
2740
|
|
2741 /* Expand atomic bit test and set. */
|
|
2742
|
|
2743 static void
|
|
2744 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
|
|
2745 {
|
|
2746 expand_ifn_atomic_compare_exchange (call);
|
|
2747 }
|
|
2748
|
|
2749 /* Expand LAUNDER to assignment, lhs = arg0. */
|
|
2750
|
|
2751 static void
|
|
2752 expand_LAUNDER (internal_fn, gcall *call)
|
|
2753 {
|
|
2754 tree lhs = gimple_call_lhs (call);
|
|
2755
|
|
2756 if (!lhs)
|
|
2757 return;
|
|
2758
|
|
2759 expand_assignment (lhs, gimple_call_arg (call, 0), false);
|
|
2760 }
|
|
2761
|
131
|
2762 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
|
|
2763
|
|
2764 static void
|
|
2765 expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
|
|
2766 {
|
|
2767 internal_fn ifn = gimple_call_internal_fn (stmt);
|
|
2768 int rhs_index = internal_fn_stored_value_index (ifn);
|
|
2769 int mask_index = internal_fn_mask_index (ifn);
|
|
2770 tree base = gimple_call_arg (stmt, 0);
|
|
2771 tree offset = gimple_call_arg (stmt, 1);
|
|
2772 tree scale = gimple_call_arg (stmt, 2);
|
|
2773 tree rhs = gimple_call_arg (stmt, rhs_index);
|
|
2774
|
|
2775 rtx base_rtx = expand_normal (base);
|
|
2776 rtx offset_rtx = expand_normal (offset);
|
|
2777 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
|
|
2778 rtx rhs_rtx = expand_normal (rhs);
|
|
2779
|
145
|
2780 class expand_operand ops[6];
|
131
|
2781 int i = 0;
|
|
2782 create_address_operand (&ops[i++], base_rtx);
|
|
2783 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
|
|
2784 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
|
|
2785 create_integer_operand (&ops[i++], scale_int);
|
|
2786 create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
|
|
2787 if (mask_index >= 0)
|
|
2788 {
|
|
2789 tree mask = gimple_call_arg (stmt, mask_index);
|
|
2790 rtx mask_rtx = expand_normal (mask);
|
|
2791 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
|
|
2792 }
|
|
2793
|
145
|
2794 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
|
|
2795 TYPE_MODE (TREE_TYPE (offset)));
|
131
|
2796 expand_insn (icode, i, ops);
|
|
2797 }
|
|
2798
|
|
2799 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
|
|
2800
|
|
2801 static void
|
|
2802 expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
|
|
2803 {
|
|
2804 tree lhs = gimple_call_lhs (stmt);
|
|
2805 tree base = gimple_call_arg (stmt, 0);
|
|
2806 tree offset = gimple_call_arg (stmt, 1);
|
|
2807 tree scale = gimple_call_arg (stmt, 2);
|
|
2808
|
|
2809 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2810 rtx base_rtx = expand_normal (base);
|
|
2811 rtx offset_rtx = expand_normal (offset);
|
|
2812 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
|
|
2813
|
|
2814 int i = 0;
|
145
|
2815 class expand_operand ops[6];
|
131
|
2816 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
|
|
2817 create_address_operand (&ops[i++], base_rtx);
|
|
2818 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
|
|
2819 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
|
|
2820 create_integer_operand (&ops[i++], scale_int);
|
|
2821 if (optab == mask_gather_load_optab)
|
|
2822 {
|
145
|
2823 tree mask = gimple_call_arg (stmt, 4);
|
131
|
2824 rtx mask_rtx = expand_normal (mask);
|
|
2825 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
|
|
2826 }
|
145
|
2827 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
|
|
2828 TYPE_MODE (TREE_TYPE (offset)));
|
131
|
2829 expand_insn (icode, i, ops);
|
|
2830 }
|
|
2831
|
111
|
2832 /* Expand DIVMOD() using:
|
|
2833 a) optab handler for udivmod/sdivmod if it is available.
|
|
2834 b) If optab_handler doesn't exist, generate call to
|
|
2835 target-specific divmod libfunc. */
|
|
2836
|
|
2837 static void
|
|
2838 expand_DIVMOD (internal_fn, gcall *call_stmt)
|
|
2839 {
|
|
2840 tree lhs = gimple_call_lhs (call_stmt);
|
|
2841 tree arg0 = gimple_call_arg (call_stmt, 0);
|
|
2842 tree arg1 = gimple_call_arg (call_stmt, 1);
|
|
2843
|
|
2844 gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
|
|
2845 tree type = TREE_TYPE (TREE_TYPE (lhs));
|
|
2846 machine_mode mode = TYPE_MODE (type);
|
|
2847 bool unsignedp = TYPE_UNSIGNED (type);
|
|
2848 optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
|
|
2849
|
|
2850 rtx op0 = expand_normal (arg0);
|
|
2851 rtx op1 = expand_normal (arg1);
|
|
2852 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2853
|
|
2854 rtx quotient, remainder, libfunc;
|
|
2855
|
|
2856 /* Check if optab_handler exists for divmod_optab for given mode. */
|
|
2857 if (optab_handler (tab, mode) != CODE_FOR_nothing)
|
|
2858 {
|
|
2859 quotient = gen_reg_rtx (mode);
|
|
2860 remainder = gen_reg_rtx (mode);
|
|
2861 expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
|
|
2862 }
|
|
2863
|
|
2864 /* Generate call to divmod libfunc if it exists. */
|
|
2865 else if ((libfunc = optab_libfunc (tab, mode)) != NULL_RTX)
|
|
2866 targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
|
|
2867 "ient, &remainder);
|
|
2868
|
|
2869 else
|
|
2870 gcc_unreachable ();
|
|
2871
|
|
2872 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
|
|
2873 expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
|
|
2874 make_tree (TREE_TYPE (arg0), quotient),
|
|
2875 make_tree (TREE_TYPE (arg1), remainder)),
|
131
|
2876 target, VOIDmode, EXPAND_NORMAL);
|
|
2877 }
|
|
2878
|
|
2879 /* Expand a NOP. */
|
|
2880
|
|
2881 static void
|
|
2882 expand_NOP (internal_fn, gcall *)
|
|
2883 {
|
|
2884 /* Nothing. But it shouldn't really prevail. */
|
111
|
2885 }
|
|
2886
|
145
|
2887 /* Coroutines, all should have been processed at this stage. */
|
|
2888
|
|
2889 static void
|
|
2890 expand_CO_FRAME (internal_fn, gcall *)
|
|
2891 {
|
|
2892 gcc_unreachable ();
|
|
2893 }
|
|
2894
|
|
2895 static void
|
|
2896 expand_CO_YIELD (internal_fn, gcall *)
|
|
2897 {
|
|
2898 gcc_unreachable ();
|
|
2899 }
|
|
2900
|
|
2901 static void
|
|
2902 expand_CO_SUSPN (internal_fn, gcall *)
|
|
2903 {
|
|
2904 gcc_unreachable ();
|
|
2905 }
|
|
2906
|
|
2907 static void
|
|
2908 expand_CO_ACTOR (internal_fn, gcall *)
|
|
2909 {
|
|
2910 gcc_unreachable ();
|
|
2911 }
|
|
2912
|
111
|
2913 /* Expand a call to FN using the operands in STMT. FN has a single
|
|
2914 output operand and NARGS input operands. */
|
|
2915
|
|
2916 static void
|
|
2917 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
|
|
2918 unsigned int nargs)
|
|
2919 {
|
|
2920 expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
|
|
2921
|
|
2922 tree_pair types = direct_internal_fn_types (fn, stmt);
|
|
2923 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
|
131
|
2924 gcc_assert (icode != CODE_FOR_nothing);
|
111
|
2925
|
|
2926 tree lhs = gimple_call_lhs (stmt);
|
131
|
2927 rtx lhs_rtx = NULL_RTX;
|
|
2928 if (lhs)
|
|
2929 lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
111
|
2930
|
|
2931 /* Do not assign directly to a promoted subreg, since there is no
|
|
2932 guarantee that the instruction will leave the upper bits of the
|
|
2933 register in the state required by SUBREG_PROMOTED_SIGN. */
|
|
2934 rtx dest = lhs_rtx;
|
131
|
2935 if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
|
111
|
2936 dest = NULL_RTX;
|
|
2937
|
|
2938 create_output_operand (&ops[0], dest, insn_data[icode].operand[0].mode);
|
|
2939
|
|
2940 for (unsigned int i = 0; i < nargs; ++i)
|
|
2941 {
|
|
2942 tree rhs = gimple_call_arg (stmt, i);
|
|
2943 tree rhs_type = TREE_TYPE (rhs);
|
|
2944 rtx rhs_rtx = expand_normal (rhs);
|
|
2945 if (INTEGRAL_TYPE_P (rhs_type))
|
|
2946 create_convert_operand_from (&ops[i + 1], rhs_rtx,
|
|
2947 TYPE_MODE (rhs_type),
|
|
2948 TYPE_UNSIGNED (rhs_type));
|
|
2949 else
|
|
2950 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
|
|
2951 }
|
|
2952
|
|
2953 expand_insn (icode, nargs + 1, ops);
|
131
|
2954 if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
|
111
|
2955 {
|
|
2956 /* If the return value has an integral type, convert the instruction
|
|
2957 result to that type. This is useful for things that return an
|
|
2958 int regardless of the size of the input. If the instruction result
|
|
2959 is smaller than required, assume that it is signed.
|
|
2960
|
|
2961 If the return value has a nonintegral type, its mode must match
|
|
2962 the instruction result. */
|
|
2963 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
|
|
2964 {
|
|
2965 /* If this is a scalar in a register that is stored in a wider
|
|
2966 mode than the declared mode, compute the result into its
|
|
2967 declared mode and then convert to the wider mode. */
|
131
|
2968 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
|
111
|
2969 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
|
|
2970 convert_move (SUBREG_REG (lhs_rtx), tmp,
|
|
2971 SUBREG_PROMOTED_SIGN (lhs_rtx));
|
|
2972 }
|
|
2973 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
|
|
2974 emit_move_insn (lhs_rtx, ops[0].value);
|
|
2975 else
|
|
2976 {
|
131
|
2977 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
|
111
|
2978 convert_move (lhs_rtx, ops[0].value, 0);
|
|
2979 }
|
|
2980 }
|
|
2981 }
|
|
2982
|
131
|
2983 /* Expand WHILE_ULT call STMT using optab OPTAB. */
|
|
2984
|
|
2985 static void
|
|
2986 expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
2987 {
|
|
2988 expand_operand ops[3];
|
|
2989 tree rhs_type[2];
|
|
2990
|
|
2991 tree lhs = gimple_call_lhs (stmt);
|
|
2992 tree lhs_type = TREE_TYPE (lhs);
|
|
2993 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
2994 create_output_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
|
|
2995
|
|
2996 for (unsigned int i = 0; i < 2; ++i)
|
|
2997 {
|
|
2998 tree rhs = gimple_call_arg (stmt, i);
|
|
2999 rhs_type[i] = TREE_TYPE (rhs);
|
|
3000 rtx rhs_rtx = expand_normal (rhs);
|
|
3001 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
|
|
3002 }
|
|
3003
|
|
3004 insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
|
|
3005 TYPE_MODE (lhs_type));
|
|
3006
|
|
3007 expand_insn (icode, 3, ops);
|
|
3008 if (!rtx_equal_p (lhs_rtx, ops[0].value))
|
|
3009 emit_move_insn (lhs_rtx, ops[0].value);
|
|
3010 }
|
|
3011
|
111
|
3012 /* Expanders for optabs that can use expand_direct_optab_fn. */
|
|
3013
|
|
3014 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
|
|
3015 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
|
|
3016
|
|
3017 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
|
|
3018 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
|
|
3019
|
131
|
3020 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
|
|
3021 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
|
|
3022
|
|
3023 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
|
|
3024 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
|
|
3025
|
|
3026 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
|
|
3027 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
|
|
3028
|
|
3029 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
|
|
3030 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
|
|
3031
|
|
3032 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
|
|
3033 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
|
|
3034
|
|
3035 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
|
|
3036 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
|
|
3037
|
145
|
3038 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
|
|
3039 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
|
|
3040
|
|
3041 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
|
|
3042 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
|
|
3043
|
111
|
3044 /* RETURN_TYPE and ARGS are a return type and argument list that are
|
|
3045 in principle compatible with FN (which satisfies direct_internal_fn_p).
|
|
3046 Return the types that should be used to determine whether the
|
|
3047 target supports FN. */
|
|
3048
|
|
3049 tree_pair
|
|
3050 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
|
|
3051 {
|
|
3052 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
3053 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
|
|
3054 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
|
|
3055 return tree_pair (type0, type1);
|
|
3056 }
|
|
3057
|
|
3058 /* CALL is a call whose return type and arguments are in principle
|
|
3059 compatible with FN (which satisfies direct_internal_fn_p). Return the
|
|
3060 types that should be used to determine whether the target supports FN. */
|
|
3061
|
|
3062 tree_pair
|
|
3063 direct_internal_fn_types (internal_fn fn, gcall *call)
|
|
3064 {
|
|
3065 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
3066 tree op0 = (info.type0 < 0
|
|
3067 ? gimple_call_lhs (call)
|
|
3068 : gimple_call_arg (call, info.type0));
|
|
3069 tree op1 = (info.type1 < 0
|
|
3070 ? gimple_call_lhs (call)
|
|
3071 : gimple_call_arg (call, info.type1));
|
|
3072 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
|
|
3073 }
|
|
3074
|
|
3075 /* Return true if OPTAB is supported for TYPES (whose modes should be
|
|
3076 the same) when the optimization type is OPT_TYPE. Used for simple
|
|
3077 direct optabs. */
|
|
3078
|
|
3079 static bool
|
|
3080 direct_optab_supported_p (direct_optab optab, tree_pair types,
|
|
3081 optimization_type opt_type)
|
|
3082 {
|
|
3083 machine_mode mode = TYPE_MODE (types.first);
|
|
3084 gcc_checking_assert (mode == TYPE_MODE (types.second));
|
|
3085 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
|
|
3086 }
|
|
3087
|
131
|
3088 /* Return true if OPTAB is supported for TYPES, where the first type
|
|
3089 is the destination and the second type is the source. Used for
|
|
3090 convert optabs. */
|
|
3091
|
|
3092 static bool
|
|
3093 convert_optab_supported_p (convert_optab optab, tree_pair types,
|
|
3094 optimization_type opt_type)
|
|
3095 {
|
|
3096 return (convert_optab_handler (optab, TYPE_MODE (types.first),
|
|
3097 TYPE_MODE (types.second), opt_type)
|
|
3098 != CODE_FOR_nothing);
|
|
3099 }
|
|
3100
|
111
|
3101 /* Return true if load/store lanes optab OPTAB is supported for
|
|
3102 array type TYPES.first when the optimization type is OPT_TYPE. */
|
|
3103
|
|
3104 static bool
|
|
3105 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
|
|
3106 optimization_type opt_type)
|
|
3107 {
|
|
3108 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
|
|
3109 machine_mode imode = TYPE_MODE (types.first);
|
|
3110 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
|
|
3111 return (convert_optab_handler (optab, imode, vmode, opt_type)
|
|
3112 != CODE_FOR_nothing);
|
|
3113 }
|
|
3114
|
|
3115 #define direct_unary_optab_supported_p direct_optab_supported_p
|
|
3116 #define direct_binary_optab_supported_p direct_optab_supported_p
|
131
|
3117 #define direct_ternary_optab_supported_p direct_optab_supported_p
|
|
3118 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
|
|
3119 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
|
|
3120 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
|
111
|
3121 #define direct_mask_load_optab_supported_p direct_optab_supported_p
|
|
3122 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
|
131
|
3123 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
|
145
|
3124 #define direct_gather_load_optab_supported_p convert_optab_supported_p
|
111
|
3125 #define direct_mask_store_optab_supported_p direct_optab_supported_p
|
|
3126 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
|
131
|
3127 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
|
145
|
3128 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
|
131
|
3129 #define direct_while_optab_supported_p convert_optab_supported_p
|
|
3130 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
|
|
3131 #define direct_fold_left_optab_supported_p direct_optab_supported_p
|
145
|
3132 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
|
|
3133 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
|
131
|
3134
|
|
3135 /* Return the optab used by internal function FN. */
|
|
3136
|
|
3137 static optab
|
|
3138 direct_internal_fn_optab (internal_fn fn, tree_pair types)
|
|
3139 {
|
|
3140 switch (fn)
|
|
3141 {
|
|
3142 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
3143 case IFN_##CODE: break;
|
|
3144 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
3145 case IFN_##CODE: return OPTAB##_optab;
|
|
3146 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
3147 UNSIGNED_OPTAB, TYPE) \
|
|
3148 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
|
|
3149 ? UNSIGNED_OPTAB ## _optab \
|
|
3150 : SIGNED_OPTAB ## _optab);
|
|
3151 #include "internal-fn.def"
|
|
3152
|
|
3153 case IFN_LAST:
|
|
3154 break;
|
|
3155 }
|
|
3156 gcc_unreachable ();
|
|
3157 }
|
|
3158
|
|
3159 /* Return the optab used by internal function FN. */
|
|
3160
|
|
3161 static optab
|
|
3162 direct_internal_fn_optab (internal_fn fn)
|
|
3163 {
|
|
3164 switch (fn)
|
|
3165 {
|
|
3166 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
3167 case IFN_##CODE: break;
|
|
3168 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
3169 case IFN_##CODE: return OPTAB##_optab;
|
|
3170 #include "internal-fn.def"
|
|
3171
|
|
3172 case IFN_LAST:
|
|
3173 break;
|
|
3174 }
|
|
3175 gcc_unreachable ();
|
|
3176 }
|
111
|
3177
|
|
3178 /* Return true if FN is supported for the types in TYPES when the
|
|
3179 optimization type is OPT_TYPE. The types are those associated with
|
|
3180 the "type0" and "type1" fields of FN's direct_internal_fn_info
|
|
3181 structure. */
|
|
3182
|
|
3183 bool
|
|
3184 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
|
|
3185 optimization_type opt_type)
|
|
3186 {
|
|
3187 switch (fn)
|
|
3188 {
|
|
3189 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
3190 case IFN_##CODE: break;
|
|
3191 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
3192 case IFN_##CODE: \
|
|
3193 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
|
|
3194 opt_type);
|
131
|
3195 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
3196 UNSIGNED_OPTAB, TYPE) \
|
|
3197 case IFN_##CODE: \
|
|
3198 { \
|
|
3199 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
|
|
3200 ? UNSIGNED_OPTAB ## _optab \
|
|
3201 : SIGNED_OPTAB ## _optab); \
|
|
3202 return direct_##TYPE##_optab_supported_p (which_optab, types, \
|
|
3203 opt_type); \
|
|
3204 }
|
111
|
3205 #include "internal-fn.def"
|
|
3206
|
|
3207 case IFN_LAST:
|
|
3208 break;
|
|
3209 }
|
|
3210 gcc_unreachable ();
|
|
3211 }
|
|
3212
|
|
3213 /* Return true if FN is supported for type TYPE when the optimization
|
|
3214 type is OPT_TYPE. The caller knows that the "type0" and "type1"
|
|
3215 fields of FN's direct_internal_fn_info structure are the same. */
|
|
3216
|
|
3217 bool
|
|
3218 direct_internal_fn_supported_p (internal_fn fn, tree type,
|
|
3219 optimization_type opt_type)
|
|
3220 {
|
|
3221 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
3222 gcc_checking_assert (info.type0 == info.type1);
|
|
3223 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
|
|
3224 }
|
|
3225
|
131
|
3226 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
|
|
3227 given that STMT is a call to a direct internal function. */
|
|
3228
|
|
3229 bool
|
|
3230 direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
|
|
3231 {
|
|
3232 internal_fn fn = gimple_call_internal_fn (stmt);
|
|
3233 tree_pair types = direct_internal_fn_types (fn, stmt);
|
|
3234 return direct_internal_fn_supported_p (fn, types, opt_type);
|
|
3235 }
|
|
3236
|
|
3237 /* If FN is commutative in two consecutive arguments, return the
|
|
3238 index of the first, otherwise return -1. */
|
|
3239
|
|
3240 int
|
|
3241 first_commutative_argument (internal_fn fn)
|
|
3242 {
|
|
3243 switch (fn)
|
|
3244 {
|
|
3245 case IFN_FMA:
|
|
3246 case IFN_FMS:
|
|
3247 case IFN_FNMA:
|
|
3248 case IFN_FNMS:
|
|
3249 case IFN_AVG_FLOOR:
|
|
3250 case IFN_AVG_CEIL:
|
145
|
3251 case IFN_MULHS:
|
|
3252 case IFN_MULHRS:
|
131
|
3253 case IFN_FMIN:
|
|
3254 case IFN_FMAX:
|
|
3255 return 0;
|
|
3256
|
|
3257 case IFN_COND_ADD:
|
|
3258 case IFN_COND_MUL:
|
|
3259 case IFN_COND_MIN:
|
|
3260 case IFN_COND_MAX:
|
|
3261 case IFN_COND_AND:
|
|
3262 case IFN_COND_IOR:
|
|
3263 case IFN_COND_XOR:
|
|
3264 case IFN_COND_FMA:
|
|
3265 case IFN_COND_FMS:
|
|
3266 case IFN_COND_FNMA:
|
|
3267 case IFN_COND_FNMS:
|
|
3268 return 1;
|
|
3269
|
|
3270 default:
|
|
3271 return -1;
|
|
3272 }
|
|
3273 }
|
|
3274
|
111
|
3275 /* Return true if IFN_SET_EDOM is supported. */
|
|
3276
|
|
3277 bool
|
|
3278 set_edom_supported_p (void)
|
|
3279 {
|
|
3280 #ifdef TARGET_EDOM
|
|
3281 return true;
|
|
3282 #else
|
|
3283 return false;
|
|
3284 #endif
|
|
3285 }
|
|
3286
|
|
3287 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
3288 static void \
|
|
3289 expand_##CODE (internal_fn fn, gcall *stmt) \
|
|
3290 { \
|
|
3291 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
|
|
3292 }
|
131
|
3293 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
3294 UNSIGNED_OPTAB, TYPE) \
|
|
3295 static void \
|
|
3296 expand_##CODE (internal_fn fn, gcall *stmt) \
|
|
3297 { \
|
|
3298 tree_pair types = direct_internal_fn_types (fn, stmt); \
|
|
3299 optab which_optab = direct_internal_fn_optab (fn, types); \
|
|
3300 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
|
|
3301 }
|
111
|
3302 #include "internal-fn.def"
|
|
3303
|
|
3304 /* Routines to expand each internal function, indexed by function number.
|
|
3305 Each routine has the prototype:
|
|
3306
|
|
3307 expand_<NAME> (gcall *stmt)
|
|
3308
|
|
3309 where STMT is the statement that performs the call. */
|
|
3310 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
|
|
3311 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
|
|
3312 #include "internal-fn.def"
|
|
3313 0
|
|
3314 };
|
|
3315
|
131
|
3316 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
|
|
3317 tree code CODE. */
|
|
3318 #define FOR_EACH_CODE_MAPPING(T) \
|
|
3319 T (PLUS_EXPR, IFN_COND_ADD) \
|
|
3320 T (MINUS_EXPR, IFN_COND_SUB) \
|
|
3321 T (MULT_EXPR, IFN_COND_MUL) \
|
|
3322 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
|
|
3323 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
|
|
3324 T (RDIV_EXPR, IFN_COND_RDIV) \
|
|
3325 T (MIN_EXPR, IFN_COND_MIN) \
|
|
3326 T (MAX_EXPR, IFN_COND_MAX) \
|
|
3327 T (BIT_AND_EXPR, IFN_COND_AND) \
|
|
3328 T (BIT_IOR_EXPR, IFN_COND_IOR) \
|
145
|
3329 T (BIT_XOR_EXPR, IFN_COND_XOR) \
|
|
3330 T (LSHIFT_EXPR, IFN_COND_SHL) \
|
|
3331 T (RSHIFT_EXPR, IFN_COND_SHR)
|
131
|
3332
|
|
3333 /* Return a function that only performs CODE when a certain condition is met
|
|
3334 and that uses a given fallback value otherwise. For example, if CODE is
|
|
3335 a binary operation associated with conditional function FN:
|
|
3336
|
|
3337 LHS = FN (COND, A, B, ELSE)
|
|
3338
|
|
3339 is equivalent to the C expression:
|
|
3340
|
|
3341 LHS = COND ? A CODE B : ELSE;
|
|
3342
|
|
3343 operating elementwise if the operands are vectors.
|
|
3344
|
|
3345 Return IFN_LAST if no such function exists. */
|
|
3346
|
|
3347 internal_fn
|
|
3348 get_conditional_internal_fn (tree_code code)
|
|
3349 {
|
|
3350 switch (code)
|
|
3351 {
|
|
3352 #define CASE(CODE, IFN) case CODE: return IFN;
|
|
3353 FOR_EACH_CODE_MAPPING(CASE)
|
|
3354 #undef CASE
|
|
3355 default:
|
|
3356 return IFN_LAST;
|
|
3357 }
|
|
3358 }
|
|
3359
|
|
3360 /* If IFN implements the conditional form of a tree code, return that
|
|
3361 tree code, otherwise return ERROR_MARK. */
|
|
3362
|
|
3363 tree_code
|
|
3364 conditional_internal_fn_code (internal_fn ifn)
|
|
3365 {
|
|
3366 switch (ifn)
|
|
3367 {
|
|
3368 #define CASE(CODE, IFN) case IFN: return CODE;
|
|
3369 FOR_EACH_CODE_MAPPING(CASE)
|
|
3370 #undef CASE
|
|
3371 default:
|
|
3372 return ERROR_MARK;
|
|
3373 }
|
|
3374 }
|
|
3375
|
|
3376 /* Invoke T(IFN) for each internal function IFN that also has an
|
|
3377 IFN_COND_* form. */
|
|
3378 #define FOR_EACH_COND_FN_PAIR(T) \
|
|
3379 T (FMA) \
|
|
3380 T (FMS) \
|
|
3381 T (FNMA) \
|
|
3382 T (FNMS)
|
|
3383
|
|
3384 /* Return a function that only performs internal function FN when a
|
|
3385 certain condition is met and that uses a given fallback value otherwise.
|
|
3386 In other words, the returned function FN' is such that:
|
|
3387
|
|
3388 LHS = FN' (COND, A1, ... An, ELSE)
|
|
3389
|
|
3390 is equivalent to the C expression:
|
|
3391
|
|
3392 LHS = COND ? FN (A1, ..., An) : ELSE;
|
|
3393
|
|
3394 operating elementwise if the operands are vectors.
|
|
3395
|
|
3396 Return IFN_LAST if no such function exists. */
|
|
3397
|
|
3398 internal_fn
|
|
3399 get_conditional_internal_fn (internal_fn fn)
|
|
3400 {
|
|
3401 switch (fn)
|
|
3402 {
|
|
3403 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
|
|
3404 FOR_EACH_COND_FN_PAIR(CASE)
|
|
3405 #undef CASE
|
|
3406 default:
|
|
3407 return IFN_LAST;
|
|
3408 }
|
|
3409 }
|
|
3410
|
|
3411 /* If IFN implements the conditional form of an unconditional internal
|
|
3412 function, return that unconditional function, otherwise return IFN_LAST. */
|
|
3413
|
|
3414 internal_fn
|
|
3415 get_unconditional_internal_fn (internal_fn ifn)
|
|
3416 {
|
|
3417 switch (ifn)
|
|
3418 {
|
|
3419 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
|
|
3420 FOR_EACH_COND_FN_PAIR(CASE)
|
|
3421 #undef CASE
|
|
3422 default:
|
|
3423 return IFN_LAST;
|
|
3424 }
|
|
3425 }
|
|
3426
|
|
3427 /* Return true if STMT can be interpreted as a conditional tree code
|
|
3428 operation of the form:
|
|
3429
|
|
3430 LHS = COND ? OP (RHS1, ...) : ELSE;
|
|
3431
|
|
3432 operating elementwise if the operands are vectors. This includes
|
|
3433 the case of an all-true COND, so that the operation always happens.
|
|
3434
|
|
3435 When returning true, set:
|
|
3436
|
|
3437 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
|
|
3438 is known to be all-true
|
|
3439 - *CODE_OUT to the tree code
|
|
3440 - OPS[I] to operand I of *CODE_OUT
|
|
3441 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
|
|
3442 condition is known to be all true. */
|
|
3443
|
|
3444 bool
|
|
3445 can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
|
|
3446 tree_code *code_out,
|
|
3447 tree (&ops)[3], tree *else_out)
|
|
3448 {
|
|
3449 if (gassign *assign = dyn_cast <gassign *> (stmt))
|
|
3450 {
|
|
3451 *cond_out = NULL_TREE;
|
|
3452 *code_out = gimple_assign_rhs_code (assign);
|
|
3453 ops[0] = gimple_assign_rhs1 (assign);
|
|
3454 ops[1] = gimple_assign_rhs2 (assign);
|
|
3455 ops[2] = gimple_assign_rhs3 (assign);
|
|
3456 *else_out = NULL_TREE;
|
|
3457 return true;
|
|
3458 }
|
|
3459 if (gcall *call = dyn_cast <gcall *> (stmt))
|
|
3460 if (gimple_call_internal_p (call))
|
|
3461 {
|
|
3462 internal_fn ifn = gimple_call_internal_fn (call);
|
|
3463 tree_code code = conditional_internal_fn_code (ifn);
|
|
3464 if (code != ERROR_MARK)
|
|
3465 {
|
|
3466 *cond_out = gimple_call_arg (call, 0);
|
|
3467 *code_out = code;
|
|
3468 unsigned int nops = gimple_call_num_args (call) - 2;
|
|
3469 for (unsigned int i = 0; i < 3; ++i)
|
|
3470 ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
|
|
3471 *else_out = gimple_call_arg (call, nops + 1);
|
|
3472 if (integer_truep (*cond_out))
|
|
3473 {
|
|
3474 *cond_out = NULL_TREE;
|
|
3475 *else_out = NULL_TREE;
|
|
3476 }
|
|
3477 return true;
|
|
3478 }
|
|
3479 }
|
|
3480 return false;
|
|
3481 }
|
|
3482
|
|
3483 /* Return true if IFN is some form of load from memory. */
|
|
3484
|
|
3485 bool
|
|
3486 internal_load_fn_p (internal_fn fn)
|
|
3487 {
|
|
3488 switch (fn)
|
|
3489 {
|
|
3490 case IFN_MASK_LOAD:
|
|
3491 case IFN_LOAD_LANES:
|
|
3492 case IFN_MASK_LOAD_LANES:
|
|
3493 case IFN_GATHER_LOAD:
|
|
3494 case IFN_MASK_GATHER_LOAD:
|
|
3495 return true;
|
|
3496
|
|
3497 default:
|
|
3498 return false;
|
|
3499 }
|
|
3500 }
|
|
3501
|
|
3502 /* Return true if IFN is some form of store to memory. */
|
|
3503
|
|
3504 bool
|
|
3505 internal_store_fn_p (internal_fn fn)
|
|
3506 {
|
|
3507 switch (fn)
|
|
3508 {
|
|
3509 case IFN_MASK_STORE:
|
|
3510 case IFN_STORE_LANES:
|
|
3511 case IFN_MASK_STORE_LANES:
|
|
3512 case IFN_SCATTER_STORE:
|
|
3513 case IFN_MASK_SCATTER_STORE:
|
|
3514 return true;
|
|
3515
|
|
3516 default:
|
|
3517 return false;
|
|
3518 }
|
|
3519 }
|
|
3520
|
|
3521 /* Return true if IFN is some form of gather load or scatter store. */
|
|
3522
|
|
3523 bool
|
|
3524 internal_gather_scatter_fn_p (internal_fn fn)
|
|
3525 {
|
|
3526 switch (fn)
|
|
3527 {
|
|
3528 case IFN_GATHER_LOAD:
|
|
3529 case IFN_MASK_GATHER_LOAD:
|
|
3530 case IFN_SCATTER_STORE:
|
|
3531 case IFN_MASK_SCATTER_STORE:
|
|
3532 return true;
|
|
3533
|
|
3534 default:
|
|
3535 return false;
|
|
3536 }
|
|
3537 }
|
|
3538
|
|
3539 /* If FN takes a vector mask argument, return the index of that argument,
|
|
3540 otherwise return -1. */
|
|
3541
|
|
3542 int
|
|
3543 internal_fn_mask_index (internal_fn fn)
|
|
3544 {
|
|
3545 switch (fn)
|
|
3546 {
|
|
3547 case IFN_MASK_LOAD:
|
|
3548 case IFN_MASK_LOAD_LANES:
|
|
3549 case IFN_MASK_STORE:
|
|
3550 case IFN_MASK_STORE_LANES:
|
|
3551 return 2;
|
|
3552
|
|
3553 case IFN_MASK_GATHER_LOAD:
|
|
3554 case IFN_MASK_SCATTER_STORE:
|
|
3555 return 4;
|
|
3556
|
|
3557 default:
|
|
3558 return (conditional_internal_fn_code (fn) != ERROR_MARK
|
|
3559 || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
|
|
3560 }
|
|
3561 }
|
|
3562
|
|
3563 /* If FN takes a value that should be stored to memory, return the index
|
|
3564 of that argument, otherwise return -1. */
|
|
3565
|
|
3566 int
|
|
3567 internal_fn_stored_value_index (internal_fn fn)
|
|
3568 {
|
|
3569 switch (fn)
|
|
3570 {
|
|
3571 case IFN_MASK_STORE:
|
|
3572 case IFN_SCATTER_STORE:
|
|
3573 case IFN_MASK_SCATTER_STORE:
|
|
3574 return 3;
|
|
3575
|
|
3576 default:
|
|
3577 return -1;
|
|
3578 }
|
|
3579 }
|
|
3580
|
|
3581 /* Return true if the target supports gather load or scatter store function
|
|
3582 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
|
|
3583 while for stores it is the vector type of the stored data argument.
|
|
3584 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
|
145
|
3585 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
|
|
3586 offset from the shared base address of each loaded or stored element.
|
|
3587 SCALE is the amount by which these offsets should be multiplied
|
|
3588 *after* they have been extended to address width. */
|
131
|
3589
|
|
3590 bool
|
|
3591 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
|
|
3592 tree memory_element_type,
|
145
|
3593 tree offset_vector_type, int scale)
|
131
|
3594 {
|
|
3595 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
|
|
3596 TYPE_SIZE (memory_element_type)))
|
|
3597 return false;
|
145
|
3598 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type),
|
|
3599 TYPE_VECTOR_SUBPARTS (offset_vector_type)))
|
|
3600 return false;
|
131
|
3601 optab optab = direct_internal_fn_optab (ifn);
|
145
|
3602 insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type),
|
|
3603 TYPE_MODE (offset_vector_type));
|
131
|
3604 int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
|
145
|
3605 bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
|
131
|
3606 return (icode != CODE_FOR_nothing
|
145
|
3607 && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
|
|
3608 && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale)));
|
|
3609 }
|
|
3610
|
|
3611 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
|
|
3612 for pointers of type TYPE when the accesses have LENGTH bytes and their
|
|
3613 common byte alignment is ALIGN. */
|
|
3614
|
|
3615 bool
|
|
3616 internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
|
|
3617 poly_uint64 length, unsigned int align)
|
|
3618 {
|
|
3619 machine_mode mode = TYPE_MODE (type);
|
|
3620 optab optab = direct_internal_fn_optab (ifn);
|
|
3621 insn_code icode = direct_optab_handler (optab, mode);
|
|
3622 if (icode == CODE_FOR_nothing)
|
|
3623 return false;
|
|
3624 rtx length_rtx = immed_wide_int_const (length, mode);
|
|
3625 return (insn_operand_matches (icode, 3, length_rtx)
|
|
3626 && insn_operand_matches (icode, 4, GEN_INT (align)));
|
131
|
3627 }
|
|
3628
|
111
|
3629 /* Expand STMT as though it were a call to internal function FN. */
|
|
3630
|
|
3631 void
|
|
3632 expand_internal_call (internal_fn fn, gcall *stmt)
|
|
3633 {
|
|
3634 internal_fn_expanders[fn] (fn, stmt);
|
|
3635 }
|
|
3636
|
|
3637 /* Expand STMT, which is a call to internal function FN. */
|
|
3638
|
|
3639 void
|
|
3640 expand_internal_call (gcall *stmt)
|
|
3641 {
|
|
3642 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
|
|
3643 }
|
|
3644
|
131
|
3645 /* If TYPE is a vector type, return true if IFN is a direct internal
|
|
3646 function that is supported for that type. If TYPE is a scalar type,
|
|
3647 return true if IFN is a direct internal function that is supported for
|
|
3648 the target's preferred vector version of TYPE. */
|
|
3649
|
|
3650 bool
|
|
3651 vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
|
|
3652 {
|
|
3653 scalar_mode smode;
|
|
3654 if (!VECTOR_TYPE_P (type) && is_a <scalar_mode> (TYPE_MODE (type), &smode))
|
|
3655 {
|
|
3656 machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
|
|
3657 if (VECTOR_MODE_P (vmode))
|
|
3658 type = build_vector_type_for_mode (type, vmode);
|
|
3659 }
|
|
3660
|
|
3661 return (VECTOR_MODE_P (TYPE_MODE (type))
|
|
3662 && direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED));
|
|
3663 }
|
|
3664
|
111
|
3665 void
|
|
3666 expand_PHI (internal_fn, gcall *)
|
|
3667 {
|
|
3668 gcc_unreachable ();
|
|
3669 }
|