|
111
|
1 /* Internal functions.
|
|
131
|
2 Copyright (C) 2011-2018 Free Software Foundation, Inc.
|
|
111
|
3
|
|
|
4 This file is part of GCC.
|
|
|
5
|
|
|
6 GCC is free software; you can redistribute it and/or modify it under
|
|
|
7 the terms of the GNU General Public License as published by the Free
|
|
|
8 Software Foundation; either version 3, or (at your option) any later
|
|
|
9 version.
|
|
|
10
|
|
|
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
14 for more details.
|
|
|
15
|
|
|
16 You should have received a copy of the GNU General Public License
|
|
|
17 along with GCC; see the file COPYING3. If not see
|
|
|
18 <http://www.gnu.org/licenses/>. */
|
|
|
19
|
|
|
20 #include "config.h"
|
|
|
21 #include "system.h"
|
|
|
22 #include "coretypes.h"
|
|
|
23 #include "backend.h"
|
|
|
24 #include "target.h"
|
|
|
25 #include "rtl.h"
|
|
|
26 #include "tree.h"
|
|
|
27 #include "gimple.h"
|
|
|
28 #include "predict.h"
|
|
|
29 #include "stringpool.h"
|
|
|
30 #include "tree-vrp.h"
|
|
|
31 #include "tree-ssanames.h"
|
|
|
32 #include "expmed.h"
|
|
|
33 #include "memmodel.h"
|
|
|
34 #include "optabs.h"
|
|
|
35 #include "emit-rtl.h"
|
|
|
36 #include "diagnostic-core.h"
|
|
|
37 #include "fold-const.h"
|
|
|
38 #include "internal-fn.h"
|
|
|
39 #include "stor-layout.h"
|
|
|
40 #include "dojump.h"
|
|
|
41 #include "expr.h"
|
|
|
42 #include "stringpool.h"
|
|
|
43 #include "attribs.h"
|
|
|
44 #include "asan.h"
|
|
|
45 #include "ubsan.h"
|
|
|
46 #include "recog.h"
|
|
|
47 #include "builtins.h"
|
|
|
48 #include "optabs-tree.h"
|
|
131
|
49 #include "gimple-ssa.h"
|
|
|
50 #include "tree-phinodes.h"
|
|
|
51 #include "ssa-iterators.h"
|
|
111
|
52
|
|
|
53 /* The names of each internal function, indexed by function number. */
|
|
|
54 const char *const internal_fn_name_array[] = {
|
|
|
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
|
|
|
56 #include "internal-fn.def"
|
|
|
57 "<invalid-fn>"
|
|
|
58 };
|
|
|
59
|
|
|
60 /* The ECF_* flags of each internal function, indexed by function number. */
|
|
|
61 const int internal_fn_flags_array[] = {
|
|
|
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
|
|
|
63 #include "internal-fn.def"
|
|
|
64 0
|
|
|
65 };
|
|
|
66
|
|
131
|
67 /* Return the internal function called NAME, or IFN_LAST if there's
|
|
|
68 no such function. */
|
|
|
69
|
|
|
70 internal_fn
|
|
|
71 lookup_internal_fn (const char *name)
|
|
|
72 {
|
|
|
73 typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
|
|
|
74 static name_to_fn_map_type *name_to_fn_map;
|
|
|
75
|
|
|
76 if (!name_to_fn_map)
|
|
|
77 {
|
|
|
78 name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
|
|
|
79 for (unsigned int i = 0; i < IFN_LAST; ++i)
|
|
|
80 name_to_fn_map->put (internal_fn_name (internal_fn (i)),
|
|
|
81 internal_fn (i));
|
|
|
82 }
|
|
|
83 internal_fn *entry = name_to_fn_map->get (name);
|
|
|
84 return entry ? *entry : IFN_LAST;
|
|
|
85 }
|
|
|
86
|
|
111
|
87 /* Fnspec of each internal function, indexed by function number. */
|
|
|
88 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
|
|
|
89
|
|
|
90 void
|
|
|
91 init_internal_fns ()
|
|
|
92 {
|
|
|
93 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
|
94 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
|
|
|
95 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
|
|
|
96 #include "internal-fn.def"
|
|
|
97 internal_fn_fnspec_array[IFN_LAST] = 0;
|
|
|
98 }
|
|
|
99
|
|
|
100 /* Create static initializers for the information returned by
|
|
|
101 direct_internal_fn. */
|
|
|
102 #define not_direct { -2, -2, false }
|
|
|
103 #define mask_load_direct { -1, 2, false }
|
|
|
104 #define load_lanes_direct { -1, -1, false }
|
|
131
|
105 #define mask_load_lanes_direct { -1, -1, false }
|
|
|
106 #define gather_load_direct { -1, -1, false }
|
|
111
|
107 #define mask_store_direct { 3, 2, false }
|
|
|
108 #define store_lanes_direct { 0, 0, false }
|
|
131
|
109 #define mask_store_lanes_direct { 0, 0, false }
|
|
|
110 #define scatter_store_direct { 3, 3, false }
|
|
111
|
111 #define unary_direct { 0, 0, true }
|
|
|
112 #define binary_direct { 0, 0, true }
|
|
131
|
113 #define ternary_direct { 0, 0, true }
|
|
|
114 #define cond_unary_direct { 1, 1, true }
|
|
|
115 #define cond_binary_direct { 1, 1, true }
|
|
|
116 #define cond_ternary_direct { 1, 1, true }
|
|
|
117 #define while_direct { 0, 2, false }
|
|
|
118 #define fold_extract_direct { 2, 2, false }
|
|
|
119 #define fold_left_direct { 1, 1, false }
|
|
111
|
120
|
|
|
121 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
|
|
|
122 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
|
|
|
123 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
|
|
131
|
124 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
|
125 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
|
|
111
|
126 #include "internal-fn.def"
|
|
|
127 not_direct
|
|
|
128 };
|
|
|
129
|
|
|
130 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
|
|
|
131 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
|
|
|
132
|
|
|
133 static enum insn_code
|
|
|
134 get_multi_vector_move (tree array_type, convert_optab optab)
|
|
|
135 {
|
|
|
136 machine_mode imode;
|
|
|
137 machine_mode vmode;
|
|
|
138
|
|
|
139 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
|
|
|
140 imode = TYPE_MODE (array_type);
|
|
|
141 vmode = TYPE_MODE (TREE_TYPE (array_type));
|
|
|
142
|
|
|
143 return convert_optab_handler (optab, imode, vmode);
|
|
|
144 }
|
|
|
145
|
|
|
146 /* Expand LOAD_LANES call STMT using optab OPTAB. */
|
|
|
147
|
|
|
148 static void
|
|
|
149 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
150 {
|
|
|
151 struct expand_operand ops[2];
|
|
|
152 tree type, lhs, rhs;
|
|
|
153 rtx target, mem;
|
|
|
154
|
|
|
155 lhs = gimple_call_lhs (stmt);
|
|
|
156 rhs = gimple_call_arg (stmt, 0);
|
|
|
157 type = TREE_TYPE (lhs);
|
|
|
158
|
|
|
159 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
160 mem = expand_normal (rhs);
|
|
|
161
|
|
|
162 gcc_assert (MEM_P (mem));
|
|
|
163 PUT_MODE (mem, TYPE_MODE (type));
|
|
|
164
|
|
|
165 create_output_operand (&ops[0], target, TYPE_MODE (type));
|
|
|
166 create_fixed_operand (&ops[1], mem);
|
|
|
167 expand_insn (get_multi_vector_move (type, optab), 2, ops);
|
|
|
168 }
|
|
|
169
|
|
|
170 /* Expand STORE_LANES call STMT using optab OPTAB. */
|
|
|
171
|
|
|
172 static void
|
|
|
173 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
174 {
|
|
|
175 struct expand_operand ops[2];
|
|
|
176 tree type, lhs, rhs;
|
|
|
177 rtx target, reg;
|
|
|
178
|
|
|
179 lhs = gimple_call_lhs (stmt);
|
|
|
180 rhs = gimple_call_arg (stmt, 0);
|
|
|
181 type = TREE_TYPE (rhs);
|
|
|
182
|
|
|
183 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
184 reg = expand_normal (rhs);
|
|
|
185
|
|
|
186 gcc_assert (MEM_P (target));
|
|
|
187 PUT_MODE (target, TYPE_MODE (type));
|
|
|
188
|
|
|
189 create_fixed_operand (&ops[0], target);
|
|
|
190 create_input_operand (&ops[1], reg, TYPE_MODE (type));
|
|
|
191 expand_insn (get_multi_vector_move (type, optab), 2, ops);
|
|
|
192 }
|
|
|
193
|
|
|
194 static void
|
|
|
195 expand_ANNOTATE (internal_fn, gcall *)
|
|
|
196 {
|
|
|
197 gcc_unreachable ();
|
|
|
198 }
|
|
|
199
|
|
|
200 /* This should get expanded in omp_device_lower pass. */
|
|
|
201
|
|
|
202 static void
|
|
|
203 expand_GOMP_USE_SIMT (internal_fn, gcall *)
|
|
|
204 {
|
|
|
205 gcc_unreachable ();
|
|
|
206 }
|
|
|
207
|
|
|
208 /* This should get expanded in omp_device_lower pass. */
|
|
|
209
|
|
|
210 static void
|
|
|
211 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
|
|
|
212 {
|
|
|
213 gcc_unreachable ();
|
|
|
214 }
|
|
|
215
|
|
|
216 /* Allocate per-lane storage and begin non-uniform execution region. */
|
|
|
217
|
|
|
218 static void
|
|
|
219 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
|
|
|
220 {
|
|
|
221 rtx target;
|
|
|
222 tree lhs = gimple_call_lhs (stmt);
|
|
|
223 if (lhs)
|
|
|
224 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
225 else
|
|
|
226 target = gen_reg_rtx (Pmode);
|
|
|
227 rtx size = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
228 rtx align = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
229 struct expand_operand ops[3];
|
|
|
230 create_output_operand (&ops[0], target, Pmode);
|
|
|
231 create_input_operand (&ops[1], size, Pmode);
|
|
|
232 create_input_operand (&ops[2], align, Pmode);
|
|
|
233 gcc_assert (targetm.have_omp_simt_enter ());
|
|
|
234 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
|
|
|
235 }
|
|
|
236
|
|
|
237 /* Deallocate per-lane storage and leave non-uniform execution region. */
|
|
|
238
|
|
|
239 static void
|
|
|
240 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
|
|
|
241 {
|
|
|
242 gcc_checking_assert (!gimple_call_lhs (stmt));
|
|
|
243 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
244 struct expand_operand ops[1];
|
|
|
245 create_input_operand (&ops[0], arg, Pmode);
|
|
|
246 gcc_assert (targetm.have_omp_simt_exit ());
|
|
|
247 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
|
|
|
248 }
|
|
|
249
|
|
|
250 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
|
|
|
251 without SIMT execution this should be expanded in omp_device_lower pass. */
|
|
|
252
|
|
|
253 static void
|
|
|
254 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
|
|
|
255 {
|
|
|
256 tree lhs = gimple_call_lhs (stmt);
|
|
|
257 if (!lhs)
|
|
|
258 return;
|
|
|
259
|
|
|
260 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
261 gcc_assert (targetm.have_omp_simt_lane ());
|
|
|
262 emit_insn (targetm.gen_omp_simt_lane (target));
|
|
|
263 }
|
|
|
264
|
|
|
265 /* This should get expanded in omp_device_lower pass. */
|
|
|
266
|
|
|
267 static void
|
|
|
268 expand_GOMP_SIMT_VF (internal_fn, gcall *)
|
|
|
269 {
|
|
|
270 gcc_unreachable ();
|
|
|
271 }
|
|
|
272
|
|
|
273 /* Lane index of the first SIMT lane that supplies a non-zero argument.
|
|
|
274 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
|
|
|
275 lane that executed the last iteration for handling OpenMP lastprivate. */
|
|
|
276
|
|
|
277 static void
|
|
|
278 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
|
|
|
279 {
|
|
|
280 tree lhs = gimple_call_lhs (stmt);
|
|
|
281 if (!lhs)
|
|
|
282 return;
|
|
|
283
|
|
|
284 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
285 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
286 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
287 struct expand_operand ops[2];
|
|
|
288 create_output_operand (&ops[0], target, mode);
|
|
|
289 create_input_operand (&ops[1], cond, mode);
|
|
|
290 gcc_assert (targetm.have_omp_simt_last_lane ());
|
|
|
291 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
|
|
|
292 }
|
|
|
293
|
|
|
294 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
|
|
|
295
|
|
|
296 static void
|
|
|
297 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
|
|
|
298 {
|
|
|
299 tree lhs = gimple_call_lhs (stmt);
|
|
|
300 if (!lhs)
|
|
|
301 return;
|
|
|
302
|
|
|
303 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
304 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
305 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
306 struct expand_operand ops[2];
|
|
|
307 create_output_operand (&ops[0], target, mode);
|
|
|
308 create_input_operand (&ops[1], ctr, mode);
|
|
|
309 gcc_assert (targetm.have_omp_simt_ordered ());
|
|
|
310 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
|
|
|
311 }
|
|
|
312
|
|
|
313 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
|
|
|
314 any lane supplies a non-zero argument. */
|
|
|
315
|
|
|
316 static void
|
|
|
317 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
|
|
|
318 {
|
|
|
319 tree lhs = gimple_call_lhs (stmt);
|
|
|
320 if (!lhs)
|
|
|
321 return;
|
|
|
322
|
|
|
323 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
324 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
325 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
326 struct expand_operand ops[2];
|
|
|
327 create_output_operand (&ops[0], target, mode);
|
|
|
328 create_input_operand (&ops[1], cond, mode);
|
|
|
329 gcc_assert (targetm.have_omp_simt_vote_any ());
|
|
|
330 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
|
|
|
331 }
|
|
|
332
|
|
|
333 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
|
|
|
334 is destination lane index XOR given offset. */
|
|
|
335
|
|
|
336 static void
|
|
|
337 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
|
|
|
338 {
|
|
|
339 tree lhs = gimple_call_lhs (stmt);
|
|
|
340 if (!lhs)
|
|
|
341 return;
|
|
|
342
|
|
|
343 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
344 rtx src = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
345 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
346 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
347 struct expand_operand ops[3];
|
|
|
348 create_output_operand (&ops[0], target, mode);
|
|
|
349 create_input_operand (&ops[1], src, mode);
|
|
|
350 create_input_operand (&ops[2], idx, SImode);
|
|
|
351 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
|
|
|
352 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
|
|
|
353 }
|
|
|
354
|
|
|
355 /* Exchange between SIMT lanes according to given source lane index. */
|
|
|
356
|
|
|
357 static void
|
|
|
358 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
|
|
|
359 {
|
|
|
360 tree lhs = gimple_call_lhs (stmt);
|
|
|
361 if (!lhs)
|
|
|
362 return;
|
|
|
363
|
|
|
364 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
365 rtx src = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
366 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
367 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
368 struct expand_operand ops[3];
|
|
|
369 create_output_operand (&ops[0], target, mode);
|
|
|
370 create_input_operand (&ops[1], src, mode);
|
|
|
371 create_input_operand (&ops[2], idx, SImode);
|
|
|
372 gcc_assert (targetm.have_omp_simt_xchg_idx ());
|
|
|
373 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
|
|
|
374 }
|
|
|
375
|
|
|
376 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
377
|
|
|
378 static void
|
|
|
379 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
|
|
|
380 {
|
|
|
381 gcc_unreachable ();
|
|
|
382 }
|
|
|
383
|
|
|
384 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
385
|
|
|
386 static void
|
|
|
387 expand_GOMP_SIMD_VF (internal_fn, gcall *)
|
|
|
388 {
|
|
|
389 gcc_unreachable ();
|
|
|
390 }
|
|
|
391
|
|
|
392 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
393
|
|
|
394 static void
|
|
|
395 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
|
|
|
396 {
|
|
|
397 gcc_unreachable ();
|
|
|
398 }
|
|
|
399
|
|
|
400 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
401
|
|
|
402 static void
|
|
|
403 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
|
|
|
404 {
|
|
|
405 gcc_unreachable ();
|
|
|
406 }
|
|
|
407
|
|
|
408 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
409
|
|
|
410 static void
|
|
|
411 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
|
|
|
412 {
|
|
|
413 gcc_unreachable ();
|
|
|
414 }
|
|
|
415
|
|
|
416 /* This should get expanded in the sanopt pass. */
|
|
|
417
|
|
|
418 static void
|
|
|
419 expand_UBSAN_NULL (internal_fn, gcall *)
|
|
|
420 {
|
|
|
421 gcc_unreachable ();
|
|
|
422 }
|
|
|
423
|
|
|
424 /* This should get expanded in the sanopt pass. */
|
|
|
425
|
|
|
426 static void
|
|
|
427 expand_UBSAN_BOUNDS (internal_fn, gcall *)
|
|
|
428 {
|
|
|
429 gcc_unreachable ();
|
|
|
430 }
|
|
|
431
|
|
|
432 /* This should get expanded in the sanopt pass. */
|
|
|
433
|
|
|
434 static void
|
|
|
435 expand_UBSAN_VPTR (internal_fn, gcall *)
|
|
|
436 {
|
|
|
437 gcc_unreachable ();
|
|
|
438 }
|
|
|
439
|
|
|
440 /* This should get expanded in the sanopt pass. */
|
|
|
441
|
|
|
442 static void
|
|
|
443 expand_UBSAN_PTR (internal_fn, gcall *)
|
|
|
444 {
|
|
|
445 gcc_unreachable ();
|
|
|
446 }
|
|
|
447
|
|
|
448 /* This should get expanded in the sanopt pass. */
|
|
|
449
|
|
|
450 static void
|
|
|
451 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
|
|
|
452 {
|
|
|
453 gcc_unreachable ();
|
|
|
454 }
|
|
|
455
|
|
|
456 /* This should get expanded in the sanopt pass. */
|
|
|
457
|
|
|
458 static void
|
|
|
459 expand_ASAN_CHECK (internal_fn, gcall *)
|
|
|
460 {
|
|
|
461 gcc_unreachable ();
|
|
|
462 }
|
|
|
463
|
|
|
464 /* This should get expanded in the sanopt pass. */
|
|
|
465
|
|
|
466 static void
|
|
|
467 expand_ASAN_MARK (internal_fn, gcall *)
|
|
|
468 {
|
|
|
469 gcc_unreachable ();
|
|
|
470 }
|
|
|
471
|
|
|
472 /* This should get expanded in the sanopt pass. */
|
|
|
473
|
|
|
474 static void
|
|
|
475 expand_ASAN_POISON (internal_fn, gcall *)
|
|
|
476 {
|
|
|
477 gcc_unreachable ();
|
|
|
478 }
|
|
|
479
|
|
|
480 /* This should get expanded in the sanopt pass. */
|
|
|
481
|
|
|
482 static void
|
|
|
483 expand_ASAN_POISON_USE (internal_fn, gcall *)
|
|
|
484 {
|
|
|
485 gcc_unreachable ();
|
|
|
486 }
|
|
|
487
|
|
|
488 /* This should get expanded in the tsan pass. */
|
|
|
489
|
|
|
490 static void
|
|
|
491 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
|
|
|
492 {
|
|
|
493 gcc_unreachable ();
|
|
|
494 }
|
|
|
495
|
|
|
496 /* This should get expanded in the lower pass. */
|
|
|
497
|
|
|
498 static void
|
|
|
499 expand_FALLTHROUGH (internal_fn, gcall *call)
|
|
|
500 {
|
|
|
501 error_at (gimple_location (call),
|
|
|
502 "invalid use of attribute %<fallthrough%>");
|
|
|
503 }
|
|
|
504
|
|
|
505 /* Return minimum precision needed to represent all values
|
|
|
506 of ARG in SIGNed integral type. */
|
|
|
507
|
|
|
508 static int
|
|
|
509 get_min_precision (tree arg, signop sign)
|
|
|
510 {
|
|
|
511 int prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
|
512 int cnt = 0;
|
|
|
513 signop orig_sign = sign;
|
|
|
514 if (TREE_CODE (arg) == INTEGER_CST)
|
|
|
515 {
|
|
|
516 int p;
|
|
|
517 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
|
|
|
518 {
|
|
|
519 widest_int w = wi::to_widest (arg);
|
|
|
520 w = wi::ext (w, prec, sign);
|
|
|
521 p = wi::min_precision (w, sign);
|
|
|
522 }
|
|
|
523 else
|
|
|
524 p = wi::min_precision (wi::to_wide (arg), sign);
|
|
|
525 return MIN (p, prec);
|
|
|
526 }
|
|
|
527 while (CONVERT_EXPR_P (arg)
|
|
|
528 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
|
|
|
529 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
|
|
|
530 {
|
|
|
531 arg = TREE_OPERAND (arg, 0);
|
|
|
532 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
|
|
|
533 {
|
|
|
534 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
|
|
|
535 sign = UNSIGNED;
|
|
|
536 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
|
|
|
537 return prec + (orig_sign != sign);
|
|
|
538 prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
|
539 }
|
|
|
540 if (++cnt > 30)
|
|
|
541 return prec + (orig_sign != sign);
|
|
|
542 }
|
|
|
543 if (TREE_CODE (arg) != SSA_NAME)
|
|
|
544 return prec + (orig_sign != sign);
|
|
|
545 wide_int arg_min, arg_max;
|
|
|
546 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
|
|
|
547 {
|
|
|
548 gimple *g = SSA_NAME_DEF_STMT (arg);
|
|
|
549 if (is_gimple_assign (g)
|
|
|
550 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
|
|
|
551 {
|
|
|
552 tree t = gimple_assign_rhs1 (g);
|
|
|
553 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
|
|
|
554 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
|
|
|
555 {
|
|
|
556 arg = t;
|
|
|
557 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
|
|
|
558 {
|
|
|
559 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
|
|
|
560 sign = UNSIGNED;
|
|
|
561 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
|
|
|
562 return prec + (orig_sign != sign);
|
|
|
563 prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
|
564 }
|
|
|
565 if (++cnt > 30)
|
|
|
566 return prec + (orig_sign != sign);
|
|
|
567 continue;
|
|
|
568 }
|
|
|
569 }
|
|
|
570 return prec + (orig_sign != sign);
|
|
|
571 }
|
|
|
572 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
|
|
|
573 {
|
|
|
574 int p1 = wi::min_precision (arg_min, sign);
|
|
|
575 int p2 = wi::min_precision (arg_max, sign);
|
|
|
576 p1 = MAX (p1, p2);
|
|
|
577 prec = MIN (prec, p1);
|
|
|
578 }
|
|
|
579 else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
|
|
|
580 {
|
|
|
581 int p = wi::min_precision (arg_max, UNSIGNED);
|
|
|
582 prec = MIN (prec, p);
|
|
|
583 }
|
|
|
584 return prec + (orig_sign != sign);
|
|
|
585 }
|
|
|
586
|
|
|
587 /* Helper for expand_*_overflow. Set the __imag__ part to true
|
|
|
588 (1 except for signed:1 type, in which case store -1). */
|
|
|
589
|
|
|
590 static void
|
|
|
591 expand_arith_set_overflow (tree lhs, rtx target)
|
|
|
592 {
|
|
|
593 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
|
|
|
594 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
|
|
|
595 write_complex_part (target, constm1_rtx, true);
|
|
|
596 else
|
|
|
597 write_complex_part (target, const1_rtx, true);
|
|
|
598 }
|
|
|
599
|
|
|
600 /* Helper for expand_*_overflow. Store RES into the __real__ part
|
|
|
601 of TARGET. If RES has larger MODE than __real__ part of TARGET,
|
|
|
602 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
|
|
|
603 if LHS has smaller precision than its mode. */
|
|
|
604
|
|
|
605 static void
|
|
|
606 expand_arith_overflow_result_store (tree lhs, rtx target,
|
|
|
607 scalar_int_mode mode, rtx res)
|
|
|
608 {
|
|
|
609 scalar_int_mode tgtmode
|
|
|
610 = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
|
|
|
611 rtx lres = res;
|
|
|
612 if (tgtmode != mode)
|
|
|
613 {
|
|
|
614 rtx_code_label *done_label = gen_label_rtx ();
|
|
|
615 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
|
|
|
616 lres = convert_modes (tgtmode, mode, res, uns);
|
|
|
617 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
|
|
|
618 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
|
|
|
619 EQ, true, mode, NULL_RTX, NULL, done_label,
|
|
|
620 profile_probability::very_likely ());
|
|
|
621 expand_arith_set_overflow (lhs, target);
|
|
|
622 emit_label (done_label);
|
|
|
623 }
|
|
|
624 int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
|
|
|
625 int tgtprec = GET_MODE_PRECISION (tgtmode);
|
|
|
626 if (prec < tgtprec)
|
|
|
627 {
|
|
|
628 rtx_code_label *done_label = gen_label_rtx ();
|
|
|
629 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
|
|
|
630 res = lres;
|
|
|
631 if (uns)
|
|
|
632 {
|
|
|
633 rtx mask
|
|
|
634 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
|
|
|
635 tgtmode);
|
|
|
636 lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
|
|
|
637 true, OPTAB_LIB_WIDEN);
|
|
|
638 }
|
|
|
639 else
|
|
|
640 {
|
|
|
641 lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
|
|
|
642 NULL_RTX, 1);
|
|
|
643 lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
|
|
|
644 NULL_RTX, 0);
|
|
|
645 }
|
|
|
646 do_compare_rtx_and_jump (res, lres,
|
|
|
647 EQ, true, tgtmode, NULL_RTX, NULL, done_label,
|
|
|
648 profile_probability::very_likely ());
|
|
|
649 expand_arith_set_overflow (lhs, target);
|
|
|
650 emit_label (done_label);
|
|
|
651 }
|
|
|
652 write_complex_part (target, lres, false);
|
|
|
653 }
|
|
|
654
|
|
|
655 /* Helper for expand_*_overflow. Store RES into TARGET. */
|
|
|
656
|
|
|
657 static void
|
|
|
658 expand_ubsan_result_store (rtx target, rtx res)
|
|
|
659 {
|
|
|
660 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
|
|
|
661 /* If this is a scalar in a register that is stored in a wider mode
|
|
|
662 than the declared mode, compute the result into its declared mode
|
|
|
663 and then convert to the wider mode. Our value is the computed
|
|
|
664 expression. */
|
|
|
665 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
|
|
|
666 else
|
|
|
667 emit_move_insn (target, res);
|
|
|
668 }
|
|
|
669
|
|
|
670 /* Add sub/add overflow checking to the statement STMT.
|
|
|
671 CODE says whether the operation is +, or -. */
|
|
|
672
|
|
|
673 static void
|
|
|
674 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
|
|
|
675 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
|
|
|
676 bool uns1_p, bool is_ubsan, tree *datap)
|
|
|
677 {
|
|
|
678 rtx res, target = NULL_RTX;
|
|
|
679 tree fn;
|
|
|
680 rtx_code_label *done_label = gen_label_rtx ();
|
|
|
681 rtx_code_label *do_error = gen_label_rtx ();
|
|
|
682 do_pending_stack_adjust ();
|
|
|
683 rtx op0 = expand_normal (arg0);
|
|
|
684 rtx op1 = expand_normal (arg1);
|
|
|
685 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
|
|
|
686 int prec = GET_MODE_PRECISION (mode);
|
|
|
687 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
|
|
|
688 bool do_xor = false;
|
|
|
689
|
|
|
690 if (is_ubsan)
|
|
|
691 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
|
|
|
692
|
|
|
693 if (lhs)
|
|
|
694 {
|
|
|
695 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
696 if (!is_ubsan)
|
|
|
697 write_complex_part (target, const0_rtx, true);
|
|
|
698 }
|
|
|
699
|
|
|
700 /* We assume both operands and result have the same precision
|
|
|
701 here (GET_MODE_BITSIZE (mode)), S stands for signed type
|
|
|
702 with that precision, U for unsigned type with that precision,
|
|
|
703 sgn for unsigned most significant bit in that precision.
|
|
|
704 s1 is signed first operand, u1 is unsigned first operand,
|
|
|
705 s2 is signed second operand, u2 is unsigned second operand,
|
|
|
706 sr is signed result, ur is unsigned result and the following
|
|
|
707 rules say how to compute result (which is always result of
|
|
|
708 the operands as if both were unsigned, cast to the right
|
|
|
709 signedness) and how to compute whether operation overflowed.
|
|
|
710
|
|
|
711 s1 + s2 -> sr
|
|
|
712 res = (S) ((U) s1 + (U) s2)
|
|
|
713 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
|
|
|
714 s1 - s2 -> sr
|
|
|
715 res = (S) ((U) s1 - (U) s2)
|
|
|
716 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
|
|
|
717 u1 + u2 -> ur
|
|
|
718 res = u1 + u2
|
|
|
719 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
|
|
|
720 u1 - u2 -> ur
|
|
|
721 res = u1 - u2
|
|
|
722 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
|
|
|
723 s1 + u2 -> sr
|
|
|
724 res = (S) ((U) s1 + u2)
|
|
|
725 ovf = ((U) res ^ sgn) < u2
|
|
|
726 s1 + u2 -> ur
|
|
|
727 t1 = (S) (u2 ^ sgn)
|
|
|
728 t2 = s1 + t1
|
|
|
729 res = (U) t2 ^ sgn
|
|
|
730 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
|
|
|
731 s1 - u2 -> sr
|
|
|
732 res = (S) ((U) s1 - u2)
|
|
|
733 ovf = u2 > ((U) s1 ^ sgn)
|
|
|
734 s1 - u2 -> ur
|
|
|
735 res = (U) s1 - u2
|
|
|
736 ovf = s1 < 0 || u2 > (U) s1
|
|
|
737 u1 - s2 -> sr
|
|
|
738 res = u1 - (U) s2
|
|
|
739 ovf = u1 >= ((U) s2 ^ sgn)
|
|
|
740 u1 - s2 -> ur
|
|
|
741 t1 = u1 ^ sgn
|
|
|
742 t2 = t1 - (U) s2
|
|
|
743 res = t2 ^ sgn
|
|
|
744 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
|
|
|
745 s1 + s2 -> ur
|
|
|
746 res = (U) s1 + (U) s2
|
|
|
747 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
|
|
|
748 u1 + u2 -> sr
|
|
|
749 res = (S) (u1 + u2)
|
|
|
750 ovf = (U) res < u2 || res < 0
|
|
|
751 u1 - u2 -> sr
|
|
|
752 res = (S) (u1 - u2)
|
|
|
753 ovf = u1 >= u2 ? res < 0 : res >= 0
|
|
|
754 s1 - s2 -> ur
|
|
|
755 res = (U) s1 - (U) s2
|
|
|
756 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
|
|
|
757
|
|
|
758 if (code == PLUS_EXPR && uns0_p && !uns1_p)
|
|
|
759 {
|
|
|
760 /* PLUS_EXPR is commutative, if operand signedness differs,
|
|
|
761 canonicalize to the first operand being signed and second
|
|
|
762 unsigned to simplify following code. */
|
|
|
763 std::swap (op0, op1);
|
|
|
764 std::swap (arg0, arg1);
|
|
|
765 uns0_p = false;
|
|
|
766 uns1_p = true;
|
|
|
767 }
|
|
|
768
|
|
|
769 /* u1 +- u2 -> ur */
|
|
|
770 if (uns0_p && uns1_p && unsr_p)
|
|
|
771 {
|
|
|
772 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
|
|
|
773 : usubv4_optab, mode);
|
|
|
774 if (icode != CODE_FOR_nothing)
|
|
|
775 {
|
|
|
776 struct expand_operand ops[4];
|
|
|
777 rtx_insn *last = get_last_insn ();
|
|
|
778
|
|
|
779 res = gen_reg_rtx (mode);
|
|
|
780 create_output_operand (&ops[0], res, mode);
|
|
|
781 create_input_operand (&ops[1], op0, mode);
|
|
|
782 create_input_operand (&ops[2], op1, mode);
|
|
|
783 create_fixed_operand (&ops[3], do_error);
|
|
|
784 if (maybe_expand_insn (icode, 4, ops))
|
|
|
785 {
|
|
|
786 last = get_last_insn ();
|
|
|
787 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
788 && JUMP_P (last)
|
|
|
789 && any_condjump_p (last)
|
|
|
790 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
791 add_reg_br_prob_note (last,
|
|
|
792 profile_probability::very_unlikely ());
|
|
|
793 emit_jump (done_label);
|
|
|
794 goto do_error_label;
|
|
|
795 }
|
|
|
796
|
|
|
797 delete_insns_since (last);
|
|
|
798 }
|
|
|
799
|
|
|
800 /* Compute the operation. On RTL level, the addition is always
|
|
|
801 unsigned. */
|
|
|
802 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
803 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
804 rtx tem = op0;
|
|
|
805 /* For PLUS_EXPR, the operation is commutative, so we can pick
|
|
|
806 operand to compare against. For prec <= BITS_PER_WORD, I think
|
|
|
807 preferring REG operand is better over CONST_INT, because
|
|
|
808 the CONST_INT might enlarge the instruction or CSE would need
|
|
|
809 to figure out we'd already loaded it into a register before.
|
|
|
810 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
|
|
|
811 as then the multi-word comparison can be perhaps simplified. */
|
|
|
812 if (code == PLUS_EXPR
|
|
|
813 && (prec <= BITS_PER_WORD
|
|
|
814 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
|
|
|
815 : CONST_SCALAR_INT_P (op1)))
|
|
|
816 tem = op1;
|
|
|
817 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
|
|
|
818 true, mode, NULL_RTX, NULL, done_label,
|
|
|
819 profile_probability::very_likely ());
|
|
|
820 goto do_error_label;
|
|
|
821 }
|
|
|
822
|
|
|
823 /* s1 +- u2 -> sr */
|
|
|
824 if (!uns0_p && uns1_p && !unsr_p)
|
|
|
825 {
|
|
|
826 /* Compute the operation. On RTL level, the addition is always
|
|
|
827 unsigned. */
|
|
|
828 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
829 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
830 rtx tem = expand_binop (mode, add_optab,
|
|
|
831 code == PLUS_EXPR ? res : op0, sgn,
|
|
|
832 NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
833 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
|
|
|
834 done_label, profile_probability::very_likely ());
|
|
|
835 goto do_error_label;
|
|
|
836 }
|
|
|
837
|
|
|
838 /* s1 + u2 -> ur */
|
|
|
839 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
|
|
|
840 {
|
|
|
841 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
|
|
|
842 OPTAB_LIB_WIDEN);
|
|
|
843 /* As we've changed op1, we have to avoid using the value range
|
|
|
844 for the original argument. */
|
|
|
845 arg1 = error_mark_node;
|
|
|
846 do_xor = true;
|
|
|
847 goto do_signed;
|
|
|
848 }
|
|
|
849
|
|
|
850 /* u1 - s2 -> ur */
|
|
|
851 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
|
|
|
852 {
|
|
|
853 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
|
|
|
854 OPTAB_LIB_WIDEN);
|
|
|
855 /* As we've changed op0, we have to avoid using the value range
|
|
|
856 for the original argument. */
|
|
|
857 arg0 = error_mark_node;
|
|
|
858 do_xor = true;
|
|
|
859 goto do_signed;
|
|
|
860 }
|
|
|
861
|
|
|
862 /* s1 - u2 -> ur */
|
|
|
863 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
|
|
|
864 {
|
|
|
865 /* Compute the operation. On RTL level, the addition is always
|
|
|
866 unsigned. */
|
|
|
867 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
|
868 OPTAB_LIB_WIDEN);
|
|
|
869 int pos_neg = get_range_pos_neg (arg0);
|
|
|
870 if (pos_neg == 2)
|
|
|
871 /* If ARG0 is known to be always negative, this is always overflow. */
|
|
|
872 emit_jump (do_error);
|
|
|
873 else if (pos_neg == 3)
|
|
|
874 /* If ARG0 is not known to be always positive, check at runtime. */
|
|
|
875 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
|
|
|
876 NULL, do_error, profile_probability::very_unlikely ());
|
|
|
877 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
|
|
|
878 done_label, profile_probability::very_likely ());
|
|
|
879 goto do_error_label;
|
|
|
880 }
|
|
|
881
|
|
|
882 /* u1 - s2 -> sr */
|
|
|
883 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
|
|
|
884 {
|
|
|
885 /* Compute the operation. On RTL level, the addition is always
|
|
|
886 unsigned. */
|
|
|
887 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
|
888 OPTAB_LIB_WIDEN);
|
|
|
889 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
|
|
|
890 OPTAB_LIB_WIDEN);
|
|
|
891 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
|
|
|
892 done_label, profile_probability::very_likely ());
|
|
|
893 goto do_error_label;
|
|
|
894 }
|
|
|
895
|
|
|
896 /* u1 + u2 -> sr */
|
|
|
897 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
|
|
|
898 {
|
|
|
899 /* Compute the operation. On RTL level, the addition is always
|
|
|
900 unsigned. */
|
|
|
901 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
|
|
|
902 OPTAB_LIB_WIDEN);
|
|
|
903 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
|
|
|
904 NULL, do_error, profile_probability::very_unlikely ());
|
|
|
905 rtx tem = op1;
|
|
|
906 /* The operation is commutative, so we can pick operand to compare
|
|
|
907 against. For prec <= BITS_PER_WORD, I think preferring REG operand
|
|
|
908 is better over CONST_INT, because the CONST_INT might enlarge the
|
|
|
909 instruction or CSE would need to figure out we'd already loaded it
|
|
|
910 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
|
|
|
911 might be more beneficial, as then the multi-word comparison can be
|
|
|
912 perhaps simplified. */
|
|
|
913 if (prec <= BITS_PER_WORD
|
|
|
914 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
|
|
|
915 : CONST_SCALAR_INT_P (op0))
|
|
|
916 tem = op0;
|
|
|
917 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
|
|
|
918 done_label, profile_probability::very_likely ());
|
|
|
919 goto do_error_label;
|
|
|
920 }
|
|
|
921
|
|
|
922 /* s1 +- s2 -> ur */
|
|
|
923 if (!uns0_p && !uns1_p && unsr_p)
|
|
|
924 {
|
|
|
925 /* Compute the operation. On RTL level, the addition is always
|
|
|
926 unsigned. */
|
|
|
927 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
928 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
929 int pos_neg = get_range_pos_neg (arg1);
|
|
|
930 if (code == PLUS_EXPR)
|
|
|
931 {
|
|
|
932 int pos_neg0 = get_range_pos_neg (arg0);
|
|
|
933 if (pos_neg0 != 3 && pos_neg == 3)
|
|
|
934 {
|
|
|
935 std::swap (op0, op1);
|
|
|
936 pos_neg = pos_neg0;
|
|
|
937 }
|
|
|
938 }
|
|
|
939 rtx tem;
|
|
|
940 if (pos_neg != 3)
|
|
|
941 {
|
|
|
942 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
|
|
|
943 ? and_optab : ior_optab,
|
|
|
944 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
945 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
|
|
|
946 NULL, done_label, profile_probability::very_likely ());
|
|
|
947 }
|
|
|
948 else
|
|
|
949 {
|
|
|
950 rtx_code_label *do_ior_label = gen_label_rtx ();
|
|
|
951 do_compare_rtx_and_jump (op1, const0_rtx,
|
|
|
952 code == MINUS_EXPR ? GE : LT, false, mode,
|
|
|
953 NULL_RTX, NULL, do_ior_label,
|
|
|
954 profile_probability::even ());
|
|
|
955 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
|
|
|
956 OPTAB_LIB_WIDEN);
|
|
|
957 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
958 NULL, done_label, profile_probability::very_likely ());
|
|
|
959 emit_jump (do_error);
|
|
|
960 emit_label (do_ior_label);
|
|
|
961 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
|
|
|
962 OPTAB_LIB_WIDEN);
|
|
|
963 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
964 NULL, done_label, profile_probability::very_likely ());
|
|
|
965 }
|
|
|
966 goto do_error_label;
|
|
|
967 }
|
|
|
968
|
|
|
969 /* u1 - u2 -> sr */
|
|
|
970 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
|
|
|
971 {
|
|
|
972 /* Compute the operation. On RTL level, the addition is always
|
|
|
973 unsigned. */
|
|
|
974 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
|
975 OPTAB_LIB_WIDEN);
|
|
|
976 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
|
|
|
977 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
|
|
|
978 op0_geu_op1, profile_probability::even ());
|
|
|
979 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
|
|
|
980 NULL, done_label, profile_probability::very_likely ());
|
|
|
981 emit_jump (do_error);
|
|
|
982 emit_label (op0_geu_op1);
|
|
|
983 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
984 NULL, done_label, profile_probability::very_likely ());
|
|
|
985 goto do_error_label;
|
|
|
986 }
|
|
|
987
|
|
|
988 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
|
|
|
989
|
|
|
990 /* s1 +- s2 -> sr */
|
|
|
991 do_signed:
|
|
|
992 {
|
|
|
993 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
|
|
|
994 : subv4_optab, mode);
|
|
|
995 if (icode != CODE_FOR_nothing)
|
|
|
996 {
|
|
|
997 struct expand_operand ops[4];
|
|
|
998 rtx_insn *last = get_last_insn ();
|
|
|
999
|
|
|
1000 res = gen_reg_rtx (mode);
|
|
|
1001 create_output_operand (&ops[0], res, mode);
|
|
|
1002 create_input_operand (&ops[1], op0, mode);
|
|
|
1003 create_input_operand (&ops[2], op1, mode);
|
|
|
1004 create_fixed_operand (&ops[3], do_error);
|
|
|
1005 if (maybe_expand_insn (icode, 4, ops))
|
|
|
1006 {
|
|
|
1007 last = get_last_insn ();
|
|
|
1008 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
1009 && JUMP_P (last)
|
|
|
1010 && any_condjump_p (last)
|
|
|
1011 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
1012 add_reg_br_prob_note (last,
|
|
|
1013 profile_probability::very_unlikely ());
|
|
|
1014 emit_jump (done_label);
|
|
|
1015 goto do_error_label;
|
|
|
1016 }
|
|
|
1017
|
|
|
1018 delete_insns_since (last);
|
|
|
1019 }
|
|
|
1020
|
|
|
1021 /* Compute the operation. On RTL level, the addition is always
|
|
|
1022 unsigned. */
|
|
|
1023 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
1024 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
1025
|
|
|
1026 /* If we can prove that one of the arguments (for MINUS_EXPR only
|
|
|
1027 the second operand, as subtraction is not commutative) is always
|
|
|
1028 non-negative or always negative, we can do just one comparison
|
|
|
1029 and conditional jump. */
|
|
|
1030 int pos_neg = get_range_pos_neg (arg1);
|
|
|
1031 if (code == PLUS_EXPR)
|
|
|
1032 {
|
|
|
1033 int pos_neg0 = get_range_pos_neg (arg0);
|
|
|
1034 if (pos_neg0 != 3 && pos_neg == 3)
|
|
|
1035 {
|
|
|
1036 std::swap (op0, op1);
|
|
|
1037 pos_neg = pos_neg0;
|
|
|
1038 }
|
|
|
1039 }
|
|
|
1040
|
|
|
1041 /* Addition overflows if and only if the two operands have the same sign,
|
|
|
1042 and the result has the opposite sign. Subtraction overflows if and
|
|
|
1043 only if the two operands have opposite sign, and the subtrahend has
|
|
|
1044 the same sign as the result. Here 0 is counted as positive. */
|
|
|
1045 if (pos_neg == 3)
|
|
|
1046 {
|
|
|
1047 /* Compute op0 ^ op1 (operands have opposite sign). */
|
|
|
1048 rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
|
|
|
1049 OPTAB_LIB_WIDEN);
|
|
|
1050
|
|
|
1051 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
|
|
|
1052 rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
|
|
|
1053 OPTAB_LIB_WIDEN);
|
|
|
1054
|
|
|
1055 rtx tem;
|
|
|
1056 if (code == PLUS_EXPR)
|
|
|
1057 {
|
|
|
1058 /* Compute (res ^ op1) & ~(op0 ^ op1). */
|
|
|
1059 tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
|
|
|
1060 tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
|
|
|
1061 OPTAB_LIB_WIDEN);
|
|
|
1062 }
|
|
|
1063 else
|
|
|
1064 {
|
|
|
1065 /* Compute (op0 ^ op1) & ~(res ^ op1). */
|
|
|
1066 tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
|
|
|
1067 tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
|
|
|
1068 OPTAB_LIB_WIDEN);
|
|
|
1069 }
|
|
|
1070
|
|
|
1071 /* No overflow if the result has bit sign cleared. */
|
|
|
1072 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1073 NULL, done_label, profile_probability::very_likely ());
|
|
|
1074 }
|
|
|
1075
|
|
|
1076 /* Compare the result of the operation with the first operand.
|
|
|
1077 No overflow for addition if second operand is positive and result
|
|
|
1078 is larger or second operand is negative and result is smaller.
|
|
|
1079 Likewise for subtraction with sign of second operand flipped. */
|
|
|
1080 else
|
|
|
1081 do_compare_rtx_and_jump (res, op0,
|
|
|
1082 (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
|
|
|
1083 false, mode, NULL_RTX, NULL, done_label,
|
|
|
1084 profile_probability::very_likely ());
|
|
|
1085 }
|
|
|
1086
|
|
|
1087 do_error_label:
|
|
|
1088 emit_label (do_error);
|
|
|
1089 if (is_ubsan)
|
|
|
1090 {
|
|
|
1091 /* Expand the ubsan builtin call. */
|
|
|
1092 push_temp_slots ();
|
|
|
1093 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
|
|
|
1094 arg0, arg1, datap);
|
|
|
1095 expand_normal (fn);
|
|
|
1096 pop_temp_slots ();
|
|
|
1097 do_pending_stack_adjust ();
|
|
|
1098 }
|
|
|
1099 else if (lhs)
|
|
|
1100 expand_arith_set_overflow (lhs, target);
|
|
|
1101
|
|
|
1102 /* We're done. */
|
|
|
1103 emit_label (done_label);
|
|
|
1104
|
|
|
1105 if (lhs)
|
|
|
1106 {
|
|
|
1107 if (is_ubsan)
|
|
|
1108 expand_ubsan_result_store (target, res);
|
|
|
1109 else
|
|
|
1110 {
|
|
|
1111 if (do_xor)
|
|
|
1112 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
|
|
|
1113 OPTAB_LIB_WIDEN);
|
|
|
1114
|
|
|
1115 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
|
1116 }
|
|
|
1117 }
|
|
|
1118 }
|
|
|
1119
|
|
|
1120 /* Add negate overflow checking to the statement STMT. */
|
|
|
1121
|
|
|
1122 static void
|
|
|
1123 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
|
|
|
1124 tree *datap)
|
|
|
1125 {
|
|
|
1126 rtx res, op1;
|
|
|
1127 tree fn;
|
|
|
1128 rtx_code_label *done_label, *do_error;
|
|
|
1129 rtx target = NULL_RTX;
|
|
|
1130
|
|
|
1131 done_label = gen_label_rtx ();
|
|
|
1132 do_error = gen_label_rtx ();
|
|
|
1133
|
|
|
1134 do_pending_stack_adjust ();
|
|
|
1135 op1 = expand_normal (arg1);
|
|
|
1136
|
|
|
1137 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
|
|
|
1138 if (lhs)
|
|
|
1139 {
|
|
|
1140 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
1141 if (!is_ubsan)
|
|
|
1142 write_complex_part (target, const0_rtx, true);
|
|
|
1143 }
|
|
|
1144
|
|
|
1145 enum insn_code icode = optab_handler (negv3_optab, mode);
|
|
|
1146 if (icode != CODE_FOR_nothing)
|
|
|
1147 {
|
|
|
1148 struct expand_operand ops[3];
|
|
|
1149 rtx_insn *last = get_last_insn ();
|
|
|
1150
|
|
|
1151 res = gen_reg_rtx (mode);
|
|
|
1152 create_output_operand (&ops[0], res, mode);
|
|
|
1153 create_input_operand (&ops[1], op1, mode);
|
|
|
1154 create_fixed_operand (&ops[2], do_error);
|
|
|
1155 if (maybe_expand_insn (icode, 3, ops))
|
|
|
1156 {
|
|
|
1157 last = get_last_insn ();
|
|
|
1158 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
1159 && JUMP_P (last)
|
|
|
1160 && any_condjump_p (last)
|
|
|
1161 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
1162 add_reg_br_prob_note (last,
|
|
|
1163 profile_probability::very_unlikely ());
|
|
|
1164 emit_jump (done_label);
|
|
|
1165 }
|
|
|
1166 else
|
|
|
1167 {
|
|
|
1168 delete_insns_since (last);
|
|
|
1169 icode = CODE_FOR_nothing;
|
|
|
1170 }
|
|
|
1171 }
|
|
|
1172
|
|
|
1173 if (icode == CODE_FOR_nothing)
|
|
|
1174 {
|
|
|
1175 /* Compute the operation. On RTL level, the addition is always
|
|
|
1176 unsigned. */
|
|
|
1177 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
|
|
|
1178
|
|
|
1179 /* Compare the operand with the most negative value. */
|
|
|
1180 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
|
|
|
1181 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
|
|
|
1182 done_label, profile_probability::very_likely ());
|
|
|
1183 }
|
|
|
1184
|
|
|
1185 emit_label (do_error);
|
|
|
1186 if (is_ubsan)
|
|
|
1187 {
|
|
|
1188 /* Expand the ubsan builtin call. */
|
|
|
1189 push_temp_slots ();
|
|
|
1190 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
|
|
|
1191 arg1, NULL_TREE, datap);
|
|
|
1192 expand_normal (fn);
|
|
|
1193 pop_temp_slots ();
|
|
|
1194 do_pending_stack_adjust ();
|
|
|
1195 }
|
|
|
1196 else if (lhs)
|
|
|
1197 expand_arith_set_overflow (lhs, target);
|
|
|
1198
|
|
|
1199 /* We're done. */
|
|
|
1200 emit_label (done_label);
|
|
|
1201
|
|
|
1202 if (lhs)
|
|
|
1203 {
|
|
|
1204 if (is_ubsan)
|
|
|
1205 expand_ubsan_result_store (target, res);
|
|
|
1206 else
|
|
|
1207 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
|
1208 }
|
|
|
1209 }
|
|
|
1210
|
|
131
|
1211 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
|
|
|
1212 mode MODE can be expanded without using a libcall. */
|
|
|
1213
|
|
|
1214 static bool
|
|
|
1215 can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
|
|
|
1216 rtx op0, rtx op1, bool uns)
|
|
|
1217 {
|
|
|
1218 if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
|
|
|
1219 != CODE_FOR_nothing)
|
|
|
1220 return true;
|
|
|
1221
|
|
|
1222 if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
|
|
|
1223 != CODE_FOR_nothing)
|
|
|
1224 return true;
|
|
|
1225
|
|
|
1226 rtx_insn *last = get_last_insn ();
|
|
|
1227 if (CONSTANT_P (op0))
|
|
|
1228 op0 = convert_modes (wmode, mode, op0, uns);
|
|
|
1229 else
|
|
|
1230 op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
|
|
|
1231 if (CONSTANT_P (op1))
|
|
|
1232 op1 = convert_modes (wmode, mode, op1, uns);
|
|
|
1233 else
|
|
|
1234 op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
|
|
|
1235 rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
|
|
|
1236 delete_insns_since (last);
|
|
|
1237 return ret != NULL_RTX;
|
|
|
1238 }
|
|
|
1239
|
|
111
|
1240 /* Add mul overflow checking to the statement STMT. */
|
|
|
1241
|
|
|
1242 static void
|
|
|
1243 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
|
|
|
1244 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
|
|
|
1245 tree *datap)
|
|
|
1246 {
|
|
|
1247 rtx res, op0, op1;
|
|
|
1248 tree fn, type;
|
|
|
1249 rtx_code_label *done_label, *do_error;
|
|
|
1250 rtx target = NULL_RTX;
|
|
|
1251 signop sign;
|
|
|
1252 enum insn_code icode;
|
|
|
1253
|
|
|
1254 done_label = gen_label_rtx ();
|
|
|
1255 do_error = gen_label_rtx ();
|
|
|
1256
|
|
|
1257 do_pending_stack_adjust ();
|
|
|
1258 op0 = expand_normal (arg0);
|
|
|
1259 op1 = expand_normal (arg1);
|
|
|
1260
|
|
|
1261 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
|
|
|
1262 bool uns = unsr_p;
|
|
|
1263 if (lhs)
|
|
|
1264 {
|
|
|
1265 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
1266 if (!is_ubsan)
|
|
|
1267 write_complex_part (target, const0_rtx, true);
|
|
|
1268 }
|
|
|
1269
|
|
|
1270 if (is_ubsan)
|
|
|
1271 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
|
|
|
1272
|
|
|
1273 /* We assume both operands and result have the same precision
|
|
|
1274 here (GET_MODE_BITSIZE (mode)), S stands for signed type
|
|
|
1275 with that precision, U for unsigned type with that precision,
|
|
|
1276 sgn for unsigned most significant bit in that precision.
|
|
|
1277 s1 is signed first operand, u1 is unsigned first operand,
|
|
|
1278 s2 is signed second operand, u2 is unsigned second operand,
|
|
|
1279 sr is signed result, ur is unsigned result and the following
|
|
|
1280 rules say how to compute result (which is always result of
|
|
|
1281 the operands as if both were unsigned, cast to the right
|
|
|
1282 signedness) and how to compute whether operation overflowed.
|
|
|
1283 main_ovf (false) stands for jump on signed multiplication
|
|
|
1284 overflow or the main algorithm with uns == false.
|
|
|
1285 main_ovf (true) stands for jump on unsigned multiplication
|
|
|
1286 overflow or the main algorithm with uns == true.
|
|
|
1287
|
|
|
1288 s1 * s2 -> sr
|
|
|
1289 res = (S) ((U) s1 * (U) s2)
|
|
|
1290 ovf = main_ovf (false)
|
|
|
1291 u1 * u2 -> ur
|
|
|
1292 res = u1 * u2
|
|
|
1293 ovf = main_ovf (true)
|
|
|
1294 s1 * u2 -> ur
|
|
|
1295 res = (U) s1 * u2
|
|
|
1296 ovf = (s1 < 0 && u2) || main_ovf (true)
|
|
|
1297 u1 * u2 -> sr
|
|
|
1298 res = (S) (u1 * u2)
|
|
|
1299 ovf = res < 0 || main_ovf (true)
|
|
|
1300 s1 * u2 -> sr
|
|
|
1301 res = (S) ((U) s1 * u2)
|
|
|
1302 ovf = (S) u2 >= 0 ? main_ovf (false)
|
|
|
1303 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
|
|
|
1304 s1 * s2 -> ur
|
|
|
1305 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
|
|
|
1306 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
|
|
|
1307 res = t1 * t2
|
|
|
1308 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
|
|
|
1309
|
|
|
1310 if (uns0_p && !uns1_p)
|
|
|
1311 {
|
|
|
1312 /* Multiplication is commutative, if operand signedness differs,
|
|
|
1313 canonicalize to the first operand being signed and second
|
|
|
1314 unsigned to simplify following code. */
|
|
|
1315 std::swap (op0, op1);
|
|
|
1316 std::swap (arg0, arg1);
|
|
|
1317 uns0_p = false;
|
|
|
1318 uns1_p = true;
|
|
|
1319 }
|
|
|
1320
|
|
|
1321 int pos_neg0 = get_range_pos_neg (arg0);
|
|
|
1322 int pos_neg1 = get_range_pos_neg (arg1);
|
|
|
1323
|
|
|
1324 /* s1 * u2 -> ur */
|
|
|
1325 if (!uns0_p && uns1_p && unsr_p)
|
|
|
1326 {
|
|
|
1327 switch (pos_neg0)
|
|
|
1328 {
|
|
|
1329 case 1:
|
|
|
1330 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
|
|
|
1331 goto do_main;
|
|
|
1332 case 2:
|
|
|
1333 /* If s1 is negative, avoid the main code, just multiply and
|
|
|
1334 signal overflow if op1 is not 0. */
|
|
|
1335 struct separate_ops ops;
|
|
|
1336 ops.code = MULT_EXPR;
|
|
|
1337 ops.type = TREE_TYPE (arg1);
|
|
|
1338 ops.op0 = make_tree (ops.type, op0);
|
|
|
1339 ops.op1 = make_tree (ops.type, op1);
|
|
|
1340 ops.op2 = NULL_TREE;
|
|
|
1341 ops.location = loc;
|
|
|
1342 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1343 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1344 NULL, done_label, profile_probability::very_likely ());
|
|
|
1345 goto do_error_label;
|
|
|
1346 case 3:
|
|
|
1347 rtx_code_label *do_main_label;
|
|
|
1348 do_main_label = gen_label_rtx ();
|
|
|
1349 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1350 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1351 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1352 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1353 expand_arith_set_overflow (lhs, target);
|
|
|
1354 emit_label (do_main_label);
|
|
|
1355 goto do_main;
|
|
|
1356 default:
|
|
|
1357 gcc_unreachable ();
|
|
|
1358 }
|
|
|
1359 }
|
|
|
1360
|
|
|
1361 /* u1 * u2 -> sr */
|
|
|
1362 if (uns0_p && uns1_p && !unsr_p)
|
|
|
1363 {
|
|
|
1364 uns = true;
|
|
|
1365 /* Rest of handling of this case after res is computed. */
|
|
|
1366 goto do_main;
|
|
|
1367 }
|
|
|
1368
|
|
|
1369 /* s1 * u2 -> sr */
|
|
|
1370 if (!uns0_p && uns1_p && !unsr_p)
|
|
|
1371 {
|
|
|
1372 switch (pos_neg1)
|
|
|
1373 {
|
|
|
1374 case 1:
|
|
|
1375 goto do_main;
|
|
|
1376 case 2:
|
|
|
1377 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
|
|
|
1378 avoid the main code, just multiply and signal overflow
|
|
|
1379 unless 0 * u2 or -1 * ((U) Smin). */
|
|
|
1380 struct separate_ops ops;
|
|
|
1381 ops.code = MULT_EXPR;
|
|
|
1382 ops.type = TREE_TYPE (arg1);
|
|
|
1383 ops.op0 = make_tree (ops.type, op0);
|
|
|
1384 ops.op1 = make_tree (ops.type, op1);
|
|
|
1385 ops.op2 = NULL_TREE;
|
|
|
1386 ops.location = loc;
|
|
|
1387 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1388 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1389 NULL, done_label, profile_probability::very_likely ());
|
|
|
1390 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
|
|
|
1391 NULL, do_error, profile_probability::very_unlikely ());
|
|
|
1392 int prec;
|
|
|
1393 prec = GET_MODE_PRECISION (mode);
|
|
|
1394 rtx sgn;
|
|
|
1395 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
|
|
|
1396 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
|
|
|
1397 NULL, done_label, profile_probability::very_likely ());
|
|
|
1398 goto do_error_label;
|
|
|
1399 case 3:
|
|
|
1400 /* Rest of handling of this case after res is computed. */
|
|
|
1401 goto do_main;
|
|
|
1402 default:
|
|
|
1403 gcc_unreachable ();
|
|
|
1404 }
|
|
|
1405 }
|
|
|
1406
|
|
|
1407 /* s1 * s2 -> ur */
|
|
|
1408 if (!uns0_p && !uns1_p && unsr_p)
|
|
|
1409 {
|
|
|
1410 rtx tem, tem2;
|
|
|
1411 switch (pos_neg0 | pos_neg1)
|
|
|
1412 {
|
|
|
1413 case 1: /* Both operands known to be non-negative. */
|
|
|
1414 goto do_main;
|
|
|
1415 case 2: /* Both operands known to be negative. */
|
|
|
1416 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
|
|
|
1417 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
|
|
|
1418 /* Avoid looking at arg0/arg1 ranges, as we've changed
|
|
|
1419 the arguments. */
|
|
|
1420 arg0 = error_mark_node;
|
|
|
1421 arg1 = error_mark_node;
|
|
|
1422 goto do_main;
|
|
|
1423 case 3:
|
|
|
1424 if ((pos_neg0 ^ pos_neg1) == 3)
|
|
|
1425 {
|
|
|
1426 /* If one operand is known to be negative and the other
|
|
|
1427 non-negative, this overflows always, unless the non-negative
|
|
|
1428 one is 0. Just do normal multiply and set overflow
|
|
|
1429 unless one of the operands is 0. */
|
|
|
1430 struct separate_ops ops;
|
|
|
1431 ops.code = MULT_EXPR;
|
|
|
1432 ops.type
|
|
|
1433 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
|
|
|
1434 1);
|
|
|
1435 ops.op0 = make_tree (ops.type, op0);
|
|
|
1436 ops.op1 = make_tree (ops.type, op1);
|
|
|
1437 ops.op2 = NULL_TREE;
|
|
|
1438 ops.location = loc;
|
|
|
1439 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1440 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
|
|
|
1441 OPTAB_LIB_WIDEN);
|
|
|
1442 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode,
|
|
|
1443 NULL_RTX, NULL, done_label,
|
|
|
1444 profile_probability::very_likely ());
|
|
|
1445 goto do_error_label;
|
|
|
1446 }
|
|
|
1447 /* The general case, do all the needed comparisons at runtime. */
|
|
|
1448 rtx_code_label *do_main_label, *after_negate_label;
|
|
|
1449 rtx rop0, rop1;
|
|
|
1450 rop0 = gen_reg_rtx (mode);
|
|
|
1451 rop1 = gen_reg_rtx (mode);
|
|
|
1452 emit_move_insn (rop0, op0);
|
|
|
1453 emit_move_insn (rop1, op1);
|
|
|
1454 op0 = rop0;
|
|
|
1455 op1 = rop1;
|
|
|
1456 do_main_label = gen_label_rtx ();
|
|
|
1457 after_negate_label = gen_label_rtx ();
|
|
|
1458 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
|
|
|
1459 OPTAB_LIB_WIDEN);
|
|
|
1460 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1461 NULL, after_negate_label, profile_probability::very_likely ());
|
|
|
1462 /* Both arguments negative here, negate them and continue with
|
|
|
1463 normal unsigned overflow checking multiplication. */
|
|
|
1464 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
|
|
|
1465 NULL_RTX, false));
|
|
|
1466 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
|
|
|
1467 NULL_RTX, false));
|
|
|
1468 /* Avoid looking at arg0/arg1 ranges, as we might have changed
|
|
|
1469 the arguments. */
|
|
|
1470 arg0 = error_mark_node;
|
|
|
1471 arg1 = error_mark_node;
|
|
|
1472 emit_jump (do_main_label);
|
|
|
1473 emit_label (after_negate_label);
|
|
|
1474 tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
|
|
|
1475 OPTAB_LIB_WIDEN);
|
|
|
1476 do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1477 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1478 /* One argument is negative here, the other positive. This
|
|
|
1479 overflows always, unless one of the arguments is 0. But
|
|
|
1480 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
|
|
|
1481 is, thus we can keep do_main code oring in overflow as is. */
|
|
|
1482 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1483 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1484 expand_arith_set_overflow (lhs, target);
|
|
|
1485 emit_label (do_main_label);
|
|
|
1486 goto do_main;
|
|
|
1487 default:
|
|
|
1488 gcc_unreachable ();
|
|
|
1489 }
|
|
|
1490 }
|
|
|
1491
|
|
|
1492 do_main:
|
|
|
1493 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
|
|
|
1494 sign = uns ? UNSIGNED : SIGNED;
|
|
|
1495 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
|
|
131
|
1496 if (uns
|
|
|
1497 && (integer_pow2p (arg0) || integer_pow2p (arg1))
|
|
|
1498 && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
|
|
|
1499 {
|
|
|
1500 /* Optimize unsigned multiplication by power of 2 constant
|
|
|
1501 using 2 shifts, one for result, one to extract the shifted
|
|
|
1502 out bits to see if they are all zero.
|
|
|
1503 Don't do this if optimizing for size and we have umulv4_optab,
|
|
|
1504 in that case assume multiplication will be shorter.
|
|
|
1505 This is heuristics based on the single target that provides
|
|
|
1506 umulv4 right now (i?86/x86_64), if further targets add it, this
|
|
|
1507 might need to be revisited.
|
|
|
1508 Cases where both operands are constant should be folded already
|
|
|
1509 during GIMPLE, and cases where one operand is constant but not
|
|
|
1510 power of 2 are questionable, either the WIDEN_MULT_EXPR case
|
|
|
1511 below can be done without multiplication, just by shifts and adds,
|
|
|
1512 or we'd need to divide the result (and hope it actually doesn't
|
|
|
1513 really divide nor multiply) and compare the result of the division
|
|
|
1514 with the original operand. */
|
|
|
1515 rtx opn0 = op0;
|
|
|
1516 rtx opn1 = op1;
|
|
|
1517 tree argn0 = arg0;
|
|
|
1518 tree argn1 = arg1;
|
|
|
1519 if (integer_pow2p (arg0))
|
|
|
1520 {
|
|
|
1521 std::swap (opn0, opn1);
|
|
|
1522 std::swap (argn0, argn1);
|
|
|
1523 }
|
|
|
1524 int cnt = tree_log2 (argn1);
|
|
|
1525 if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
|
|
|
1526 {
|
|
|
1527 rtx upper = const0_rtx;
|
|
|
1528 res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
|
|
|
1529 if (cnt != 0)
|
|
|
1530 upper = expand_shift (RSHIFT_EXPR, mode, opn0,
|
|
|
1531 GET_MODE_PRECISION (mode) - cnt,
|
|
|
1532 NULL_RTX, uns);
|
|
|
1533 do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
|
|
|
1534 NULL_RTX, NULL, done_label,
|
|
|
1535 profile_probability::very_likely ());
|
|
|
1536 goto do_error_label;
|
|
|
1537 }
|
|
|
1538 }
|
|
111
|
1539 if (icode != CODE_FOR_nothing)
|
|
|
1540 {
|
|
|
1541 struct expand_operand ops[4];
|
|
|
1542 rtx_insn *last = get_last_insn ();
|
|
|
1543
|
|
|
1544 res = gen_reg_rtx (mode);
|
|
|
1545 create_output_operand (&ops[0], res, mode);
|
|
|
1546 create_input_operand (&ops[1], op0, mode);
|
|
|
1547 create_input_operand (&ops[2], op1, mode);
|
|
|
1548 create_fixed_operand (&ops[3], do_error);
|
|
|
1549 if (maybe_expand_insn (icode, 4, ops))
|
|
|
1550 {
|
|
|
1551 last = get_last_insn ();
|
|
|
1552 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
1553 && JUMP_P (last)
|
|
|
1554 && any_condjump_p (last)
|
|
|
1555 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
1556 add_reg_br_prob_note (last,
|
|
|
1557 profile_probability::very_unlikely ());
|
|
|
1558 emit_jump (done_label);
|
|
|
1559 }
|
|
|
1560 else
|
|
|
1561 {
|
|
|
1562 delete_insns_since (last);
|
|
|
1563 icode = CODE_FOR_nothing;
|
|
|
1564 }
|
|
|
1565 }
|
|
|
1566
|
|
|
1567 if (icode == CODE_FOR_nothing)
|
|
|
1568 {
|
|
|
1569 struct separate_ops ops;
|
|
|
1570 int prec = GET_MODE_PRECISION (mode);
|
|
|
1571 scalar_int_mode hmode, wmode;
|
|
|
1572 ops.op0 = make_tree (type, op0);
|
|
|
1573 ops.op1 = make_tree (type, op1);
|
|
|
1574 ops.op2 = NULL_TREE;
|
|
|
1575 ops.location = loc;
|
|
131
|
1576
|
|
|
1577 /* Optimize unsigned overflow check where we don't use the
|
|
|
1578 multiplication result, just whether overflow happened.
|
|
|
1579 If we can do MULT_HIGHPART_EXPR, that followed by
|
|
|
1580 comparison of the result against zero is cheapest.
|
|
|
1581 We'll still compute res, but it should be DCEd later. */
|
|
|
1582 use_operand_p use;
|
|
|
1583 gimple *use_stmt;
|
|
|
1584 if (!is_ubsan
|
|
|
1585 && lhs
|
|
|
1586 && uns
|
|
|
1587 && !(uns0_p && uns1_p && !unsr_p)
|
|
|
1588 && can_mult_highpart_p (mode, uns) == 1
|
|
|
1589 && single_imm_use (lhs, &use, &use_stmt)
|
|
|
1590 && is_gimple_assign (use_stmt)
|
|
|
1591 && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
|
|
|
1592 goto highpart;
|
|
|
1593
|
|
111
|
1594 if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
|
|
131
|
1595 && targetm.scalar_mode_supported_p (wmode)
|
|
|
1596 && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
|
|
111
|
1597 {
|
|
131
|
1598 twoxwider:
|
|
111
|
1599 ops.code = WIDEN_MULT_EXPR;
|
|
|
1600 ops.type
|
|
|
1601 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
|
|
|
1602
|
|
|
1603 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
|
|
|
1604 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
|
|
|
1605 NULL_RTX, uns);
|
|
|
1606 hipart = convert_modes (mode, wmode, hipart, uns);
|
|
|
1607 res = convert_modes (mode, wmode, res, uns);
|
|
|
1608 if (uns)
|
|
|
1609 /* For the unsigned multiplication, there was overflow if
|
|
|
1610 HIPART is non-zero. */
|
|
|
1611 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
|
|
|
1612 NULL_RTX, NULL, done_label,
|
|
|
1613 profile_probability::very_likely ());
|
|
|
1614 else
|
|
|
1615 {
|
|
|
1616 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
|
|
|
1617 NULL_RTX, 0);
|
|
|
1618 /* RES is low half of the double width result, HIPART
|
|
|
1619 the high half. There was overflow if
|
|
|
1620 HIPART is different from RES < 0 ? -1 : 0. */
|
|
|
1621 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
|
|
|
1622 NULL_RTX, NULL, done_label,
|
|
|
1623 profile_probability::very_likely ());
|
|
|
1624 }
|
|
|
1625 }
|
|
131
|
1626 else if (can_mult_highpart_p (mode, uns) == 1)
|
|
|
1627 {
|
|
|
1628 highpart:
|
|
|
1629 ops.code = MULT_HIGHPART_EXPR;
|
|
|
1630 ops.type = type;
|
|
|
1631
|
|
|
1632 rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
|
|
|
1633 EXPAND_NORMAL);
|
|
|
1634 ops.code = MULT_EXPR;
|
|
|
1635 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1636 if (uns)
|
|
|
1637 /* For the unsigned multiplication, there was overflow if
|
|
|
1638 HIPART is non-zero. */
|
|
|
1639 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
|
|
|
1640 NULL_RTX, NULL, done_label,
|
|
|
1641 profile_probability::very_likely ());
|
|
|
1642 else
|
|
|
1643 {
|
|
|
1644 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
|
|
|
1645 NULL_RTX, 0);
|
|
|
1646 /* RES is low half of the double width result, HIPART
|
|
|
1647 the high half. There was overflow if
|
|
|
1648 HIPART is different from RES < 0 ? -1 : 0. */
|
|
|
1649 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
|
|
|
1650 NULL_RTX, NULL, done_label,
|
|
|
1651 profile_probability::very_likely ());
|
|
|
1652 }
|
|
|
1653
|
|
|
1654 }
|
|
111
|
1655 else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
|
|
|
1656 && 2 * GET_MODE_PRECISION (hmode) == prec)
|
|
|
1657 {
|
|
|
1658 rtx_code_label *large_op0 = gen_label_rtx ();
|
|
|
1659 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
|
|
|
1660 rtx_code_label *one_small_one_large = gen_label_rtx ();
|
|
|
1661 rtx_code_label *both_ops_large = gen_label_rtx ();
|
|
|
1662 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
|
|
|
1663 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
|
|
|
1664 rtx_code_label *do_overflow = gen_label_rtx ();
|
|
|
1665 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
|
|
|
1666
|
|
|
1667 unsigned int hprec = GET_MODE_PRECISION (hmode);
|
|
|
1668 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
|
|
|
1669 NULL_RTX, uns);
|
|
|
1670 hipart0 = convert_modes (hmode, mode, hipart0, uns);
|
|
|
1671 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
|
|
|
1672 rtx signbit0 = const0_rtx;
|
|
|
1673 if (!uns)
|
|
|
1674 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
|
|
|
1675 NULL_RTX, 0);
|
|
|
1676 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
|
|
|
1677 NULL_RTX, uns);
|
|
|
1678 hipart1 = convert_modes (hmode, mode, hipart1, uns);
|
|
|
1679 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
|
|
|
1680 rtx signbit1 = const0_rtx;
|
|
|
1681 if (!uns)
|
|
|
1682 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
|
|
|
1683 NULL_RTX, 0);
|
|
|
1684
|
|
|
1685 res = gen_reg_rtx (mode);
|
|
|
1686
|
|
|
1687 /* True if op0 resp. op1 are known to be in the range of
|
|
|
1688 halfstype. */
|
|
|
1689 bool op0_small_p = false;
|
|
|
1690 bool op1_small_p = false;
|
|
|
1691 /* True if op0 resp. op1 are known to have all zeros or all ones
|
|
|
1692 in the upper half of bits, but are not known to be
|
|
|
1693 op{0,1}_small_p. */
|
|
|
1694 bool op0_medium_p = false;
|
|
|
1695 bool op1_medium_p = false;
|
|
|
1696 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
|
|
|
1697 nonnegative, 1 if unknown. */
|
|
|
1698 int op0_sign = 1;
|
|
|
1699 int op1_sign = 1;
|
|
|
1700
|
|
|
1701 if (pos_neg0 == 1)
|
|
|
1702 op0_sign = 0;
|
|
|
1703 else if (pos_neg0 == 2)
|
|
|
1704 op0_sign = -1;
|
|
|
1705 if (pos_neg1 == 1)
|
|
|
1706 op1_sign = 0;
|
|
|
1707 else if (pos_neg1 == 2)
|
|
|
1708 op1_sign = -1;
|
|
|
1709
|
|
|
1710 unsigned int mprec0 = prec;
|
|
|
1711 if (arg0 != error_mark_node)
|
|
|
1712 mprec0 = get_min_precision (arg0, sign);
|
|
|
1713 if (mprec0 <= hprec)
|
|
|
1714 op0_small_p = true;
|
|
|
1715 else if (!uns && mprec0 <= hprec + 1)
|
|
|
1716 op0_medium_p = true;
|
|
|
1717 unsigned int mprec1 = prec;
|
|
|
1718 if (arg1 != error_mark_node)
|
|
|
1719 mprec1 = get_min_precision (arg1, sign);
|
|
|
1720 if (mprec1 <= hprec)
|
|
|
1721 op1_small_p = true;
|
|
|
1722 else if (!uns && mprec1 <= hprec + 1)
|
|
|
1723 op1_medium_p = true;
|
|
|
1724
|
|
|
1725 int smaller_sign = 1;
|
|
|
1726 int larger_sign = 1;
|
|
|
1727 if (op0_small_p)
|
|
|
1728 {
|
|
|
1729 smaller_sign = op0_sign;
|
|
|
1730 larger_sign = op1_sign;
|
|
|
1731 }
|
|
|
1732 else if (op1_small_p)
|
|
|
1733 {
|
|
|
1734 smaller_sign = op1_sign;
|
|
|
1735 larger_sign = op0_sign;
|
|
|
1736 }
|
|
|
1737 else if (op0_sign == op1_sign)
|
|
|
1738 {
|
|
|
1739 smaller_sign = op0_sign;
|
|
|
1740 larger_sign = op0_sign;
|
|
|
1741 }
|
|
|
1742
|
|
|
1743 if (!op0_small_p)
|
|
|
1744 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
|
|
|
1745 NULL_RTX, NULL, large_op0,
|
|
|
1746 profile_probability::unlikely ());
|
|
|
1747
|
|
|
1748 if (!op1_small_p)
|
|
|
1749 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
|
|
|
1750 NULL_RTX, NULL, small_op0_large_op1,
|
|
|
1751 profile_probability::unlikely ());
|
|
|
1752
|
|
|
1753 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
|
|
|
1754 hmode to mode, the multiplication will never overflow. We can
|
|
|
1755 do just one hmode x hmode => mode widening multiplication. */
|
|
|
1756 rtx lopart0s = lopart0, lopart1s = lopart1;
|
|
|
1757 if (GET_CODE (lopart0) == SUBREG)
|
|
|
1758 {
|
|
|
1759 lopart0s = shallow_copy_rtx (lopart0);
|
|
|
1760 SUBREG_PROMOTED_VAR_P (lopart0s) = 1;
|
|
|
1761 SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED);
|
|
|
1762 }
|
|
|
1763 if (GET_CODE (lopart1) == SUBREG)
|
|
|
1764 {
|
|
|
1765 lopart1s = shallow_copy_rtx (lopart1);
|
|
|
1766 SUBREG_PROMOTED_VAR_P (lopart1s) = 1;
|
|
|
1767 SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED);
|
|
|
1768 }
|
|
|
1769 tree halfstype = build_nonstandard_integer_type (hprec, uns);
|
|
|
1770 ops.op0 = make_tree (halfstype, lopart0s);
|
|
|
1771 ops.op1 = make_tree (halfstype, lopart1s);
|
|
|
1772 ops.code = WIDEN_MULT_EXPR;
|
|
|
1773 ops.type = type;
|
|
|
1774 rtx thisres
|
|
|
1775 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1776 emit_move_insn (res, thisres);
|
|
|
1777 emit_jump (done_label);
|
|
|
1778
|
|
|
1779 emit_label (small_op0_large_op1);
|
|
|
1780
|
|
|
1781 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
|
|
|
1782 but op1 is not, just swap the arguments and handle it as op1
|
|
|
1783 sign/zero extended, op0 not. */
|
|
|
1784 rtx larger = gen_reg_rtx (mode);
|
|
|
1785 rtx hipart = gen_reg_rtx (hmode);
|
|
|
1786 rtx lopart = gen_reg_rtx (hmode);
|
|
|
1787 emit_move_insn (larger, op1);
|
|
|
1788 emit_move_insn (hipart, hipart1);
|
|
|
1789 emit_move_insn (lopart, lopart0);
|
|
|
1790 emit_jump (one_small_one_large);
|
|
|
1791
|
|
|
1792 emit_label (large_op0);
|
|
|
1793
|
|
|
1794 if (!op1_small_p)
|
|
|
1795 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
|
|
|
1796 NULL_RTX, NULL, both_ops_large,
|
|
|
1797 profile_probability::unlikely ());
|
|
|
1798
|
|
|
1799 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
|
|
|
1800 but op0 is not, prepare larger, hipart and lopart pseudos and
|
|
|
1801 handle it together with small_op0_large_op1. */
|
|
|
1802 emit_move_insn (larger, op0);
|
|
|
1803 emit_move_insn (hipart, hipart0);
|
|
|
1804 emit_move_insn (lopart, lopart1);
|
|
|
1805
|
|
|
1806 emit_label (one_small_one_large);
|
|
|
1807
|
|
|
1808 /* lopart is the low part of the operand that is sign extended
|
|
|
1809 to mode, larger is the other operand, hipart is the
|
|
|
1810 high part of larger and lopart0 and lopart1 are the low parts
|
|
|
1811 of both operands.
|
|
|
1812 We perform lopart0 * lopart1 and lopart * hipart widening
|
|
|
1813 multiplications. */
|
|
|
1814 tree halfutype = build_nonstandard_integer_type (hprec, 1);
|
|
|
1815 ops.op0 = make_tree (halfutype, lopart0);
|
|
|
1816 ops.op1 = make_tree (halfutype, lopart1);
|
|
|
1817 rtx lo0xlo1
|
|
|
1818 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1819
|
|
|
1820 ops.op0 = make_tree (halfutype, lopart);
|
|
|
1821 ops.op1 = make_tree (halfutype, hipart);
|
|
|
1822 rtx loxhi = gen_reg_rtx (mode);
|
|
|
1823 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1824 emit_move_insn (loxhi, tem);
|
|
|
1825
|
|
|
1826 if (!uns)
|
|
|
1827 {
|
|
|
1828 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
|
|
|
1829 if (larger_sign == 0)
|
|
|
1830 emit_jump (after_hipart_neg);
|
|
|
1831 else if (larger_sign != -1)
|
|
|
1832 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
|
|
|
1833 NULL_RTX, NULL, after_hipart_neg,
|
|
|
1834 profile_probability::even ());
|
|
|
1835
|
|
|
1836 tem = convert_modes (mode, hmode, lopart, 1);
|
|
|
1837 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
|
|
|
1838 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
|
|
131
|
1839 1, OPTAB_WIDEN);
|
|
111
|
1840 emit_move_insn (loxhi, tem);
|
|
|
1841
|
|
|
1842 emit_label (after_hipart_neg);
|
|
|
1843
|
|
|
1844 /* if (lopart < 0) loxhi -= larger; */
|
|
|
1845 if (smaller_sign == 0)
|
|
|
1846 emit_jump (after_lopart_neg);
|
|
|
1847 else if (smaller_sign != -1)
|
|
|
1848 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
|
|
|
1849 NULL_RTX, NULL, after_lopart_neg,
|
|
|
1850 profile_probability::even ());
|
|
|
1851
|
|
|
1852 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
|
|
131
|
1853 1, OPTAB_WIDEN);
|
|
111
|
1854 emit_move_insn (loxhi, tem);
|
|
|
1855
|
|
|
1856 emit_label (after_lopart_neg);
|
|
|
1857 }
|
|
|
1858
|
|
|
1859 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
|
|
|
1860 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
|
|
|
1861 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
|
|
131
|
1862 1, OPTAB_WIDEN);
|
|
111
|
1863 emit_move_insn (loxhi, tem);
|
|
|
1864
|
|
|
1865 /* if (loxhi >> (bitsize / 2)
|
|
|
1866 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
|
|
|
1867 if (loxhi >> (bitsize / 2) == 0 (if uns). */
|
|
|
1868 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
|
|
|
1869 NULL_RTX, 0);
|
|
|
1870 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
|
|
|
1871 rtx signbitloxhi = const0_rtx;
|
|
|
1872 if (!uns)
|
|
|
1873 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
|
|
|
1874 convert_modes (hmode, mode,
|
|
|
1875 loxhi, 0),
|
|
|
1876 hprec - 1, NULL_RTX, 0);
|
|
|
1877
|
|
|
1878 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
|
|
|
1879 NULL_RTX, NULL, do_overflow,
|
|
|
1880 profile_probability::very_unlikely ());
|
|
|
1881
|
|
|
1882 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
|
|
|
1883 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
|
|
|
1884 NULL_RTX, 1);
|
|
|
1885 tem = convert_modes (mode, hmode,
|
|
|
1886 convert_modes (hmode, mode, lo0xlo1, 1), 1);
|
|
|
1887
|
|
|
1888 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
|
|
131
|
1889 1, OPTAB_WIDEN);
|
|
111
|
1890 if (tem != res)
|
|
|
1891 emit_move_insn (res, tem);
|
|
|
1892 emit_jump (done_label);
|
|
|
1893
|
|
|
1894 emit_label (both_ops_large);
|
|
|
1895
|
|
|
1896 /* If both operands are large (not sign (!uns) or zero (uns)
|
|
|
1897 extended from hmode), then perform the full multiplication
|
|
|
1898 which will be the result of the operation.
|
|
|
1899 The only cases which don't overflow are for signed multiplication
|
|
|
1900 some cases where both hipart0 and highpart1 are 0 or -1.
|
|
|
1901 For unsigned multiplication when high parts are both non-zero
|
|
|
1902 this overflows always. */
|
|
|
1903 ops.code = MULT_EXPR;
|
|
|
1904 ops.op0 = make_tree (type, op0);
|
|
|
1905 ops.op1 = make_tree (type, op1);
|
|
|
1906 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1907 emit_move_insn (res, tem);
|
|
|
1908
|
|
|
1909 if (!uns)
|
|
|
1910 {
|
|
|
1911 if (!op0_medium_p)
|
|
|
1912 {
|
|
|
1913 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
|
|
131
|
1914 NULL_RTX, 1, OPTAB_WIDEN);
|
|
111
|
1915 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
|
|
|
1916 NULL_RTX, NULL, do_error,
|
|
|
1917 profile_probability::very_unlikely ());
|
|
|
1918 }
|
|
|
1919
|
|
|
1920 if (!op1_medium_p)
|
|
|
1921 {
|
|
|
1922 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
|
|
131
|
1923 NULL_RTX, 1, OPTAB_WIDEN);
|
|
111
|
1924 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
|
|
|
1925 NULL_RTX, NULL, do_error,
|
|
|
1926 profile_probability::very_unlikely ());
|
|
|
1927 }
|
|
|
1928
|
|
|
1929 /* At this point hipart{0,1} are both in [-1, 0]. If they are
|
|
|
1930 the same, overflow happened if res is non-positive, if they
|
|
|
1931 are different, overflow happened if res is positive. */
|
|
|
1932 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
|
|
|
1933 emit_jump (hipart_different);
|
|
|
1934 else if (op0_sign == 1 || op1_sign == 1)
|
|
|
1935 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
|
|
|
1936 NULL_RTX, NULL, hipart_different,
|
|
|
1937 profile_probability::even ());
|
|
|
1938
|
|
|
1939 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
|
|
|
1940 NULL_RTX, NULL, do_error,
|
|
|
1941 profile_probability::very_unlikely ());
|
|
|
1942 emit_jump (done_label);
|
|
|
1943
|
|
|
1944 emit_label (hipart_different);
|
|
|
1945
|
|
|
1946 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
|
|
|
1947 NULL_RTX, NULL, do_error,
|
|
|
1948 profile_probability::very_unlikely ());
|
|
|
1949 emit_jump (done_label);
|
|
|
1950 }
|
|
|
1951
|
|
|
1952 emit_label (do_overflow);
|
|
|
1953
|
|
|
1954 /* Overflow, do full multiplication and fallthru into do_error. */
|
|
|
1955 ops.op0 = make_tree (type, op0);
|
|
|
1956 ops.op1 = make_tree (type, op1);
|
|
|
1957 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1958 emit_move_insn (res, tem);
|
|
|
1959 }
|
|
131
|
1960 else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
|
|
|
1961 && targetm.scalar_mode_supported_p (wmode))
|
|
|
1962 /* Even emitting a libcall is better than not detecting overflow
|
|
|
1963 at all. */
|
|
|
1964 goto twoxwider;
|
|
111
|
1965 else
|
|
|
1966 {
|
|
|
1967 gcc_assert (!is_ubsan);
|
|
|
1968 ops.code = MULT_EXPR;
|
|
|
1969 ops.type = type;
|
|
|
1970 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1971 emit_jump (done_label);
|
|
|
1972 }
|
|
|
1973 }
|
|
|
1974
|
|
|
1975 do_error_label:
|
|
|
1976 emit_label (do_error);
|
|
|
1977 if (is_ubsan)
|
|
|
1978 {
|
|
|
1979 /* Expand the ubsan builtin call. */
|
|
|
1980 push_temp_slots ();
|
|
|
1981 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
|
|
|
1982 arg0, arg1, datap);
|
|
|
1983 expand_normal (fn);
|
|
|
1984 pop_temp_slots ();
|
|
|
1985 do_pending_stack_adjust ();
|
|
|
1986 }
|
|
|
1987 else if (lhs)
|
|
|
1988 expand_arith_set_overflow (lhs, target);
|
|
|
1989
|
|
|
1990 /* We're done. */
|
|
|
1991 emit_label (done_label);
|
|
|
1992
|
|
|
1993 /* u1 * u2 -> sr */
|
|
|
1994 if (uns0_p && uns1_p && !unsr_p)
|
|
|
1995 {
|
|
|
1996 rtx_code_label *all_done_label = gen_label_rtx ();
|
|
|
1997 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1998 NULL, all_done_label, profile_probability::very_likely ());
|
|
|
1999 expand_arith_set_overflow (lhs, target);
|
|
|
2000 emit_label (all_done_label);
|
|
|
2001 }
|
|
|
2002
|
|
|
2003 /* s1 * u2 -> sr */
|
|
|
2004 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
|
|
|
2005 {
|
|
|
2006 rtx_code_label *all_done_label = gen_label_rtx ();
|
|
|
2007 rtx_code_label *set_noovf = gen_label_rtx ();
|
|
|
2008 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
2009 NULL, all_done_label, profile_probability::very_likely ());
|
|
|
2010 expand_arith_set_overflow (lhs, target);
|
|
|
2011 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
2012 NULL, set_noovf, profile_probability::very_likely ());
|
|
|
2013 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
|
|
|
2014 NULL, all_done_label, profile_probability::very_unlikely ());
|
|
|
2015 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
|
|
|
2016 all_done_label, profile_probability::very_unlikely ());
|
|
|
2017 emit_label (set_noovf);
|
|
|
2018 write_complex_part (target, const0_rtx, true);
|
|
|
2019 emit_label (all_done_label);
|
|
|
2020 }
|
|
|
2021
|
|
|
2022 if (lhs)
|
|
|
2023 {
|
|
|
2024 if (is_ubsan)
|
|
|
2025 expand_ubsan_result_store (target, res);
|
|
|
2026 else
|
|
|
2027 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
|
2028 }
|
|
|
2029 }
|
|
|
2030
|
|
|
2031 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
|
|
|
2032
|
|
|
2033 static void
|
|
|
2034 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
|
|
|
2035 tree arg0, tree arg1)
|
|
|
2036 {
|
|
131
|
2037 poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
|
|
111
|
2038 rtx_code_label *loop_lab = NULL;
|
|
|
2039 rtx cntvar = NULL_RTX;
|
|
|
2040 tree cntv = NULL_TREE;
|
|
|
2041 tree eltype = TREE_TYPE (TREE_TYPE (arg0));
|
|
|
2042 tree sz = TYPE_SIZE (eltype);
|
|
|
2043 tree data = NULL_TREE;
|
|
|
2044 tree resv = NULL_TREE;
|
|
|
2045 rtx lhsr = NULL_RTX;
|
|
|
2046 rtx resvr = NULL_RTX;
|
|
131
|
2047 unsigned HOST_WIDE_INT const_cnt = 0;
|
|
|
2048 bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
|
|
111
|
2049
|
|
|
2050 if (lhs)
|
|
|
2051 {
|
|
|
2052 optab op;
|
|
|
2053 lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2054 if (!VECTOR_MODE_P (GET_MODE (lhsr))
|
|
|
2055 || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
|
|
|
2056 optab_default)) == unknown_optab
|
|
|
2057 || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
|
|
|
2058 == CODE_FOR_nothing))
|
|
|
2059 {
|
|
|
2060 if (MEM_P (lhsr))
|
|
|
2061 resv = make_tree (TREE_TYPE (lhs), lhsr);
|
|
|
2062 else
|
|
|
2063 {
|
|
|
2064 resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
|
|
|
2065 resv = make_tree (TREE_TYPE (lhs), resvr);
|
|
|
2066 }
|
|
|
2067 }
|
|
|
2068 }
|
|
131
|
2069 if (use_loop_p)
|
|
111
|
2070 {
|
|
|
2071 do_pending_stack_adjust ();
|
|
|
2072 loop_lab = gen_label_rtx ();
|
|
|
2073 cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
|
|
|
2074 cntv = make_tree (sizetype, cntvar);
|
|
|
2075 emit_move_insn (cntvar, const0_rtx);
|
|
|
2076 emit_label (loop_lab);
|
|
|
2077 }
|
|
|
2078 if (TREE_CODE (arg0) != VECTOR_CST)
|
|
|
2079 {
|
|
|
2080 rtx arg0r = expand_normal (arg0);
|
|
|
2081 arg0 = make_tree (TREE_TYPE (arg0), arg0r);
|
|
|
2082 }
|
|
|
2083 if (TREE_CODE (arg1) != VECTOR_CST)
|
|
|
2084 {
|
|
|
2085 rtx arg1r = expand_normal (arg1);
|
|
|
2086 arg1 = make_tree (TREE_TYPE (arg1), arg1r);
|
|
|
2087 }
|
|
131
|
2088 for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
|
|
111
|
2089 {
|
|
|
2090 tree op0, op1, res = NULL_TREE;
|
|
131
|
2091 if (use_loop_p)
|
|
111
|
2092 {
|
|
|
2093 tree atype = build_array_type_nelts (eltype, cnt);
|
|
|
2094 op0 = uniform_vector_p (arg0);
|
|
|
2095 if (op0 == NULL_TREE)
|
|
|
2096 {
|
|
|
2097 op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
|
|
|
2098 op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
|
|
|
2099 NULL_TREE, NULL_TREE);
|
|
|
2100 }
|
|
|
2101 op1 = uniform_vector_p (arg1);
|
|
|
2102 if (op1 == NULL_TREE)
|
|
|
2103 {
|
|
|
2104 op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
|
|
|
2105 op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
|
|
|
2106 NULL_TREE, NULL_TREE);
|
|
|
2107 }
|
|
|
2108 if (resv)
|
|
|
2109 {
|
|
|
2110 res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
|
|
|
2111 res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
|
|
|
2112 NULL_TREE, NULL_TREE);
|
|
|
2113 }
|
|
|
2114 }
|
|
|
2115 else
|
|
|
2116 {
|
|
|
2117 tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
|
|
|
2118 op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
|
|
|
2119 op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
|
|
|
2120 if (resv)
|
|
|
2121 res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
|
|
|
2122 bitpos);
|
|
|
2123 }
|
|
|
2124 switch (code)
|
|
|
2125 {
|
|
|
2126 case PLUS_EXPR:
|
|
|
2127 expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
|
|
|
2128 false, false, false, true, &data);
|
|
|
2129 break;
|
|
|
2130 case MINUS_EXPR:
|
|
131
|
2131 if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
|
|
111
|
2132 expand_neg_overflow (loc, res, op1, true, &data);
|
|
|
2133 else
|
|
|
2134 expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
|
|
|
2135 false, false, false, true, &data);
|
|
|
2136 break;
|
|
|
2137 case MULT_EXPR:
|
|
|
2138 expand_mul_overflow (loc, res, op0, op1, false, false, false,
|
|
|
2139 true, &data);
|
|
|
2140 break;
|
|
|
2141 default:
|
|
|
2142 gcc_unreachable ();
|
|
|
2143 }
|
|
|
2144 }
|
|
131
|
2145 if (use_loop_p)
|
|
111
|
2146 {
|
|
|
2147 struct separate_ops ops;
|
|
|
2148 ops.code = PLUS_EXPR;
|
|
|
2149 ops.type = TREE_TYPE (cntv);
|
|
|
2150 ops.op0 = cntv;
|
|
|
2151 ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
|
|
|
2152 ops.op2 = NULL_TREE;
|
|
|
2153 ops.location = loc;
|
|
|
2154 rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
|
|
|
2155 EXPAND_NORMAL);
|
|
|
2156 if (ret != cntvar)
|
|
|
2157 emit_move_insn (cntvar, ret);
|
|
131
|
2158 rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
|
|
|
2159 do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
|
|
111
|
2160 TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
|
|
|
2161 profile_probability::very_likely ());
|
|
|
2162 }
|
|
|
2163 if (lhs && resv == NULL_TREE)
|
|
|
2164 {
|
|
|
2165 struct separate_ops ops;
|
|
|
2166 ops.code = code;
|
|
|
2167 ops.type = TREE_TYPE (arg0);
|
|
|
2168 ops.op0 = arg0;
|
|
|
2169 ops.op1 = arg1;
|
|
|
2170 ops.op2 = NULL_TREE;
|
|
|
2171 ops.location = loc;
|
|
|
2172 rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
|
|
|
2173 EXPAND_NORMAL);
|
|
|
2174 if (ret != lhsr)
|
|
|
2175 emit_move_insn (lhsr, ret);
|
|
|
2176 }
|
|
|
2177 else if (resvr)
|
|
|
2178 emit_move_insn (lhsr, resvr);
|
|
|
2179 }
|
|
|
2180
|
|
|
2181 /* Expand UBSAN_CHECK_ADD call STMT. */
|
|
|
2182
|
|
|
2183 static void
|
|
|
2184 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
|
|
|
2185 {
|
|
|
2186 location_t loc = gimple_location (stmt);
|
|
|
2187 tree lhs = gimple_call_lhs (stmt);
|
|
|
2188 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2189 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2190 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
|
2191 expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
|
|
|
2192 else
|
|
|
2193 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
|
|
|
2194 false, false, false, true, NULL);
|
|
|
2195 }
|
|
|
2196
|
|
|
2197 /* Expand UBSAN_CHECK_SUB call STMT. */
|
|
|
2198
|
|
|
2199 static void
|
|
|
2200 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
|
|
|
2201 {
|
|
|
2202 location_t loc = gimple_location (stmt);
|
|
|
2203 tree lhs = gimple_call_lhs (stmt);
|
|
|
2204 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2205 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2206 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
|
2207 expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
|
|
|
2208 else if (integer_zerop (arg0))
|
|
|
2209 expand_neg_overflow (loc, lhs, arg1, true, NULL);
|
|
|
2210 else
|
|
|
2211 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
|
|
|
2212 false, false, false, true, NULL);
|
|
|
2213 }
|
|
|
2214
|
|
|
2215 /* Expand UBSAN_CHECK_MUL call STMT. */
|
|
|
2216
|
|
|
2217 static void
|
|
|
2218 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
|
|
|
2219 {
|
|
|
2220 location_t loc = gimple_location (stmt);
|
|
|
2221 tree lhs = gimple_call_lhs (stmt);
|
|
|
2222 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2223 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2224 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
|
2225 expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
|
|
|
2226 else
|
|
|
2227 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
|
|
|
2228 NULL);
|
|
|
2229 }
|
|
|
2230
|
|
|
2231 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
|
|
|
2232
|
|
|
2233 static void
|
|
|
2234 expand_arith_overflow (enum tree_code code, gimple *stmt)
|
|
|
2235 {
|
|
|
2236 tree lhs = gimple_call_lhs (stmt);
|
|
|
2237 if (lhs == NULL_TREE)
|
|
|
2238 return;
|
|
|
2239 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2240 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2241 tree type = TREE_TYPE (TREE_TYPE (lhs));
|
|
|
2242 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
|
|
|
2243 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
|
|
|
2244 int unsr_p = TYPE_UNSIGNED (type);
|
|
|
2245 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
|
|
|
2246 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
|
|
|
2247 int precres = TYPE_PRECISION (type);
|
|
|
2248 location_t loc = gimple_location (stmt);
|
|
|
2249 if (!uns0_p && get_range_pos_neg (arg0) == 1)
|
|
|
2250 uns0_p = true;
|
|
|
2251 if (!uns1_p && get_range_pos_neg (arg1) == 1)
|
|
|
2252 uns1_p = true;
|
|
|
2253 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
|
|
|
2254 prec0 = MIN (prec0, pr);
|
|
|
2255 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
|
|
|
2256 prec1 = MIN (prec1, pr);
|
|
|
2257
|
|
|
2258 /* If uns0_p && uns1_p, precop is minimum needed precision
|
|
|
2259 of unsigned type to hold the exact result, otherwise
|
|
|
2260 precop is minimum needed precision of signed type to
|
|
|
2261 hold the exact result. */
|
|
|
2262 int precop;
|
|
|
2263 if (code == MULT_EXPR)
|
|
|
2264 precop = prec0 + prec1 + (uns0_p != uns1_p);
|
|
|
2265 else
|
|
|
2266 {
|
|
|
2267 if (uns0_p == uns1_p)
|
|
|
2268 precop = MAX (prec0, prec1) + 1;
|
|
|
2269 else if (uns0_p)
|
|
|
2270 precop = MAX (prec0 + 1, prec1) + 1;
|
|
|
2271 else
|
|
|
2272 precop = MAX (prec0, prec1 + 1) + 1;
|
|
|
2273 }
|
|
|
2274 int orig_precres = precres;
|
|
|
2275
|
|
|
2276 do
|
|
|
2277 {
|
|
|
2278 if ((uns0_p && uns1_p)
|
|
|
2279 ? ((precop + !unsr_p) <= precres
|
|
|
2280 /* u1 - u2 -> ur can overflow, no matter what precision
|
|
|
2281 the result has. */
|
|
|
2282 && (code != MINUS_EXPR || !unsr_p))
|
|
|
2283 : (!unsr_p && precop <= precres))
|
|
|
2284 {
|
|
|
2285 /* The infinity precision result will always fit into result. */
|
|
|
2286 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2287 write_complex_part (target, const0_rtx, true);
|
|
|
2288 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
|
|
|
2289 struct separate_ops ops;
|
|
|
2290 ops.code = code;
|
|
|
2291 ops.type = type;
|
|
|
2292 ops.op0 = fold_convert_loc (loc, type, arg0);
|
|
|
2293 ops.op1 = fold_convert_loc (loc, type, arg1);
|
|
|
2294 ops.op2 = NULL_TREE;
|
|
|
2295 ops.location = loc;
|
|
|
2296 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
2297 expand_arith_overflow_result_store (lhs, target, mode, tem);
|
|
|
2298 return;
|
|
|
2299 }
|
|
|
2300
|
|
|
2301 /* For operations with low precision, if target doesn't have them, start
|
|
|
2302 with precres widening right away, otherwise do it only if the most
|
|
|
2303 simple cases can't be used. */
|
|
|
2304 const int min_precision = targetm.min_arithmetic_precision ();
|
|
|
2305 if (orig_precres == precres && precres < min_precision)
|
|
|
2306 ;
|
|
|
2307 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
|
|
|
2308 && prec1 <= precres)
|
|
|
2309 || ((!uns0_p || !uns1_p) && !unsr_p
|
|
|
2310 && prec0 + uns0_p <= precres
|
|
|
2311 && prec1 + uns1_p <= precres))
|
|
|
2312 {
|
|
|
2313 arg0 = fold_convert_loc (loc, type, arg0);
|
|
|
2314 arg1 = fold_convert_loc (loc, type, arg1);
|
|
|
2315 switch (code)
|
|
|
2316 {
|
|
|
2317 case MINUS_EXPR:
|
|
|
2318 if (integer_zerop (arg0) && !unsr_p)
|
|
|
2319 {
|
|
|
2320 expand_neg_overflow (loc, lhs, arg1, false, NULL);
|
|
|
2321 return;
|
|
|
2322 }
|
|
|
2323 /* FALLTHRU */
|
|
|
2324 case PLUS_EXPR:
|
|
|
2325 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
|
|
|
2326 unsr_p, unsr_p, false, NULL);
|
|
|
2327 return;
|
|
|
2328 case MULT_EXPR:
|
|
|
2329 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
|
|
|
2330 unsr_p, unsr_p, false, NULL);
|
|
|
2331 return;
|
|
|
2332 default:
|
|
|
2333 gcc_unreachable ();
|
|
|
2334 }
|
|
|
2335 }
|
|
|
2336
|
|
|
2337 /* For sub-word operations, retry with a wider type first. */
|
|
|
2338 if (orig_precres == precres && precop <= BITS_PER_WORD)
|
|
|
2339 {
|
|
|
2340 int p = MAX (min_precision, precop);
|
|
|
2341 scalar_int_mode m = smallest_int_mode_for_size (p);
|
|
|
2342 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
|
|
|
2343 uns0_p && uns1_p
|
|
|
2344 && unsr_p);
|
|
|
2345 p = TYPE_PRECISION (optype);
|
|
|
2346 if (p > precres)
|
|
|
2347 {
|
|
|
2348 precres = p;
|
|
|
2349 unsr_p = TYPE_UNSIGNED (optype);
|
|
|
2350 type = optype;
|
|
|
2351 continue;
|
|
|
2352 }
|
|
|
2353 }
|
|
|
2354
|
|
|
2355 if (prec0 <= precres && prec1 <= precres)
|
|
|
2356 {
|
|
|
2357 tree types[2];
|
|
|
2358 if (unsr_p)
|
|
|
2359 {
|
|
|
2360 types[0] = build_nonstandard_integer_type (precres, 0);
|
|
|
2361 types[1] = type;
|
|
|
2362 }
|
|
|
2363 else
|
|
|
2364 {
|
|
|
2365 types[0] = type;
|
|
|
2366 types[1] = build_nonstandard_integer_type (precres, 1);
|
|
|
2367 }
|
|
|
2368 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
|
|
|
2369 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
|
|
|
2370 if (code != MULT_EXPR)
|
|
|
2371 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
|
|
|
2372 uns0_p, uns1_p, false, NULL);
|
|
|
2373 else
|
|
|
2374 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
|
|
|
2375 uns0_p, uns1_p, false, NULL);
|
|
|
2376 return;
|
|
|
2377 }
|
|
|
2378
|
|
|
2379 /* Retry with a wider type. */
|
|
|
2380 if (orig_precres == precres)
|
|
|
2381 {
|
|
|
2382 int p = MAX (prec0, prec1);
|
|
|
2383 scalar_int_mode m = smallest_int_mode_for_size (p);
|
|
|
2384 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
|
|
|
2385 uns0_p && uns1_p
|
|
|
2386 && unsr_p);
|
|
|
2387 p = TYPE_PRECISION (optype);
|
|
|
2388 if (p > precres)
|
|
|
2389 {
|
|
|
2390 precres = p;
|
|
|
2391 unsr_p = TYPE_UNSIGNED (optype);
|
|
|
2392 type = optype;
|
|
|
2393 continue;
|
|
|
2394 }
|
|
|
2395 }
|
|
|
2396
|
|
|
2397 gcc_unreachable ();
|
|
|
2398 }
|
|
|
2399 while (1);
|
|
|
2400 }
|
|
|
2401
|
|
|
2402 /* Expand ADD_OVERFLOW STMT. */
|
|
|
2403
|
|
|
2404 static void
|
|
|
2405 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
|
|
|
2406 {
|
|
|
2407 expand_arith_overflow (PLUS_EXPR, stmt);
|
|
|
2408 }
|
|
|
2409
|
|
|
2410 /* Expand SUB_OVERFLOW STMT. */
|
|
|
2411
|
|
|
2412 static void
|
|
|
2413 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
|
|
|
2414 {
|
|
|
2415 expand_arith_overflow (MINUS_EXPR, stmt);
|
|
|
2416 }
|
|
|
2417
|
|
|
2418 /* Expand MUL_OVERFLOW STMT. */
|
|
|
2419
|
|
|
2420 static void
|
|
|
2421 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
|
|
|
2422 {
|
|
|
2423 expand_arith_overflow (MULT_EXPR, stmt);
|
|
|
2424 }
|
|
|
2425
|
|
|
2426 /* This should get folded in tree-vectorizer.c. */
|
|
|
2427
|
|
|
2428 static void
|
|
|
2429 expand_LOOP_VECTORIZED (internal_fn, gcall *)
|
|
|
2430 {
|
|
|
2431 gcc_unreachable ();
|
|
|
2432 }
|
|
|
2433
|
|
|
2434 /* This should get folded in tree-vectorizer.c. */
|
|
|
2435
|
|
|
2436 static void
|
|
|
2437 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
|
|
|
2438 {
|
|
|
2439 gcc_unreachable ();
|
|
|
2440 }
|
|
|
2441
|
|
131
|
2442 /* Return a memory reference of type TYPE for argument INDEX of STMT.
|
|
|
2443 Use argument INDEX + 1 to derive the second (TBAA) operand. */
|
|
|
2444
|
|
|
2445 static tree
|
|
|
2446 expand_call_mem_ref (tree type, gcall *stmt, int index)
|
|
|
2447 {
|
|
|
2448 tree addr = gimple_call_arg (stmt, index);
|
|
|
2449 tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
|
|
|
2450 unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
|
|
|
2451 if (TYPE_ALIGN (type) != align)
|
|
|
2452 type = build_aligned_type (type, align);
|
|
|
2453
|
|
|
2454 tree tmp = addr;
|
|
|
2455 if (TREE_CODE (tmp) == SSA_NAME)
|
|
|
2456 {
|
|
|
2457 gimple *def = SSA_NAME_DEF_STMT (tmp);
|
|
|
2458 if (gimple_assign_single_p (def))
|
|
|
2459 tmp = gimple_assign_rhs1 (def);
|
|
|
2460 }
|
|
|
2461
|
|
|
2462 if (TREE_CODE (tmp) == ADDR_EXPR)
|
|
|
2463 {
|
|
|
2464 tree mem = TREE_OPERAND (tmp, 0);
|
|
|
2465 if (TREE_CODE (mem) == TARGET_MEM_REF
|
|
|
2466 && types_compatible_p (TREE_TYPE (mem), type))
|
|
|
2467 {
|
|
|
2468 tree offset = TMR_OFFSET (mem);
|
|
|
2469 if (type != TREE_TYPE (mem)
|
|
|
2470 || alias_ptr_type != TREE_TYPE (offset)
|
|
|
2471 || !integer_zerop (offset))
|
|
|
2472 {
|
|
|
2473 mem = copy_node (mem);
|
|
|
2474 TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
|
|
|
2475 wi::to_poly_wide (offset));
|
|
|
2476 TREE_TYPE (mem) = type;
|
|
|
2477 }
|
|
|
2478 return mem;
|
|
|
2479 }
|
|
|
2480 }
|
|
|
2481
|
|
|
2482 return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
|
|
|
2483 }
|
|
|
2484
|
|
|
2485 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
|
|
111
|
2486
|
|
|
2487 static void
|
|
|
2488 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
2489 {
|
|
|
2490 struct expand_operand ops[3];
|
|
131
|
2491 tree type, lhs, rhs, maskt;
|
|
111
|
2492 rtx mem, target, mask;
|
|
131
|
2493 insn_code icode;
|
|
111
|
2494
|
|
|
2495 maskt = gimple_call_arg (stmt, 2);
|
|
|
2496 lhs = gimple_call_lhs (stmt);
|
|
|
2497 if (lhs == NULL_TREE)
|
|
|
2498 return;
|
|
|
2499 type = TREE_TYPE (lhs);
|
|
131
|
2500 rhs = expand_call_mem_ref (type, stmt, 0);
|
|
|
2501
|
|
|
2502 if (optab == vec_mask_load_lanes_optab)
|
|
|
2503 icode = get_multi_vector_move (type, optab);
|
|
|
2504 else
|
|
|
2505 icode = convert_optab_handler (optab, TYPE_MODE (type),
|
|
|
2506 TYPE_MODE (TREE_TYPE (maskt)));
|
|
111
|
2507
|
|
|
2508 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2509 gcc_assert (MEM_P (mem));
|
|
|
2510 mask = expand_normal (maskt);
|
|
|
2511 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2512 create_output_operand (&ops[0], target, TYPE_MODE (type));
|
|
|
2513 create_fixed_operand (&ops[1], mem);
|
|
|
2514 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
|
|
131
|
2515 expand_insn (icode, 3, ops);
|
|
111
|
2516 }
|
|
|
2517
|
|
131
|
2518 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
|
|
|
2519
|
|
|
2520 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
|
|
111
|
2521
|
|
|
2522 static void
|
|
|
2523 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
2524 {
|
|
|
2525 struct expand_operand ops[3];
|
|
131
|
2526 tree type, lhs, rhs, maskt;
|
|
111
|
2527 rtx mem, reg, mask;
|
|
131
|
2528 insn_code icode;
|
|
111
|
2529
|
|
|
2530 maskt = gimple_call_arg (stmt, 2);
|
|
|
2531 rhs = gimple_call_arg (stmt, 3);
|
|
|
2532 type = TREE_TYPE (rhs);
|
|
131
|
2533 lhs = expand_call_mem_ref (type, stmt, 0);
|
|
|
2534
|
|
|
2535 if (optab == vec_mask_store_lanes_optab)
|
|
|
2536 icode = get_multi_vector_move (type, optab);
|
|
|
2537 else
|
|
|
2538 icode = convert_optab_handler (optab, TYPE_MODE (type),
|
|
|
2539 TYPE_MODE (TREE_TYPE (maskt)));
|
|
111
|
2540
|
|
|
2541 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2542 gcc_assert (MEM_P (mem));
|
|
|
2543 mask = expand_normal (maskt);
|
|
|
2544 reg = expand_normal (rhs);
|
|
|
2545 create_fixed_operand (&ops[0], mem);
|
|
|
2546 create_input_operand (&ops[1], reg, TYPE_MODE (type));
|
|
|
2547 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
|
|
131
|
2548 expand_insn (icode, 3, ops);
|
|
111
|
2549 }
|
|
|
2550
|
|
131
|
2551 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
|
|
|
2552
|
|
111
|
2553 static void
|
|
|
2554 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
|
|
|
2555 {
|
|
|
2556 }
|
|
|
2557
|
|
|
2558 static void
|
|
|
2559 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
|
|
|
2560 {
|
|
|
2561 /* When guessing was done, the hints should be already stripped away. */
|
|
|
2562 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
|
|
|
2563
|
|
|
2564 rtx target;
|
|
|
2565 tree lhs = gimple_call_lhs (stmt);
|
|
|
2566 if (lhs)
|
|
|
2567 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2568 else
|
|
|
2569 target = const0_rtx;
|
|
|
2570 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
|
|
|
2571 if (lhs && val != target)
|
|
|
2572 emit_move_insn (target, val);
|
|
|
2573 }
|
|
|
2574
|
|
|
2575 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
|
|
|
2576 should never be called. */
|
|
|
2577
|
|
|
2578 static void
|
|
|
2579 expand_VA_ARG (internal_fn, gcall *)
|
|
|
2580 {
|
|
|
2581 gcc_unreachable ();
|
|
|
2582 }
|
|
|
2583
|
|
|
2584 /* Expand the IFN_UNIQUE function according to its first argument. */
|
|
|
2585
|
|
|
2586 static void
|
|
|
2587 expand_UNIQUE (internal_fn, gcall *stmt)
|
|
|
2588 {
|
|
|
2589 rtx pattern = NULL_RTX;
|
|
|
2590 enum ifn_unique_kind kind
|
|
|
2591 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
|
|
|
2592
|
|
|
2593 switch (kind)
|
|
|
2594 {
|
|
|
2595 default:
|
|
|
2596 gcc_unreachable ();
|
|
|
2597
|
|
|
2598 case IFN_UNIQUE_UNSPEC:
|
|
|
2599 if (targetm.have_unique ())
|
|
|
2600 pattern = targetm.gen_unique ();
|
|
|
2601 break;
|
|
|
2602
|
|
|
2603 case IFN_UNIQUE_OACC_FORK:
|
|
|
2604 case IFN_UNIQUE_OACC_JOIN:
|
|
|
2605 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
|
|
|
2606 {
|
|
|
2607 tree lhs = gimple_call_lhs (stmt);
|
|
|
2608 rtx target = const0_rtx;
|
|
|
2609
|
|
|
2610 if (lhs)
|
|
|
2611 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2612
|
|
|
2613 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
2614 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
|
|
|
2615
|
|
|
2616 if (kind == IFN_UNIQUE_OACC_FORK)
|
|
|
2617 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
|
|
|
2618 else
|
|
|
2619 pattern = targetm.gen_oacc_join (target, data_dep, axis);
|
|
|
2620 }
|
|
|
2621 else
|
|
|
2622 gcc_unreachable ();
|
|
|
2623 break;
|
|
|
2624 }
|
|
|
2625
|
|
|
2626 if (pattern)
|
|
|
2627 emit_insn (pattern);
|
|
|
2628 }
|
|
|
2629
|
|
|
2630 /* The size of an OpenACC compute dimension. */
|
|
|
2631
|
|
|
2632 static void
|
|
|
2633 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
|
|
|
2634 {
|
|
|
2635 tree lhs = gimple_call_lhs (stmt);
|
|
|
2636
|
|
|
2637 if (!lhs)
|
|
|
2638 return;
|
|
|
2639
|
|
|
2640 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2641 if (targetm.have_oacc_dim_size ())
|
|
|
2642 {
|
|
|
2643 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
|
|
|
2644 VOIDmode, EXPAND_NORMAL);
|
|
|
2645 emit_insn (targetm.gen_oacc_dim_size (target, dim));
|
|
|
2646 }
|
|
|
2647 else
|
|
|
2648 emit_move_insn (target, GEN_INT (1));
|
|
|
2649 }
|
|
|
2650
|
|
|
2651 /* The position of an OpenACC execution engine along one compute axis. */
|
|
|
2652
|
|
|
2653 static void
|
|
|
2654 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
|
|
|
2655 {
|
|
|
2656 tree lhs = gimple_call_lhs (stmt);
|
|
|
2657
|
|
|
2658 if (!lhs)
|
|
|
2659 return;
|
|
|
2660
|
|
|
2661 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2662 if (targetm.have_oacc_dim_pos ())
|
|
|
2663 {
|
|
|
2664 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
|
|
|
2665 VOIDmode, EXPAND_NORMAL);
|
|
|
2666 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
|
|
|
2667 }
|
|
|
2668 else
|
|
|
2669 emit_move_insn (target, const0_rtx);
|
|
|
2670 }
|
|
|
2671
|
|
|
2672 /* This is expanded by oacc_device_lower pass. */
|
|
|
2673
|
|
|
2674 static void
|
|
|
2675 expand_GOACC_LOOP (internal_fn, gcall *)
|
|
|
2676 {
|
|
|
2677 gcc_unreachable ();
|
|
|
2678 }
|
|
|
2679
|
|
|
2680 /* This is expanded by oacc_device_lower pass. */
|
|
|
2681
|
|
|
2682 static void
|
|
|
2683 expand_GOACC_REDUCTION (internal_fn, gcall *)
|
|
|
2684 {
|
|
|
2685 gcc_unreachable ();
|
|
|
2686 }
|
|
|
2687
|
|
|
2688 /* This is expanded by oacc_device_lower pass. */
|
|
|
2689
|
|
|
2690 static void
|
|
|
2691 expand_GOACC_TILE (internal_fn, gcall *)
|
|
|
2692 {
|
|
|
2693 gcc_unreachable ();
|
|
|
2694 }
|
|
|
2695
|
|
|
2696 /* Set errno to EDOM. */
|
|
|
2697
|
|
|
2698 static void
|
|
|
2699 expand_SET_EDOM (internal_fn, gcall *)
|
|
|
2700 {
|
|
|
2701 #ifdef TARGET_EDOM
|
|
|
2702 #ifdef GEN_ERRNO_RTX
|
|
|
2703 rtx errno_rtx = GEN_ERRNO_RTX;
|
|
|
2704 #else
|
|
|
2705 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
|
|
|
2706 #endif
|
|
|
2707 emit_move_insn (errno_rtx,
|
|
|
2708 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
|
|
|
2709 #else
|
|
|
2710 gcc_unreachable ();
|
|
|
2711 #endif
|
|
|
2712 }
|
|
|
2713
|
|
|
2714 /* Expand atomic bit test and set. */
|
|
|
2715
|
|
|
2716 static void
|
|
|
2717 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
|
|
|
2718 {
|
|
|
2719 expand_ifn_atomic_bit_test_and (call);
|
|
|
2720 }
|
|
|
2721
|
|
|
2722 /* Expand atomic bit test and complement. */
|
|
|
2723
|
|
|
2724 static void
|
|
|
2725 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
|
|
|
2726 {
|
|
|
2727 expand_ifn_atomic_bit_test_and (call);
|
|
|
2728 }
|
|
|
2729
|
|
|
2730 /* Expand atomic bit test and reset. */
|
|
|
2731
|
|
|
2732 static void
|
|
|
2733 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
|
|
|
2734 {
|
|
|
2735 expand_ifn_atomic_bit_test_and (call);
|
|
|
2736 }
|
|
|
2737
|
|
|
2738 /* Expand atomic bit test and set. */
|
|
|
2739
|
|
|
2740 static void
|
|
|
2741 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
|
|
|
2742 {
|
|
|
2743 expand_ifn_atomic_compare_exchange (call);
|
|
|
2744 }
|
|
|
2745
|
|
|
2746 /* Expand LAUNDER to assignment, lhs = arg0. */
|
|
|
2747
|
|
|
2748 static void
|
|
|
2749 expand_LAUNDER (internal_fn, gcall *call)
|
|
|
2750 {
|
|
|
2751 tree lhs = gimple_call_lhs (call);
|
|
|
2752
|
|
|
2753 if (!lhs)
|
|
|
2754 return;
|
|
|
2755
|
|
|
2756 expand_assignment (lhs, gimple_call_arg (call, 0), false);
|
|
|
2757 }
|
|
|
2758
|
|
131
|
2759 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
|
|
|
2760
|
|
|
2761 static void
|
|
|
2762 expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
|
|
|
2763 {
|
|
|
2764 internal_fn ifn = gimple_call_internal_fn (stmt);
|
|
|
2765 int rhs_index = internal_fn_stored_value_index (ifn);
|
|
|
2766 int mask_index = internal_fn_mask_index (ifn);
|
|
|
2767 tree base = gimple_call_arg (stmt, 0);
|
|
|
2768 tree offset = gimple_call_arg (stmt, 1);
|
|
|
2769 tree scale = gimple_call_arg (stmt, 2);
|
|
|
2770 tree rhs = gimple_call_arg (stmt, rhs_index);
|
|
|
2771
|
|
|
2772 rtx base_rtx = expand_normal (base);
|
|
|
2773 rtx offset_rtx = expand_normal (offset);
|
|
|
2774 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
|
|
|
2775 rtx rhs_rtx = expand_normal (rhs);
|
|
|
2776
|
|
|
2777 struct expand_operand ops[6];
|
|
|
2778 int i = 0;
|
|
|
2779 create_address_operand (&ops[i++], base_rtx);
|
|
|
2780 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
|
|
|
2781 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
|
|
|
2782 create_integer_operand (&ops[i++], scale_int);
|
|
|
2783 create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
|
|
|
2784 if (mask_index >= 0)
|
|
|
2785 {
|
|
|
2786 tree mask = gimple_call_arg (stmt, mask_index);
|
|
|
2787 rtx mask_rtx = expand_normal (mask);
|
|
|
2788 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
|
|
|
2789 }
|
|
|
2790
|
|
|
2791 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)));
|
|
|
2792 expand_insn (icode, i, ops);
|
|
|
2793 }
|
|
|
2794
|
|
|
2795 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
|
|
|
2796
|
|
|
2797 static void
|
|
|
2798 expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
|
|
|
2799 {
|
|
|
2800 tree lhs = gimple_call_lhs (stmt);
|
|
|
2801 tree base = gimple_call_arg (stmt, 0);
|
|
|
2802 tree offset = gimple_call_arg (stmt, 1);
|
|
|
2803 tree scale = gimple_call_arg (stmt, 2);
|
|
|
2804
|
|
|
2805 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2806 rtx base_rtx = expand_normal (base);
|
|
|
2807 rtx offset_rtx = expand_normal (offset);
|
|
|
2808 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
|
|
|
2809
|
|
|
2810 int i = 0;
|
|
|
2811 struct expand_operand ops[6];
|
|
|
2812 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
|
|
|
2813 create_address_operand (&ops[i++], base_rtx);
|
|
|
2814 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
|
|
|
2815 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
|
|
|
2816 create_integer_operand (&ops[i++], scale_int);
|
|
|
2817 if (optab == mask_gather_load_optab)
|
|
|
2818 {
|
|
|
2819 tree mask = gimple_call_arg (stmt, 3);
|
|
|
2820 rtx mask_rtx = expand_normal (mask);
|
|
|
2821 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
|
|
|
2822 }
|
|
|
2823 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)));
|
|
|
2824 expand_insn (icode, i, ops);
|
|
|
2825 }
|
|
|
2826
|
|
111
|
2827 /* Expand DIVMOD() using:
|
|
|
2828 a) optab handler for udivmod/sdivmod if it is available.
|
|
|
2829 b) If optab_handler doesn't exist, generate call to
|
|
|
2830 target-specific divmod libfunc. */
|
|
|
2831
|
|
|
2832 static void
|
|
|
2833 expand_DIVMOD (internal_fn, gcall *call_stmt)
|
|
|
2834 {
|
|
|
2835 tree lhs = gimple_call_lhs (call_stmt);
|
|
|
2836 tree arg0 = gimple_call_arg (call_stmt, 0);
|
|
|
2837 tree arg1 = gimple_call_arg (call_stmt, 1);
|
|
|
2838
|
|
|
2839 gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
|
|
|
2840 tree type = TREE_TYPE (TREE_TYPE (lhs));
|
|
|
2841 machine_mode mode = TYPE_MODE (type);
|
|
|
2842 bool unsignedp = TYPE_UNSIGNED (type);
|
|
|
2843 optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
|
|
|
2844
|
|
|
2845 rtx op0 = expand_normal (arg0);
|
|
|
2846 rtx op1 = expand_normal (arg1);
|
|
|
2847 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2848
|
|
|
2849 rtx quotient, remainder, libfunc;
|
|
|
2850
|
|
|
2851 /* Check if optab_handler exists for divmod_optab for given mode. */
|
|
|
2852 if (optab_handler (tab, mode) != CODE_FOR_nothing)
|
|
|
2853 {
|
|
|
2854 quotient = gen_reg_rtx (mode);
|
|
|
2855 remainder = gen_reg_rtx (mode);
|
|
|
2856 expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
|
|
|
2857 }
|
|
|
2858
|
|
|
2859 /* Generate call to divmod libfunc if it exists. */
|
|
|
2860 else if ((libfunc = optab_libfunc (tab, mode)) != NULL_RTX)
|
|
|
2861 targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
|
|
|
2862 "ient, &remainder);
|
|
|
2863
|
|
|
2864 else
|
|
|
2865 gcc_unreachable ();
|
|
|
2866
|
|
|
2867 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
|
|
|
2868 expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
|
|
|
2869 make_tree (TREE_TYPE (arg0), quotient),
|
|
|
2870 make_tree (TREE_TYPE (arg1), remainder)),
|
|
131
|
2871 target, VOIDmode, EXPAND_NORMAL);
|
|
|
2872 }
|
|
|
2873
|
|
|
2874 /* Expand a NOP. */
|
|
|
2875
|
|
|
2876 static void
|
|
|
2877 expand_NOP (internal_fn, gcall *)
|
|
|
2878 {
|
|
|
2879 /* Nothing. But it shouldn't really prevail. */
|
|
111
|
2880 }
|
|
|
2881
|
|
|
2882 /* Expand a call to FN using the operands in STMT. FN has a single
|
|
|
2883 output operand and NARGS input operands. */
|
|
|
2884
|
|
|
2885 static void
|
|
|
2886 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
|
|
|
2887 unsigned int nargs)
|
|
|
2888 {
|
|
|
2889 expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
|
|
|
2890
|
|
|
2891 tree_pair types = direct_internal_fn_types (fn, stmt);
|
|
|
2892 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
|
|
131
|
2893 gcc_assert (icode != CODE_FOR_nothing);
|
|
111
|
2894
|
|
|
2895 tree lhs = gimple_call_lhs (stmt);
|
|
131
|
2896 rtx lhs_rtx = NULL_RTX;
|
|
|
2897 if (lhs)
|
|
|
2898 lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
111
|
2899
|
|
|
2900 /* Do not assign directly to a promoted subreg, since there is no
|
|
|
2901 guarantee that the instruction will leave the upper bits of the
|
|
|
2902 register in the state required by SUBREG_PROMOTED_SIGN. */
|
|
|
2903 rtx dest = lhs_rtx;
|
|
131
|
2904 if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
|
|
111
|
2905 dest = NULL_RTX;
|
|
|
2906
|
|
|
2907 create_output_operand (&ops[0], dest, insn_data[icode].operand[0].mode);
|
|
|
2908
|
|
|
2909 for (unsigned int i = 0; i < nargs; ++i)
|
|
|
2910 {
|
|
|
2911 tree rhs = gimple_call_arg (stmt, i);
|
|
|
2912 tree rhs_type = TREE_TYPE (rhs);
|
|
|
2913 rtx rhs_rtx = expand_normal (rhs);
|
|
|
2914 if (INTEGRAL_TYPE_P (rhs_type))
|
|
|
2915 create_convert_operand_from (&ops[i + 1], rhs_rtx,
|
|
|
2916 TYPE_MODE (rhs_type),
|
|
|
2917 TYPE_UNSIGNED (rhs_type));
|
|
|
2918 else
|
|
|
2919 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
|
|
|
2920 }
|
|
|
2921
|
|
|
2922 expand_insn (icode, nargs + 1, ops);
|
|
131
|
2923 if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
|
|
111
|
2924 {
|
|
|
2925 /* If the return value has an integral type, convert the instruction
|
|
|
2926 result to that type. This is useful for things that return an
|
|
|
2927 int regardless of the size of the input. If the instruction result
|
|
|
2928 is smaller than required, assume that it is signed.
|
|
|
2929
|
|
|
2930 If the return value has a nonintegral type, its mode must match
|
|
|
2931 the instruction result. */
|
|
|
2932 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
|
|
|
2933 {
|
|
|
2934 /* If this is a scalar in a register that is stored in a wider
|
|
|
2935 mode than the declared mode, compute the result into its
|
|
|
2936 declared mode and then convert to the wider mode. */
|
|
131
|
2937 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
|
|
111
|
2938 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
|
|
|
2939 convert_move (SUBREG_REG (lhs_rtx), tmp,
|
|
|
2940 SUBREG_PROMOTED_SIGN (lhs_rtx));
|
|
|
2941 }
|
|
|
2942 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
|
|
|
2943 emit_move_insn (lhs_rtx, ops[0].value);
|
|
|
2944 else
|
|
|
2945 {
|
|
131
|
2946 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
|
|
111
|
2947 convert_move (lhs_rtx, ops[0].value, 0);
|
|
|
2948 }
|
|
|
2949 }
|
|
|
2950 }
|
|
|
2951
|
|
131
|
2952 /* Expand WHILE_ULT call STMT using optab OPTAB. */
|
|
|
2953
|
|
|
2954 static void
|
|
|
2955 expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
2956 {
|
|
|
2957 expand_operand ops[3];
|
|
|
2958 tree rhs_type[2];
|
|
|
2959
|
|
|
2960 tree lhs = gimple_call_lhs (stmt);
|
|
|
2961 tree lhs_type = TREE_TYPE (lhs);
|
|
|
2962 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2963 create_output_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
|
|
|
2964
|
|
|
2965 for (unsigned int i = 0; i < 2; ++i)
|
|
|
2966 {
|
|
|
2967 tree rhs = gimple_call_arg (stmt, i);
|
|
|
2968 rhs_type[i] = TREE_TYPE (rhs);
|
|
|
2969 rtx rhs_rtx = expand_normal (rhs);
|
|
|
2970 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
|
|
|
2971 }
|
|
|
2972
|
|
|
2973 insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
|
|
|
2974 TYPE_MODE (lhs_type));
|
|
|
2975
|
|
|
2976 expand_insn (icode, 3, ops);
|
|
|
2977 if (!rtx_equal_p (lhs_rtx, ops[0].value))
|
|
|
2978 emit_move_insn (lhs_rtx, ops[0].value);
|
|
|
2979 }
|
|
|
2980
|
|
111
|
2981 /* Expanders for optabs that can use expand_direct_optab_fn. */
|
|
|
2982
|
|
|
2983 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2984 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
|
|
|
2985
|
|
|
2986 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2987 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
|
|
|
2988
|
|
131
|
2989 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2990 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
|
|
|
2991
|
|
|
2992 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2993 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
|
|
|
2994
|
|
|
2995 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2996 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
|
|
|
2997
|
|
|
2998 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2999 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
|
|
|
3000
|
|
|
3001 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
|
|
|
3002 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
|
|
|
3003
|
|
|
3004 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
|
|
|
3005 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
|
|
|
3006
|
|
111
|
3007 /* RETURN_TYPE and ARGS are a return type and argument list that are
|
|
|
3008 in principle compatible with FN (which satisfies direct_internal_fn_p).
|
|
|
3009 Return the types that should be used to determine whether the
|
|
|
3010 target supports FN. */
|
|
|
3011
|
|
|
3012 tree_pair
|
|
|
3013 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
|
|
|
3014 {
|
|
|
3015 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
|
3016 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
|
|
|
3017 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
|
|
|
3018 return tree_pair (type0, type1);
|
|
|
3019 }
|
|
|
3020
|
|
|
3021 /* CALL is a call whose return type and arguments are in principle
|
|
|
3022 compatible with FN (which satisfies direct_internal_fn_p). Return the
|
|
|
3023 types that should be used to determine whether the target supports FN. */
|
|
|
3024
|
|
|
3025 tree_pair
|
|
|
3026 direct_internal_fn_types (internal_fn fn, gcall *call)
|
|
|
3027 {
|
|
|
3028 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
|
3029 tree op0 = (info.type0 < 0
|
|
|
3030 ? gimple_call_lhs (call)
|
|
|
3031 : gimple_call_arg (call, info.type0));
|
|
|
3032 tree op1 = (info.type1 < 0
|
|
|
3033 ? gimple_call_lhs (call)
|
|
|
3034 : gimple_call_arg (call, info.type1));
|
|
|
3035 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
|
|
|
3036 }
|
|
|
3037
|
|
|
3038 /* Return true if OPTAB is supported for TYPES (whose modes should be
|
|
|
3039 the same) when the optimization type is OPT_TYPE. Used for simple
|
|
|
3040 direct optabs. */
|
|
|
3041
|
|
|
3042 static bool
|
|
|
3043 direct_optab_supported_p (direct_optab optab, tree_pair types,
|
|
|
3044 optimization_type opt_type)
|
|
|
3045 {
|
|
|
3046 machine_mode mode = TYPE_MODE (types.first);
|
|
|
3047 gcc_checking_assert (mode == TYPE_MODE (types.second));
|
|
|
3048 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
|
|
|
3049 }
|
|
|
3050
|
|
131
|
3051 /* Return true if OPTAB is supported for TYPES, where the first type
|
|
|
3052 is the destination and the second type is the source. Used for
|
|
|
3053 convert optabs. */
|
|
|
3054
|
|
|
3055 static bool
|
|
|
3056 convert_optab_supported_p (convert_optab optab, tree_pair types,
|
|
|
3057 optimization_type opt_type)
|
|
|
3058 {
|
|
|
3059 return (convert_optab_handler (optab, TYPE_MODE (types.first),
|
|
|
3060 TYPE_MODE (types.second), opt_type)
|
|
|
3061 != CODE_FOR_nothing);
|
|
|
3062 }
|
|
|
3063
|
|
111
|
3064 /* Return true if load/store lanes optab OPTAB is supported for
|
|
|
3065 array type TYPES.first when the optimization type is OPT_TYPE. */
|
|
|
3066
|
|
|
3067 static bool
|
|
|
3068 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
|
|
|
3069 optimization_type opt_type)
|
|
|
3070 {
|
|
|
3071 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
|
|
|
3072 machine_mode imode = TYPE_MODE (types.first);
|
|
|
3073 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
|
|
|
3074 return (convert_optab_handler (optab, imode, vmode, opt_type)
|
|
|
3075 != CODE_FOR_nothing);
|
|
|
3076 }
|
|
|
3077
|
|
|
3078 #define direct_unary_optab_supported_p direct_optab_supported_p
|
|
|
3079 #define direct_binary_optab_supported_p direct_optab_supported_p
|
|
131
|
3080 #define direct_ternary_optab_supported_p direct_optab_supported_p
|
|
|
3081 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
|
|
|
3082 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
|
|
|
3083 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
|
|
111
|
3084 #define direct_mask_load_optab_supported_p direct_optab_supported_p
|
|
|
3085 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
|
|
131
|
3086 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
|
|
|
3087 #define direct_gather_load_optab_supported_p direct_optab_supported_p
|
|
111
|
3088 #define direct_mask_store_optab_supported_p direct_optab_supported_p
|
|
|
3089 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
|
|
131
|
3090 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
|
|
|
3091 #define direct_scatter_store_optab_supported_p direct_optab_supported_p
|
|
|
3092 #define direct_while_optab_supported_p convert_optab_supported_p
|
|
|
3093 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
|
|
|
3094 #define direct_fold_left_optab_supported_p direct_optab_supported_p
|
|
|
3095
|
|
|
3096 /* Return the optab used by internal function FN. */
|
|
|
3097
|
|
|
3098 static optab
|
|
|
3099 direct_internal_fn_optab (internal_fn fn, tree_pair types)
|
|
|
3100 {
|
|
|
3101 switch (fn)
|
|
|
3102 {
|
|
|
3103 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
|
3104 case IFN_##CODE: break;
|
|
|
3105 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
|
3106 case IFN_##CODE: return OPTAB##_optab;
|
|
|
3107 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
|
3108 UNSIGNED_OPTAB, TYPE) \
|
|
|
3109 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
|
|
|
3110 ? UNSIGNED_OPTAB ## _optab \
|
|
|
3111 : SIGNED_OPTAB ## _optab);
|
|
|
3112 #include "internal-fn.def"
|
|
|
3113
|
|
|
3114 case IFN_LAST:
|
|
|
3115 break;
|
|
|
3116 }
|
|
|
3117 gcc_unreachable ();
|
|
|
3118 }
|
|
|
3119
|
|
|
3120 /* Return the optab used by internal function FN. */
|
|
|
3121
|
|
|
3122 static optab
|
|
|
3123 direct_internal_fn_optab (internal_fn fn)
|
|
|
3124 {
|
|
|
3125 switch (fn)
|
|
|
3126 {
|
|
|
3127 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
|
3128 case IFN_##CODE: break;
|
|
|
3129 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
|
3130 case IFN_##CODE: return OPTAB##_optab;
|
|
|
3131 #include "internal-fn.def"
|
|
|
3132
|
|
|
3133 case IFN_LAST:
|
|
|
3134 break;
|
|
|
3135 }
|
|
|
3136 gcc_unreachable ();
|
|
|
3137 }
|
|
111
|
3138
|
|
|
3139 /* Return true if FN is supported for the types in TYPES when the
|
|
|
3140 optimization type is OPT_TYPE. The types are those associated with
|
|
|
3141 the "type0" and "type1" fields of FN's direct_internal_fn_info
|
|
|
3142 structure. */
|
|
|
3143
|
|
|
3144 bool
|
|
|
3145 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
|
|
|
3146 optimization_type opt_type)
|
|
|
3147 {
|
|
|
3148 switch (fn)
|
|
|
3149 {
|
|
|
3150 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
|
3151 case IFN_##CODE: break;
|
|
|
3152 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
|
3153 case IFN_##CODE: \
|
|
|
3154 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
|
|
|
3155 opt_type);
|
|
131
|
3156 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
|
3157 UNSIGNED_OPTAB, TYPE) \
|
|
|
3158 case IFN_##CODE: \
|
|
|
3159 { \
|
|
|
3160 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
|
|
|
3161 ? UNSIGNED_OPTAB ## _optab \
|
|
|
3162 : SIGNED_OPTAB ## _optab); \
|
|
|
3163 return direct_##TYPE##_optab_supported_p (which_optab, types, \
|
|
|
3164 opt_type); \
|
|
|
3165 }
|
|
111
|
3166 #include "internal-fn.def"
|
|
|
3167
|
|
|
3168 case IFN_LAST:
|
|
|
3169 break;
|
|
|
3170 }
|
|
|
3171 gcc_unreachable ();
|
|
|
3172 }
|
|
|
3173
|
|
|
3174 /* Return true if FN is supported for type TYPE when the optimization
|
|
|
3175 type is OPT_TYPE. The caller knows that the "type0" and "type1"
|
|
|
3176 fields of FN's direct_internal_fn_info structure are the same. */
|
|
|
3177
|
|
|
3178 bool
|
|
|
3179 direct_internal_fn_supported_p (internal_fn fn, tree type,
|
|
|
3180 optimization_type opt_type)
|
|
|
3181 {
|
|
|
3182 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
|
3183 gcc_checking_assert (info.type0 == info.type1);
|
|
|
3184 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
|
|
|
3185 }
|
|
|
3186
|
|
131
|
3187 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
|
|
|
3188 given that STMT is a call to a direct internal function. */
|
|
|
3189
|
|
|
3190 bool
|
|
|
3191 direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
|
|
|
3192 {
|
|
|
3193 internal_fn fn = gimple_call_internal_fn (stmt);
|
|
|
3194 tree_pair types = direct_internal_fn_types (fn, stmt);
|
|
|
3195 return direct_internal_fn_supported_p (fn, types, opt_type);
|
|
|
3196 }
|
|
|
3197
|
|
|
3198 /* If FN is commutative in two consecutive arguments, return the
|
|
|
3199 index of the first, otherwise return -1. */
|
|
|
3200
|
|
|
3201 int
|
|
|
3202 first_commutative_argument (internal_fn fn)
|
|
|
3203 {
|
|
|
3204 switch (fn)
|
|
|
3205 {
|
|
|
3206 case IFN_FMA:
|
|
|
3207 case IFN_FMS:
|
|
|
3208 case IFN_FNMA:
|
|
|
3209 case IFN_FNMS:
|
|
|
3210 case IFN_AVG_FLOOR:
|
|
|
3211 case IFN_AVG_CEIL:
|
|
|
3212 case IFN_FMIN:
|
|
|
3213 case IFN_FMAX:
|
|
|
3214 return 0;
|
|
|
3215
|
|
|
3216 case IFN_COND_ADD:
|
|
|
3217 case IFN_COND_MUL:
|
|
|
3218 case IFN_COND_MIN:
|
|
|
3219 case IFN_COND_MAX:
|
|
|
3220 case IFN_COND_AND:
|
|
|
3221 case IFN_COND_IOR:
|
|
|
3222 case IFN_COND_XOR:
|
|
|
3223 case IFN_COND_FMA:
|
|
|
3224 case IFN_COND_FMS:
|
|
|
3225 case IFN_COND_FNMA:
|
|
|
3226 case IFN_COND_FNMS:
|
|
|
3227 return 1;
|
|
|
3228
|
|
|
3229 default:
|
|
|
3230 return -1;
|
|
|
3231 }
|
|
|
3232 }
|
|
|
3233
|
|
111
|
3234 /* Return true if IFN_SET_EDOM is supported. */
|
|
|
3235
|
|
|
3236 bool
|
|
|
3237 set_edom_supported_p (void)
|
|
|
3238 {
|
|
|
3239 #ifdef TARGET_EDOM
|
|
|
3240 return true;
|
|
|
3241 #else
|
|
|
3242 return false;
|
|
|
3243 #endif
|
|
|
3244 }
|
|
|
3245
|
|
|
3246 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
|
3247 static void \
|
|
|
3248 expand_##CODE (internal_fn fn, gcall *stmt) \
|
|
|
3249 { \
|
|
|
3250 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
|
|
|
3251 }
|
|
131
|
3252 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
|
|
3253 UNSIGNED_OPTAB, TYPE) \
|
|
|
3254 static void \
|
|
|
3255 expand_##CODE (internal_fn fn, gcall *stmt) \
|
|
|
3256 { \
|
|
|
3257 tree_pair types = direct_internal_fn_types (fn, stmt); \
|
|
|
3258 optab which_optab = direct_internal_fn_optab (fn, types); \
|
|
|
3259 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
|
|
|
3260 }
|
|
111
|
3261 #include "internal-fn.def"
|
|
|
3262
|
|
|
3263 /* Routines to expand each internal function, indexed by function number.
|
|
|
3264 Each routine has the prototype:
|
|
|
3265
|
|
|
3266 expand_<NAME> (gcall *stmt)
|
|
|
3267
|
|
|
3268 where STMT is the statement that performs the call. */
|
|
|
3269 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
|
|
|
3270 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
|
|
|
3271 #include "internal-fn.def"
|
|
|
3272 0
|
|
|
3273 };
|
|
|
3274
|
|
131
|
3275 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
|
|
|
3276 tree code CODE. */
|
|
|
3277 #define FOR_EACH_CODE_MAPPING(T) \
|
|
|
3278 T (PLUS_EXPR, IFN_COND_ADD) \
|
|
|
3279 T (MINUS_EXPR, IFN_COND_SUB) \
|
|
|
3280 T (MULT_EXPR, IFN_COND_MUL) \
|
|
|
3281 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
|
|
|
3282 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
|
|
|
3283 T (RDIV_EXPR, IFN_COND_RDIV) \
|
|
|
3284 T (MIN_EXPR, IFN_COND_MIN) \
|
|
|
3285 T (MAX_EXPR, IFN_COND_MAX) \
|
|
|
3286 T (BIT_AND_EXPR, IFN_COND_AND) \
|
|
|
3287 T (BIT_IOR_EXPR, IFN_COND_IOR) \
|
|
|
3288 T (BIT_XOR_EXPR, IFN_COND_XOR)
|
|
|
3289
|
|
|
3290 /* Return a function that only performs CODE when a certain condition is met
|
|
|
3291 and that uses a given fallback value otherwise. For example, if CODE is
|
|
|
3292 a binary operation associated with conditional function FN:
|
|
|
3293
|
|
|
3294 LHS = FN (COND, A, B, ELSE)
|
|
|
3295
|
|
|
3296 is equivalent to the C expression:
|
|
|
3297
|
|
|
3298 LHS = COND ? A CODE B : ELSE;
|
|
|
3299
|
|
|
3300 operating elementwise if the operands are vectors.
|
|
|
3301
|
|
|
3302 Return IFN_LAST if no such function exists. */
|
|
|
3303
|
|
|
3304 internal_fn
|
|
|
3305 get_conditional_internal_fn (tree_code code)
|
|
|
3306 {
|
|
|
3307 switch (code)
|
|
|
3308 {
|
|
|
3309 #define CASE(CODE, IFN) case CODE: return IFN;
|
|
|
3310 FOR_EACH_CODE_MAPPING(CASE)
|
|
|
3311 #undef CASE
|
|
|
3312 default:
|
|
|
3313 return IFN_LAST;
|
|
|
3314 }
|
|
|
3315 }
|
|
|
3316
|
|
|
3317 /* If IFN implements the conditional form of a tree code, return that
|
|
|
3318 tree code, otherwise return ERROR_MARK. */
|
|
|
3319
|
|
|
3320 tree_code
|
|
|
3321 conditional_internal_fn_code (internal_fn ifn)
|
|
|
3322 {
|
|
|
3323 switch (ifn)
|
|
|
3324 {
|
|
|
3325 #define CASE(CODE, IFN) case IFN: return CODE;
|
|
|
3326 FOR_EACH_CODE_MAPPING(CASE)
|
|
|
3327 #undef CASE
|
|
|
3328 default:
|
|
|
3329 return ERROR_MARK;
|
|
|
3330 }
|
|
|
3331 }
|
|
|
3332
|
|
|
3333 /* Invoke T(IFN) for each internal function IFN that also has an
|
|
|
3334 IFN_COND_* form. */
|
|
|
3335 #define FOR_EACH_COND_FN_PAIR(T) \
|
|
|
3336 T (FMA) \
|
|
|
3337 T (FMS) \
|
|
|
3338 T (FNMA) \
|
|
|
3339 T (FNMS)
|
|
|
3340
|
|
|
3341 /* Return a function that only performs internal function FN when a
|
|
|
3342 certain condition is met and that uses a given fallback value otherwise.
|
|
|
3343 In other words, the returned function FN' is such that:
|
|
|
3344
|
|
|
3345 LHS = FN' (COND, A1, ... An, ELSE)
|
|
|
3346
|
|
|
3347 is equivalent to the C expression:
|
|
|
3348
|
|
|
3349 LHS = COND ? FN (A1, ..., An) : ELSE;
|
|
|
3350
|
|
|
3351 operating elementwise if the operands are vectors.
|
|
|
3352
|
|
|
3353 Return IFN_LAST if no such function exists. */
|
|
|
3354
|
|
|
3355 internal_fn
|
|
|
3356 get_conditional_internal_fn (internal_fn fn)
|
|
|
3357 {
|
|
|
3358 switch (fn)
|
|
|
3359 {
|
|
|
3360 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
|
|
|
3361 FOR_EACH_COND_FN_PAIR(CASE)
|
|
|
3362 #undef CASE
|
|
|
3363 default:
|
|
|
3364 return IFN_LAST;
|
|
|
3365 }
|
|
|
3366 }
|
|
|
3367
|
|
|
3368 /* If IFN implements the conditional form of an unconditional internal
|
|
|
3369 function, return that unconditional function, otherwise return IFN_LAST. */
|
|
|
3370
|
|
|
3371 internal_fn
|
|
|
3372 get_unconditional_internal_fn (internal_fn ifn)
|
|
|
3373 {
|
|
|
3374 switch (ifn)
|
|
|
3375 {
|
|
|
3376 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
|
|
|
3377 FOR_EACH_COND_FN_PAIR(CASE)
|
|
|
3378 #undef CASE
|
|
|
3379 default:
|
|
|
3380 return IFN_LAST;
|
|
|
3381 }
|
|
|
3382 }
|
|
|
3383
|
|
|
3384 /* Return true if STMT can be interpreted as a conditional tree code
|
|
|
3385 operation of the form:
|
|
|
3386
|
|
|
3387 LHS = COND ? OP (RHS1, ...) : ELSE;
|
|
|
3388
|
|
|
3389 operating elementwise if the operands are vectors. This includes
|
|
|
3390 the case of an all-true COND, so that the operation always happens.
|
|
|
3391
|
|
|
3392 When returning true, set:
|
|
|
3393
|
|
|
3394 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
|
|
|
3395 is known to be all-true
|
|
|
3396 - *CODE_OUT to the tree code
|
|
|
3397 - OPS[I] to operand I of *CODE_OUT
|
|
|
3398 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
|
|
|
3399 condition is known to be all true. */
|
|
|
3400
|
|
|
3401 bool
|
|
|
3402 can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
|
|
|
3403 tree_code *code_out,
|
|
|
3404 tree (&ops)[3], tree *else_out)
|
|
|
3405 {
|
|
|
3406 if (gassign *assign = dyn_cast <gassign *> (stmt))
|
|
|
3407 {
|
|
|
3408 *cond_out = NULL_TREE;
|
|
|
3409 *code_out = gimple_assign_rhs_code (assign);
|
|
|
3410 ops[0] = gimple_assign_rhs1 (assign);
|
|
|
3411 ops[1] = gimple_assign_rhs2 (assign);
|
|
|
3412 ops[2] = gimple_assign_rhs3 (assign);
|
|
|
3413 *else_out = NULL_TREE;
|
|
|
3414 return true;
|
|
|
3415 }
|
|
|
3416 if (gcall *call = dyn_cast <gcall *> (stmt))
|
|
|
3417 if (gimple_call_internal_p (call))
|
|
|
3418 {
|
|
|
3419 internal_fn ifn = gimple_call_internal_fn (call);
|
|
|
3420 tree_code code = conditional_internal_fn_code (ifn);
|
|
|
3421 if (code != ERROR_MARK)
|
|
|
3422 {
|
|
|
3423 *cond_out = gimple_call_arg (call, 0);
|
|
|
3424 *code_out = code;
|
|
|
3425 unsigned int nops = gimple_call_num_args (call) - 2;
|
|
|
3426 for (unsigned int i = 0; i < 3; ++i)
|
|
|
3427 ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
|
|
|
3428 *else_out = gimple_call_arg (call, nops + 1);
|
|
|
3429 if (integer_truep (*cond_out))
|
|
|
3430 {
|
|
|
3431 *cond_out = NULL_TREE;
|
|
|
3432 *else_out = NULL_TREE;
|
|
|
3433 }
|
|
|
3434 return true;
|
|
|
3435 }
|
|
|
3436 }
|
|
|
3437 return false;
|
|
|
3438 }
|
|
|
3439
|
|
|
3440 /* Return true if IFN is some form of load from memory. */
|
|
|
3441
|
|
|
3442 bool
|
|
|
3443 internal_load_fn_p (internal_fn fn)
|
|
|
3444 {
|
|
|
3445 switch (fn)
|
|
|
3446 {
|
|
|
3447 case IFN_MASK_LOAD:
|
|
|
3448 case IFN_LOAD_LANES:
|
|
|
3449 case IFN_MASK_LOAD_LANES:
|
|
|
3450 case IFN_GATHER_LOAD:
|
|
|
3451 case IFN_MASK_GATHER_LOAD:
|
|
|
3452 return true;
|
|
|
3453
|
|
|
3454 default:
|
|
|
3455 return false;
|
|
|
3456 }
|
|
|
3457 }
|
|
|
3458
|
|
|
3459 /* Return true if IFN is some form of store to memory. */
|
|
|
3460
|
|
|
3461 bool
|
|
|
3462 internal_store_fn_p (internal_fn fn)
|
|
|
3463 {
|
|
|
3464 switch (fn)
|
|
|
3465 {
|
|
|
3466 case IFN_MASK_STORE:
|
|
|
3467 case IFN_STORE_LANES:
|
|
|
3468 case IFN_MASK_STORE_LANES:
|
|
|
3469 case IFN_SCATTER_STORE:
|
|
|
3470 case IFN_MASK_SCATTER_STORE:
|
|
|
3471 return true;
|
|
|
3472
|
|
|
3473 default:
|
|
|
3474 return false;
|
|
|
3475 }
|
|
|
3476 }
|
|
|
3477
|
|
|
3478 /* Return true if IFN is some form of gather load or scatter store. */
|
|
|
3479
|
|
|
3480 bool
|
|
|
3481 internal_gather_scatter_fn_p (internal_fn fn)
|
|
|
3482 {
|
|
|
3483 switch (fn)
|
|
|
3484 {
|
|
|
3485 case IFN_GATHER_LOAD:
|
|
|
3486 case IFN_MASK_GATHER_LOAD:
|
|
|
3487 case IFN_SCATTER_STORE:
|
|
|
3488 case IFN_MASK_SCATTER_STORE:
|
|
|
3489 return true;
|
|
|
3490
|
|
|
3491 default:
|
|
|
3492 return false;
|
|
|
3493 }
|
|
|
3494 }
|
|
|
3495
|
|
|
3496 /* If FN takes a vector mask argument, return the index of that argument,
|
|
|
3497 otherwise return -1. */
|
|
|
3498
|
|
|
3499 int
|
|
|
3500 internal_fn_mask_index (internal_fn fn)
|
|
|
3501 {
|
|
|
3502 switch (fn)
|
|
|
3503 {
|
|
|
3504 case IFN_MASK_LOAD:
|
|
|
3505 case IFN_MASK_LOAD_LANES:
|
|
|
3506 case IFN_MASK_STORE:
|
|
|
3507 case IFN_MASK_STORE_LANES:
|
|
|
3508 return 2;
|
|
|
3509
|
|
|
3510 case IFN_MASK_GATHER_LOAD:
|
|
|
3511 return 3;
|
|
|
3512
|
|
|
3513 case IFN_MASK_SCATTER_STORE:
|
|
|
3514 return 4;
|
|
|
3515
|
|
|
3516 default:
|
|
|
3517 return (conditional_internal_fn_code (fn) != ERROR_MARK
|
|
|
3518 || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
|
|
|
3519 }
|
|
|
3520 }
|
|
|
3521
|
|
|
3522 /* If FN takes a value that should be stored to memory, return the index
|
|
|
3523 of that argument, otherwise return -1. */
|
|
|
3524
|
|
|
3525 int
|
|
|
3526 internal_fn_stored_value_index (internal_fn fn)
|
|
|
3527 {
|
|
|
3528 switch (fn)
|
|
|
3529 {
|
|
|
3530 case IFN_MASK_STORE:
|
|
|
3531 case IFN_SCATTER_STORE:
|
|
|
3532 case IFN_MASK_SCATTER_STORE:
|
|
|
3533 return 3;
|
|
|
3534
|
|
|
3535 default:
|
|
|
3536 return -1;
|
|
|
3537 }
|
|
|
3538 }
|
|
|
3539
|
|
|
3540 /* Return true if the target supports gather load or scatter store function
|
|
|
3541 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
|
|
|
3542 while for stores it is the vector type of the stored data argument.
|
|
|
3543 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
|
|
|
3544 or stored. OFFSET_SIGN is the sign of the offset argument, which is
|
|
|
3545 only relevant when the offset is narrower than an address. SCALE is
|
|
|
3546 the amount by which the offset should be multiplied *after* it has
|
|
|
3547 been extended to address width. */
|
|
|
3548
|
|
|
3549 bool
|
|
|
3550 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
|
|
|
3551 tree memory_element_type,
|
|
|
3552 signop offset_sign, int scale)
|
|
|
3553 {
|
|
|
3554 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
|
|
|
3555 TYPE_SIZE (memory_element_type)))
|
|
|
3556 return false;
|
|
|
3557 optab optab = direct_internal_fn_optab (ifn);
|
|
|
3558 insn_code icode = direct_optab_handler (optab, TYPE_MODE (vector_type));
|
|
|
3559 int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
|
|
|
3560 return (icode != CODE_FOR_nothing
|
|
|
3561 && insn_operand_matches (icode, 2 + output_ops,
|
|
|
3562 GEN_INT (offset_sign == UNSIGNED))
|
|
|
3563 && insn_operand_matches (icode, 3 + output_ops,
|
|
|
3564 GEN_INT (scale)));
|
|
|
3565 }
|
|
|
3566
|
|
111
|
3567 /* Expand STMT as though it were a call to internal function FN. */
|
|
|
3568
|
|
|
3569 void
|
|
|
3570 expand_internal_call (internal_fn fn, gcall *stmt)
|
|
|
3571 {
|
|
|
3572 internal_fn_expanders[fn] (fn, stmt);
|
|
|
3573 }
|
|
|
3574
|
|
|
3575 /* Expand STMT, which is a call to internal function FN. */
|
|
|
3576
|
|
|
3577 void
|
|
|
3578 expand_internal_call (gcall *stmt)
|
|
|
3579 {
|
|
|
3580 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
|
|
|
3581 }
|
|
|
3582
|
|
131
|
3583 /* If TYPE is a vector type, return true if IFN is a direct internal
|
|
|
3584 function that is supported for that type. If TYPE is a scalar type,
|
|
|
3585 return true if IFN is a direct internal function that is supported for
|
|
|
3586 the target's preferred vector version of TYPE. */
|
|
|
3587
|
|
|
3588 bool
|
|
|
3589 vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
|
|
|
3590 {
|
|
|
3591 scalar_mode smode;
|
|
|
3592 if (!VECTOR_TYPE_P (type) && is_a <scalar_mode> (TYPE_MODE (type), &smode))
|
|
|
3593 {
|
|
|
3594 machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
|
|
|
3595 if (VECTOR_MODE_P (vmode))
|
|
|
3596 type = build_vector_type_for_mode (type, vmode);
|
|
|
3597 }
|
|
|
3598
|
|
|
3599 return (VECTOR_MODE_P (TYPE_MODE (type))
|
|
|
3600 && direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED));
|
|
|
3601 }
|
|
|
3602
|
|
111
|
3603 void
|
|
|
3604 expand_PHI (internal_fn, gcall *)
|
|
|
3605 {
|
|
|
3606 gcc_unreachable ();
|
|
|
3607 }
|
