|
111
|
1 /* Internal functions.
|
|
|
2 Copyright (C) 2011-2017 Free Software Foundation, Inc.
|
|
|
3
|
|
|
4 This file is part of GCC.
|
|
|
5
|
|
|
6 GCC is free software; you can redistribute it and/or modify it under
|
|
|
7 the terms of the GNU General Public License as published by the Free
|
|
|
8 Software Foundation; either version 3, or (at your option) any later
|
|
|
9 version.
|
|
|
10
|
|
|
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
14 for more details.
|
|
|
15
|
|
|
16 You should have received a copy of the GNU General Public License
|
|
|
17 along with GCC; see the file COPYING3. If not see
|
|
|
18 <http://www.gnu.org/licenses/>. */
|
|
|
19
|
|
|
20 #include "config.h"
|
|
|
21 #include "system.h"
|
|
|
22 #include "coretypes.h"
|
|
|
23 #include "backend.h"
|
|
|
24 #include "target.h"
|
|
|
25 #include "rtl.h"
|
|
|
26 #include "tree.h"
|
|
|
27 #include "gimple.h"
|
|
|
28 #include "predict.h"
|
|
|
29 #include "stringpool.h"
|
|
|
30 #include "tree-vrp.h"
|
|
|
31 #include "tree-ssanames.h"
|
|
|
32 #include "expmed.h"
|
|
|
33 #include "memmodel.h"
|
|
|
34 #include "optabs.h"
|
|
|
35 #include "emit-rtl.h"
|
|
|
36 #include "diagnostic-core.h"
|
|
|
37 #include "fold-const.h"
|
|
|
38 #include "internal-fn.h"
|
|
|
39 #include "stor-layout.h"
|
|
|
40 #include "dojump.h"
|
|
|
41 #include "expr.h"
|
|
|
42 #include "stringpool.h"
|
|
|
43 #include "attribs.h"
|
|
|
44 #include "asan.h"
|
|
|
45 #include "ubsan.h"
|
|
|
46 #include "recog.h"
|
|
|
47 #include "builtins.h"
|
|
|
48 #include "optabs-tree.h"
|
|
|
49
|
|
|
50 /* The names of each internal function, indexed by function number. */
|
|
|
51 const char *const internal_fn_name_array[] = {
|
|
|
52 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
|
|
|
53 #include "internal-fn.def"
|
|
|
54 "<invalid-fn>"
|
|
|
55 };
|
|
|
56
|
|
|
57 /* The ECF_* flags of each internal function, indexed by function number. */
|
|
|
58 const int internal_fn_flags_array[] = {
|
|
|
59 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
|
|
|
60 #include "internal-fn.def"
|
|
|
61 0
|
|
|
62 };
|
|
|
63
|
|
|
64 /* Fnspec of each internal function, indexed by function number. */
|
|
|
65 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
|
|
|
66
|
|
|
67 void
|
|
|
68 init_internal_fns ()
|
|
|
69 {
|
|
|
70 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
|
71 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
|
|
|
72 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
|
|
|
73 #include "internal-fn.def"
|
|
|
74 internal_fn_fnspec_array[IFN_LAST] = 0;
|
|
|
75 }
|
|
|
76
|
|
|
77 /* Create static initializers for the information returned by
|
|
|
78 direct_internal_fn. */
|
|
|
79 #define not_direct { -2, -2, false }
|
|
|
80 #define mask_load_direct { -1, 2, false }
|
|
|
81 #define load_lanes_direct { -1, -1, false }
|
|
|
82 #define mask_store_direct { 3, 2, false }
|
|
|
83 #define store_lanes_direct { 0, 0, false }
|
|
|
84 #define unary_direct { 0, 0, true }
|
|
|
85 #define binary_direct { 0, 0, true }
|
|
|
86
|
|
|
87 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
|
|
|
88 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
|
|
|
89 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
|
|
|
90 #include "internal-fn.def"
|
|
|
91 not_direct
|
|
|
92 };
|
|
|
93
|
|
|
94 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
|
|
|
95 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
|
|
|
96
|
|
|
97 static enum insn_code
|
|
|
98 get_multi_vector_move (tree array_type, convert_optab optab)
|
|
|
99 {
|
|
|
100 machine_mode imode;
|
|
|
101 machine_mode vmode;
|
|
|
102
|
|
|
103 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
|
|
|
104 imode = TYPE_MODE (array_type);
|
|
|
105 vmode = TYPE_MODE (TREE_TYPE (array_type));
|
|
|
106
|
|
|
107 return convert_optab_handler (optab, imode, vmode);
|
|
|
108 }
|
|
|
109
|
|
|
110 /* Expand LOAD_LANES call STMT using optab OPTAB. */
|
|
|
111
|
|
|
112 static void
|
|
|
113 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
114 {
|
|
|
115 struct expand_operand ops[2];
|
|
|
116 tree type, lhs, rhs;
|
|
|
117 rtx target, mem;
|
|
|
118
|
|
|
119 lhs = gimple_call_lhs (stmt);
|
|
|
120 rhs = gimple_call_arg (stmt, 0);
|
|
|
121 type = TREE_TYPE (lhs);
|
|
|
122
|
|
|
123 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
124 mem = expand_normal (rhs);
|
|
|
125
|
|
|
126 gcc_assert (MEM_P (mem));
|
|
|
127 PUT_MODE (mem, TYPE_MODE (type));
|
|
|
128
|
|
|
129 create_output_operand (&ops[0], target, TYPE_MODE (type));
|
|
|
130 create_fixed_operand (&ops[1], mem);
|
|
|
131 expand_insn (get_multi_vector_move (type, optab), 2, ops);
|
|
|
132 }
|
|
|
133
|
|
|
134 /* Expand STORE_LANES call STMT using optab OPTAB. */
|
|
|
135
|
|
|
136 static void
|
|
|
137 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
138 {
|
|
|
139 struct expand_operand ops[2];
|
|
|
140 tree type, lhs, rhs;
|
|
|
141 rtx target, reg;
|
|
|
142
|
|
|
143 lhs = gimple_call_lhs (stmt);
|
|
|
144 rhs = gimple_call_arg (stmt, 0);
|
|
|
145 type = TREE_TYPE (rhs);
|
|
|
146
|
|
|
147 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
148 reg = expand_normal (rhs);
|
|
|
149
|
|
|
150 gcc_assert (MEM_P (target));
|
|
|
151 PUT_MODE (target, TYPE_MODE (type));
|
|
|
152
|
|
|
153 create_fixed_operand (&ops[0], target);
|
|
|
154 create_input_operand (&ops[1], reg, TYPE_MODE (type));
|
|
|
155 expand_insn (get_multi_vector_move (type, optab), 2, ops);
|
|
|
156 }
|
|
|
157
|
|
|
158 static void
|
|
|
159 expand_ANNOTATE (internal_fn, gcall *)
|
|
|
160 {
|
|
|
161 gcc_unreachable ();
|
|
|
162 }
|
|
|
163
|
|
|
164 /* This should get expanded in omp_device_lower pass. */
|
|
|
165
|
|
|
166 static void
|
|
|
167 expand_GOMP_USE_SIMT (internal_fn, gcall *)
|
|
|
168 {
|
|
|
169 gcc_unreachable ();
|
|
|
170 }
|
|
|
171
|
|
|
172 /* This should get expanded in omp_device_lower pass. */
|
|
|
173
|
|
|
174 static void
|
|
|
175 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
|
|
|
176 {
|
|
|
177 gcc_unreachable ();
|
|
|
178 }
|
|
|
179
|
|
|
180 /* Allocate per-lane storage and begin non-uniform execution region. */
|
|
|
181
|
|
|
182 static void
|
|
|
183 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
|
|
|
184 {
|
|
|
185 rtx target;
|
|
|
186 tree lhs = gimple_call_lhs (stmt);
|
|
|
187 if (lhs)
|
|
|
188 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
189 else
|
|
|
190 target = gen_reg_rtx (Pmode);
|
|
|
191 rtx size = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
192 rtx align = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
193 struct expand_operand ops[3];
|
|
|
194 create_output_operand (&ops[0], target, Pmode);
|
|
|
195 create_input_operand (&ops[1], size, Pmode);
|
|
|
196 create_input_operand (&ops[2], align, Pmode);
|
|
|
197 gcc_assert (targetm.have_omp_simt_enter ());
|
|
|
198 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
|
|
|
199 }
|
|
|
200
|
|
|
201 /* Deallocate per-lane storage and leave non-uniform execution region. */
|
|
|
202
|
|
|
203 static void
|
|
|
204 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
|
|
|
205 {
|
|
|
206 gcc_checking_assert (!gimple_call_lhs (stmt));
|
|
|
207 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
208 struct expand_operand ops[1];
|
|
|
209 create_input_operand (&ops[0], arg, Pmode);
|
|
|
210 gcc_assert (targetm.have_omp_simt_exit ());
|
|
|
211 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
|
|
|
212 }
|
|
|
213
|
|
|
214 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
|
|
|
215 without SIMT execution this should be expanded in omp_device_lower pass. */
|
|
|
216
|
|
|
217 static void
|
|
|
218 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
|
|
|
219 {
|
|
|
220 tree lhs = gimple_call_lhs (stmt);
|
|
|
221 if (!lhs)
|
|
|
222 return;
|
|
|
223
|
|
|
224 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
225 gcc_assert (targetm.have_omp_simt_lane ());
|
|
|
226 emit_insn (targetm.gen_omp_simt_lane (target));
|
|
|
227 }
|
|
|
228
|
|
|
229 /* This should get expanded in omp_device_lower pass. */
|
|
|
230
|
|
|
231 static void
|
|
|
232 expand_GOMP_SIMT_VF (internal_fn, gcall *)
|
|
|
233 {
|
|
|
234 gcc_unreachable ();
|
|
|
235 }
|
|
|
236
|
|
|
237 /* Lane index of the first SIMT lane that supplies a non-zero argument.
|
|
|
238 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
|
|
|
239 lane that executed the last iteration for handling OpenMP lastprivate. */
|
|
|
240
|
|
|
241 static void
|
|
|
242 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
|
|
|
243 {
|
|
|
244 tree lhs = gimple_call_lhs (stmt);
|
|
|
245 if (!lhs)
|
|
|
246 return;
|
|
|
247
|
|
|
248 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
249 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
250 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
251 struct expand_operand ops[2];
|
|
|
252 create_output_operand (&ops[0], target, mode);
|
|
|
253 create_input_operand (&ops[1], cond, mode);
|
|
|
254 gcc_assert (targetm.have_omp_simt_last_lane ());
|
|
|
255 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
|
|
|
256 }
|
|
|
257
|
|
|
258 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
|
|
|
259
|
|
|
260 static void
|
|
|
261 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
|
|
|
262 {
|
|
|
263 tree lhs = gimple_call_lhs (stmt);
|
|
|
264 if (!lhs)
|
|
|
265 return;
|
|
|
266
|
|
|
267 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
268 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
269 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
270 struct expand_operand ops[2];
|
|
|
271 create_output_operand (&ops[0], target, mode);
|
|
|
272 create_input_operand (&ops[1], ctr, mode);
|
|
|
273 gcc_assert (targetm.have_omp_simt_ordered ());
|
|
|
274 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
|
|
|
275 }
|
|
|
276
|
|
|
277 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
|
|
|
278 any lane supplies a non-zero argument. */
|
|
|
279
|
|
|
280 static void
|
|
|
281 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
|
|
|
282 {
|
|
|
283 tree lhs = gimple_call_lhs (stmt);
|
|
|
284 if (!lhs)
|
|
|
285 return;
|
|
|
286
|
|
|
287 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
288 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
289 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
290 struct expand_operand ops[2];
|
|
|
291 create_output_operand (&ops[0], target, mode);
|
|
|
292 create_input_operand (&ops[1], cond, mode);
|
|
|
293 gcc_assert (targetm.have_omp_simt_vote_any ());
|
|
|
294 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
|
|
|
295 }
|
|
|
296
|
|
|
297 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
|
|
|
298 is destination lane index XOR given offset. */
|
|
|
299
|
|
|
300 static void
|
|
|
301 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
|
|
|
302 {
|
|
|
303 tree lhs = gimple_call_lhs (stmt);
|
|
|
304 if (!lhs)
|
|
|
305 return;
|
|
|
306
|
|
|
307 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
308 rtx src = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
309 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
310 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
311 struct expand_operand ops[3];
|
|
|
312 create_output_operand (&ops[0], target, mode);
|
|
|
313 create_input_operand (&ops[1], src, mode);
|
|
|
314 create_input_operand (&ops[2], idx, SImode);
|
|
|
315 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
|
|
|
316 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
|
|
|
317 }
|
|
|
318
|
|
|
319 /* Exchange between SIMT lanes according to given source lane index. */
|
|
|
320
|
|
|
321 static void
|
|
|
322 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
|
|
|
323 {
|
|
|
324 tree lhs = gimple_call_lhs (stmt);
|
|
|
325 if (!lhs)
|
|
|
326 return;
|
|
|
327
|
|
|
328 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
329 rtx src = expand_normal (gimple_call_arg (stmt, 0));
|
|
|
330 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
331 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
|
|
|
332 struct expand_operand ops[3];
|
|
|
333 create_output_operand (&ops[0], target, mode);
|
|
|
334 create_input_operand (&ops[1], src, mode);
|
|
|
335 create_input_operand (&ops[2], idx, SImode);
|
|
|
336 gcc_assert (targetm.have_omp_simt_xchg_idx ());
|
|
|
337 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
|
|
|
338 }
|
|
|
339
|
|
|
340 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
341
|
|
|
342 static void
|
|
|
343 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
|
|
|
344 {
|
|
|
345 gcc_unreachable ();
|
|
|
346 }
|
|
|
347
|
|
|
348 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
349
|
|
|
350 static void
|
|
|
351 expand_GOMP_SIMD_VF (internal_fn, gcall *)
|
|
|
352 {
|
|
|
353 gcc_unreachable ();
|
|
|
354 }
|
|
|
355
|
|
|
356 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
357
|
|
|
358 static void
|
|
|
359 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
|
|
|
360 {
|
|
|
361 gcc_unreachable ();
|
|
|
362 }
|
|
|
363
|
|
|
364 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
365
|
|
|
366 static void
|
|
|
367 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
|
|
|
368 {
|
|
|
369 gcc_unreachable ();
|
|
|
370 }
|
|
|
371
|
|
|
372 /* This should get expanded in adjust_simduid_builtins. */
|
|
|
373
|
|
|
374 static void
|
|
|
375 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
|
|
|
376 {
|
|
|
377 gcc_unreachable ();
|
|
|
378 }
|
|
|
379
|
|
|
380 /* This should get expanded in the sanopt pass. */
|
|
|
381
|
|
|
382 static void
|
|
|
383 expand_UBSAN_NULL (internal_fn, gcall *)
|
|
|
384 {
|
|
|
385 gcc_unreachable ();
|
|
|
386 }
|
|
|
387
|
|
|
388 /* This should get expanded in the sanopt pass. */
|
|
|
389
|
|
|
390 static void
|
|
|
391 expand_UBSAN_BOUNDS (internal_fn, gcall *)
|
|
|
392 {
|
|
|
393 gcc_unreachable ();
|
|
|
394 }
|
|
|
395
|
|
|
396 /* This should get expanded in the sanopt pass. */
|
|
|
397
|
|
|
398 static void
|
|
|
399 expand_UBSAN_VPTR (internal_fn, gcall *)
|
|
|
400 {
|
|
|
401 gcc_unreachable ();
|
|
|
402 }
|
|
|
403
|
|
|
404 /* This should get expanded in the sanopt pass. */
|
|
|
405
|
|
|
406 static void
|
|
|
407 expand_UBSAN_PTR (internal_fn, gcall *)
|
|
|
408 {
|
|
|
409 gcc_unreachable ();
|
|
|
410 }
|
|
|
411
|
|
|
412 /* This should get expanded in the sanopt pass. */
|
|
|
413
|
|
|
414 static void
|
|
|
415 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
|
|
|
416 {
|
|
|
417 gcc_unreachable ();
|
|
|
418 }
|
|
|
419
|
|
|
420 /* This should get expanded in the sanopt pass. */
|
|
|
421
|
|
|
422 static void
|
|
|
423 expand_ASAN_CHECK (internal_fn, gcall *)
|
|
|
424 {
|
|
|
425 gcc_unreachable ();
|
|
|
426 }
|
|
|
427
|
|
|
428 /* This should get expanded in the sanopt pass. */
|
|
|
429
|
|
|
430 static void
|
|
|
431 expand_ASAN_MARK (internal_fn, gcall *)
|
|
|
432 {
|
|
|
433 gcc_unreachable ();
|
|
|
434 }
|
|
|
435
|
|
|
436 /* This should get expanded in the sanopt pass. */
|
|
|
437
|
|
|
438 static void
|
|
|
439 expand_ASAN_POISON (internal_fn, gcall *)
|
|
|
440 {
|
|
|
441 gcc_unreachable ();
|
|
|
442 }
|
|
|
443
|
|
|
444 /* This should get expanded in the sanopt pass. */
|
|
|
445
|
|
|
446 static void
|
|
|
447 expand_ASAN_POISON_USE (internal_fn, gcall *)
|
|
|
448 {
|
|
|
449 gcc_unreachable ();
|
|
|
450 }
|
|
|
451
|
|
|
452 /* This should get expanded in the tsan pass. */
|
|
|
453
|
|
|
454 static void
|
|
|
455 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
|
|
|
456 {
|
|
|
457 gcc_unreachable ();
|
|
|
458 }
|
|
|
459
|
|
|
460 /* This should get expanded in the lower pass. */
|
|
|
461
|
|
|
462 static void
|
|
|
463 expand_FALLTHROUGH (internal_fn, gcall *call)
|
|
|
464 {
|
|
|
465 error_at (gimple_location (call),
|
|
|
466 "invalid use of attribute %<fallthrough%>");
|
|
|
467 }
|
|
|
468
|
|
|
469 /* Return minimum precision needed to represent all values
|
|
|
470 of ARG in SIGNed integral type. */
|
|
|
471
|
|
|
472 static int
|
|
|
473 get_min_precision (tree arg, signop sign)
|
|
|
474 {
|
|
|
475 int prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
|
476 int cnt = 0;
|
|
|
477 signop orig_sign = sign;
|
|
|
478 if (TREE_CODE (arg) == INTEGER_CST)
|
|
|
479 {
|
|
|
480 int p;
|
|
|
481 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
|
|
|
482 {
|
|
|
483 widest_int w = wi::to_widest (arg);
|
|
|
484 w = wi::ext (w, prec, sign);
|
|
|
485 p = wi::min_precision (w, sign);
|
|
|
486 }
|
|
|
487 else
|
|
|
488 p = wi::min_precision (wi::to_wide (arg), sign);
|
|
|
489 return MIN (p, prec);
|
|
|
490 }
|
|
|
491 while (CONVERT_EXPR_P (arg)
|
|
|
492 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
|
|
|
493 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
|
|
|
494 {
|
|
|
495 arg = TREE_OPERAND (arg, 0);
|
|
|
496 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
|
|
|
497 {
|
|
|
498 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
|
|
|
499 sign = UNSIGNED;
|
|
|
500 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
|
|
|
501 return prec + (orig_sign != sign);
|
|
|
502 prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
|
503 }
|
|
|
504 if (++cnt > 30)
|
|
|
505 return prec + (orig_sign != sign);
|
|
|
506 }
|
|
|
507 if (TREE_CODE (arg) != SSA_NAME)
|
|
|
508 return prec + (orig_sign != sign);
|
|
|
509 wide_int arg_min, arg_max;
|
|
|
510 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
|
|
|
511 {
|
|
|
512 gimple *g = SSA_NAME_DEF_STMT (arg);
|
|
|
513 if (is_gimple_assign (g)
|
|
|
514 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
|
|
|
515 {
|
|
|
516 tree t = gimple_assign_rhs1 (g);
|
|
|
517 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
|
|
|
518 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
|
|
|
519 {
|
|
|
520 arg = t;
|
|
|
521 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
|
|
|
522 {
|
|
|
523 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
|
|
|
524 sign = UNSIGNED;
|
|
|
525 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
|
|
|
526 return prec + (orig_sign != sign);
|
|
|
527 prec = TYPE_PRECISION (TREE_TYPE (arg));
|
|
|
528 }
|
|
|
529 if (++cnt > 30)
|
|
|
530 return prec + (orig_sign != sign);
|
|
|
531 continue;
|
|
|
532 }
|
|
|
533 }
|
|
|
534 return prec + (orig_sign != sign);
|
|
|
535 }
|
|
|
536 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
|
|
|
537 {
|
|
|
538 int p1 = wi::min_precision (arg_min, sign);
|
|
|
539 int p2 = wi::min_precision (arg_max, sign);
|
|
|
540 p1 = MAX (p1, p2);
|
|
|
541 prec = MIN (prec, p1);
|
|
|
542 }
|
|
|
543 else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
|
|
|
544 {
|
|
|
545 int p = wi::min_precision (arg_max, UNSIGNED);
|
|
|
546 prec = MIN (prec, p);
|
|
|
547 }
|
|
|
548 return prec + (orig_sign != sign);
|
|
|
549 }
|
|
|
550
|
|
|
551 /* Helper for expand_*_overflow. Set the __imag__ part to true
|
|
|
552 (1 except for signed:1 type, in which case store -1). */
|
|
|
553
|
|
|
554 static void
|
|
|
555 expand_arith_set_overflow (tree lhs, rtx target)
|
|
|
556 {
|
|
|
557 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
|
|
|
558 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
|
|
|
559 write_complex_part (target, constm1_rtx, true);
|
|
|
560 else
|
|
|
561 write_complex_part (target, const1_rtx, true);
|
|
|
562 }
|
|
|
563
|
|
|
564 /* Helper for expand_*_overflow. Store RES into the __real__ part
|
|
|
565 of TARGET. If RES has larger MODE than __real__ part of TARGET,
|
|
|
566 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
|
|
|
567 if LHS has smaller precision than its mode. */
|
|
|
568
|
|
|
569 static void
|
|
|
570 expand_arith_overflow_result_store (tree lhs, rtx target,
|
|
|
571 scalar_int_mode mode, rtx res)
|
|
|
572 {
|
|
|
573 scalar_int_mode tgtmode
|
|
|
574 = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
|
|
|
575 rtx lres = res;
|
|
|
576 if (tgtmode != mode)
|
|
|
577 {
|
|
|
578 rtx_code_label *done_label = gen_label_rtx ();
|
|
|
579 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
|
|
|
580 lres = convert_modes (tgtmode, mode, res, uns);
|
|
|
581 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
|
|
|
582 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
|
|
|
583 EQ, true, mode, NULL_RTX, NULL, done_label,
|
|
|
584 profile_probability::very_likely ());
|
|
|
585 expand_arith_set_overflow (lhs, target);
|
|
|
586 emit_label (done_label);
|
|
|
587 }
|
|
|
588 int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
|
|
|
589 int tgtprec = GET_MODE_PRECISION (tgtmode);
|
|
|
590 if (prec < tgtprec)
|
|
|
591 {
|
|
|
592 rtx_code_label *done_label = gen_label_rtx ();
|
|
|
593 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
|
|
|
594 res = lres;
|
|
|
595 if (uns)
|
|
|
596 {
|
|
|
597 rtx mask
|
|
|
598 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
|
|
|
599 tgtmode);
|
|
|
600 lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
|
|
|
601 true, OPTAB_LIB_WIDEN);
|
|
|
602 }
|
|
|
603 else
|
|
|
604 {
|
|
|
605 lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
|
|
|
606 NULL_RTX, 1);
|
|
|
607 lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
|
|
|
608 NULL_RTX, 0);
|
|
|
609 }
|
|
|
610 do_compare_rtx_and_jump (res, lres,
|
|
|
611 EQ, true, tgtmode, NULL_RTX, NULL, done_label,
|
|
|
612 profile_probability::very_likely ());
|
|
|
613 expand_arith_set_overflow (lhs, target);
|
|
|
614 emit_label (done_label);
|
|
|
615 }
|
|
|
616 write_complex_part (target, lres, false);
|
|
|
617 }
|
|
|
618
|
|
|
619 /* Helper for expand_*_overflow. Store RES into TARGET. */
|
|
|
620
|
|
|
621 static void
|
|
|
622 expand_ubsan_result_store (rtx target, rtx res)
|
|
|
623 {
|
|
|
624 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
|
|
|
625 /* If this is a scalar in a register that is stored in a wider mode
|
|
|
626 than the declared mode, compute the result into its declared mode
|
|
|
627 and then convert to the wider mode. Our value is the computed
|
|
|
628 expression. */
|
|
|
629 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
|
|
|
630 else
|
|
|
631 emit_move_insn (target, res);
|
|
|
632 }
|
|
|
633
|
|
|
634 /* Add sub/add overflow checking to the statement STMT.
|
|
|
635 CODE says whether the operation is +, or -. */
|
|
|
636
|
|
|
637 static void
|
|
|
638 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
|
|
|
639 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
|
|
|
640 bool uns1_p, bool is_ubsan, tree *datap)
|
|
|
641 {
|
|
|
642 rtx res, target = NULL_RTX;
|
|
|
643 tree fn;
|
|
|
644 rtx_code_label *done_label = gen_label_rtx ();
|
|
|
645 rtx_code_label *do_error = gen_label_rtx ();
|
|
|
646 do_pending_stack_adjust ();
|
|
|
647 rtx op0 = expand_normal (arg0);
|
|
|
648 rtx op1 = expand_normal (arg1);
|
|
|
649 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
|
|
|
650 int prec = GET_MODE_PRECISION (mode);
|
|
|
651 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
|
|
|
652 bool do_xor = false;
|
|
|
653
|
|
|
654 if (is_ubsan)
|
|
|
655 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
|
|
|
656
|
|
|
657 if (lhs)
|
|
|
658 {
|
|
|
659 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
660 if (!is_ubsan)
|
|
|
661 write_complex_part (target, const0_rtx, true);
|
|
|
662 }
|
|
|
663
|
|
|
664 /* We assume both operands and result have the same precision
|
|
|
665 here (GET_MODE_BITSIZE (mode)), S stands for signed type
|
|
|
666 with that precision, U for unsigned type with that precision,
|
|
|
667 sgn for unsigned most significant bit in that precision.
|
|
|
668 s1 is signed first operand, u1 is unsigned first operand,
|
|
|
669 s2 is signed second operand, u2 is unsigned second operand,
|
|
|
670 sr is signed result, ur is unsigned result and the following
|
|
|
671 rules say how to compute result (which is always result of
|
|
|
672 the operands as if both were unsigned, cast to the right
|
|
|
673 signedness) and how to compute whether operation overflowed.
|
|
|
674
|
|
|
675 s1 + s2 -> sr
|
|
|
676 res = (S) ((U) s1 + (U) s2)
|
|
|
677 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
|
|
|
678 s1 - s2 -> sr
|
|
|
679 res = (S) ((U) s1 - (U) s2)
|
|
|
680 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
|
|
|
681 u1 + u2 -> ur
|
|
|
682 res = u1 + u2
|
|
|
683 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
|
|
|
684 u1 - u2 -> ur
|
|
|
685 res = u1 - u2
|
|
|
686 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
|
|
|
687 s1 + u2 -> sr
|
|
|
688 res = (S) ((U) s1 + u2)
|
|
|
689 ovf = ((U) res ^ sgn) < u2
|
|
|
690 s1 + u2 -> ur
|
|
|
691 t1 = (S) (u2 ^ sgn)
|
|
|
692 t2 = s1 + t1
|
|
|
693 res = (U) t2 ^ sgn
|
|
|
694 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
|
|
|
695 s1 - u2 -> sr
|
|
|
696 res = (S) ((U) s1 - u2)
|
|
|
697 ovf = u2 > ((U) s1 ^ sgn)
|
|
|
698 s1 - u2 -> ur
|
|
|
699 res = (U) s1 - u2
|
|
|
700 ovf = s1 < 0 || u2 > (U) s1
|
|
|
701 u1 - s2 -> sr
|
|
|
702 res = u1 - (U) s2
|
|
|
703 ovf = u1 >= ((U) s2 ^ sgn)
|
|
|
704 u1 - s2 -> ur
|
|
|
705 t1 = u1 ^ sgn
|
|
|
706 t2 = t1 - (U) s2
|
|
|
707 res = t2 ^ sgn
|
|
|
708 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
|
|
|
709 s1 + s2 -> ur
|
|
|
710 res = (U) s1 + (U) s2
|
|
|
711 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
|
|
|
712 u1 + u2 -> sr
|
|
|
713 res = (S) (u1 + u2)
|
|
|
714 ovf = (U) res < u2 || res < 0
|
|
|
715 u1 - u2 -> sr
|
|
|
716 res = (S) (u1 - u2)
|
|
|
717 ovf = u1 >= u2 ? res < 0 : res >= 0
|
|
|
718 s1 - s2 -> ur
|
|
|
719 res = (U) s1 - (U) s2
|
|
|
720 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
|
|
|
721
|
|
|
722 if (code == PLUS_EXPR && uns0_p && !uns1_p)
|
|
|
723 {
|
|
|
724 /* PLUS_EXPR is commutative, if operand signedness differs,
|
|
|
725 canonicalize to the first operand being signed and second
|
|
|
726 unsigned to simplify following code. */
|
|
|
727 std::swap (op0, op1);
|
|
|
728 std::swap (arg0, arg1);
|
|
|
729 uns0_p = false;
|
|
|
730 uns1_p = true;
|
|
|
731 }
|
|
|
732
|
|
|
733 /* u1 +- u2 -> ur */
|
|
|
734 if (uns0_p && uns1_p && unsr_p)
|
|
|
735 {
|
|
|
736 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
|
|
|
737 : usubv4_optab, mode);
|
|
|
738 if (icode != CODE_FOR_nothing)
|
|
|
739 {
|
|
|
740 struct expand_operand ops[4];
|
|
|
741 rtx_insn *last = get_last_insn ();
|
|
|
742
|
|
|
743 res = gen_reg_rtx (mode);
|
|
|
744 create_output_operand (&ops[0], res, mode);
|
|
|
745 create_input_operand (&ops[1], op0, mode);
|
|
|
746 create_input_operand (&ops[2], op1, mode);
|
|
|
747 create_fixed_operand (&ops[3], do_error);
|
|
|
748 if (maybe_expand_insn (icode, 4, ops))
|
|
|
749 {
|
|
|
750 last = get_last_insn ();
|
|
|
751 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
752 && JUMP_P (last)
|
|
|
753 && any_condjump_p (last)
|
|
|
754 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
755 add_reg_br_prob_note (last,
|
|
|
756 profile_probability::very_unlikely ());
|
|
|
757 emit_jump (done_label);
|
|
|
758 goto do_error_label;
|
|
|
759 }
|
|
|
760
|
|
|
761 delete_insns_since (last);
|
|
|
762 }
|
|
|
763
|
|
|
764 /* Compute the operation. On RTL level, the addition is always
|
|
|
765 unsigned. */
|
|
|
766 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
767 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
768 rtx tem = op0;
|
|
|
769 /* For PLUS_EXPR, the operation is commutative, so we can pick
|
|
|
770 operand to compare against. For prec <= BITS_PER_WORD, I think
|
|
|
771 preferring REG operand is better over CONST_INT, because
|
|
|
772 the CONST_INT might enlarge the instruction or CSE would need
|
|
|
773 to figure out we'd already loaded it into a register before.
|
|
|
774 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
|
|
|
775 as then the multi-word comparison can be perhaps simplified. */
|
|
|
776 if (code == PLUS_EXPR
|
|
|
777 && (prec <= BITS_PER_WORD
|
|
|
778 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
|
|
|
779 : CONST_SCALAR_INT_P (op1)))
|
|
|
780 tem = op1;
|
|
|
781 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
|
|
|
782 true, mode, NULL_RTX, NULL, done_label,
|
|
|
783 profile_probability::very_likely ());
|
|
|
784 goto do_error_label;
|
|
|
785 }
|
|
|
786
|
|
|
787 /* s1 +- u2 -> sr */
|
|
|
788 if (!uns0_p && uns1_p && !unsr_p)
|
|
|
789 {
|
|
|
790 /* Compute the operation. On RTL level, the addition is always
|
|
|
791 unsigned. */
|
|
|
792 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
793 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
794 rtx tem = expand_binop (mode, add_optab,
|
|
|
795 code == PLUS_EXPR ? res : op0, sgn,
|
|
|
796 NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
797 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
|
|
|
798 done_label, profile_probability::very_likely ());
|
|
|
799 goto do_error_label;
|
|
|
800 }
|
|
|
801
|
|
|
802 /* s1 + u2 -> ur */
|
|
|
803 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
|
|
|
804 {
|
|
|
805 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
|
|
|
806 OPTAB_LIB_WIDEN);
|
|
|
807 /* As we've changed op1, we have to avoid using the value range
|
|
|
808 for the original argument. */
|
|
|
809 arg1 = error_mark_node;
|
|
|
810 do_xor = true;
|
|
|
811 goto do_signed;
|
|
|
812 }
|
|
|
813
|
|
|
814 /* u1 - s2 -> ur */
|
|
|
815 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
|
|
|
816 {
|
|
|
817 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
|
|
|
818 OPTAB_LIB_WIDEN);
|
|
|
819 /* As we've changed op0, we have to avoid using the value range
|
|
|
820 for the original argument. */
|
|
|
821 arg0 = error_mark_node;
|
|
|
822 do_xor = true;
|
|
|
823 goto do_signed;
|
|
|
824 }
|
|
|
825
|
|
|
826 /* s1 - u2 -> ur */
|
|
|
827 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
|
|
|
828 {
|
|
|
829 /* Compute the operation. On RTL level, the addition is always
|
|
|
830 unsigned. */
|
|
|
831 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
|
832 OPTAB_LIB_WIDEN);
|
|
|
833 int pos_neg = get_range_pos_neg (arg0);
|
|
|
834 if (pos_neg == 2)
|
|
|
835 /* If ARG0 is known to be always negative, this is always overflow. */
|
|
|
836 emit_jump (do_error);
|
|
|
837 else if (pos_neg == 3)
|
|
|
838 /* If ARG0 is not known to be always positive, check at runtime. */
|
|
|
839 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
|
|
|
840 NULL, do_error, profile_probability::very_unlikely ());
|
|
|
841 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
|
|
|
842 done_label, profile_probability::very_likely ());
|
|
|
843 goto do_error_label;
|
|
|
844 }
|
|
|
845
|
|
|
846 /* u1 - s2 -> sr */
|
|
|
847 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
|
|
|
848 {
|
|
|
849 /* Compute the operation. On RTL level, the addition is always
|
|
|
850 unsigned. */
|
|
|
851 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
|
852 OPTAB_LIB_WIDEN);
|
|
|
853 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
|
|
|
854 OPTAB_LIB_WIDEN);
|
|
|
855 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
|
|
|
856 done_label, profile_probability::very_likely ());
|
|
|
857 goto do_error_label;
|
|
|
858 }
|
|
|
859
|
|
|
860 /* u1 + u2 -> sr */
|
|
|
861 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
|
|
|
862 {
|
|
|
863 /* Compute the operation. On RTL level, the addition is always
|
|
|
864 unsigned. */
|
|
|
865 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
|
|
|
866 OPTAB_LIB_WIDEN);
|
|
|
867 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
|
|
|
868 NULL, do_error, profile_probability::very_unlikely ());
|
|
|
869 rtx tem = op1;
|
|
|
870 /* The operation is commutative, so we can pick operand to compare
|
|
|
871 against. For prec <= BITS_PER_WORD, I think preferring REG operand
|
|
|
872 is better over CONST_INT, because the CONST_INT might enlarge the
|
|
|
873 instruction or CSE would need to figure out we'd already loaded it
|
|
|
874 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
|
|
|
875 might be more beneficial, as then the multi-word comparison can be
|
|
|
876 perhaps simplified. */
|
|
|
877 if (prec <= BITS_PER_WORD
|
|
|
878 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
|
|
|
879 : CONST_SCALAR_INT_P (op0))
|
|
|
880 tem = op0;
|
|
|
881 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
|
|
|
882 done_label, profile_probability::very_likely ());
|
|
|
883 goto do_error_label;
|
|
|
884 }
|
|
|
885
|
|
|
886 /* s1 +- s2 -> ur */
|
|
|
887 if (!uns0_p && !uns1_p && unsr_p)
|
|
|
888 {
|
|
|
889 /* Compute the operation. On RTL level, the addition is always
|
|
|
890 unsigned. */
|
|
|
891 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
892 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
893 int pos_neg = get_range_pos_neg (arg1);
|
|
|
894 if (code == PLUS_EXPR)
|
|
|
895 {
|
|
|
896 int pos_neg0 = get_range_pos_neg (arg0);
|
|
|
897 if (pos_neg0 != 3 && pos_neg == 3)
|
|
|
898 {
|
|
|
899 std::swap (op0, op1);
|
|
|
900 pos_neg = pos_neg0;
|
|
|
901 }
|
|
|
902 }
|
|
|
903 rtx tem;
|
|
|
904 if (pos_neg != 3)
|
|
|
905 {
|
|
|
906 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
|
|
|
907 ? and_optab : ior_optab,
|
|
|
908 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
909 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
|
|
|
910 NULL, done_label, profile_probability::very_likely ());
|
|
|
911 }
|
|
|
912 else
|
|
|
913 {
|
|
|
914 rtx_code_label *do_ior_label = gen_label_rtx ();
|
|
|
915 do_compare_rtx_and_jump (op1, const0_rtx,
|
|
|
916 code == MINUS_EXPR ? GE : LT, false, mode,
|
|
|
917 NULL_RTX, NULL, do_ior_label,
|
|
|
918 profile_probability::even ());
|
|
|
919 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
|
|
|
920 OPTAB_LIB_WIDEN);
|
|
|
921 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
922 NULL, done_label, profile_probability::very_likely ());
|
|
|
923 emit_jump (do_error);
|
|
|
924 emit_label (do_ior_label);
|
|
|
925 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
|
|
|
926 OPTAB_LIB_WIDEN);
|
|
|
927 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
928 NULL, done_label, profile_probability::very_likely ());
|
|
|
929 }
|
|
|
930 goto do_error_label;
|
|
|
931 }
|
|
|
932
|
|
|
933 /* u1 - u2 -> sr */
|
|
|
934 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
|
|
|
935 {
|
|
|
936 /* Compute the operation. On RTL level, the addition is always
|
|
|
937 unsigned. */
|
|
|
938 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
|
|
|
939 OPTAB_LIB_WIDEN);
|
|
|
940 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
|
|
|
941 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
|
|
|
942 op0_geu_op1, profile_probability::even ());
|
|
|
943 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
|
|
|
944 NULL, done_label, profile_probability::very_likely ());
|
|
|
945 emit_jump (do_error);
|
|
|
946 emit_label (op0_geu_op1);
|
|
|
947 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
948 NULL, done_label, profile_probability::very_likely ());
|
|
|
949 goto do_error_label;
|
|
|
950 }
|
|
|
951
|
|
|
952 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
|
|
|
953
|
|
|
954 /* s1 +- s2 -> sr */
|
|
|
955 do_signed:
|
|
|
956 {
|
|
|
957 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
|
|
|
958 : subv4_optab, mode);
|
|
|
959 if (icode != CODE_FOR_nothing)
|
|
|
960 {
|
|
|
961 struct expand_operand ops[4];
|
|
|
962 rtx_insn *last = get_last_insn ();
|
|
|
963
|
|
|
964 res = gen_reg_rtx (mode);
|
|
|
965 create_output_operand (&ops[0], res, mode);
|
|
|
966 create_input_operand (&ops[1], op0, mode);
|
|
|
967 create_input_operand (&ops[2], op1, mode);
|
|
|
968 create_fixed_operand (&ops[3], do_error);
|
|
|
969 if (maybe_expand_insn (icode, 4, ops))
|
|
|
970 {
|
|
|
971 last = get_last_insn ();
|
|
|
972 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
973 && JUMP_P (last)
|
|
|
974 && any_condjump_p (last)
|
|
|
975 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
976 add_reg_br_prob_note (last,
|
|
|
977 profile_probability::very_unlikely ());
|
|
|
978 emit_jump (done_label);
|
|
|
979 goto do_error_label;
|
|
|
980 }
|
|
|
981
|
|
|
982 delete_insns_since (last);
|
|
|
983 }
|
|
|
984
|
|
|
985 /* Compute the operation. On RTL level, the addition is always
|
|
|
986 unsigned. */
|
|
|
987 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
|
|
|
988 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
|
|
|
989
|
|
|
990 /* If we can prove that one of the arguments (for MINUS_EXPR only
|
|
|
991 the second operand, as subtraction is not commutative) is always
|
|
|
992 non-negative or always negative, we can do just one comparison
|
|
|
993 and conditional jump. */
|
|
|
994 int pos_neg = get_range_pos_neg (arg1);
|
|
|
995 if (code == PLUS_EXPR)
|
|
|
996 {
|
|
|
997 int pos_neg0 = get_range_pos_neg (arg0);
|
|
|
998 if (pos_neg0 != 3 && pos_neg == 3)
|
|
|
999 {
|
|
|
1000 std::swap (op0, op1);
|
|
|
1001 pos_neg = pos_neg0;
|
|
|
1002 }
|
|
|
1003 }
|
|
|
1004
|
|
|
1005 /* Addition overflows if and only if the two operands have the same sign,
|
|
|
1006 and the result has the opposite sign. Subtraction overflows if and
|
|
|
1007 only if the two operands have opposite sign, and the subtrahend has
|
|
|
1008 the same sign as the result. Here 0 is counted as positive. */
|
|
|
1009 if (pos_neg == 3)
|
|
|
1010 {
|
|
|
1011 /* Compute op0 ^ op1 (operands have opposite sign). */
|
|
|
1012 rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
|
|
|
1013 OPTAB_LIB_WIDEN);
|
|
|
1014
|
|
|
1015 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
|
|
|
1016 rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
|
|
|
1017 OPTAB_LIB_WIDEN);
|
|
|
1018
|
|
|
1019 rtx tem;
|
|
|
1020 if (code == PLUS_EXPR)
|
|
|
1021 {
|
|
|
1022 /* Compute (res ^ op1) & ~(op0 ^ op1). */
|
|
|
1023 tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
|
|
|
1024 tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
|
|
|
1025 OPTAB_LIB_WIDEN);
|
|
|
1026 }
|
|
|
1027 else
|
|
|
1028 {
|
|
|
1029 /* Compute (op0 ^ op1) & ~(res ^ op1). */
|
|
|
1030 tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
|
|
|
1031 tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
|
|
|
1032 OPTAB_LIB_WIDEN);
|
|
|
1033 }
|
|
|
1034
|
|
|
1035 /* No overflow if the result has bit sign cleared. */
|
|
|
1036 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1037 NULL, done_label, profile_probability::very_likely ());
|
|
|
1038 }
|
|
|
1039
|
|
|
1040 /* Compare the result of the operation with the first operand.
|
|
|
1041 No overflow for addition if second operand is positive and result
|
|
|
1042 is larger or second operand is negative and result is smaller.
|
|
|
1043 Likewise for subtraction with sign of second operand flipped. */
|
|
|
1044 else
|
|
|
1045 do_compare_rtx_and_jump (res, op0,
|
|
|
1046 (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
|
|
|
1047 false, mode, NULL_RTX, NULL, done_label,
|
|
|
1048 profile_probability::very_likely ());
|
|
|
1049 }
|
|
|
1050
|
|
|
1051 do_error_label:
|
|
|
1052 emit_label (do_error);
|
|
|
1053 if (is_ubsan)
|
|
|
1054 {
|
|
|
1055 /* Expand the ubsan builtin call. */
|
|
|
1056 push_temp_slots ();
|
|
|
1057 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
|
|
|
1058 arg0, arg1, datap);
|
|
|
1059 expand_normal (fn);
|
|
|
1060 pop_temp_slots ();
|
|
|
1061 do_pending_stack_adjust ();
|
|
|
1062 }
|
|
|
1063 else if (lhs)
|
|
|
1064 expand_arith_set_overflow (lhs, target);
|
|
|
1065
|
|
|
1066 /* We're done. */
|
|
|
1067 emit_label (done_label);
|
|
|
1068
|
|
|
1069 if (lhs)
|
|
|
1070 {
|
|
|
1071 if (is_ubsan)
|
|
|
1072 expand_ubsan_result_store (target, res);
|
|
|
1073 else
|
|
|
1074 {
|
|
|
1075 if (do_xor)
|
|
|
1076 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
|
|
|
1077 OPTAB_LIB_WIDEN);
|
|
|
1078
|
|
|
1079 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
|
1080 }
|
|
|
1081 }
|
|
|
1082 }
|
|
|
1083
|
|
|
1084 /* Add negate overflow checking to the statement STMT. */
|
|
|
1085
|
|
|
1086 static void
|
|
|
1087 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
|
|
|
1088 tree *datap)
|
|
|
1089 {
|
|
|
1090 rtx res, op1;
|
|
|
1091 tree fn;
|
|
|
1092 rtx_code_label *done_label, *do_error;
|
|
|
1093 rtx target = NULL_RTX;
|
|
|
1094
|
|
|
1095 done_label = gen_label_rtx ();
|
|
|
1096 do_error = gen_label_rtx ();
|
|
|
1097
|
|
|
1098 do_pending_stack_adjust ();
|
|
|
1099 op1 = expand_normal (arg1);
|
|
|
1100
|
|
|
1101 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
|
|
|
1102 if (lhs)
|
|
|
1103 {
|
|
|
1104 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
1105 if (!is_ubsan)
|
|
|
1106 write_complex_part (target, const0_rtx, true);
|
|
|
1107 }
|
|
|
1108
|
|
|
1109 enum insn_code icode = optab_handler (negv3_optab, mode);
|
|
|
1110 if (icode != CODE_FOR_nothing)
|
|
|
1111 {
|
|
|
1112 struct expand_operand ops[3];
|
|
|
1113 rtx_insn *last = get_last_insn ();
|
|
|
1114
|
|
|
1115 res = gen_reg_rtx (mode);
|
|
|
1116 create_output_operand (&ops[0], res, mode);
|
|
|
1117 create_input_operand (&ops[1], op1, mode);
|
|
|
1118 create_fixed_operand (&ops[2], do_error);
|
|
|
1119 if (maybe_expand_insn (icode, 3, ops))
|
|
|
1120 {
|
|
|
1121 last = get_last_insn ();
|
|
|
1122 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
1123 && JUMP_P (last)
|
|
|
1124 && any_condjump_p (last)
|
|
|
1125 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
1126 add_reg_br_prob_note (last,
|
|
|
1127 profile_probability::very_unlikely ());
|
|
|
1128 emit_jump (done_label);
|
|
|
1129 }
|
|
|
1130 else
|
|
|
1131 {
|
|
|
1132 delete_insns_since (last);
|
|
|
1133 icode = CODE_FOR_nothing;
|
|
|
1134 }
|
|
|
1135 }
|
|
|
1136
|
|
|
1137 if (icode == CODE_FOR_nothing)
|
|
|
1138 {
|
|
|
1139 /* Compute the operation. On RTL level, the addition is always
|
|
|
1140 unsigned. */
|
|
|
1141 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
|
|
|
1142
|
|
|
1143 /* Compare the operand with the most negative value. */
|
|
|
1144 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
|
|
|
1145 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
|
|
|
1146 done_label, profile_probability::very_likely ());
|
|
|
1147 }
|
|
|
1148
|
|
|
1149 emit_label (do_error);
|
|
|
1150 if (is_ubsan)
|
|
|
1151 {
|
|
|
1152 /* Expand the ubsan builtin call. */
|
|
|
1153 push_temp_slots ();
|
|
|
1154 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
|
|
|
1155 arg1, NULL_TREE, datap);
|
|
|
1156 expand_normal (fn);
|
|
|
1157 pop_temp_slots ();
|
|
|
1158 do_pending_stack_adjust ();
|
|
|
1159 }
|
|
|
1160 else if (lhs)
|
|
|
1161 expand_arith_set_overflow (lhs, target);
|
|
|
1162
|
|
|
1163 /* We're done. */
|
|
|
1164 emit_label (done_label);
|
|
|
1165
|
|
|
1166 if (lhs)
|
|
|
1167 {
|
|
|
1168 if (is_ubsan)
|
|
|
1169 expand_ubsan_result_store (target, res);
|
|
|
1170 else
|
|
|
1171 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
|
1172 }
|
|
|
1173 }
|
|
|
1174
|
|
|
1175 /* Add mul overflow checking to the statement STMT. */
|
|
|
1176
|
|
|
1177 static void
|
|
|
1178 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
|
|
|
1179 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
|
|
|
1180 tree *datap)
|
|
|
1181 {
|
|
|
1182 rtx res, op0, op1;
|
|
|
1183 tree fn, type;
|
|
|
1184 rtx_code_label *done_label, *do_error;
|
|
|
1185 rtx target = NULL_RTX;
|
|
|
1186 signop sign;
|
|
|
1187 enum insn_code icode;
|
|
|
1188
|
|
|
1189 done_label = gen_label_rtx ();
|
|
|
1190 do_error = gen_label_rtx ();
|
|
|
1191
|
|
|
1192 do_pending_stack_adjust ();
|
|
|
1193 op0 = expand_normal (arg0);
|
|
|
1194 op1 = expand_normal (arg1);
|
|
|
1195
|
|
|
1196 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
|
|
|
1197 bool uns = unsr_p;
|
|
|
1198 if (lhs)
|
|
|
1199 {
|
|
|
1200 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
1201 if (!is_ubsan)
|
|
|
1202 write_complex_part (target, const0_rtx, true);
|
|
|
1203 }
|
|
|
1204
|
|
|
1205 if (is_ubsan)
|
|
|
1206 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
|
|
|
1207
|
|
|
1208 /* We assume both operands and result have the same precision
|
|
|
1209 here (GET_MODE_BITSIZE (mode)), S stands for signed type
|
|
|
1210 with that precision, U for unsigned type with that precision,
|
|
|
1211 sgn for unsigned most significant bit in that precision.
|
|
|
1212 s1 is signed first operand, u1 is unsigned first operand,
|
|
|
1213 s2 is signed second operand, u2 is unsigned second operand,
|
|
|
1214 sr is signed result, ur is unsigned result and the following
|
|
|
1215 rules say how to compute result (which is always result of
|
|
|
1216 the operands as if both were unsigned, cast to the right
|
|
|
1217 signedness) and how to compute whether operation overflowed.
|
|
|
1218 main_ovf (false) stands for jump on signed multiplication
|
|
|
1219 overflow or the main algorithm with uns == false.
|
|
|
1220 main_ovf (true) stands for jump on unsigned multiplication
|
|
|
1221 overflow or the main algorithm with uns == true.
|
|
|
1222
|
|
|
1223 s1 * s2 -> sr
|
|
|
1224 res = (S) ((U) s1 * (U) s2)
|
|
|
1225 ovf = main_ovf (false)
|
|
|
1226 u1 * u2 -> ur
|
|
|
1227 res = u1 * u2
|
|
|
1228 ovf = main_ovf (true)
|
|
|
1229 s1 * u2 -> ur
|
|
|
1230 res = (U) s1 * u2
|
|
|
1231 ovf = (s1 < 0 && u2) || main_ovf (true)
|
|
|
1232 u1 * u2 -> sr
|
|
|
1233 res = (S) (u1 * u2)
|
|
|
1234 ovf = res < 0 || main_ovf (true)
|
|
|
1235 s1 * u2 -> sr
|
|
|
1236 res = (S) ((U) s1 * u2)
|
|
|
1237 ovf = (S) u2 >= 0 ? main_ovf (false)
|
|
|
1238 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
|
|
|
1239 s1 * s2 -> ur
|
|
|
1240 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
|
|
|
1241 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
|
|
|
1242 res = t1 * t2
|
|
|
1243 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
|
|
|
1244
|
|
|
1245 if (uns0_p && !uns1_p)
|
|
|
1246 {
|
|
|
1247 /* Multiplication is commutative, if operand signedness differs,
|
|
|
1248 canonicalize to the first operand being signed and second
|
|
|
1249 unsigned to simplify following code. */
|
|
|
1250 std::swap (op0, op1);
|
|
|
1251 std::swap (arg0, arg1);
|
|
|
1252 uns0_p = false;
|
|
|
1253 uns1_p = true;
|
|
|
1254 }
|
|
|
1255
|
|
|
1256 int pos_neg0 = get_range_pos_neg (arg0);
|
|
|
1257 int pos_neg1 = get_range_pos_neg (arg1);
|
|
|
1258
|
|
|
1259 /* s1 * u2 -> ur */
|
|
|
1260 if (!uns0_p && uns1_p && unsr_p)
|
|
|
1261 {
|
|
|
1262 switch (pos_neg0)
|
|
|
1263 {
|
|
|
1264 case 1:
|
|
|
1265 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
|
|
|
1266 goto do_main;
|
|
|
1267 case 2:
|
|
|
1268 /* If s1 is negative, avoid the main code, just multiply and
|
|
|
1269 signal overflow if op1 is not 0. */
|
|
|
1270 struct separate_ops ops;
|
|
|
1271 ops.code = MULT_EXPR;
|
|
|
1272 ops.type = TREE_TYPE (arg1);
|
|
|
1273 ops.op0 = make_tree (ops.type, op0);
|
|
|
1274 ops.op1 = make_tree (ops.type, op1);
|
|
|
1275 ops.op2 = NULL_TREE;
|
|
|
1276 ops.location = loc;
|
|
|
1277 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1278 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1279 NULL, done_label, profile_probability::very_likely ());
|
|
|
1280 goto do_error_label;
|
|
|
1281 case 3:
|
|
|
1282 rtx_code_label *do_main_label;
|
|
|
1283 do_main_label = gen_label_rtx ();
|
|
|
1284 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1285 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1286 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1287 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1288 expand_arith_set_overflow (lhs, target);
|
|
|
1289 emit_label (do_main_label);
|
|
|
1290 goto do_main;
|
|
|
1291 default:
|
|
|
1292 gcc_unreachable ();
|
|
|
1293 }
|
|
|
1294 }
|
|
|
1295
|
|
|
1296 /* u1 * u2 -> sr */
|
|
|
1297 if (uns0_p && uns1_p && !unsr_p)
|
|
|
1298 {
|
|
|
1299 uns = true;
|
|
|
1300 /* Rest of handling of this case after res is computed. */
|
|
|
1301 goto do_main;
|
|
|
1302 }
|
|
|
1303
|
|
|
1304 /* s1 * u2 -> sr */
|
|
|
1305 if (!uns0_p && uns1_p && !unsr_p)
|
|
|
1306 {
|
|
|
1307 switch (pos_neg1)
|
|
|
1308 {
|
|
|
1309 case 1:
|
|
|
1310 goto do_main;
|
|
|
1311 case 2:
|
|
|
1312 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
|
|
|
1313 avoid the main code, just multiply and signal overflow
|
|
|
1314 unless 0 * u2 or -1 * ((U) Smin). */
|
|
|
1315 struct separate_ops ops;
|
|
|
1316 ops.code = MULT_EXPR;
|
|
|
1317 ops.type = TREE_TYPE (arg1);
|
|
|
1318 ops.op0 = make_tree (ops.type, op0);
|
|
|
1319 ops.op1 = make_tree (ops.type, op1);
|
|
|
1320 ops.op2 = NULL_TREE;
|
|
|
1321 ops.location = loc;
|
|
|
1322 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1323 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1324 NULL, done_label, profile_probability::very_likely ());
|
|
|
1325 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
|
|
|
1326 NULL, do_error, profile_probability::very_unlikely ());
|
|
|
1327 int prec;
|
|
|
1328 prec = GET_MODE_PRECISION (mode);
|
|
|
1329 rtx sgn;
|
|
|
1330 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
|
|
|
1331 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
|
|
|
1332 NULL, done_label, profile_probability::very_likely ());
|
|
|
1333 goto do_error_label;
|
|
|
1334 case 3:
|
|
|
1335 /* Rest of handling of this case after res is computed. */
|
|
|
1336 goto do_main;
|
|
|
1337 default:
|
|
|
1338 gcc_unreachable ();
|
|
|
1339 }
|
|
|
1340 }
|
|
|
1341
|
|
|
1342 /* s1 * s2 -> ur */
|
|
|
1343 if (!uns0_p && !uns1_p && unsr_p)
|
|
|
1344 {
|
|
|
1345 rtx tem, tem2;
|
|
|
1346 switch (pos_neg0 | pos_neg1)
|
|
|
1347 {
|
|
|
1348 case 1: /* Both operands known to be non-negative. */
|
|
|
1349 goto do_main;
|
|
|
1350 case 2: /* Both operands known to be negative. */
|
|
|
1351 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
|
|
|
1352 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
|
|
|
1353 /* Avoid looking at arg0/arg1 ranges, as we've changed
|
|
|
1354 the arguments. */
|
|
|
1355 arg0 = error_mark_node;
|
|
|
1356 arg1 = error_mark_node;
|
|
|
1357 goto do_main;
|
|
|
1358 case 3:
|
|
|
1359 if ((pos_neg0 ^ pos_neg1) == 3)
|
|
|
1360 {
|
|
|
1361 /* If one operand is known to be negative and the other
|
|
|
1362 non-negative, this overflows always, unless the non-negative
|
|
|
1363 one is 0. Just do normal multiply and set overflow
|
|
|
1364 unless one of the operands is 0. */
|
|
|
1365 struct separate_ops ops;
|
|
|
1366 ops.code = MULT_EXPR;
|
|
|
1367 ops.type
|
|
|
1368 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
|
|
|
1369 1);
|
|
|
1370 ops.op0 = make_tree (ops.type, op0);
|
|
|
1371 ops.op1 = make_tree (ops.type, op1);
|
|
|
1372 ops.op2 = NULL_TREE;
|
|
|
1373 ops.location = loc;
|
|
|
1374 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1375 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
|
|
|
1376 OPTAB_LIB_WIDEN);
|
|
|
1377 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode,
|
|
|
1378 NULL_RTX, NULL, done_label,
|
|
|
1379 profile_probability::very_likely ());
|
|
|
1380 goto do_error_label;
|
|
|
1381 }
|
|
|
1382 /* The general case, do all the needed comparisons at runtime. */
|
|
|
1383 rtx_code_label *do_main_label, *after_negate_label;
|
|
|
1384 rtx rop0, rop1;
|
|
|
1385 rop0 = gen_reg_rtx (mode);
|
|
|
1386 rop1 = gen_reg_rtx (mode);
|
|
|
1387 emit_move_insn (rop0, op0);
|
|
|
1388 emit_move_insn (rop1, op1);
|
|
|
1389 op0 = rop0;
|
|
|
1390 op1 = rop1;
|
|
|
1391 do_main_label = gen_label_rtx ();
|
|
|
1392 after_negate_label = gen_label_rtx ();
|
|
|
1393 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
|
|
|
1394 OPTAB_LIB_WIDEN);
|
|
|
1395 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1396 NULL, after_negate_label, profile_probability::very_likely ());
|
|
|
1397 /* Both arguments negative here, negate them and continue with
|
|
|
1398 normal unsigned overflow checking multiplication. */
|
|
|
1399 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
|
|
|
1400 NULL_RTX, false));
|
|
|
1401 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
|
|
|
1402 NULL_RTX, false));
|
|
|
1403 /* Avoid looking at arg0/arg1 ranges, as we might have changed
|
|
|
1404 the arguments. */
|
|
|
1405 arg0 = error_mark_node;
|
|
|
1406 arg1 = error_mark_node;
|
|
|
1407 emit_jump (do_main_label);
|
|
|
1408 emit_label (after_negate_label);
|
|
|
1409 tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
|
|
|
1410 OPTAB_LIB_WIDEN);
|
|
|
1411 do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1412 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1413 /* One argument is negative here, the other positive. This
|
|
|
1414 overflows always, unless one of the arguments is 0. But
|
|
|
1415 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
|
|
|
1416 is, thus we can keep do_main code oring in overflow as is. */
|
|
|
1417 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1418 NULL, do_main_label, profile_probability::very_likely ());
|
|
|
1419 expand_arith_set_overflow (lhs, target);
|
|
|
1420 emit_label (do_main_label);
|
|
|
1421 goto do_main;
|
|
|
1422 default:
|
|
|
1423 gcc_unreachable ();
|
|
|
1424 }
|
|
|
1425 }
|
|
|
1426
|
|
|
1427 do_main:
|
|
|
1428 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
|
|
|
1429 sign = uns ? UNSIGNED : SIGNED;
|
|
|
1430 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
|
|
|
1431 if (icode != CODE_FOR_nothing)
|
|
|
1432 {
|
|
|
1433 struct expand_operand ops[4];
|
|
|
1434 rtx_insn *last = get_last_insn ();
|
|
|
1435
|
|
|
1436 res = gen_reg_rtx (mode);
|
|
|
1437 create_output_operand (&ops[0], res, mode);
|
|
|
1438 create_input_operand (&ops[1], op0, mode);
|
|
|
1439 create_input_operand (&ops[2], op1, mode);
|
|
|
1440 create_fixed_operand (&ops[3], do_error);
|
|
|
1441 if (maybe_expand_insn (icode, 4, ops))
|
|
|
1442 {
|
|
|
1443 last = get_last_insn ();
|
|
|
1444 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
|
|
|
1445 && JUMP_P (last)
|
|
|
1446 && any_condjump_p (last)
|
|
|
1447 && !find_reg_note (last, REG_BR_PROB, 0))
|
|
|
1448 add_reg_br_prob_note (last,
|
|
|
1449 profile_probability::very_unlikely ());
|
|
|
1450 emit_jump (done_label);
|
|
|
1451 }
|
|
|
1452 else
|
|
|
1453 {
|
|
|
1454 delete_insns_since (last);
|
|
|
1455 icode = CODE_FOR_nothing;
|
|
|
1456 }
|
|
|
1457 }
|
|
|
1458
|
|
|
1459 if (icode == CODE_FOR_nothing)
|
|
|
1460 {
|
|
|
1461 struct separate_ops ops;
|
|
|
1462 int prec = GET_MODE_PRECISION (mode);
|
|
|
1463 scalar_int_mode hmode, wmode;
|
|
|
1464 ops.op0 = make_tree (type, op0);
|
|
|
1465 ops.op1 = make_tree (type, op1);
|
|
|
1466 ops.op2 = NULL_TREE;
|
|
|
1467 ops.location = loc;
|
|
|
1468 if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
|
|
|
1469 && targetm.scalar_mode_supported_p (wmode))
|
|
|
1470 {
|
|
|
1471 ops.code = WIDEN_MULT_EXPR;
|
|
|
1472 ops.type
|
|
|
1473 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
|
|
|
1474
|
|
|
1475 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
|
|
|
1476 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
|
|
|
1477 NULL_RTX, uns);
|
|
|
1478 hipart = convert_modes (mode, wmode, hipart, uns);
|
|
|
1479 res = convert_modes (mode, wmode, res, uns);
|
|
|
1480 if (uns)
|
|
|
1481 /* For the unsigned multiplication, there was overflow if
|
|
|
1482 HIPART is non-zero. */
|
|
|
1483 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
|
|
|
1484 NULL_RTX, NULL, done_label,
|
|
|
1485 profile_probability::very_likely ());
|
|
|
1486 else
|
|
|
1487 {
|
|
|
1488 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
|
|
|
1489 NULL_RTX, 0);
|
|
|
1490 /* RES is low half of the double width result, HIPART
|
|
|
1491 the high half. There was overflow if
|
|
|
1492 HIPART is different from RES < 0 ? -1 : 0. */
|
|
|
1493 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
|
|
|
1494 NULL_RTX, NULL, done_label,
|
|
|
1495 profile_probability::very_likely ());
|
|
|
1496 }
|
|
|
1497 }
|
|
|
1498 else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
|
|
|
1499 && 2 * GET_MODE_PRECISION (hmode) == prec)
|
|
|
1500 {
|
|
|
1501 rtx_code_label *large_op0 = gen_label_rtx ();
|
|
|
1502 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
|
|
|
1503 rtx_code_label *one_small_one_large = gen_label_rtx ();
|
|
|
1504 rtx_code_label *both_ops_large = gen_label_rtx ();
|
|
|
1505 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
|
|
|
1506 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
|
|
|
1507 rtx_code_label *do_overflow = gen_label_rtx ();
|
|
|
1508 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
|
|
|
1509
|
|
|
1510 unsigned int hprec = GET_MODE_PRECISION (hmode);
|
|
|
1511 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
|
|
|
1512 NULL_RTX, uns);
|
|
|
1513 hipart0 = convert_modes (hmode, mode, hipart0, uns);
|
|
|
1514 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
|
|
|
1515 rtx signbit0 = const0_rtx;
|
|
|
1516 if (!uns)
|
|
|
1517 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
|
|
|
1518 NULL_RTX, 0);
|
|
|
1519 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
|
|
|
1520 NULL_RTX, uns);
|
|
|
1521 hipart1 = convert_modes (hmode, mode, hipart1, uns);
|
|
|
1522 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
|
|
|
1523 rtx signbit1 = const0_rtx;
|
|
|
1524 if (!uns)
|
|
|
1525 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
|
|
|
1526 NULL_RTX, 0);
|
|
|
1527
|
|
|
1528 res = gen_reg_rtx (mode);
|
|
|
1529
|
|
|
1530 /* True if op0 resp. op1 are known to be in the range of
|
|
|
1531 halfstype. */
|
|
|
1532 bool op0_small_p = false;
|
|
|
1533 bool op1_small_p = false;
|
|
|
1534 /* True if op0 resp. op1 are known to have all zeros or all ones
|
|
|
1535 in the upper half of bits, but are not known to be
|
|
|
1536 op{0,1}_small_p. */
|
|
|
1537 bool op0_medium_p = false;
|
|
|
1538 bool op1_medium_p = false;
|
|
|
1539 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
|
|
|
1540 nonnegative, 1 if unknown. */
|
|
|
1541 int op0_sign = 1;
|
|
|
1542 int op1_sign = 1;
|
|
|
1543
|
|
|
1544 if (pos_neg0 == 1)
|
|
|
1545 op0_sign = 0;
|
|
|
1546 else if (pos_neg0 == 2)
|
|
|
1547 op0_sign = -1;
|
|
|
1548 if (pos_neg1 == 1)
|
|
|
1549 op1_sign = 0;
|
|
|
1550 else if (pos_neg1 == 2)
|
|
|
1551 op1_sign = -1;
|
|
|
1552
|
|
|
1553 unsigned int mprec0 = prec;
|
|
|
1554 if (arg0 != error_mark_node)
|
|
|
1555 mprec0 = get_min_precision (arg0, sign);
|
|
|
1556 if (mprec0 <= hprec)
|
|
|
1557 op0_small_p = true;
|
|
|
1558 else if (!uns && mprec0 <= hprec + 1)
|
|
|
1559 op0_medium_p = true;
|
|
|
1560 unsigned int mprec1 = prec;
|
|
|
1561 if (arg1 != error_mark_node)
|
|
|
1562 mprec1 = get_min_precision (arg1, sign);
|
|
|
1563 if (mprec1 <= hprec)
|
|
|
1564 op1_small_p = true;
|
|
|
1565 else if (!uns && mprec1 <= hprec + 1)
|
|
|
1566 op1_medium_p = true;
|
|
|
1567
|
|
|
1568 int smaller_sign = 1;
|
|
|
1569 int larger_sign = 1;
|
|
|
1570 if (op0_small_p)
|
|
|
1571 {
|
|
|
1572 smaller_sign = op0_sign;
|
|
|
1573 larger_sign = op1_sign;
|
|
|
1574 }
|
|
|
1575 else if (op1_small_p)
|
|
|
1576 {
|
|
|
1577 smaller_sign = op1_sign;
|
|
|
1578 larger_sign = op0_sign;
|
|
|
1579 }
|
|
|
1580 else if (op0_sign == op1_sign)
|
|
|
1581 {
|
|
|
1582 smaller_sign = op0_sign;
|
|
|
1583 larger_sign = op0_sign;
|
|
|
1584 }
|
|
|
1585
|
|
|
1586 if (!op0_small_p)
|
|
|
1587 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
|
|
|
1588 NULL_RTX, NULL, large_op0,
|
|
|
1589 profile_probability::unlikely ());
|
|
|
1590
|
|
|
1591 if (!op1_small_p)
|
|
|
1592 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
|
|
|
1593 NULL_RTX, NULL, small_op0_large_op1,
|
|
|
1594 profile_probability::unlikely ());
|
|
|
1595
|
|
|
1596 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
|
|
|
1597 hmode to mode, the multiplication will never overflow. We can
|
|
|
1598 do just one hmode x hmode => mode widening multiplication. */
|
|
|
1599 rtx lopart0s = lopart0, lopart1s = lopart1;
|
|
|
1600 if (GET_CODE (lopart0) == SUBREG)
|
|
|
1601 {
|
|
|
1602 lopart0s = shallow_copy_rtx (lopart0);
|
|
|
1603 SUBREG_PROMOTED_VAR_P (lopart0s) = 1;
|
|
|
1604 SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED);
|
|
|
1605 }
|
|
|
1606 if (GET_CODE (lopart1) == SUBREG)
|
|
|
1607 {
|
|
|
1608 lopart1s = shallow_copy_rtx (lopart1);
|
|
|
1609 SUBREG_PROMOTED_VAR_P (lopart1s) = 1;
|
|
|
1610 SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED);
|
|
|
1611 }
|
|
|
1612 tree halfstype = build_nonstandard_integer_type (hprec, uns);
|
|
|
1613 ops.op0 = make_tree (halfstype, lopart0s);
|
|
|
1614 ops.op1 = make_tree (halfstype, lopart1s);
|
|
|
1615 ops.code = WIDEN_MULT_EXPR;
|
|
|
1616 ops.type = type;
|
|
|
1617 rtx thisres
|
|
|
1618 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1619 emit_move_insn (res, thisres);
|
|
|
1620 emit_jump (done_label);
|
|
|
1621
|
|
|
1622 emit_label (small_op0_large_op1);
|
|
|
1623
|
|
|
1624 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
|
|
|
1625 but op1 is not, just swap the arguments and handle it as op1
|
|
|
1626 sign/zero extended, op0 not. */
|
|
|
1627 rtx larger = gen_reg_rtx (mode);
|
|
|
1628 rtx hipart = gen_reg_rtx (hmode);
|
|
|
1629 rtx lopart = gen_reg_rtx (hmode);
|
|
|
1630 emit_move_insn (larger, op1);
|
|
|
1631 emit_move_insn (hipart, hipart1);
|
|
|
1632 emit_move_insn (lopart, lopart0);
|
|
|
1633 emit_jump (one_small_one_large);
|
|
|
1634
|
|
|
1635 emit_label (large_op0);
|
|
|
1636
|
|
|
1637 if (!op1_small_p)
|
|
|
1638 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
|
|
|
1639 NULL_RTX, NULL, both_ops_large,
|
|
|
1640 profile_probability::unlikely ());
|
|
|
1641
|
|
|
1642 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
|
|
|
1643 but op0 is not, prepare larger, hipart and lopart pseudos and
|
|
|
1644 handle it together with small_op0_large_op1. */
|
|
|
1645 emit_move_insn (larger, op0);
|
|
|
1646 emit_move_insn (hipart, hipart0);
|
|
|
1647 emit_move_insn (lopart, lopart1);
|
|
|
1648
|
|
|
1649 emit_label (one_small_one_large);
|
|
|
1650
|
|
|
1651 /* lopart is the low part of the operand that is sign extended
|
|
|
1652 to mode, larger is the other operand, hipart is the
|
|
|
1653 high part of larger and lopart0 and lopart1 are the low parts
|
|
|
1654 of both operands.
|
|
|
1655 We perform lopart0 * lopart1 and lopart * hipart widening
|
|
|
1656 multiplications. */
|
|
|
1657 tree halfutype = build_nonstandard_integer_type (hprec, 1);
|
|
|
1658 ops.op0 = make_tree (halfutype, lopart0);
|
|
|
1659 ops.op1 = make_tree (halfutype, lopart1);
|
|
|
1660 rtx lo0xlo1
|
|
|
1661 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1662
|
|
|
1663 ops.op0 = make_tree (halfutype, lopart);
|
|
|
1664 ops.op1 = make_tree (halfutype, hipart);
|
|
|
1665 rtx loxhi = gen_reg_rtx (mode);
|
|
|
1666 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1667 emit_move_insn (loxhi, tem);
|
|
|
1668
|
|
|
1669 if (!uns)
|
|
|
1670 {
|
|
|
1671 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
|
|
|
1672 if (larger_sign == 0)
|
|
|
1673 emit_jump (after_hipart_neg);
|
|
|
1674 else if (larger_sign != -1)
|
|
|
1675 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
|
|
|
1676 NULL_RTX, NULL, after_hipart_neg,
|
|
|
1677 profile_probability::even ());
|
|
|
1678
|
|
|
1679 tem = convert_modes (mode, hmode, lopart, 1);
|
|
|
1680 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
|
|
|
1681 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
|
|
|
1682 1, OPTAB_DIRECT);
|
|
|
1683 emit_move_insn (loxhi, tem);
|
|
|
1684
|
|
|
1685 emit_label (after_hipart_neg);
|
|
|
1686
|
|
|
1687 /* if (lopart < 0) loxhi -= larger; */
|
|
|
1688 if (smaller_sign == 0)
|
|
|
1689 emit_jump (after_lopart_neg);
|
|
|
1690 else if (smaller_sign != -1)
|
|
|
1691 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
|
|
|
1692 NULL_RTX, NULL, after_lopart_neg,
|
|
|
1693 profile_probability::even ());
|
|
|
1694
|
|
|
1695 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
|
|
|
1696 1, OPTAB_DIRECT);
|
|
|
1697 emit_move_insn (loxhi, tem);
|
|
|
1698
|
|
|
1699 emit_label (after_lopart_neg);
|
|
|
1700 }
|
|
|
1701
|
|
|
1702 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
|
|
|
1703 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
|
|
|
1704 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
|
|
|
1705 1, OPTAB_DIRECT);
|
|
|
1706 emit_move_insn (loxhi, tem);
|
|
|
1707
|
|
|
1708 /* if (loxhi >> (bitsize / 2)
|
|
|
1709 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
|
|
|
1710 if (loxhi >> (bitsize / 2) == 0 (if uns). */
|
|
|
1711 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
|
|
|
1712 NULL_RTX, 0);
|
|
|
1713 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
|
|
|
1714 rtx signbitloxhi = const0_rtx;
|
|
|
1715 if (!uns)
|
|
|
1716 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
|
|
|
1717 convert_modes (hmode, mode,
|
|
|
1718 loxhi, 0),
|
|
|
1719 hprec - 1, NULL_RTX, 0);
|
|
|
1720
|
|
|
1721 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
|
|
|
1722 NULL_RTX, NULL, do_overflow,
|
|
|
1723 profile_probability::very_unlikely ());
|
|
|
1724
|
|
|
1725 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
|
|
|
1726 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
|
|
|
1727 NULL_RTX, 1);
|
|
|
1728 tem = convert_modes (mode, hmode,
|
|
|
1729 convert_modes (hmode, mode, lo0xlo1, 1), 1);
|
|
|
1730
|
|
|
1731 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
|
|
|
1732 1, OPTAB_DIRECT);
|
|
|
1733 if (tem != res)
|
|
|
1734 emit_move_insn (res, tem);
|
|
|
1735 emit_jump (done_label);
|
|
|
1736
|
|
|
1737 emit_label (both_ops_large);
|
|
|
1738
|
|
|
1739 /* If both operands are large (not sign (!uns) or zero (uns)
|
|
|
1740 extended from hmode), then perform the full multiplication
|
|
|
1741 which will be the result of the operation.
|
|
|
1742 The only cases which don't overflow are for signed multiplication
|
|
|
1743 some cases where both hipart0 and highpart1 are 0 or -1.
|
|
|
1744 For unsigned multiplication when high parts are both non-zero
|
|
|
1745 this overflows always. */
|
|
|
1746 ops.code = MULT_EXPR;
|
|
|
1747 ops.op0 = make_tree (type, op0);
|
|
|
1748 ops.op1 = make_tree (type, op1);
|
|
|
1749 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1750 emit_move_insn (res, tem);
|
|
|
1751
|
|
|
1752 if (!uns)
|
|
|
1753 {
|
|
|
1754 if (!op0_medium_p)
|
|
|
1755 {
|
|
|
1756 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
|
|
|
1757 NULL_RTX, 1, OPTAB_DIRECT);
|
|
|
1758 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
|
|
|
1759 NULL_RTX, NULL, do_error,
|
|
|
1760 profile_probability::very_unlikely ());
|
|
|
1761 }
|
|
|
1762
|
|
|
1763 if (!op1_medium_p)
|
|
|
1764 {
|
|
|
1765 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
|
|
|
1766 NULL_RTX, 1, OPTAB_DIRECT);
|
|
|
1767 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
|
|
|
1768 NULL_RTX, NULL, do_error,
|
|
|
1769 profile_probability::very_unlikely ());
|
|
|
1770 }
|
|
|
1771
|
|
|
1772 /* At this point hipart{0,1} are both in [-1, 0]. If they are
|
|
|
1773 the same, overflow happened if res is non-positive, if they
|
|
|
1774 are different, overflow happened if res is positive. */
|
|
|
1775 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
|
|
|
1776 emit_jump (hipart_different);
|
|
|
1777 else if (op0_sign == 1 || op1_sign == 1)
|
|
|
1778 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
|
|
|
1779 NULL_RTX, NULL, hipart_different,
|
|
|
1780 profile_probability::even ());
|
|
|
1781
|
|
|
1782 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
|
|
|
1783 NULL_RTX, NULL, do_error,
|
|
|
1784 profile_probability::very_unlikely ());
|
|
|
1785 emit_jump (done_label);
|
|
|
1786
|
|
|
1787 emit_label (hipart_different);
|
|
|
1788
|
|
|
1789 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
|
|
|
1790 NULL_RTX, NULL, do_error,
|
|
|
1791 profile_probability::very_unlikely ());
|
|
|
1792 emit_jump (done_label);
|
|
|
1793 }
|
|
|
1794
|
|
|
1795 emit_label (do_overflow);
|
|
|
1796
|
|
|
1797 /* Overflow, do full multiplication and fallthru into do_error. */
|
|
|
1798 ops.op0 = make_tree (type, op0);
|
|
|
1799 ops.op1 = make_tree (type, op1);
|
|
|
1800 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1801 emit_move_insn (res, tem);
|
|
|
1802 }
|
|
|
1803 else
|
|
|
1804 {
|
|
|
1805 gcc_assert (!is_ubsan);
|
|
|
1806 ops.code = MULT_EXPR;
|
|
|
1807 ops.type = type;
|
|
|
1808 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
1809 emit_jump (done_label);
|
|
|
1810 }
|
|
|
1811 }
|
|
|
1812
|
|
|
1813 do_error_label:
|
|
|
1814 emit_label (do_error);
|
|
|
1815 if (is_ubsan)
|
|
|
1816 {
|
|
|
1817 /* Expand the ubsan builtin call. */
|
|
|
1818 push_temp_slots ();
|
|
|
1819 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
|
|
|
1820 arg0, arg1, datap);
|
|
|
1821 expand_normal (fn);
|
|
|
1822 pop_temp_slots ();
|
|
|
1823 do_pending_stack_adjust ();
|
|
|
1824 }
|
|
|
1825 else if (lhs)
|
|
|
1826 expand_arith_set_overflow (lhs, target);
|
|
|
1827
|
|
|
1828 /* We're done. */
|
|
|
1829 emit_label (done_label);
|
|
|
1830
|
|
|
1831 /* u1 * u2 -> sr */
|
|
|
1832 if (uns0_p && uns1_p && !unsr_p)
|
|
|
1833 {
|
|
|
1834 rtx_code_label *all_done_label = gen_label_rtx ();
|
|
|
1835 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1836 NULL, all_done_label, profile_probability::very_likely ());
|
|
|
1837 expand_arith_set_overflow (lhs, target);
|
|
|
1838 emit_label (all_done_label);
|
|
|
1839 }
|
|
|
1840
|
|
|
1841 /* s1 * u2 -> sr */
|
|
|
1842 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
|
|
|
1843 {
|
|
|
1844 rtx_code_label *all_done_label = gen_label_rtx ();
|
|
|
1845 rtx_code_label *set_noovf = gen_label_rtx ();
|
|
|
1846 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
|
|
|
1847 NULL, all_done_label, profile_probability::very_likely ());
|
|
|
1848 expand_arith_set_overflow (lhs, target);
|
|
|
1849 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
|
|
|
1850 NULL, set_noovf, profile_probability::very_likely ());
|
|
|
1851 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
|
|
|
1852 NULL, all_done_label, profile_probability::very_unlikely ());
|
|
|
1853 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
|
|
|
1854 all_done_label, profile_probability::very_unlikely ());
|
|
|
1855 emit_label (set_noovf);
|
|
|
1856 write_complex_part (target, const0_rtx, true);
|
|
|
1857 emit_label (all_done_label);
|
|
|
1858 }
|
|
|
1859
|
|
|
1860 if (lhs)
|
|
|
1861 {
|
|
|
1862 if (is_ubsan)
|
|
|
1863 expand_ubsan_result_store (target, res);
|
|
|
1864 else
|
|
|
1865 expand_arith_overflow_result_store (lhs, target, mode, res);
|
|
|
1866 }
|
|
|
1867 }
|
|
|
1868
|
|
|
1869 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
|
|
|
1870
|
|
|
1871 static void
|
|
|
1872 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
|
|
|
1873 tree arg0, tree arg1)
|
|
|
1874 {
|
|
|
1875 int cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
|
|
|
1876 rtx_code_label *loop_lab = NULL;
|
|
|
1877 rtx cntvar = NULL_RTX;
|
|
|
1878 tree cntv = NULL_TREE;
|
|
|
1879 tree eltype = TREE_TYPE (TREE_TYPE (arg0));
|
|
|
1880 tree sz = TYPE_SIZE (eltype);
|
|
|
1881 tree data = NULL_TREE;
|
|
|
1882 tree resv = NULL_TREE;
|
|
|
1883 rtx lhsr = NULL_RTX;
|
|
|
1884 rtx resvr = NULL_RTX;
|
|
|
1885
|
|
|
1886 if (lhs)
|
|
|
1887 {
|
|
|
1888 optab op;
|
|
|
1889 lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
1890 if (!VECTOR_MODE_P (GET_MODE (lhsr))
|
|
|
1891 || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
|
|
|
1892 optab_default)) == unknown_optab
|
|
|
1893 || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
|
|
|
1894 == CODE_FOR_nothing))
|
|
|
1895 {
|
|
|
1896 if (MEM_P (lhsr))
|
|
|
1897 resv = make_tree (TREE_TYPE (lhs), lhsr);
|
|
|
1898 else
|
|
|
1899 {
|
|
|
1900 resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
|
|
|
1901 resv = make_tree (TREE_TYPE (lhs), resvr);
|
|
|
1902 }
|
|
|
1903 }
|
|
|
1904 }
|
|
|
1905 if (cnt > 4)
|
|
|
1906 {
|
|
|
1907 do_pending_stack_adjust ();
|
|
|
1908 loop_lab = gen_label_rtx ();
|
|
|
1909 cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
|
|
|
1910 cntv = make_tree (sizetype, cntvar);
|
|
|
1911 emit_move_insn (cntvar, const0_rtx);
|
|
|
1912 emit_label (loop_lab);
|
|
|
1913 }
|
|
|
1914 if (TREE_CODE (arg0) != VECTOR_CST)
|
|
|
1915 {
|
|
|
1916 rtx arg0r = expand_normal (arg0);
|
|
|
1917 arg0 = make_tree (TREE_TYPE (arg0), arg0r);
|
|
|
1918 }
|
|
|
1919 if (TREE_CODE (arg1) != VECTOR_CST)
|
|
|
1920 {
|
|
|
1921 rtx arg1r = expand_normal (arg1);
|
|
|
1922 arg1 = make_tree (TREE_TYPE (arg1), arg1r);
|
|
|
1923 }
|
|
|
1924 for (int i = 0; i < (cnt > 4 ? 1 : cnt); i++)
|
|
|
1925 {
|
|
|
1926 tree op0, op1, res = NULL_TREE;
|
|
|
1927 if (cnt > 4)
|
|
|
1928 {
|
|
|
1929 tree atype = build_array_type_nelts (eltype, cnt);
|
|
|
1930 op0 = uniform_vector_p (arg0);
|
|
|
1931 if (op0 == NULL_TREE)
|
|
|
1932 {
|
|
|
1933 op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
|
|
|
1934 op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
|
|
|
1935 NULL_TREE, NULL_TREE);
|
|
|
1936 }
|
|
|
1937 op1 = uniform_vector_p (arg1);
|
|
|
1938 if (op1 == NULL_TREE)
|
|
|
1939 {
|
|
|
1940 op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
|
|
|
1941 op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
|
|
|
1942 NULL_TREE, NULL_TREE);
|
|
|
1943 }
|
|
|
1944 if (resv)
|
|
|
1945 {
|
|
|
1946 res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
|
|
|
1947 res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
|
|
|
1948 NULL_TREE, NULL_TREE);
|
|
|
1949 }
|
|
|
1950 }
|
|
|
1951 else
|
|
|
1952 {
|
|
|
1953 tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
|
|
|
1954 op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
|
|
|
1955 op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
|
|
|
1956 if (resv)
|
|
|
1957 res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
|
|
|
1958 bitpos);
|
|
|
1959 }
|
|
|
1960 switch (code)
|
|
|
1961 {
|
|
|
1962 case PLUS_EXPR:
|
|
|
1963 expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
|
|
|
1964 false, false, false, true, &data);
|
|
|
1965 break;
|
|
|
1966 case MINUS_EXPR:
|
|
|
1967 if (cnt > 4 ? integer_zerop (arg0) : integer_zerop (op0))
|
|
|
1968 expand_neg_overflow (loc, res, op1, true, &data);
|
|
|
1969 else
|
|
|
1970 expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
|
|
|
1971 false, false, false, true, &data);
|
|
|
1972 break;
|
|
|
1973 case MULT_EXPR:
|
|
|
1974 expand_mul_overflow (loc, res, op0, op1, false, false, false,
|
|
|
1975 true, &data);
|
|
|
1976 break;
|
|
|
1977 default:
|
|
|
1978 gcc_unreachable ();
|
|
|
1979 }
|
|
|
1980 }
|
|
|
1981 if (cnt > 4)
|
|
|
1982 {
|
|
|
1983 struct separate_ops ops;
|
|
|
1984 ops.code = PLUS_EXPR;
|
|
|
1985 ops.type = TREE_TYPE (cntv);
|
|
|
1986 ops.op0 = cntv;
|
|
|
1987 ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
|
|
|
1988 ops.op2 = NULL_TREE;
|
|
|
1989 ops.location = loc;
|
|
|
1990 rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
|
|
|
1991 EXPAND_NORMAL);
|
|
|
1992 if (ret != cntvar)
|
|
|
1993 emit_move_insn (cntvar, ret);
|
|
|
1994 do_compare_rtx_and_jump (cntvar, GEN_INT (cnt), NE, false,
|
|
|
1995 TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
|
|
|
1996 profile_probability::very_likely ());
|
|
|
1997 }
|
|
|
1998 if (lhs && resv == NULL_TREE)
|
|
|
1999 {
|
|
|
2000 struct separate_ops ops;
|
|
|
2001 ops.code = code;
|
|
|
2002 ops.type = TREE_TYPE (arg0);
|
|
|
2003 ops.op0 = arg0;
|
|
|
2004 ops.op1 = arg1;
|
|
|
2005 ops.op2 = NULL_TREE;
|
|
|
2006 ops.location = loc;
|
|
|
2007 rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
|
|
|
2008 EXPAND_NORMAL);
|
|
|
2009 if (ret != lhsr)
|
|
|
2010 emit_move_insn (lhsr, ret);
|
|
|
2011 }
|
|
|
2012 else if (resvr)
|
|
|
2013 emit_move_insn (lhsr, resvr);
|
|
|
2014 }
|
|
|
2015
|
|
|
2016 /* Expand UBSAN_CHECK_ADD call STMT. */
|
|
|
2017
|
|
|
2018 static void
|
|
|
2019 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
|
|
|
2020 {
|
|
|
2021 location_t loc = gimple_location (stmt);
|
|
|
2022 tree lhs = gimple_call_lhs (stmt);
|
|
|
2023 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2024 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2025 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
|
2026 expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
|
|
|
2027 else
|
|
|
2028 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
|
|
|
2029 false, false, false, true, NULL);
|
|
|
2030 }
|
|
|
2031
|
|
|
2032 /* Expand UBSAN_CHECK_SUB call STMT. */
|
|
|
2033
|
|
|
2034 static void
|
|
|
2035 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
|
|
|
2036 {
|
|
|
2037 location_t loc = gimple_location (stmt);
|
|
|
2038 tree lhs = gimple_call_lhs (stmt);
|
|
|
2039 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2040 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2041 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
|
2042 expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
|
|
|
2043 else if (integer_zerop (arg0))
|
|
|
2044 expand_neg_overflow (loc, lhs, arg1, true, NULL);
|
|
|
2045 else
|
|
|
2046 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
|
|
|
2047 false, false, false, true, NULL);
|
|
|
2048 }
|
|
|
2049
|
|
|
2050 /* Expand UBSAN_CHECK_MUL call STMT. */
|
|
|
2051
|
|
|
2052 static void
|
|
|
2053 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
|
|
|
2054 {
|
|
|
2055 location_t loc = gimple_location (stmt);
|
|
|
2056 tree lhs = gimple_call_lhs (stmt);
|
|
|
2057 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2058 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2059 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
|
|
|
2060 expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
|
|
|
2061 else
|
|
|
2062 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
|
|
|
2063 NULL);
|
|
|
2064 }
|
|
|
2065
|
|
|
2066 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
|
|
|
2067
|
|
|
2068 static void
|
|
|
2069 expand_arith_overflow (enum tree_code code, gimple *stmt)
|
|
|
2070 {
|
|
|
2071 tree lhs = gimple_call_lhs (stmt);
|
|
|
2072 if (lhs == NULL_TREE)
|
|
|
2073 return;
|
|
|
2074 tree arg0 = gimple_call_arg (stmt, 0);
|
|
|
2075 tree arg1 = gimple_call_arg (stmt, 1);
|
|
|
2076 tree type = TREE_TYPE (TREE_TYPE (lhs));
|
|
|
2077 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
|
|
|
2078 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
|
|
|
2079 int unsr_p = TYPE_UNSIGNED (type);
|
|
|
2080 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
|
|
|
2081 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
|
|
|
2082 int precres = TYPE_PRECISION (type);
|
|
|
2083 location_t loc = gimple_location (stmt);
|
|
|
2084 if (!uns0_p && get_range_pos_neg (arg0) == 1)
|
|
|
2085 uns0_p = true;
|
|
|
2086 if (!uns1_p && get_range_pos_neg (arg1) == 1)
|
|
|
2087 uns1_p = true;
|
|
|
2088 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
|
|
|
2089 prec0 = MIN (prec0, pr);
|
|
|
2090 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
|
|
|
2091 prec1 = MIN (prec1, pr);
|
|
|
2092
|
|
|
2093 /* If uns0_p && uns1_p, precop is minimum needed precision
|
|
|
2094 of unsigned type to hold the exact result, otherwise
|
|
|
2095 precop is minimum needed precision of signed type to
|
|
|
2096 hold the exact result. */
|
|
|
2097 int precop;
|
|
|
2098 if (code == MULT_EXPR)
|
|
|
2099 precop = prec0 + prec1 + (uns0_p != uns1_p);
|
|
|
2100 else
|
|
|
2101 {
|
|
|
2102 if (uns0_p == uns1_p)
|
|
|
2103 precop = MAX (prec0, prec1) + 1;
|
|
|
2104 else if (uns0_p)
|
|
|
2105 precop = MAX (prec0 + 1, prec1) + 1;
|
|
|
2106 else
|
|
|
2107 precop = MAX (prec0, prec1 + 1) + 1;
|
|
|
2108 }
|
|
|
2109 int orig_precres = precres;
|
|
|
2110
|
|
|
2111 do
|
|
|
2112 {
|
|
|
2113 if ((uns0_p && uns1_p)
|
|
|
2114 ? ((precop + !unsr_p) <= precres
|
|
|
2115 /* u1 - u2 -> ur can overflow, no matter what precision
|
|
|
2116 the result has. */
|
|
|
2117 && (code != MINUS_EXPR || !unsr_p))
|
|
|
2118 : (!unsr_p && precop <= precres))
|
|
|
2119 {
|
|
|
2120 /* The infinity precision result will always fit into result. */
|
|
|
2121 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2122 write_complex_part (target, const0_rtx, true);
|
|
|
2123 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
|
|
|
2124 struct separate_ops ops;
|
|
|
2125 ops.code = code;
|
|
|
2126 ops.type = type;
|
|
|
2127 ops.op0 = fold_convert_loc (loc, type, arg0);
|
|
|
2128 ops.op1 = fold_convert_loc (loc, type, arg1);
|
|
|
2129 ops.op2 = NULL_TREE;
|
|
|
2130 ops.location = loc;
|
|
|
2131 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
|
|
|
2132 expand_arith_overflow_result_store (lhs, target, mode, tem);
|
|
|
2133 return;
|
|
|
2134 }
|
|
|
2135
|
|
|
2136 /* For operations with low precision, if target doesn't have them, start
|
|
|
2137 with precres widening right away, otherwise do it only if the most
|
|
|
2138 simple cases can't be used. */
|
|
|
2139 const int min_precision = targetm.min_arithmetic_precision ();
|
|
|
2140 if (orig_precres == precres && precres < min_precision)
|
|
|
2141 ;
|
|
|
2142 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
|
|
|
2143 && prec1 <= precres)
|
|
|
2144 || ((!uns0_p || !uns1_p) && !unsr_p
|
|
|
2145 && prec0 + uns0_p <= precres
|
|
|
2146 && prec1 + uns1_p <= precres))
|
|
|
2147 {
|
|
|
2148 arg0 = fold_convert_loc (loc, type, arg0);
|
|
|
2149 arg1 = fold_convert_loc (loc, type, arg1);
|
|
|
2150 switch (code)
|
|
|
2151 {
|
|
|
2152 case MINUS_EXPR:
|
|
|
2153 if (integer_zerop (arg0) && !unsr_p)
|
|
|
2154 {
|
|
|
2155 expand_neg_overflow (loc, lhs, arg1, false, NULL);
|
|
|
2156 return;
|
|
|
2157 }
|
|
|
2158 /* FALLTHRU */
|
|
|
2159 case PLUS_EXPR:
|
|
|
2160 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
|
|
|
2161 unsr_p, unsr_p, false, NULL);
|
|
|
2162 return;
|
|
|
2163 case MULT_EXPR:
|
|
|
2164 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
|
|
|
2165 unsr_p, unsr_p, false, NULL);
|
|
|
2166 return;
|
|
|
2167 default:
|
|
|
2168 gcc_unreachable ();
|
|
|
2169 }
|
|
|
2170 }
|
|
|
2171
|
|
|
2172 /* For sub-word operations, retry with a wider type first. */
|
|
|
2173 if (orig_precres == precres && precop <= BITS_PER_WORD)
|
|
|
2174 {
|
|
|
2175 int p = MAX (min_precision, precop);
|
|
|
2176 scalar_int_mode m = smallest_int_mode_for_size (p);
|
|
|
2177 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
|
|
|
2178 uns0_p && uns1_p
|
|
|
2179 && unsr_p);
|
|
|
2180 p = TYPE_PRECISION (optype);
|
|
|
2181 if (p > precres)
|
|
|
2182 {
|
|
|
2183 precres = p;
|
|
|
2184 unsr_p = TYPE_UNSIGNED (optype);
|
|
|
2185 type = optype;
|
|
|
2186 continue;
|
|
|
2187 }
|
|
|
2188 }
|
|
|
2189
|
|
|
2190 if (prec0 <= precres && prec1 <= precres)
|
|
|
2191 {
|
|
|
2192 tree types[2];
|
|
|
2193 if (unsr_p)
|
|
|
2194 {
|
|
|
2195 types[0] = build_nonstandard_integer_type (precres, 0);
|
|
|
2196 types[1] = type;
|
|
|
2197 }
|
|
|
2198 else
|
|
|
2199 {
|
|
|
2200 types[0] = type;
|
|
|
2201 types[1] = build_nonstandard_integer_type (precres, 1);
|
|
|
2202 }
|
|
|
2203 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
|
|
|
2204 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
|
|
|
2205 if (code != MULT_EXPR)
|
|
|
2206 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
|
|
|
2207 uns0_p, uns1_p, false, NULL);
|
|
|
2208 else
|
|
|
2209 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
|
|
|
2210 uns0_p, uns1_p, false, NULL);
|
|
|
2211 return;
|
|
|
2212 }
|
|
|
2213
|
|
|
2214 /* Retry with a wider type. */
|
|
|
2215 if (orig_precres == precres)
|
|
|
2216 {
|
|
|
2217 int p = MAX (prec0, prec1);
|
|
|
2218 scalar_int_mode m = smallest_int_mode_for_size (p);
|
|
|
2219 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
|
|
|
2220 uns0_p && uns1_p
|
|
|
2221 && unsr_p);
|
|
|
2222 p = TYPE_PRECISION (optype);
|
|
|
2223 if (p > precres)
|
|
|
2224 {
|
|
|
2225 precres = p;
|
|
|
2226 unsr_p = TYPE_UNSIGNED (optype);
|
|
|
2227 type = optype;
|
|
|
2228 continue;
|
|
|
2229 }
|
|
|
2230 }
|
|
|
2231
|
|
|
2232 gcc_unreachable ();
|
|
|
2233 }
|
|
|
2234 while (1);
|
|
|
2235 }
|
|
|
2236
|
|
|
2237 /* Expand ADD_OVERFLOW STMT. */
|
|
|
2238
|
|
|
2239 static void
|
|
|
2240 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
|
|
|
2241 {
|
|
|
2242 expand_arith_overflow (PLUS_EXPR, stmt);
|
|
|
2243 }
|
|
|
2244
|
|
|
2245 /* Expand SUB_OVERFLOW STMT. */
|
|
|
2246
|
|
|
2247 static void
|
|
|
2248 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
|
|
|
2249 {
|
|
|
2250 expand_arith_overflow (MINUS_EXPR, stmt);
|
|
|
2251 }
|
|
|
2252
|
|
|
2253 /* Expand MUL_OVERFLOW STMT. */
|
|
|
2254
|
|
|
2255 static void
|
|
|
2256 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
|
|
|
2257 {
|
|
|
2258 expand_arith_overflow (MULT_EXPR, stmt);
|
|
|
2259 }
|
|
|
2260
|
|
|
2261 /* This should get folded in tree-vectorizer.c. */
|
|
|
2262
|
|
|
2263 static void
|
|
|
2264 expand_LOOP_VECTORIZED (internal_fn, gcall *)
|
|
|
2265 {
|
|
|
2266 gcc_unreachable ();
|
|
|
2267 }
|
|
|
2268
|
|
|
2269 /* This should get folded in tree-vectorizer.c. */
|
|
|
2270
|
|
|
2271 static void
|
|
|
2272 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
|
|
|
2273 {
|
|
|
2274 gcc_unreachable ();
|
|
|
2275 }
|
|
|
2276
|
|
|
2277 /* Expand MASK_LOAD call STMT using optab OPTAB. */
|
|
|
2278
|
|
|
2279 static void
|
|
|
2280 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
2281 {
|
|
|
2282 struct expand_operand ops[3];
|
|
|
2283 tree type, lhs, rhs, maskt, ptr;
|
|
|
2284 rtx mem, target, mask;
|
|
|
2285 unsigned align;
|
|
|
2286
|
|
|
2287 maskt = gimple_call_arg (stmt, 2);
|
|
|
2288 lhs = gimple_call_lhs (stmt);
|
|
|
2289 if (lhs == NULL_TREE)
|
|
|
2290 return;
|
|
|
2291 type = TREE_TYPE (lhs);
|
|
|
2292 ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)), 0);
|
|
|
2293 align = tree_to_shwi (gimple_call_arg (stmt, 1));
|
|
|
2294 if (TYPE_ALIGN (type) != align)
|
|
|
2295 type = build_aligned_type (type, align);
|
|
|
2296 rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), ptr);
|
|
|
2297
|
|
|
2298 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2299 gcc_assert (MEM_P (mem));
|
|
|
2300 mask = expand_normal (maskt);
|
|
|
2301 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2302 create_output_operand (&ops[0], target, TYPE_MODE (type));
|
|
|
2303 create_fixed_operand (&ops[1], mem);
|
|
|
2304 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
|
|
|
2305 expand_insn (convert_optab_handler (optab, TYPE_MODE (type),
|
|
|
2306 TYPE_MODE (TREE_TYPE (maskt))),
|
|
|
2307 3, ops);
|
|
|
2308 }
|
|
|
2309
|
|
|
2310 /* Expand MASK_STORE call STMT using optab OPTAB. */
|
|
|
2311
|
|
|
2312 static void
|
|
|
2313 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
|
|
|
2314 {
|
|
|
2315 struct expand_operand ops[3];
|
|
|
2316 tree type, lhs, rhs, maskt, ptr;
|
|
|
2317 rtx mem, reg, mask;
|
|
|
2318 unsigned align;
|
|
|
2319
|
|
|
2320 maskt = gimple_call_arg (stmt, 2);
|
|
|
2321 rhs = gimple_call_arg (stmt, 3);
|
|
|
2322 type = TREE_TYPE (rhs);
|
|
|
2323 ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)), 0);
|
|
|
2324 align = tree_to_shwi (gimple_call_arg (stmt, 1));
|
|
|
2325 if (TYPE_ALIGN (type) != align)
|
|
|
2326 type = build_aligned_type (type, align);
|
|
|
2327 lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), ptr);
|
|
|
2328
|
|
|
2329 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2330 gcc_assert (MEM_P (mem));
|
|
|
2331 mask = expand_normal (maskt);
|
|
|
2332 reg = expand_normal (rhs);
|
|
|
2333 create_fixed_operand (&ops[0], mem);
|
|
|
2334 create_input_operand (&ops[1], reg, TYPE_MODE (type));
|
|
|
2335 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
|
|
|
2336 expand_insn (convert_optab_handler (optab, TYPE_MODE (type),
|
|
|
2337 TYPE_MODE (TREE_TYPE (maskt))),
|
|
|
2338 3, ops);
|
|
|
2339 }
|
|
|
2340
|
|
|
2341 static void
|
|
|
2342 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
|
|
|
2343 {
|
|
|
2344 }
|
|
|
2345
|
|
|
2346 static void
|
|
|
2347 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
|
|
|
2348 {
|
|
|
2349 /* When guessing was done, the hints should be already stripped away. */
|
|
|
2350 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
|
|
|
2351
|
|
|
2352 rtx target;
|
|
|
2353 tree lhs = gimple_call_lhs (stmt);
|
|
|
2354 if (lhs)
|
|
|
2355 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2356 else
|
|
|
2357 target = const0_rtx;
|
|
|
2358 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
|
|
|
2359 if (lhs && val != target)
|
|
|
2360 emit_move_insn (target, val);
|
|
|
2361 }
|
|
|
2362
|
|
|
2363 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
|
|
|
2364 should never be called. */
|
|
|
2365
|
|
|
2366 static void
|
|
|
2367 expand_VA_ARG (internal_fn, gcall *)
|
|
|
2368 {
|
|
|
2369 gcc_unreachable ();
|
|
|
2370 }
|
|
|
2371
|
|
|
2372 /* Expand the IFN_UNIQUE function according to its first argument. */
|
|
|
2373
|
|
|
2374 static void
|
|
|
2375 expand_UNIQUE (internal_fn, gcall *stmt)
|
|
|
2376 {
|
|
|
2377 rtx pattern = NULL_RTX;
|
|
|
2378 enum ifn_unique_kind kind
|
|
|
2379 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
|
|
|
2380
|
|
|
2381 switch (kind)
|
|
|
2382 {
|
|
|
2383 default:
|
|
|
2384 gcc_unreachable ();
|
|
|
2385
|
|
|
2386 case IFN_UNIQUE_UNSPEC:
|
|
|
2387 if (targetm.have_unique ())
|
|
|
2388 pattern = targetm.gen_unique ();
|
|
|
2389 break;
|
|
|
2390
|
|
|
2391 case IFN_UNIQUE_OACC_FORK:
|
|
|
2392 case IFN_UNIQUE_OACC_JOIN:
|
|
|
2393 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
|
|
|
2394 {
|
|
|
2395 tree lhs = gimple_call_lhs (stmt);
|
|
|
2396 rtx target = const0_rtx;
|
|
|
2397
|
|
|
2398 if (lhs)
|
|
|
2399 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2400
|
|
|
2401 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
|
|
|
2402 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
|
|
|
2403
|
|
|
2404 if (kind == IFN_UNIQUE_OACC_FORK)
|
|
|
2405 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
|
|
|
2406 else
|
|
|
2407 pattern = targetm.gen_oacc_join (target, data_dep, axis);
|
|
|
2408 }
|
|
|
2409 else
|
|
|
2410 gcc_unreachable ();
|
|
|
2411 break;
|
|
|
2412 }
|
|
|
2413
|
|
|
2414 if (pattern)
|
|
|
2415 emit_insn (pattern);
|
|
|
2416 }
|
|
|
2417
|
|
|
2418 /* The size of an OpenACC compute dimension. */
|
|
|
2419
|
|
|
2420 static void
|
|
|
2421 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
|
|
|
2422 {
|
|
|
2423 tree lhs = gimple_call_lhs (stmt);
|
|
|
2424
|
|
|
2425 if (!lhs)
|
|
|
2426 return;
|
|
|
2427
|
|
|
2428 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2429 if (targetm.have_oacc_dim_size ())
|
|
|
2430 {
|
|
|
2431 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
|
|
|
2432 VOIDmode, EXPAND_NORMAL);
|
|
|
2433 emit_insn (targetm.gen_oacc_dim_size (target, dim));
|
|
|
2434 }
|
|
|
2435 else
|
|
|
2436 emit_move_insn (target, GEN_INT (1));
|
|
|
2437 }
|
|
|
2438
|
|
|
2439 /* The position of an OpenACC execution engine along one compute axis. */
|
|
|
2440
|
|
|
2441 static void
|
|
|
2442 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
|
|
|
2443 {
|
|
|
2444 tree lhs = gimple_call_lhs (stmt);
|
|
|
2445
|
|
|
2446 if (!lhs)
|
|
|
2447 return;
|
|
|
2448
|
|
|
2449 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2450 if (targetm.have_oacc_dim_pos ())
|
|
|
2451 {
|
|
|
2452 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
|
|
|
2453 VOIDmode, EXPAND_NORMAL);
|
|
|
2454 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
|
|
|
2455 }
|
|
|
2456 else
|
|
|
2457 emit_move_insn (target, const0_rtx);
|
|
|
2458 }
|
|
|
2459
|
|
|
2460 /* This is expanded by oacc_device_lower pass. */
|
|
|
2461
|
|
|
2462 static void
|
|
|
2463 expand_GOACC_LOOP (internal_fn, gcall *)
|
|
|
2464 {
|
|
|
2465 gcc_unreachable ();
|
|
|
2466 }
|
|
|
2467
|
|
|
2468 /* This is expanded by oacc_device_lower pass. */
|
|
|
2469
|
|
|
2470 static void
|
|
|
2471 expand_GOACC_REDUCTION (internal_fn, gcall *)
|
|
|
2472 {
|
|
|
2473 gcc_unreachable ();
|
|
|
2474 }
|
|
|
2475
|
|
|
2476 /* This is expanded by oacc_device_lower pass. */
|
|
|
2477
|
|
|
2478 static void
|
|
|
2479 expand_GOACC_TILE (internal_fn, gcall *)
|
|
|
2480 {
|
|
|
2481 gcc_unreachable ();
|
|
|
2482 }
|
|
|
2483
|
|
|
2484 /* Set errno to EDOM. */
|
|
|
2485
|
|
|
2486 static void
|
|
|
2487 expand_SET_EDOM (internal_fn, gcall *)
|
|
|
2488 {
|
|
|
2489 #ifdef TARGET_EDOM
|
|
|
2490 #ifdef GEN_ERRNO_RTX
|
|
|
2491 rtx errno_rtx = GEN_ERRNO_RTX;
|
|
|
2492 #else
|
|
|
2493 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
|
|
|
2494 #endif
|
|
|
2495 emit_move_insn (errno_rtx,
|
|
|
2496 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
|
|
|
2497 #else
|
|
|
2498 gcc_unreachable ();
|
|
|
2499 #endif
|
|
|
2500 }
|
|
|
2501
|
|
|
2502 /* Expand atomic bit test and set. */
|
|
|
2503
|
|
|
2504 static void
|
|
|
2505 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
|
|
|
2506 {
|
|
|
2507 expand_ifn_atomic_bit_test_and (call);
|
|
|
2508 }
|
|
|
2509
|
|
|
2510 /* Expand atomic bit test and complement. */
|
|
|
2511
|
|
|
2512 static void
|
|
|
2513 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
|
|
|
2514 {
|
|
|
2515 expand_ifn_atomic_bit_test_and (call);
|
|
|
2516 }
|
|
|
2517
|
|
|
2518 /* Expand atomic bit test and reset. */
|
|
|
2519
|
|
|
2520 static void
|
|
|
2521 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
|
|
|
2522 {
|
|
|
2523 expand_ifn_atomic_bit_test_and (call);
|
|
|
2524 }
|
|
|
2525
|
|
|
2526 /* Expand atomic bit test and set. */
|
|
|
2527
|
|
|
2528 static void
|
|
|
2529 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
|
|
|
2530 {
|
|
|
2531 expand_ifn_atomic_compare_exchange (call);
|
|
|
2532 }
|
|
|
2533
|
|
|
2534 /* Expand LAUNDER to assignment, lhs = arg0. */
|
|
|
2535
|
|
|
2536 static void
|
|
|
2537 expand_LAUNDER (internal_fn, gcall *call)
|
|
|
2538 {
|
|
|
2539 tree lhs = gimple_call_lhs (call);
|
|
|
2540
|
|
|
2541 if (!lhs)
|
|
|
2542 return;
|
|
|
2543
|
|
|
2544 expand_assignment (lhs, gimple_call_arg (call, 0), false);
|
|
|
2545 }
|
|
|
2546
|
|
|
2547 /* Expand DIVMOD() using:
|
|
|
2548 a) optab handler for udivmod/sdivmod if it is available.
|
|
|
2549 b) If optab_handler doesn't exist, generate call to
|
|
|
2550 target-specific divmod libfunc. */
|
|
|
2551
|
|
|
2552 static void
|
|
|
2553 expand_DIVMOD (internal_fn, gcall *call_stmt)
|
|
|
2554 {
|
|
|
2555 tree lhs = gimple_call_lhs (call_stmt);
|
|
|
2556 tree arg0 = gimple_call_arg (call_stmt, 0);
|
|
|
2557 tree arg1 = gimple_call_arg (call_stmt, 1);
|
|
|
2558
|
|
|
2559 gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
|
|
|
2560 tree type = TREE_TYPE (TREE_TYPE (lhs));
|
|
|
2561 machine_mode mode = TYPE_MODE (type);
|
|
|
2562 bool unsignedp = TYPE_UNSIGNED (type);
|
|
|
2563 optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
|
|
|
2564
|
|
|
2565 rtx op0 = expand_normal (arg0);
|
|
|
2566 rtx op1 = expand_normal (arg1);
|
|
|
2567 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2568
|
|
|
2569 rtx quotient, remainder, libfunc;
|
|
|
2570
|
|
|
2571 /* Check if optab_handler exists for divmod_optab for given mode. */
|
|
|
2572 if (optab_handler (tab, mode) != CODE_FOR_nothing)
|
|
|
2573 {
|
|
|
2574 quotient = gen_reg_rtx (mode);
|
|
|
2575 remainder = gen_reg_rtx (mode);
|
|
|
2576 expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
|
|
|
2577 }
|
|
|
2578
|
|
|
2579 /* Generate call to divmod libfunc if it exists. */
|
|
|
2580 else if ((libfunc = optab_libfunc (tab, mode)) != NULL_RTX)
|
|
|
2581 targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
|
|
|
2582 "ient, &remainder);
|
|
|
2583
|
|
|
2584 else
|
|
|
2585 gcc_unreachable ();
|
|
|
2586
|
|
|
2587 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
|
|
|
2588 expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
|
|
|
2589 make_tree (TREE_TYPE (arg0), quotient),
|
|
|
2590 make_tree (TREE_TYPE (arg1), remainder)),
|
|
|
2591 target, VOIDmode, EXPAND_NORMAL);
|
|
|
2592 }
|
|
|
2593
|
|
|
2594 /* Expand a call to FN using the operands in STMT. FN has a single
|
|
|
2595 output operand and NARGS input operands. */
|
|
|
2596
|
|
|
2597 static void
|
|
|
2598 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
|
|
|
2599 unsigned int nargs)
|
|
|
2600 {
|
|
|
2601 expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
|
|
|
2602
|
|
|
2603 tree_pair types = direct_internal_fn_types (fn, stmt);
|
|
|
2604 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
|
|
|
2605
|
|
|
2606 tree lhs = gimple_call_lhs (stmt);
|
|
|
2607 tree lhs_type = TREE_TYPE (lhs);
|
|
|
2608 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
|
|
|
2609
|
|
|
2610 /* Do not assign directly to a promoted subreg, since there is no
|
|
|
2611 guarantee that the instruction will leave the upper bits of the
|
|
|
2612 register in the state required by SUBREG_PROMOTED_SIGN. */
|
|
|
2613 rtx dest = lhs_rtx;
|
|
|
2614 if (GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
|
|
|
2615 dest = NULL_RTX;
|
|
|
2616
|
|
|
2617 create_output_operand (&ops[0], dest, insn_data[icode].operand[0].mode);
|
|
|
2618
|
|
|
2619 for (unsigned int i = 0; i < nargs; ++i)
|
|
|
2620 {
|
|
|
2621 tree rhs = gimple_call_arg (stmt, i);
|
|
|
2622 tree rhs_type = TREE_TYPE (rhs);
|
|
|
2623 rtx rhs_rtx = expand_normal (rhs);
|
|
|
2624 if (INTEGRAL_TYPE_P (rhs_type))
|
|
|
2625 create_convert_operand_from (&ops[i + 1], rhs_rtx,
|
|
|
2626 TYPE_MODE (rhs_type),
|
|
|
2627 TYPE_UNSIGNED (rhs_type));
|
|
|
2628 else
|
|
|
2629 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
|
|
|
2630 }
|
|
|
2631
|
|
|
2632 expand_insn (icode, nargs + 1, ops);
|
|
|
2633 if (!rtx_equal_p (lhs_rtx, ops[0].value))
|
|
|
2634 {
|
|
|
2635 /* If the return value has an integral type, convert the instruction
|
|
|
2636 result to that type. This is useful for things that return an
|
|
|
2637 int regardless of the size of the input. If the instruction result
|
|
|
2638 is smaller than required, assume that it is signed.
|
|
|
2639
|
|
|
2640 If the return value has a nonintegral type, its mode must match
|
|
|
2641 the instruction result. */
|
|
|
2642 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
|
|
|
2643 {
|
|
|
2644 /* If this is a scalar in a register that is stored in a wider
|
|
|
2645 mode than the declared mode, compute the result into its
|
|
|
2646 declared mode and then convert to the wider mode. */
|
|
|
2647 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type));
|
|
|
2648 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
|
|
|
2649 convert_move (SUBREG_REG (lhs_rtx), tmp,
|
|
|
2650 SUBREG_PROMOTED_SIGN (lhs_rtx));
|
|
|
2651 }
|
|
|
2652 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
|
|
|
2653 emit_move_insn (lhs_rtx, ops[0].value);
|
|
|
2654 else
|
|
|
2655 {
|
|
|
2656 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type));
|
|
|
2657 convert_move (lhs_rtx, ops[0].value, 0);
|
|
|
2658 }
|
|
|
2659 }
|
|
|
2660 }
|
|
|
2661
|
|
|
2662 /* Expanders for optabs that can use expand_direct_optab_fn. */
|
|
|
2663
|
|
|
2664 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2665 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
|
|
|
2666
|
|
|
2667 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
|
|
|
2668 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
|
|
|
2669
|
|
|
2670 /* RETURN_TYPE and ARGS are a return type and argument list that are
|
|
|
2671 in principle compatible with FN (which satisfies direct_internal_fn_p).
|
|
|
2672 Return the types that should be used to determine whether the
|
|
|
2673 target supports FN. */
|
|
|
2674
|
|
|
2675 tree_pair
|
|
|
2676 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
|
|
|
2677 {
|
|
|
2678 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
|
2679 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
|
|
|
2680 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
|
|
|
2681 return tree_pair (type0, type1);
|
|
|
2682 }
|
|
|
2683
|
|
|
2684 /* CALL is a call whose return type and arguments are in principle
|
|
|
2685 compatible with FN (which satisfies direct_internal_fn_p). Return the
|
|
|
2686 types that should be used to determine whether the target supports FN. */
|
|
|
2687
|
|
|
2688 tree_pair
|
|
|
2689 direct_internal_fn_types (internal_fn fn, gcall *call)
|
|
|
2690 {
|
|
|
2691 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
|
2692 tree op0 = (info.type0 < 0
|
|
|
2693 ? gimple_call_lhs (call)
|
|
|
2694 : gimple_call_arg (call, info.type0));
|
|
|
2695 tree op1 = (info.type1 < 0
|
|
|
2696 ? gimple_call_lhs (call)
|
|
|
2697 : gimple_call_arg (call, info.type1));
|
|
|
2698 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
|
|
|
2699 }
|
|
|
2700
|
|
|
2701 /* Return true if OPTAB is supported for TYPES (whose modes should be
|
|
|
2702 the same) when the optimization type is OPT_TYPE. Used for simple
|
|
|
2703 direct optabs. */
|
|
|
2704
|
|
|
2705 static bool
|
|
|
2706 direct_optab_supported_p (direct_optab optab, tree_pair types,
|
|
|
2707 optimization_type opt_type)
|
|
|
2708 {
|
|
|
2709 machine_mode mode = TYPE_MODE (types.first);
|
|
|
2710 gcc_checking_assert (mode == TYPE_MODE (types.second));
|
|
|
2711 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
|
|
|
2712 }
|
|
|
2713
|
|
|
2714 /* Return true if load/store lanes optab OPTAB is supported for
|
|
|
2715 array type TYPES.first when the optimization type is OPT_TYPE. */
|
|
|
2716
|
|
|
2717 static bool
|
|
|
2718 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
|
|
|
2719 optimization_type opt_type)
|
|
|
2720 {
|
|
|
2721 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
|
|
|
2722 machine_mode imode = TYPE_MODE (types.first);
|
|
|
2723 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
|
|
|
2724 return (convert_optab_handler (optab, imode, vmode, opt_type)
|
|
|
2725 != CODE_FOR_nothing);
|
|
|
2726 }
|
|
|
2727
|
|
|
2728 #define direct_unary_optab_supported_p direct_optab_supported_p
|
|
|
2729 #define direct_binary_optab_supported_p direct_optab_supported_p
|
|
|
2730 #define direct_mask_load_optab_supported_p direct_optab_supported_p
|
|
|
2731 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
|
|
|
2732 #define direct_mask_store_optab_supported_p direct_optab_supported_p
|
|
|
2733 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
|
|
|
2734
|
|
|
2735 /* Return true if FN is supported for the types in TYPES when the
|
|
|
2736 optimization type is OPT_TYPE. The types are those associated with
|
|
|
2737 the "type0" and "type1" fields of FN's direct_internal_fn_info
|
|
|
2738 structure. */
|
|
|
2739
|
|
|
2740 bool
|
|
|
2741 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
|
|
|
2742 optimization_type opt_type)
|
|
|
2743 {
|
|
|
2744 switch (fn)
|
|
|
2745 {
|
|
|
2746 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
|
|
|
2747 case IFN_##CODE: break;
|
|
|
2748 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
|
2749 case IFN_##CODE: \
|
|
|
2750 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
|
|
|
2751 opt_type);
|
|
|
2752 #include "internal-fn.def"
|
|
|
2753
|
|
|
2754 case IFN_LAST:
|
|
|
2755 break;
|
|
|
2756 }
|
|
|
2757 gcc_unreachable ();
|
|
|
2758 }
|
|
|
2759
|
|
|
2760 /* Return true if FN is supported for type TYPE when the optimization
|
|
|
2761 type is OPT_TYPE. The caller knows that the "type0" and "type1"
|
|
|
2762 fields of FN's direct_internal_fn_info structure are the same. */
|
|
|
2763
|
|
|
2764 bool
|
|
|
2765 direct_internal_fn_supported_p (internal_fn fn, tree type,
|
|
|
2766 optimization_type opt_type)
|
|
|
2767 {
|
|
|
2768 const direct_internal_fn_info &info = direct_internal_fn (fn);
|
|
|
2769 gcc_checking_assert (info.type0 == info.type1);
|
|
|
2770 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
|
|
|
2771 }
|
|
|
2772
|
|
|
2773 /* Return true if IFN_SET_EDOM is supported. */
|
|
|
2774
|
|
|
2775 bool
|
|
|
2776 set_edom_supported_p (void)
|
|
|
2777 {
|
|
|
2778 #ifdef TARGET_EDOM
|
|
|
2779 return true;
|
|
|
2780 #else
|
|
|
2781 return false;
|
|
|
2782 #endif
|
|
|
2783 }
|
|
|
2784
|
|
|
2785 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
|
|
|
2786 static void \
|
|
|
2787 expand_##CODE (internal_fn fn, gcall *stmt) \
|
|
|
2788 { \
|
|
|
2789 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
|
|
|
2790 }
|
|
|
2791 #include "internal-fn.def"
|
|
|
2792
|
|
|
2793 /* Routines to expand each internal function, indexed by function number.
|
|
|
2794 Each routine has the prototype:
|
|
|
2795
|
|
|
2796 expand_<NAME> (gcall *stmt)
|
|
|
2797
|
|
|
2798 where STMT is the statement that performs the call. */
|
|
|
2799 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
|
|
|
2800 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
|
|
|
2801 #include "internal-fn.def"
|
|
|
2802 0
|
|
|
2803 };
|
|
|
2804
|
|
|
2805 /* Expand STMT as though it were a call to internal function FN. */
|
|
|
2806
|
|
|
2807 void
|
|
|
2808 expand_internal_call (internal_fn fn, gcall *stmt)
|
|
|
2809 {
|
|
|
2810 internal_fn_expanders[fn] (fn, stmt);
|
|
|
2811 }
|
|
|
2812
|
|
|
2813 /* Expand STMT, which is a call to internal function FN. */
|
|
|
2814
|
|
|
2815 void
|
|
|
2816 expand_internal_call (gcall *stmt)
|
|
|
2817 {
|
|
|
2818 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
|
|
|
2819 }
|
|
|
2820
|
|
|
2821 void
|
|
|
2822 expand_PHI (internal_fn, gcall *)
|
|
|
2823 {
|
|
|
2824 gcc_unreachable ();
|
|
|
2825 }
|
