|
111
|
1 /* -----------------------------------------------------------------------
|
|
|
2 ffi64.c - Copyright (c) 2013 The Written Word, Inc.
|
|
|
3 Copyright (c) 2011 Anthony Green
|
|
|
4 Copyright (c) 2008, 2010 Red Hat, Inc.
|
|
|
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
|
|
|
6
|
|
|
7 x86-64 Foreign Function Interface
|
|
|
8
|
|
|
9 Permission is hereby granted, free of charge, to any person obtaining
|
|
|
10 a copy of this software and associated documentation files (the
|
|
|
11 ``Software''), to deal in the Software without restriction, including
|
|
|
12 without limitation the rights to use, copy, modify, merge, publish,
|
|
|
13 distribute, sublicense, and/or sell copies of the Software, and to
|
|
|
14 permit persons to whom the Software is furnished to do so, subject to
|
|
|
15 the following conditions:
|
|
|
16
|
|
|
17 The above copyright notice and this permission notice shall be included
|
|
|
18 in all copies or substantial portions of the Software.
|
|
|
19
|
|
|
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
|
|
|
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
|
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
|
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
|
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
|
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
|
27 DEALINGS IN THE SOFTWARE.
|
|
|
28 ----------------------------------------------------------------------- */
|
|
|
29
|
|
|
30 #include <ffi.h>
|
|
|
31 #include <ffi_common.h>
|
|
|
32
|
|
|
33 #include <stdlib.h>
|
|
|
34 #include <stdarg.h>
|
|
|
35 #include <stdint.h>
|
|
|
36 #include "internal64.h"
|
|
|
37
|
|
|
38 #ifdef __x86_64__
|
|
|
39
|
|
|
40 #define MAX_GPR_REGS 6
|
|
|
41 #define MAX_SSE_REGS 8
|
|
|
42
|
|
|
43 #if defined(__INTEL_COMPILER)
|
|
|
44 #include "xmmintrin.h"
|
|
|
45 #define UINT128 __m128
|
|
|
46 #else
|
|
|
47 #if defined(__SUNPRO_C)
|
|
|
48 #include <sunmedia_types.h>
|
|
|
49 #define UINT128 __m128i
|
|
|
50 #else
|
|
|
51 #define UINT128 __int128_t
|
|
|
52 #endif
|
|
|
53 #endif
|
|
|
54
|
|
|
55 union big_int_union
|
|
|
56 {
|
|
|
57 UINT32 i32;
|
|
|
58 UINT64 i64;
|
|
|
59 UINT128 i128;
|
|
|
60 };
|
|
|
61
|
|
|
62 struct register_args
|
|
|
63 {
|
|
|
64 /* Registers for argument passing. */
|
|
|
65 UINT64 gpr[MAX_GPR_REGS];
|
|
|
66 union big_int_union sse[MAX_SSE_REGS];
|
|
|
67 UINT64 rax; /* ssecount */
|
|
|
68 UINT64 r10; /* static chain */
|
|
|
69 };
|
|
|
70
|
|
|
71 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
|
|
|
72 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
|
|
|
73
|
|
|
74 /* All reference to register classes here is identical to the code in
|
|
|
75 gcc/config/i386/i386.c. Do *not* change one without the other. */
|
|
|
76
|
|
|
77 /* Register class used for passing given 64bit part of the argument.
|
|
|
78 These represent classes as documented by the PS ABI, with the
|
|
|
79 exception of SSESF, SSEDF classes, that are basically SSE class,
|
|
|
80 just gcc will use SF or DFmode move instead of DImode to avoid
|
|
|
81 reformatting penalties.
|
|
|
82
|
|
|
83 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
|
|
|
84 whenever possible (upper half does contain padding). */
|
|
|
85 enum x86_64_reg_class
|
|
|
86 {
|
|
|
87 X86_64_NO_CLASS,
|
|
|
88 X86_64_INTEGER_CLASS,
|
|
|
89 X86_64_INTEGERSI_CLASS,
|
|
|
90 X86_64_SSE_CLASS,
|
|
|
91 X86_64_SSESF_CLASS,
|
|
|
92 X86_64_SSEDF_CLASS,
|
|
|
93 X86_64_SSEUP_CLASS,
|
|
|
94 X86_64_X87_CLASS,
|
|
|
95 X86_64_X87UP_CLASS,
|
|
|
96 X86_64_COMPLEX_X87_CLASS,
|
|
|
97 X86_64_MEMORY_CLASS
|
|
|
98 };
|
|
|
99
|
|
|
100 #define MAX_CLASSES 4
|
|
|
101
|
|
|
102 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
|
|
|
103
|
|
|
104 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
|
|
|
105 of this code is to classify each 8bytes of incoming argument by the register
|
|
|
106 class and assign registers accordingly. */
|
|
|
107
|
|
|
108 /* Return the union class of CLASS1 and CLASS2.
|
|
|
109 See the x86-64 PS ABI for details. */
|
|
|
110
|
|
|
111 static enum x86_64_reg_class
|
|
|
112 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
|
|
|
113 {
|
|
|
114 /* Rule #1: If both classes are equal, this is the resulting class. */
|
|
|
115 if (class1 == class2)
|
|
|
116 return class1;
|
|
|
117
|
|
|
118 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
|
|
|
119 the other class. */
|
|
|
120 if (class1 == X86_64_NO_CLASS)
|
|
|
121 return class2;
|
|
|
122 if (class2 == X86_64_NO_CLASS)
|
|
|
123 return class1;
|
|
|
124
|
|
|
125 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
|
|
|
126 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
|
|
|
127 return X86_64_MEMORY_CLASS;
|
|
|
128
|
|
|
129 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
|
|
|
130 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
|
|
|
131 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
|
|
|
132 return X86_64_INTEGERSI_CLASS;
|
|
|
133 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
|
|
|
134 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
|
|
|
135 return X86_64_INTEGER_CLASS;
|
|
|
136
|
|
|
137 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
|
|
|
138 MEMORY is used. */
|
|
|
139 if (class1 == X86_64_X87_CLASS
|
|
|
140 || class1 == X86_64_X87UP_CLASS
|
|
|
141 || class1 == X86_64_COMPLEX_X87_CLASS
|
|
|
142 || class2 == X86_64_X87_CLASS
|
|
|
143 || class2 == X86_64_X87UP_CLASS
|
|
|
144 || class2 == X86_64_COMPLEX_X87_CLASS)
|
|
|
145 return X86_64_MEMORY_CLASS;
|
|
|
146
|
|
|
147 /* Rule #6: Otherwise class SSE is used. */
|
|
|
148 return X86_64_SSE_CLASS;
|
|
|
149 }
|
|
|
150
|
|
|
151 /* Classify the argument of type TYPE and mode MODE.
|
|
|
152 CLASSES will be filled by the register class used to pass each word
|
|
|
153 of the operand. The number of words is returned. In case the parameter
|
|
|
154 should be passed in memory, 0 is returned. As a special case for zero
|
|
|
155 sized containers, classes[0] will be NO_CLASS and 1 is returned.
|
|
|
156
|
|
|
157 See the x86-64 PS ABI for details.
|
|
|
158 */
|
|
|
159 static size_t
|
|
|
160 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
|
|
|
161 size_t byte_offset)
|
|
|
162 {
|
|
|
163 switch (type->type)
|
|
|
164 {
|
|
|
165 case FFI_TYPE_UINT8:
|
|
|
166 case FFI_TYPE_SINT8:
|
|
|
167 case FFI_TYPE_UINT16:
|
|
|
168 case FFI_TYPE_SINT16:
|
|
|
169 case FFI_TYPE_UINT32:
|
|
|
170 case FFI_TYPE_SINT32:
|
|
|
171 case FFI_TYPE_UINT64:
|
|
|
172 case FFI_TYPE_SINT64:
|
|
|
173 case FFI_TYPE_POINTER:
|
|
|
174 do_integer:
|
|
|
175 {
|
|
|
176 size_t size = byte_offset + type->size;
|
|
|
177
|
|
|
178 if (size <= 4)
|
|
|
179 {
|
|
|
180 classes[0] = X86_64_INTEGERSI_CLASS;
|
|
|
181 return 1;
|
|
|
182 }
|
|
|
183 else if (size <= 8)
|
|
|
184 {
|
|
|
185 classes[0] = X86_64_INTEGER_CLASS;
|
|
|
186 return 1;
|
|
|
187 }
|
|
|
188 else if (size <= 12)
|
|
|
189 {
|
|
|
190 classes[0] = X86_64_INTEGER_CLASS;
|
|
|
191 classes[1] = X86_64_INTEGERSI_CLASS;
|
|
|
192 return 2;
|
|
|
193 }
|
|
|
194 else if (size <= 16)
|
|
|
195 {
|
|
|
196 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
|
|
|
197 return 2;
|
|
|
198 }
|
|
|
199 else
|
|
|
200 FFI_ASSERT (0);
|
|
|
201 }
|
|
|
202 case FFI_TYPE_FLOAT:
|
|
|
203 if (!(byte_offset % 8))
|
|
|
204 classes[0] = X86_64_SSESF_CLASS;
|
|
|
205 else
|
|
|
206 classes[0] = X86_64_SSE_CLASS;
|
|
|
207 return 1;
|
|
|
208 case FFI_TYPE_DOUBLE:
|
|
|
209 classes[0] = X86_64_SSEDF_CLASS;
|
|
|
210 return 1;
|
|
|
211 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
|
|
|
212 case FFI_TYPE_LONGDOUBLE:
|
|
|
213 classes[0] = X86_64_X87_CLASS;
|
|
|
214 classes[1] = X86_64_X87UP_CLASS;
|
|
|
215 return 2;
|
|
|
216 #endif
|
|
|
217 case FFI_TYPE_STRUCT:
|
|
|
218 {
|
|
|
219 const size_t UNITS_PER_WORD = 8;
|
|
|
220 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
|
|
|
221 ffi_type **ptr;
|
|
|
222 int i;
|
|
|
223 enum x86_64_reg_class subclasses[MAX_CLASSES];
|
|
|
224
|
|
|
225 /* If the struct is larger than 32 bytes, pass it on the stack. */
|
|
|
226 if (type->size > 32)
|
|
|
227 return 0;
|
|
|
228
|
|
|
229 for (i = 0; i < words; i++)
|
|
|
230 classes[i] = X86_64_NO_CLASS;
|
|
|
231
|
|
|
232 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
|
|
|
233 signalize memory class, so handle it as special case. */
|
|
|
234 if (!words)
|
|
|
235 {
|
|
|
236 case FFI_TYPE_VOID:
|
|
|
237 classes[0] = X86_64_NO_CLASS;
|
|
|
238 return 1;
|
|
|
239 }
|
|
|
240
|
|
|
241 /* Merge the fields of structure. */
|
|
|
242 for (ptr = type->elements; *ptr != NULL; ptr++)
|
|
|
243 {
|
|
|
244 size_t num;
|
|
|
245
|
|
|
246 byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
|
|
|
247
|
|
|
248 num = classify_argument (*ptr, subclasses, byte_offset % 8);
|
|
|
249 if (num == 0)
|
|
|
250 return 0;
|
|
|
251 for (i = 0; i < num; i++)
|
|
|
252 {
|
|
|
253 size_t pos = byte_offset / 8;
|
|
|
254 classes[i + pos] =
|
|
|
255 merge_classes (subclasses[i], classes[i + pos]);
|
|
|
256 }
|
|
|
257
|
|
|
258 byte_offset += (*ptr)->size;
|
|
|
259 }
|
|
|
260
|
|
|
261 if (words > 2)
|
|
|
262 {
|
|
|
263 /* When size > 16 bytes, if the first one isn't
|
|
|
264 X86_64_SSE_CLASS or any other ones aren't
|
|
|
265 X86_64_SSEUP_CLASS, everything should be passed in
|
|
|
266 memory. */
|
|
|
267 if (classes[0] != X86_64_SSE_CLASS)
|
|
|
268 return 0;
|
|
|
269
|
|
|
270 for (i = 1; i < words; i++)
|
|
|
271 if (classes[i] != X86_64_SSEUP_CLASS)
|
|
|
272 return 0;
|
|
|
273 }
|
|
|
274
|
|
|
275 /* Final merger cleanup. */
|
|
|
276 for (i = 0; i < words; i++)
|
|
|
277 {
|
|
|
278 /* If one class is MEMORY, everything should be passed in
|
|
|
279 memory. */
|
|
|
280 if (classes[i] == X86_64_MEMORY_CLASS)
|
|
|
281 return 0;
|
|
|
282
|
|
|
283 /* The X86_64_SSEUP_CLASS should be always preceded by
|
|
|
284 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
|
|
|
285 if (classes[i] == X86_64_SSEUP_CLASS
|
|
|
286 && classes[i - 1] != X86_64_SSE_CLASS
|
|
|
287 && classes[i - 1] != X86_64_SSEUP_CLASS)
|
|
|
288 {
|
|
|
289 /* The first one should never be X86_64_SSEUP_CLASS. */
|
|
|
290 FFI_ASSERT (i != 0);
|
|
|
291 classes[i] = X86_64_SSE_CLASS;
|
|
|
292 }
|
|
|
293
|
|
|
294 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
|
|
|
295 everything should be passed in memory. */
|
|
|
296 if (classes[i] == X86_64_X87UP_CLASS
|
|
|
297 && (classes[i - 1] != X86_64_X87_CLASS))
|
|
|
298 {
|
|
|
299 /* The first one should never be X86_64_X87UP_CLASS. */
|
|
|
300 FFI_ASSERT (i != 0);
|
|
|
301 return 0;
|
|
|
302 }
|
|
|
303 }
|
|
|
304 return words;
|
|
|
305 }
|
|
|
306 case FFI_TYPE_COMPLEX:
|
|
|
307 {
|
|
|
308 ffi_type *inner = type->elements[0];
|
|
|
309 switch (inner->type)
|
|
|
310 {
|
|
|
311 case FFI_TYPE_INT:
|
|
|
312 case FFI_TYPE_UINT8:
|
|
|
313 case FFI_TYPE_SINT8:
|
|
|
314 case FFI_TYPE_UINT16:
|
|
|
315 case FFI_TYPE_SINT16:
|
|
|
316 case FFI_TYPE_UINT32:
|
|
|
317 case FFI_TYPE_SINT32:
|
|
|
318 case FFI_TYPE_UINT64:
|
|
|
319 case FFI_TYPE_SINT64:
|
|
|
320 goto do_integer;
|
|
|
321
|
|
|
322 case FFI_TYPE_FLOAT:
|
|
|
323 classes[0] = X86_64_SSE_CLASS;
|
|
|
324 if (byte_offset % 8)
|
|
|
325 {
|
|
|
326 classes[1] = X86_64_SSESF_CLASS;
|
|
|
327 return 2;
|
|
|
328 }
|
|
|
329 return 1;
|
|
|
330 case FFI_TYPE_DOUBLE:
|
|
|
331 classes[0] = classes[1] = X86_64_SSEDF_CLASS;
|
|
|
332 return 2;
|
|
|
333 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
|
|
|
334 case FFI_TYPE_LONGDOUBLE:
|
|
|
335 classes[0] = X86_64_COMPLEX_X87_CLASS;
|
|
|
336 return 1;
|
|
|
337 #endif
|
|
|
338 }
|
|
|
339 }
|
|
|
340 }
|
|
|
341 abort();
|
|
|
342 }
|
|
|
343
|
|
|
344 /* Examine the argument and return set number of register required in each
|
|
|
345 class. Return zero iff parameter should be passed in memory, otherwise
|
|
|
346 the number of registers. */
|
|
|
347
|
|
|
348 static size_t
|
|
|
349 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
|
|
|
350 _Bool in_return, int *pngpr, int *pnsse)
|
|
|
351 {
|
|
|
352 size_t n;
|
|
|
353 int i, ngpr, nsse;
|
|
|
354
|
|
|
355 n = classify_argument (type, classes, 0);
|
|
|
356 if (n == 0)
|
|
|
357 return 0;
|
|
|
358
|
|
|
359 ngpr = nsse = 0;
|
|
|
360 for (i = 0; i < n; ++i)
|
|
|
361 switch (classes[i])
|
|
|
362 {
|
|
|
363 case X86_64_INTEGER_CLASS:
|
|
|
364 case X86_64_INTEGERSI_CLASS:
|
|
|
365 ngpr++;
|
|
|
366 break;
|
|
|
367 case X86_64_SSE_CLASS:
|
|
|
368 case X86_64_SSESF_CLASS:
|
|
|
369 case X86_64_SSEDF_CLASS:
|
|
|
370 nsse++;
|
|
|
371 break;
|
|
|
372 case X86_64_NO_CLASS:
|
|
|
373 case X86_64_SSEUP_CLASS:
|
|
|
374 break;
|
|
|
375 case X86_64_X87_CLASS:
|
|
|
376 case X86_64_X87UP_CLASS:
|
|
|
377 case X86_64_COMPLEX_X87_CLASS:
|
|
|
378 return in_return != 0;
|
|
|
379 default:
|
|
|
380 abort ();
|
|
|
381 }
|
|
|
382
|
|
|
383 *pngpr = ngpr;
|
|
|
384 *pnsse = nsse;
|
|
|
385
|
|
|
386 return n;
|
|
|
387 }
|
|
|
388
|
|
|
389 /* Perform machine dependent cif processing. */
|
|
|
390
|
|
|
391 ffi_status
|
|
|
392 ffi_prep_cif_machdep (ffi_cif *cif)
|
|
|
393 {
|
|
|
394 int gprcount, ssecount, i, avn, ngpr, nsse, flags;
|
|
|
395 enum x86_64_reg_class classes[MAX_CLASSES];
|
|
|
396 size_t bytes, n, rtype_size;
|
|
|
397 ffi_type *rtype;
|
|
|
398
|
|
|
399 if (cif->abi != FFI_UNIX64)
|
|
|
400 return FFI_BAD_ABI;
|
|
|
401
|
|
|
402 gprcount = ssecount = 0;
|
|
|
403
|
|
|
404 rtype = cif->rtype;
|
|
|
405 rtype_size = rtype->size;
|
|
|
406 switch (rtype->type)
|
|
|
407 {
|
|
|
408 case FFI_TYPE_VOID:
|
|
|
409 flags = UNIX64_RET_VOID;
|
|
|
410 break;
|
|
|
411 case FFI_TYPE_UINT8:
|
|
|
412 flags = UNIX64_RET_UINT8;
|
|
|
413 break;
|
|
|
414 case FFI_TYPE_SINT8:
|
|
|
415 flags = UNIX64_RET_SINT8;
|
|
|
416 break;
|
|
|
417 case FFI_TYPE_UINT16:
|
|
|
418 flags = UNIX64_RET_UINT16;
|
|
|
419 break;
|
|
|
420 case FFI_TYPE_SINT16:
|
|
|
421 flags = UNIX64_RET_SINT16;
|
|
|
422 break;
|
|
|
423 case FFI_TYPE_UINT32:
|
|
|
424 flags = UNIX64_RET_UINT32;
|
|
|
425 break;
|
|
|
426 case FFI_TYPE_INT:
|
|
|
427 case FFI_TYPE_SINT32:
|
|
|
428 flags = UNIX64_RET_SINT32;
|
|
|
429 break;
|
|
|
430 case FFI_TYPE_UINT64:
|
|
|
431 case FFI_TYPE_SINT64:
|
|
|
432 flags = UNIX64_RET_INT64;
|
|
|
433 break;
|
|
|
434 case FFI_TYPE_POINTER:
|
|
|
435 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
|
|
|
436 break;
|
|
|
437 case FFI_TYPE_FLOAT:
|
|
|
438 flags = UNIX64_RET_XMM32;
|
|
|
439 break;
|
|
|
440 case FFI_TYPE_DOUBLE:
|
|
|
441 flags = UNIX64_RET_XMM64;
|
|
|
442 break;
|
|
|
443 case FFI_TYPE_LONGDOUBLE:
|
|
|
444 flags = UNIX64_RET_X87;
|
|
|
445 break;
|
|
|
446 case FFI_TYPE_STRUCT:
|
|
|
447 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
|
|
|
448 if (n == 0)
|
|
|
449 {
|
|
|
450 /* The return value is passed in memory. A pointer to that
|
|
|
451 memory is the first argument. Allocate a register for it. */
|
|
|
452 gprcount++;
|
|
|
453 /* We don't have to do anything in asm for the return. */
|
|
|
454 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
|
|
|
455 }
|
|
|
456 else
|
|
|
457 {
|
|
|
458 _Bool sse0 = SSE_CLASS_P (classes[0]);
|
|
|
459
|
|
|
460 if (rtype_size == 4 && sse0)
|
|
|
461 flags = UNIX64_RET_XMM32;
|
|
|
462 else if (rtype_size == 8)
|
|
|
463 flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
|
|
|
464 else
|
|
|
465 {
|
|
|
466 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
|
|
|
467 if (sse0 && sse1)
|
|
|
468 flags = UNIX64_RET_ST_XMM0_XMM1;
|
|
|
469 else if (sse0)
|
|
|
470 flags = UNIX64_RET_ST_XMM0_RAX;
|
|
|
471 else if (sse1)
|
|
|
472 flags = UNIX64_RET_ST_RAX_XMM0;
|
|
|
473 else
|
|
|
474 flags = UNIX64_RET_ST_RAX_RDX;
|
|
|
475 flags |= rtype_size << UNIX64_SIZE_SHIFT;
|
|
|
476 }
|
|
|
477 }
|
|
|
478 break;
|
|
|
479 case FFI_TYPE_COMPLEX:
|
|
|
480 switch (rtype->elements[0]->type)
|
|
|
481 {
|
|
|
482 case FFI_TYPE_UINT8:
|
|
|
483 case FFI_TYPE_SINT8:
|
|
|
484 case FFI_TYPE_UINT16:
|
|
|
485 case FFI_TYPE_SINT16:
|
|
|
486 case FFI_TYPE_INT:
|
|
|
487 case FFI_TYPE_UINT32:
|
|
|
488 case FFI_TYPE_SINT32:
|
|
|
489 case FFI_TYPE_UINT64:
|
|
|
490 case FFI_TYPE_SINT64:
|
|
|
491 flags = UNIX64_RET_ST_RAX_RDX | (rtype_size << UNIX64_SIZE_SHIFT);
|
|
|
492 break;
|
|
|
493 case FFI_TYPE_FLOAT:
|
|
|
494 flags = UNIX64_RET_XMM64;
|
|
|
495 break;
|
|
|
496 case FFI_TYPE_DOUBLE:
|
|
|
497 flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
|
|
|
498 break;
|
|
|
499 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
|
|
|
500 case FFI_TYPE_LONGDOUBLE:
|
|
|
501 flags = UNIX64_RET_X87_2;
|
|
|
502 break;
|
|
|
503 #endif
|
|
|
504 default:
|
|
|
505 return FFI_BAD_TYPEDEF;
|
|
|
506 }
|
|
|
507 break;
|
|
|
508 default:
|
|
|
509 return FFI_BAD_TYPEDEF;
|
|
|
510 }
|
|
|
511
|
|
|
512 /* Go over all arguments and determine the way they should be passed.
|
|
|
513 If it's in a register and there is space for it, let that be so. If
|
|
|
514 not, add it's size to the stack byte count. */
|
|
|
515 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
|
|
|
516 {
|
|
|
517 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
|
|
|
518 || gprcount + ngpr > MAX_GPR_REGS
|
|
|
519 || ssecount + nsse > MAX_SSE_REGS)
|
|
|
520 {
|
|
|
521 long align = cif->arg_types[i]->alignment;
|
|
|
522
|
|
|
523 if (align < 8)
|
|
|
524 align = 8;
|
|
|
525
|
|
|
526 bytes = ALIGN (bytes, align);
|
|
|
527 bytes += cif->arg_types[i]->size;
|
|
|
528 }
|
|
|
529 else
|
|
|
530 {
|
|
|
531 gprcount += ngpr;
|
|
|
532 ssecount += nsse;
|
|
|
533 }
|
|
|
534 }
|
|
|
535 if (ssecount)
|
|
|
536 flags |= UNIX64_FLAG_XMM_ARGS;
|
|
|
537
|
|
|
538 cif->flags = flags;
|
|
|
539 cif->bytes = ALIGN (bytes, 8);
|
|
|
540
|
|
|
541 return FFI_OK;
|
|
|
542 }
|
|
|
543
|
|
|
544 static void
|
|
|
545 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
|
|
|
546 void **avalue, void *closure)
|
|
|
547 {
|
|
|
548 enum x86_64_reg_class classes[MAX_CLASSES];
|
|
|
549 char *stack, *argp;
|
|
|
550 ffi_type **arg_types;
|
|
|
551 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
|
|
|
552 struct register_args *reg_args;
|
|
|
553
|
|
|
554 /* Can't call 32-bit mode from 64-bit mode. */
|
|
|
555 FFI_ASSERT (cif->abi == FFI_UNIX64);
|
|
|
556
|
|
|
557 /* If the return value is a struct and we don't have a return value
|
|
|
558 address then we need to make one. Otherwise we can ignore it. */
|
|
|
559 flags = cif->flags;
|
|
|
560 if (rvalue == NULL)
|
|
|
561 {
|
|
|
562 if (flags & UNIX64_FLAG_RET_IN_MEM)
|
|
|
563 rvalue = alloca (cif->rtype->size);
|
|
|
564 else
|
|
|
565 flags = UNIX64_RET_VOID;
|
|
|
566 }
|
|
|
567
|
|
|
568 /* Allocate the space for the arguments, plus 4 words of temp space. */
|
|
|
569 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
|
|
|
570 reg_args = (struct register_args *) stack;
|
|
|
571 argp = stack + sizeof (struct register_args);
|
|
|
572
|
|
|
573 reg_args->r10 = (uintptr_t) closure;
|
|
|
574
|
|
|
575 gprcount = ssecount = 0;
|
|
|
576
|
|
|
577 /* If the return value is passed in memory, add the pointer as the
|
|
|
578 first integer argument. */
|
|
|
579 if (flags & UNIX64_FLAG_RET_IN_MEM)
|
|
|
580 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
|
|
|
581
|
|
|
582 avn = cif->nargs;
|
|
|
583 arg_types = cif->arg_types;
|
|
|
584
|
|
|
585 for (i = 0; i < avn; ++i)
|
|
|
586 {
|
|
|
587 size_t n, size = arg_types[i]->size;
|
|
|
588
|
|
|
589 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
|
|
|
590 if (n == 0
|
|
|
591 || gprcount + ngpr > MAX_GPR_REGS
|
|
|
592 || ssecount + nsse > MAX_SSE_REGS)
|
|
|
593 {
|
|
|
594 long align = arg_types[i]->alignment;
|
|
|
595
|
|
|
596 /* Stack arguments are *always* at least 8 byte aligned. */
|
|
|
597 if (align < 8)
|
|
|
598 align = 8;
|
|
|
599
|
|
|
600 /* Pass this argument in memory. */
|
|
|
601 argp = (void *) ALIGN (argp, align);
|
|
|
602 memcpy (argp, avalue[i], size);
|
|
|
603 argp += size;
|
|
|
604 }
|
|
|
605 else
|
|
|
606 {
|
|
|
607 /* The argument is passed entirely in registers. */
|
|
|
608 char *a = (char *) avalue[i];
|
|
|
609 int j;
|
|
|
610
|
|
|
611 for (j = 0; j < n; j++, a += 8, size -= 8)
|
|
|
612 {
|
|
|
613 switch (classes[j])
|
|
|
614 {
|
|
|
615 case X86_64_NO_CLASS:
|
|
|
616 case X86_64_SSEUP_CLASS:
|
|
|
617 break;
|
|
|
618 case X86_64_INTEGER_CLASS:
|
|
|
619 case X86_64_INTEGERSI_CLASS:
|
|
|
620 /* Sign-extend integer arguments passed in general
|
|
|
621 purpose registers, to cope with the fact that
|
|
|
622 LLVM incorrectly assumes that this will be done
|
|
|
623 (the x86-64 PS ABI does not specify this). */
|
|
|
624 switch (arg_types[i]->type)
|
|
|
625 {
|
|
|
626 case FFI_TYPE_SINT8:
|
|
|
627 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
|
|
|
628 break;
|
|
|
629 case FFI_TYPE_SINT16:
|
|
|
630 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
|
|
|
631 break;
|
|
|
632 case FFI_TYPE_SINT32:
|
|
|
633 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
|
|
|
634 break;
|
|
|
635 default:
|
|
|
636 reg_args->gpr[gprcount] = 0;
|
|
|
637 memcpy (®_args->gpr[gprcount], a, size);
|
|
|
638 }
|
|
|
639 gprcount++;
|
|
|
640 break;
|
|
|
641 case X86_64_SSE_CLASS:
|
|
|
642 case X86_64_SSEDF_CLASS:
|
|
|
643 reg_args->sse[ssecount++].i64 = *(UINT64 *) a;
|
|
|
644 break;
|
|
|
645 case X86_64_SSESF_CLASS:
|
|
|
646 reg_args->sse[ssecount++].i32 = *(UINT32 *) a;
|
|
|
647 break;
|
|
|
648 default:
|
|
|
649 abort();
|
|
|
650 }
|
|
|
651 }
|
|
|
652 }
|
|
|
653 }
|
|
|
654 reg_args->rax = ssecount;
|
|
|
655
|
|
|
656 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
|
|
|
657 flags, rvalue, fn);
|
|
|
658 }
|
|
|
659
|
|
|
660 void
|
|
|
661 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
|
|
|
662 {
|
|
|
663 ffi_call_int (cif, fn, rvalue, avalue, NULL);
|
|
|
664 }
|
|
|
665
|
|
|
666 void
|
|
|
667 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
|
|
|
668 void **avalue, void *closure)
|
|
|
669 {
|
|
|
670 ffi_call_int (cif, fn, rvalue, avalue, closure);
|
|
|
671 }
|
|
|
672
|
|
|
673 extern void ffi_closure_unix64(void) FFI_HIDDEN;
|
|
|
674 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
|
|
|
675
|
|
|
676 ffi_status
|
|
|
677 ffi_prep_closure_loc (ffi_closure* closure,
|
|
|
678 ffi_cif* cif,
|
|
|
679 void (*fun)(ffi_cif*, void*, void**, void*),
|
|
|
680 void *user_data,
|
|
|
681 void *codeloc)
|
|
|
682 {
|
|
|
683 static const unsigned char trampoline[16] = {
|
|
|
684 /* leaq -0x7(%rip),%r10 # 0x0 */
|
|
|
685 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff,
|
|
|
686 /* jmpq *0x3(%rip) # 0x10 */
|
|
|
687 0xff, 0x25, 0x03, 0x00, 0x00, 0x00,
|
|
|
688 /* nopl (%rax) */
|
|
|
689 0x0f, 0x1f, 0x00
|
|
|
690 };
|
|
|
691 void (*dest)(void);
|
|
|
692 char *tramp = closure->tramp;
|
|
|
693
|
|
|
694 if (cif->abi != FFI_UNIX64)
|
|
|
695 return FFI_BAD_ABI;
|
|
|
696
|
|
|
697 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
|
|
|
698 dest = ffi_closure_unix64_sse;
|
|
|
699 else
|
|
|
700 dest = ffi_closure_unix64;
|
|
|
701
|
|
|
702 memcpy (tramp, trampoline, sizeof(trampoline));
|
|
|
703 *(UINT64 *)(tramp + 16) = (uintptr_t)dest;
|
|
|
704
|
|
|
705 closure->cif = cif;
|
|
|
706 closure->fun = fun;
|
|
|
707 closure->user_data = user_data;
|
|
|
708
|
|
|
709 return FFI_OK;
|
|
|
710 }
|
|
|
711
|
|
|
712 int FFI_HIDDEN
|
|
|
713 ffi_closure_unix64_inner(ffi_cif *cif,
|
|
|
714 void (*fun)(ffi_cif*, void*, void**, void*),
|
|
|
715 void *user_data,
|
|
|
716 void *rvalue,
|
|
|
717 struct register_args *reg_args,
|
|
|
718 char *argp)
|
|
|
719 {
|
|
|
720 void **avalue;
|
|
|
721 ffi_type **arg_types;
|
|
|
722 long i, avn;
|
|
|
723 int gprcount, ssecount, ngpr, nsse;
|
|
|
724 int flags;
|
|
|
725
|
|
|
726 avn = cif->nargs;
|
|
|
727 flags = cif->flags;
|
|
|
728 avalue = alloca(avn * sizeof(void *));
|
|
|
729 gprcount = ssecount = 0;
|
|
|
730
|
|
|
731 if (flags & UNIX64_FLAG_RET_IN_MEM)
|
|
|
732 {
|
|
|
733 /* On return, %rax will contain the address that was passed
|
|
|
734 by the caller in %rdi. */
|
|
|
735 void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
|
|
|
736 *(void **)rvalue = r;
|
|
|
737 rvalue = r;
|
|
|
738 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
|
|
|
739 }
|
|
|
740
|
|
|
741 arg_types = cif->arg_types;
|
|
|
742 for (i = 0; i < avn; ++i)
|
|
|
743 {
|
|
|
744 enum x86_64_reg_class classes[MAX_CLASSES];
|
|
|
745 size_t n;
|
|
|
746
|
|
|
747 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
|
|
|
748 if (n == 0
|
|
|
749 || gprcount + ngpr > MAX_GPR_REGS
|
|
|
750 || ssecount + nsse > MAX_SSE_REGS)
|
|
|
751 {
|
|
|
752 long align = arg_types[i]->alignment;
|
|
|
753
|
|
|
754 /* Stack arguments are *always* at least 8 byte aligned. */
|
|
|
755 if (align < 8)
|
|
|
756 align = 8;
|
|
|
757
|
|
|
758 /* Pass this argument in memory. */
|
|
|
759 argp = (void *) ALIGN (argp, align);
|
|
|
760 avalue[i] = argp;
|
|
|
761 argp += arg_types[i]->size;
|
|
|
762 }
|
|
|
763 /* If the argument is in a single register, or two consecutive
|
|
|
764 integer registers, then we can use that address directly. */
|
|
|
765 else if (n == 1
|
|
|
766 || (n == 2 && !(SSE_CLASS_P (classes[0])
|
|
|
767 || SSE_CLASS_P (classes[1]))))
|
|
|
768 {
|
|
|
769 /* The argument is in a single register. */
|
|
|
770 if (SSE_CLASS_P (classes[0]))
|
|
|
771 {
|
|
|
772 avalue[i] = ®_args->sse[ssecount];
|
|
|
773 ssecount += n;
|
|
|
774 }
|
|
|
775 else
|
|
|
776 {
|
|
|
777 avalue[i] = ®_args->gpr[gprcount];
|
|
|
778 gprcount += n;
|
|
|
779 }
|
|
|
780 }
|
|
|
781 /* Otherwise, allocate space to make them consecutive. */
|
|
|
782 else
|
|
|
783 {
|
|
|
784 char *a = alloca (16);
|
|
|
785 int j;
|
|
|
786
|
|
|
787 avalue[i] = a;
|
|
|
788 for (j = 0; j < n; j++, a += 8)
|
|
|
789 {
|
|
|
790 if (SSE_CLASS_P (classes[j]))
|
|
|
791 memcpy (a, ®_args->sse[ssecount++], 8);
|
|
|
792 else
|
|
|
793 memcpy (a, ®_args->gpr[gprcount++], 8);
|
|
|
794 }
|
|
|
795 }
|
|
|
796 }
|
|
|
797
|
|
|
798 /* Invoke the closure. */
|
|
|
799 fun (cif, rvalue, avalue, user_data);
|
|
|
800
|
|
|
801 /* Tell assembly how to perform return type promotions. */
|
|
|
802 return flags;
|
|
|
803 }
|
|
|
804
|
|
|
805 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
|
|
|
806 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
|
|
|
807
|
|
|
808 ffi_status
|
|
|
809 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
|
|
|
810 void (*fun)(ffi_cif*, void*, void**, void*))
|
|
|
811 {
|
|
|
812 if (cif->abi != FFI_UNIX64)
|
|
|
813 return FFI_BAD_ABI;
|
|
|
814
|
|
|
815 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
|
|
|
816 ? ffi_go_closure_unix64_sse
|
|
|
817 : ffi_go_closure_unix64);
|
|
|
818 closure->cif = cif;
|
|
|
819 closure->fun = fun;
|
|
|
820
|
|
|
821 return FFI_OK;
|
|
|
822 }
|
|
|
823
|
|
|
824 #endif /* __x86_64__ */
|
