Initial import of compiler-rt.
-
git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@74292 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/i386/Makefile.mk b/lib/i386/Makefile.mk
new file mode 100644
index 0000000..140ee99
--- /dev/null
+++ b/lib/i386/Makefile.mk
@@ -0,0 +1,22 @@
+#===- lib/i386/Makefile.mk ---------------------------------*- Makefile -*--===#
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===------------------------------------------------------------------------===#
+
+Dir := lib/i386
+SubDirs :=
+OnlyArchs := i386
+
+AsmSources := $(foreach file,$(wildcard $(Dir)/*.s),$(notdir $(file)))
+Sources := $(foreach file,$(wildcard $(Dir)/*.c),$(notdir $(file)))
+ObjNames := $(Sources:%.c=%.o) $(AsmSources:%.s=%.o)
+Target := Optimized
+
+# FIXME: use automatic dependencies?
+Dependencies := $(wildcard $(Dir)/*.h)
+
+include make/subdir.mk
diff --git a/lib/i386/ashldi3.s b/lib/i386/ashldi3.s
new file mode 100644
index 0000000..3de2dfc
--- /dev/null
+++ b/lib/i386/ashldi3.s
@@ -0,0 +1,65 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// di_int __ashldi3(di_int input, int count);
+
+// This routine has some extra memory traffic, loading the 64-bit input via two
+// 32-bit loads, then immediately storing it back to the stack via a single 64-bit
+// store. This is to avoid a write-small, read-large stall.
+// However, if callers of this routine can be safely assumed to store the argument
+// via a 64-bt store, this is unnecessary memory traffic, and should be avoided.
+// It can be turned off by defining the TRUST_CALLERS_USE_64_BIT_STORES macro.
+
+#ifdef __i386__
+#ifdef __SSE2__
+
+.text
+.align 4
+.globl ___ashldi3
+___ashldi3:
+ movd 12(%esp), %xmm2 // Load count
+#ifndef TRUST_CALLERS_USE_64_BIT_STORES
+ movd 4(%esp), %xmm0
+ movd 8(%esp), %xmm1
+ punpckldq %xmm1, %xmm0 // Load input
+#else
+ movq 4(%esp), %xmm0 // Load input
+#endif
+ psllq %xmm2, %xmm0 // shift input by count
+ movd %xmm0, %eax
+ psrlq $32, %xmm0
+ movd %xmm0, %edx
+ ret
+
+#else // Use GPRs instead of SSE2 instructions, if they aren't available.
+
+.text
+.align 4
+.globl ___ashldi3
+___ashldi3:
+ movl 12(%esp), %ecx // Load count
+ movl 8(%esp), %edx // Load high
+ movl 4(%esp), %eax // Load low
+
+ testl $0x20, %ecx // If count >= 32
+ jnz 2f // goto 2
+ testl $0x1f, %ecx // If count == 0
+ jz 1f // goto 1
+
+ pushl %ebx
+ movl %eax, %ebx // copy low
+ shll %cl, %eax // left shift low by count
+ shll %cl, %edx // left shift high by count
+ neg %cl
+ shrl %cl, %ebx // right shift low by 32 - count
+ orl %ebx, %edx // or the result into the high word
+ popl %ebx
+1: ret
+
+2: movl %eax, %edx // Move low to high
+ xorl %eax, %eax // clear low
+ shll %cl, %edx // shift high by count - 32
+ ret
+
+#endif // __SSE2__
+#endif // __i386__
diff --git a/lib/i386/ashrdi3.s b/lib/i386/ashrdi3.s
new file mode 100644
index 0000000..db01f21
--- /dev/null
+++ b/lib/i386/ashrdi3.s
@@ -0,0 +1,75 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// di_int __ashrdi3(di_int input, int count);
+
+#ifdef __i386__
+#ifdef __SSE2__
+
+.text
+.align 4
+.globl ___ashrdi3
+___ashrdi3:
+ movd 12(%esp), %xmm2 // Load count
+ movl 8(%esp), %eax
+#ifndef TRUST_CALLERS_USE_64_BIT_STORES
+ movd 4(%esp), %xmm0
+ movd 8(%esp), %xmm1
+ punpckldq %xmm1, %xmm0 // Load input
+#else
+ movq 4(%esp), %xmm0 // Load input
+#endif
+
+ psrlq %xmm2, %xmm0 // unsigned shift input by count
+
+ testl %eax, %eax // check the sign-bit of the input
+ jns 1f // early out for positive inputs
+
+ // If the input is negative, we need to construct the shifted sign bit
+ // to or into the result, as xmm does not have a signed right shift.
+ pcmpeqb %xmm1, %xmm1 // -1ULL
+ psrlq $58, %xmm1 // 0x3f
+ pandn %xmm1, %xmm2 // 63 - count
+ pcmpeqb %xmm1, %xmm1 // -1ULL
+ psubq %xmm1, %xmm2 // 64 - count
+ psllq %xmm2, %xmm1 // -1 << (64 - count) = leading sign bits
+ por %xmm1, %xmm0
+
+ // Move the result back to the general purpose registers and return
+1: movd %xmm0, %eax
+ psrlq $32, %xmm0
+ movd %xmm0, %edx
+ ret
+
+#else // Use GPRs instead of SSE2 instructions, if they aren't available.
+
+.text
+.align 4
+.globl ___ashrdi3
+___ashrdi3:
+ movl 12(%esp), %ecx // Load count
+ movl 8(%esp), %edx // Load high
+ movl 4(%esp), %eax // Load low
+
+ testl $0x20, %ecx // If count >= 32
+ jnz 2f // goto 2
+ testl $0x1f, %ecx // If count == 0
+ jz 1f // goto 1
+
+ pushl %ebx
+ movl %edx, %ebx // copy high
+ shrl %cl, %eax // right shift low by count
+ sarl %cl, %edx // right shift high by count
+ neg %cl
+ shll %cl, %ebx // left shift high by 32 - count
+ orl %ebx, %eax // or the result into the low word
+ popl %ebx
+1: ret
+
+2: movl %edx, %eax // Move high to low
+ sarl $31, %edx // clear high
+ sarl %cl, %eax // shift low by count - 32
+ ret
+
+#endif // __SSE2__
+#endif // __i386__
diff --git a/lib/i386/divdi3.s b/lib/i386/divdi3.s
new file mode 100644
index 0000000..3b50266
--- /dev/null
+++ b/lib/i386/divdi3.s
@@ -0,0 +1,160 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// di_int __divdi3(di_int a, di_int b);
+
+// result = a / b.
+// both inputs and the output are 64-bit signed integers.
+// This will do whatever the underlying hardware is set to do on division by zero.
+// No other exceptions are generated, as the divide cannot overflow.
+//
+// This is targeted at 32-bit x86 *only*, as this can be done directly in hardware
+// on x86_64. The performance goal is ~40 cycles per divide, which is faster than
+// currently possible via simulation of integer divides on the x87 unit.
+//
+// Stephen Canon, December 2008
+
+#ifdef __i386__
+
+.text
+.align 4
+.globl ___divdi3
+___divdi3:
+
+/* This is currently implemented by wrapping the unsigned divide up in an absolute
+ value, then restoring the correct sign at the end of the computation. This could
+ certainly be improved upon. */
+
+ pushl %esi
+ movl 20(%esp), %edx // high word of b
+ movl 16(%esp), %eax // low word of b
+ movl %edx, %ecx
+ sarl $31, %ecx // (b < 0) ? -1 : 0
+ xorl %ecx, %eax
+ xorl %ecx, %edx // EDX:EAX = (b < 0) ? not(b) : b
+ subl %ecx, %eax
+ sbbl %ecx, %edx // EDX:EAX = abs(b)
+ movl %edx, 20(%esp)
+ movl %eax, 16(%esp) // store abs(b) back to stack
+ movl %ecx, %esi // set aside sign of b
+
+ movl 12(%esp), %edx // high word of b
+ movl 8(%esp), %eax // low word of b
+ movl %edx, %ecx
+ sarl $31, %ecx // (a < 0) ? -1 : 0
+ xorl %ecx, %eax
+ xorl %ecx, %edx // EDX:EAX = (a < 0) ? not(a) : a
+ subl %ecx, %eax
+ sbbl %ecx, %edx // EDX:EAX = abs(a)
+ movl %edx, 12(%esp)
+ movl %eax, 8(%esp) // store abs(a) back to stack
+ xorl %ecx, %esi // sign of result = (sign of a) ^ (sign of b)
+
+ pushl %ebx
+ movl 24(%esp), %ebx // Find the index i of the leading bit in b.
+ bsrl %ebx, %ecx // If the high word of b is zero, jump to
+ jz 9f // the code to handle that special case [9].
+
+ /* High word of b is known to be non-zero on this branch */
+
+ movl 20(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b
+
+ shrl %cl, %eax // Practically, this means that bhi is given by:
+ shrl %eax //
+ notl %ecx // bhi = (high word of b) << (31 - i) |
+ shll %cl, %ebx // (low word of b) >> (1 + i)
+ orl %eax, %ebx //
+ movl 16(%esp), %edx // Load the high and low words of a, and jump
+ movl 12(%esp), %eax // to [1] if the high word is larger than bhi
+ cmpl %ebx, %edx // to avoid overflowing the upcoming divide.
+ jae 1f
+
+ /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ shrl %cl, %eax // q = qs >> (1 + i)
+ movl %eax, %edi
+ mull 24(%esp) // q*blo
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 28(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+ sbbl $0, %edi // decrement q if remainder is negative
+ xorl %edx, %edx
+ movl %edi, %eax
+
+ addl %esi, %eax // Restore correct sign to result
+ adcl %esi, %edx
+ xorl %esi, %eax
+ xorl %esi, %edx
+ popl %edi // Restore callee-save registers
+ popl %ebx
+ popl %esi
+ retl // Return
+
+
+1: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ subl %ebx, %edx // subtract bhi from ahi so that divide will not
+ divl %ebx // overflow, and find q and r such that
+ //
+ // ahi:alo = (1:q)*bhi + r
+ //
+ // Note that q is a number in (31-i).(1+i)
+ // fix point.
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ orl $0x80000000, %eax
+ shrl %cl, %eax // q = (1:qs) >> (1 + i)
+ movl %eax, %edi
+ mull 24(%esp) // q*blo
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 28(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+ sbbl $0, %edi // decrement q if remainder is negative
+ xorl %edx, %edx
+ movl %edi, %eax
+
+ addl %esi, %eax // Restore correct sign to result
+ adcl %esi, %edx
+ xorl %esi, %eax
+ xorl %esi, %edx
+ popl %edi // Restore callee-save registers
+ popl %ebx
+ popl %esi
+ retl // Return
+
+
+9: /* High word of b is zero on this branch */
+
+ movl 16(%esp), %eax // Find qhi and rhi such that
+ movl 20(%esp), %ecx //
+ xorl %edx, %edx // ahi = qhi*b + rhi with 0 ≤ rhi < b
+ divl %ecx //
+ movl %eax, %ebx //
+ movl 12(%esp), %eax // Find qlo such that
+ divl %ecx //
+ movl %ebx, %edx // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b
+
+ addl %esi, %eax // Restore correct sign to result
+ adcl %esi, %edx
+ xorl %esi, %eax
+ xorl %esi, %edx
+ popl %ebx // Restore callee-save registers
+ popl %esi
+ retl // Return
+
+#endif // __i386__
diff --git a/lib/i386/floatdidf.s b/lib/i386/floatdidf.s
new file mode 100644
index 0000000..34736ac
--- /dev/null
+++ b/lib/i386/floatdidf.s
@@ -0,0 +1,32 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// double __floatundidf(du_int a);
+
+#ifdef __i386__
+
+.const
+.align 4
+twop52: .quad 0x4330000000000000
+twop32: .quad 0x41f0000000000000
+
+#define REL_ADDR(_a) (_a)-0b(%eax)
+
+.text
+.align 4
+.globl ___floatdidf
+___floatdidf:
+ cvtsi2sd 8(%esp), %xmm1
+ movss 4(%esp), %xmm0 // low 32 bits of a
+ calll 0f
+0: popl %eax
+ mulsd REL_ADDR(twop32), %xmm1 // a_hi as a double (without rounding)
+ movsd REL_ADDR(twop52), %xmm2 // 0x1.0p52
+ subsd %xmm2, %xmm1 // a_hi - 0x1p52 (no rounding occurs)
+ orpd %xmm2, %xmm0 // 0x1p52 + a_lo (no rounding occurs)
+ addsd %xmm1, %xmm0 // a_hi + a_lo (round happens here)
+ movsd %xmm0, 4(%esp)
+ fldl 4(%esp)
+ ret
+
+#endif // __i386__
diff --git a/lib/i386/floatdisf.s b/lib/i386/floatdisf.s
new file mode 100644
index 0000000..20b80d1
--- /dev/null
+++ b/lib/i386/floatdisf.s
@@ -0,0 +1,30 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// float __floatdisf(di_int a);
+
+// This routine has some extra memory traffic, loading the 64-bit input via two
+// 32-bit loads, then immediately storing it back to the stack via a single 64-bit
+// store. This is to avoid a write-small, read-large stall.
+// However, if callers of this routine can be safely assumed to store the argument
+// via a 64-bt store, this is unnecessary memory traffic, and should be avoided.
+// It can be turned off by defining the TRUST_CALLERS_USE_64_BIT_STORES macro.
+
+#ifdef __i386__
+
+.text
+.align 4
+.globl ___floatdisf
+___floatdisf:
+#ifndef TRUST_CALLERS_USE_64_BIT_STORES
+ movd 4(%esp), %xmm0
+ movd 8(%esp), %xmm1
+ punpckldq %xmm1, %xmm0
+ movq %xmm0, 4(%esp)
+#endif
+ fildll 4(%esp)
+ fstps 4(%esp)
+ flds 4(%esp)
+ ret
+
+#endif // __i386__
diff --git a/lib/i386/floatdixf.s b/lib/i386/floatdixf.s
new file mode 100644
index 0000000..71e2d2a
--- /dev/null
+++ b/lib/i386/floatdixf.s
@@ -0,0 +1,28 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// float __floatdixf(di_int a);
+
+#ifdef __i386__
+
+// This routine has some extra memory traffic, loading the 64-bit input via two
+// 32-bit loads, then immediately storing it back to the stack via a single 64-bit
+// store. This is to avoid a write-small, read-large stall.
+// However, if callers of this routine can be safely assumed to store the argument
+// via a 64-bt store, this is unnecessary memory traffic, and should be avoided.
+// It can be turned off by defining the TRUST_CALLERS_USE_64_BIT_STORES macro.
+
+.text
+.align 4
+.globl ___floatdixf
+___floatdixf:
+#ifndef TRUST_CALLERS_USE_64_BIT_STORES
+ movd 4(%esp), %xmm0
+ movd 8(%esp), %xmm1
+ punpckldq %xmm1, %xmm0
+ movq %xmm0, 4(%esp)
+#endif
+ fildll 4(%esp)
+ ret
+
+#endif // __i386__
\ No newline at end of file
diff --git a/lib/i386/floatundidf.s b/lib/i386/floatundidf.s
new file mode 100644
index 0000000..d9be853
--- /dev/null
+++ b/lib/i386/floatundidf.s
@@ -0,0 +1,43 @@
+//===-- floatundidf.s - Implement __floatundidf for i386 ------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements __floatundidf for the compiler_rt library.
+//
+//===----------------------------------------------------------------------===//
+
+// double __floatundidf(du_int a);
+
+#ifdef __i386__
+
+.const
+.align 4
+twop52: .quad 0x4330000000000000
+twop84_plus_twop52:
+ .quad 0x4530000000100000
+twop84: .quad 0x4530000000000000
+
+#define REL_ADDR(_a) (_a)-0b(%eax)
+
+.text
+.align 4
+.globl ___floatundidf
+___floatundidf:
+ movss 8(%esp), %xmm1 // high 32 bits of a
+ movss 4(%esp), %xmm0 // low 32 bits of a
+ calll 0f
+0: popl %eax
+ orpd REL_ADDR(twop84), %xmm1 // 0x1p84 + a_hi (no rounding occurs)
+ subsd REL_ADDR(twop84_plus_twop52), %xmm1 // a_hi - 0x1p52 (no rounding occurs)
+ orpd REL_ADDR(twop52), %xmm0 // 0x1p52 + a_lo (no rounding occurs)
+ addsd %xmm1, %xmm0 // a_hi + a_lo (round happens here)
+ movsd %xmm0, 4(%esp)
+ fldl 4(%esp)
+ ret
+
+#endif // __i386__
diff --git a/lib/i386/floatundisf.s b/lib/i386/floatundisf.s
new file mode 100644
index 0000000..a1b2966
--- /dev/null
+++ b/lib/i386/floatundisf.s
@@ -0,0 +1,95 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// float __floatundisf(du_int a);
+
+// Note that there is a hardware instruction, fildll, that does most of what
+// this function needs to do. However, because of our ia32 ABI, it will take
+// a write-small read-large stall, so the software implementation here is
+// actually several cycles faster.
+
+// This is a branch-free implementation. A branchy implementation might be
+// faster for the common case if you know something a priori about the input
+// distribution.
+
+/* branch-free x87 implementation - one cycle slower than without x87.
+
+#ifdef __i386__
+
+.const
+.align 3
+
+ .quad 0x43f0000000000000
+twop64: .quad 0x0000000000000000
+
+#define TWOp64 twop64-0b(%ecx,%eax,8)
+
+.text
+.align 4
+.globl ___floatundisf
+___floatundisf:
+ movl 8(%esp), %eax
+ movd 8(%esp), %xmm1
+ movd 4(%esp), %xmm0
+ punpckldq %xmm1, %xmm0
+ calll 0f
+0: popl %ecx
+ sarl $31, %eax
+ movq %xmm0, 4(%esp)
+ fildll 4(%esp)
+ faddl TWOp64
+ fstps 4(%esp)
+ flds 4(%esp)
+ ret
+
+#endif // __i386__
+
+*/
+
+/* branch-free, x87-free implementation - faster at the expense of code size */
+
+#ifdef __i386__
+
+.const
+.align 3
+twop52: .quad 0x4330000000000000
+ .quad 0x0000000000000fff
+sticky: .quad 0x0000000000000000
+ .long 0x00000012
+twelve: .long 0x00000000
+
+#define TWOp52 twop52-0b(%ecx)
+#define STICKY sticky-0b(%ecx,%eax,8)
+
+.text
+.align 4
+.globl ___floatundisf
+___floatundisf:
+ movl 8(%esp), %eax
+ movd 8(%esp), %xmm1
+ movd 4(%esp), %xmm0
+ punpckldq %xmm1, %xmm0
+
+ calll 0f
+0: popl %ecx
+ shrl %eax // high 31 bits of input as sint32
+ addl $0x7ff80000, %eax
+ sarl $31, %eax // (big input) ? -1 : 0
+ movsd STICKY, %xmm1 // (big input) ? 0xfff : 0
+ movl $12, %edx
+ andl %eax, %edx // (big input) ? 12 : 0
+ movd %edx, %xmm3
+ andpd %xmm0, %xmm1 // (big input) ? input & 0xfff : 0
+ movsd TWOp52, %xmm2 // 0x1.0p52
+ psrlq %xmm3, %xmm0 // (big input) ? input >> 12 : input
+ orpd %xmm2, %xmm1 // 0x1.0p52 + ((big input) ? input & 0xfff : input)
+ orpd %xmm1, %xmm0 // 0x1.0p52 + ((big input) ? (input >> 12 | input & 0xfff) : input)
+ subsd %xmm2, %xmm0 // (double)((big input) ? (input >> 12 | input & 0xfff) : input)
+ cvtsd2ss %xmm0, %xmm0 // (float)((big input) ? (input >> 12 | input & 0xfff) : input)
+ pslld $23, %xmm3
+ paddd %xmm3, %xmm0 // (float)input
+ movd %xmm0, 4(%esp)
+ flds 4(%esp)
+ ret
+
+#endif // __i386__
diff --git a/lib/i386/floatundixf.s b/lib/i386/floatundixf.s
new file mode 100644
index 0000000..c24689a
--- /dev/null
+++ b/lib/i386/floatundixf.s
@@ -0,0 +1,34 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// long double __floatundixf(du_int a);16
+
+#ifdef __i386__
+
+.const
+.align 4
+twop52: .quad 0x4330000000000000
+twop84_plus_twop52_neg:
+ .quad 0xc530000000100000
+twop84: .quad 0x4530000000000000
+
+#define REL_ADDR(_a) (_a)-0b(%eax)
+
+.text
+.align 4
+.globl ___floatundixf
+___floatundixf:
+ calll 0f
+0: popl %eax
+ movss 8(%esp), %xmm0 // hi 32 bits of input
+ movss 4(%esp), %xmm1 // lo 32 bits of input
+ orpd REL_ADDR(twop84), %xmm0 // 2^84 + hi (as a double)
+ orpd REL_ADDR(twop52), %xmm1 // 2^52 + lo (as a double)
+ addsd REL_ADDR(twop84_plus_twop52_neg), %xmm0 // hi - 2^52 (no rounding occurs)
+ movsd %xmm1, 4(%esp)
+ fldl 4(%esp)
+ movsd %xmm0, 4(%esp)
+ faddl 4(%esp)
+ ret
+
+#endif // __i386__
\ No newline at end of file
diff --git a/lib/i386/lshrdi3.s b/lib/i386/lshrdi3.s
new file mode 100644
index 0000000..5992c21
--- /dev/null
+++ b/lib/i386/lshrdi3.s
@@ -0,0 +1,65 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// di_int __lshrdi3(di_int input, int count);
+
+// This routine has some extra memory traffic, loading the 64-bit input via two
+// 32-bit loads, then immediately storing it back to the stack via a single 64-bit
+// store. This is to avoid a write-small, read-large stall.
+// However, if callers of this routine can be safely assumed to store the argument
+// via a 64-bt store, this is unnecessary memory traffic, and should be avoided.
+// It can be turned off by defining the TRUST_CALLERS_USE_64_BIT_STORES macro.
+
+#ifdef __i386__
+#ifdef __SSE2__
+
+.text
+.align 4
+.globl ___lshrdi3
+___lshrdi3:
+ movd 12(%esp), %xmm2 // Load count
+#ifndef TRUST_CALLERS_USE_64_BIT_STORES
+ movd 4(%esp), %xmm0
+ movd 8(%esp), %xmm1
+ punpckldq %xmm1, %xmm0 // Load input
+#else
+ movq 4(%esp), %xmm0 // Load input
+#endif
+ psrlq %xmm2, %xmm0 // shift input by count
+ movd %xmm0, %eax
+ psrlq $32, %xmm0
+ movd %xmm0, %edx
+ ret
+
+#else // Use GPRs instead of SSE2 instructions, if they aren't available.
+
+.text
+.align 4
+.globl ___lshrdi3
+___lshrdi3:
+ movl 12(%esp), %ecx // Load count
+ movl 8(%esp), %edx // Load high
+ movl 4(%esp), %eax // Load low
+
+ testl $0x20, %ecx // If count >= 32
+ jnz 2f // goto 2
+ testl $0x1f, %ecx // If count == 0
+ jz 1f // goto 1
+
+ pushl %ebx
+ movl %edx, %ebx // copy high
+ shrl %cl, %eax // right shift low by count
+ shrl %cl, %edx // right shift high by count
+ neg %cl
+ shll %cl, %ebx // left shift high by 32 - count
+ orl %ebx, %eax // or the result into the low word
+ popl %ebx
+1: ret
+
+2: movl %edx, %eax // Move high to low
+ xorl %edx, %edx // clear high
+ shrl %cl, %eax // shift low by count - 32
+ ret
+
+#endif // __SSE2__
+#endif // __i386__
diff --git a/lib/i386/moddi3.s b/lib/i386/moddi3.s
new file mode 100644
index 0000000..af1f38a
--- /dev/null
+++ b/lib/i386/moddi3.s
@@ -0,0 +1,165 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// di_int __moddi3(di_int a, di_int b);
+
+// result = remainder of a / b.
+// both inputs and the output are 64-bit signed integers.
+// This will do whatever the underlying hardware is set to do on division by zero.
+// No other exceptions are generated, as the divide cannot overflow.
+//
+// This is targeted at 32-bit x86 *only*, as this can be done directly in hardware
+// on x86_64. The performance goal is ~40 cycles per divide, which is faster than
+// currently possible via simulation of integer divides on the x87 unit.
+//
+
+// Stephen Canon, December 2008
+
+#ifdef __i386__
+
+.text
+.align 4
+.globl ___moddi3
+___moddi3:
+
+/* This is currently implemented by wrapping the unsigned modulus up in an absolute
+ value. This could certainly be improved upon. */
+
+ pushl %esi
+ movl 20(%esp), %edx // high word of b
+ movl 16(%esp), %eax // low word of b
+ movl %edx, %ecx
+ sarl $31, %ecx // (b < 0) ? -1 : 0
+ xorl %ecx, %eax
+ xorl %ecx, %edx // EDX:EAX = (b < 0) ? not(b) : b
+ subl %ecx, %eax
+ sbbl %ecx, %edx // EDX:EAX = abs(b)
+ movl %edx, 20(%esp)
+ movl %eax, 16(%esp) // store abs(b) back to stack
+
+ movl 12(%esp), %edx // high word of b
+ movl 8(%esp), %eax // low word of b
+ movl %edx, %ecx
+ sarl $31, %ecx // (a < 0) ? -1 : 0
+ xorl %ecx, %eax
+ xorl %ecx, %edx // EDX:EAX = (a < 0) ? not(a) : a
+ subl %ecx, %eax
+ sbbl %ecx, %edx // EDX:EAX = abs(a)
+ movl %edx, 12(%esp)
+ movl %eax, 8(%esp) // store abs(a) back to stack
+ movl %ecx, %esi // set aside sign of a
+
+ pushl %ebx
+ movl 24(%esp), %ebx // Find the index i of the leading bit in b.
+ bsrl %ebx, %ecx // If the high word of b is zero, jump to
+ jz 9f // the code to handle that special case [9].
+
+ /* High word of b is known to be non-zero on this branch */
+
+ movl 20(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b
+
+ shrl %cl, %eax // Practically, this means that bhi is given by:
+ shrl %eax //
+ notl %ecx // bhi = (high word of b) << (31 - i) |
+ shll %cl, %ebx // (low word of b) >> (1 + i)
+ orl %eax, %ebx //
+ movl 16(%esp), %edx // Load the high and low words of a, and jump
+ movl 12(%esp), %eax // to [2] if the high word is larger than bhi
+ cmpl %ebx, %edx // to avoid overflowing the upcoming divide.
+ jae 2f
+
+ /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ shrl %cl, %eax // q = qs >> (1 + i)
+ movl %eax, %edi
+ mull 24(%esp) // q*blo
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 28(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+
+ jnc 1f // if positive, this is the result.
+ addl 24(%esp), %ebx // otherwise
+ adcl 28(%esp), %ecx // ECX:EBX = a - (q-1)*b = result
+1: movl %ebx, %eax
+ movl %ecx, %edx
+
+ addl %esi, %eax // Restore correct sign to result
+ adcl %esi, %edx
+ xorl %esi, %eax
+ xorl %esi, %edx
+ popl %edi // Restore callee-save registers
+ popl %ebx
+ popl %esi
+ retl // Return
+
+2: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ subl %ebx, %edx // subtract bhi from ahi so that divide will not
+ divl %ebx // overflow, and find q and r such that
+ //
+ // ahi:alo = (1:q)*bhi + r
+ //
+ // Note that q is a number in (31-i).(1+i)
+ // fix point.
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ orl $0x80000000, %eax
+ shrl %cl, %eax // q = (1:qs) >> (1 + i)
+ movl %eax, %edi
+ mull 24(%esp) // q*blo
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 28(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+
+ jnc 3f // if positive, this is the result.
+ addl 24(%esp), %ebx // otherwise
+ adcl 28(%esp), %ecx // ECX:EBX = a - (q-1)*b = result
+3: movl %ebx, %eax
+ movl %ecx, %edx
+
+ addl %esi, %eax // Restore correct sign to result
+ adcl %esi, %edx
+ xorl %esi, %eax
+ xorl %esi, %edx
+ popl %edi // Restore callee-save registers
+ popl %ebx
+ popl %esi
+ retl // Return
+
+9: /* High word of b is zero on this branch */
+
+ movl 16(%esp), %eax // Find qhi and rhi such that
+ movl 20(%esp), %ecx //
+ xorl %edx, %edx // ahi = qhi*b + rhi with 0 ≤ rhi < b
+ divl %ecx //
+ movl %eax, %ebx //
+ movl 12(%esp), %eax // Find rlo such that
+ divl %ecx //
+ movl %edx, %eax // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b
+ popl %ebx //
+ xorl %edx, %edx // and return 0:rlo
+
+ addl %esi, %eax // Restore correct sign to result
+ adcl %esi, %edx
+ xorl %esi, %eax
+ xorl %esi, %edx
+ popl %esi
+ retl // Return
+
+
+#endif // __i386__
diff --git a/lib/i386/muldi3.s b/lib/i386/muldi3.s
new file mode 100644
index 0000000..9f29c3c
--- /dev/null
+++ b/lib/i386/muldi3.s
@@ -0,0 +1,28 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// di_int __muldi3(di_int a, di_int b);
+
+#ifdef __i386__
+
+.text
+.align 4
+.globl ___muldi3
+___muldi3:
+ pushl %ebx
+ movl 16(%esp), %eax // b.lo
+ movl 12(%esp), %ecx // a.hi
+ imull %eax, %ecx // b.lo * a.hi
+
+ movl 8(%esp), %edx // a.lo
+ movl 20(%esp), %ebx // b.hi
+ imull %edx, %ebx // a.lo * b.hi
+
+ mull %edx // EDX:EAX = a.lo * b.lo
+ addl %ecx, %ebx // EBX = (a.lo*b.hi + a.hi*b.lo)
+ addl %ebx, %edx
+
+ popl %ebx
+ retl
+
+#endif // __i386__
diff --git a/lib/i386/udivdi3.s b/lib/i386/udivdi3.s
new file mode 100644
index 0000000..977cee3
--- /dev/null
+++ b/lib/i386/udivdi3.s
@@ -0,0 +1,113 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// du_int __udivdi3(du_int a, du_int b);
+
+// result = a / b.
+// both inputs and the output are 64-bit unsigned integers.
+// This will do whatever the underlying hardware is set to do on division by zero.
+// No other exceptions are generated, as the divide cannot overflow.
+//
+// This is targeted at 32-bit x86 *only*, as this can be done directly in hardware
+// on x86_64. The performance goal is ~40 cycles per divide, which is faster than
+// currently possible via simulation of integer divides on the x87 unit.
+//
+// Stephen Canon, December 2008
+
+#ifdef __i386__
+
+.text
+.align 4
+.globl ___udivdi3
+___udivdi3:
+
+ pushl %ebx
+ movl 20(%esp), %ebx // Find the index i of the leading bit in b.
+ bsrl %ebx, %ecx // If the high word of b is zero, jump to
+ jz 9f // the code to handle that special case [9].
+
+ /* High word of b is known to be non-zero on this branch */
+
+ movl 16(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b
+
+ shrl %cl, %eax // Practically, this means that bhi is given by:
+ shrl %eax //
+ notl %ecx // bhi = (high word of b) << (31 - i) |
+ shll %cl, %ebx // (low word of b) >> (1 + i)
+ orl %eax, %ebx //
+ movl 12(%esp), %edx // Load the high and low words of a, and jump
+ movl 8(%esp), %eax // to [1] if the high word is larger than bhi
+ cmpl %ebx, %edx // to avoid overflowing the upcoming divide.
+ jae 1f
+
+ /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ shrl %cl, %eax // q = qs >> (1 + i)
+ movl %eax, %edi
+ mull 20(%esp) // q*blo
+ movl 12(%esp), %ebx
+ movl 16(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 24(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+ sbbl $0, %edi // decrement q if remainder is negative
+ xorl %edx, %edx
+ movl %edi, %eax
+ popl %edi
+ popl %ebx
+ retl
+
+
+1: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ subl %ebx, %edx // subtract bhi from ahi so that divide will not
+ divl %ebx // overflow, and find q and r such that
+ //
+ // ahi:alo = (1:q)*bhi + r
+ //
+ // Note that q is a number in (31-i).(1+i)
+ // fix point.
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ orl $0x80000000, %eax
+ shrl %cl, %eax // q = (1:qs) >> (1 + i)
+ movl %eax, %edi
+ mull 20(%esp) // q*blo
+ movl 12(%esp), %ebx
+ movl 16(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 24(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+ sbbl $0, %edi // decrement q if remainder is negative
+ xorl %edx, %edx
+ movl %edi, %eax
+ popl %edi
+ popl %ebx
+ retl
+
+
+9: /* High word of b is zero on this branch */
+
+ movl 12(%esp), %eax // Find qhi and rhi such that
+ movl 16(%esp), %ecx //
+ xorl %edx, %edx // ahi = qhi*b + rhi with 0 ≤ rhi < b
+ divl %ecx //
+ movl %eax, %ebx //
+ movl 8(%esp), %eax // Find qlo such that
+ divl %ecx //
+ movl %ebx, %edx // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b
+ popl %ebx //
+ retl // and return qhi:qlo
+
+#endif // __i386__
diff --git a/lib/i386/umoddi3.s b/lib/i386/umoddi3.s
new file mode 100644
index 0000000..2ddd710
--- /dev/null
+++ b/lib/i386/umoddi3.s
@@ -0,0 +1,124 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// du_int __umoddi3(du_int a, du_int b);
+
+// result = remainder of a / b.
+// both inputs and the output are 64-bit unsigned integers.
+// This will do whatever the underlying hardware is set to do on division by zero.
+// No other exceptions are generated, as the divide cannot overflow.
+//
+// This is targeted at 32-bit x86 *only*, as this can be done directly in hardware
+// on x86_64. The performance goal is ~40 cycles per divide, which is faster than
+// currently possible via simulation of integer divides on the x87 unit.
+//
+
+// Stephen Canon, December 2008
+
+#ifdef __i386__
+
+.text
+.align 4
+.globl ___umoddi3
+___umoddi3:
+
+ pushl %ebx
+ movl 20(%esp), %ebx // Find the index i of the leading bit in b.
+ bsrl %ebx, %ecx // If the high word of b is zero, jump to
+ jz 9f // the code to handle that special case [9].
+
+ /* High word of b is known to be non-zero on this branch */
+
+ movl 16(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b
+
+ shrl %cl, %eax // Practically, this means that bhi is given by:
+ shrl %eax //
+ notl %ecx // bhi = (high word of b) << (31 - i) |
+ shll %cl, %ebx // (low word of b) >> (1 + i)
+ orl %eax, %ebx //
+ movl 12(%esp), %edx // Load the high and low words of a, and jump
+ movl 8(%esp), %eax // to [2] if the high word is larger than bhi
+ cmpl %ebx, %edx // to avoid overflowing the upcoming divide.
+ jae 2f
+
+ /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ shrl %cl, %eax // q = qs >> (1 + i)
+ movl %eax, %edi
+ mull 20(%esp) // q*blo
+ movl 12(%esp), %ebx
+ movl 16(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 24(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+
+ jnc 1f // if positive, this is the result.
+ addl 20(%esp), %ebx // otherwise
+ adcl 24(%esp), %ecx // ECX:EBX = a - (q-1)*b = result
+1: movl %ebx, %eax
+ movl %ecx, %edx
+
+ popl %edi
+ popl %ebx
+ retl
+
+
+2: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */
+
+ subl %ebx, %edx // subtract bhi from ahi so that divide will not
+ divl %ebx // overflow, and find q and r such that
+ //
+ // ahi:alo = (1:q)*bhi + r
+ //
+ // Note that q is a number in (31-i).(1+i)
+ // fix point.
+
+ pushl %edi
+ notl %ecx
+ shrl %eax
+ orl $0x80000000, %eax
+ shrl %cl, %eax // q = (1:qs) >> (1 + i)
+ movl %eax, %edi
+ mull 20(%esp) // q*blo
+ movl 12(%esp), %ebx
+ movl 16(%esp), %ecx // ECX:EBX = a
+ subl %eax, %ebx
+ sbbl %edx, %ecx // ECX:EBX = a - q*blo
+ movl 24(%esp), %eax
+ imull %edi, %eax // q*bhi
+ subl %eax, %ecx // ECX:EBX = a - q*b
+
+ jnc 3f // if positive, this is the result.
+ addl 20(%esp), %ebx // otherwise
+ adcl 24(%esp), %ecx // ECX:EBX = a - (q-1)*b = result
+3: movl %ebx, %eax
+ movl %ecx, %edx
+
+ popl %edi
+ popl %ebx
+ retl
+
+
+
+9: /* High word of b is zero on this branch */
+
+ movl 12(%esp), %eax // Find qhi and rhi such that
+ movl 16(%esp), %ecx //
+ xorl %edx, %edx // ahi = qhi*b + rhi with 0 ≤ rhi < b
+ divl %ecx //
+ movl %eax, %ebx //
+ movl 8(%esp), %eax // Find rlo such that
+ divl %ecx //
+ movl %edx, %eax // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b
+ popl %ebx //
+ xorl %edx, %edx // and return 0:rlo
+ retl //
+
+#endif // __i386__