blob: d0ece2aeb70dfc380d689a85a141a28a5917ec20 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/arm/lib/uaccess.S
3 *
4 * Copyright (C) 1995, 1996,1997,1998 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Routines to block copy data to/from user memory
11 * These are highly optimised both for the 4k page size
12 * and for various alignments.
13 */
14#include <linux/linkage.h>
15#include <asm/assembler.h>
16#include <asm/errno.h>
Catalin Marinas247055a2010-09-13 16:03:21 +010017#include <asm/domain.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
19 .text
20
21#define PAGE_SHIFT 12
22
Russell King02fcb972006-06-21 14:44:52 +010023/* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 * Purpose : copy a block to user memory from kernel memory
25 * Params : to - user memory
26 * : from - kernel memory
27 * : n - number of bytes to copy
28 * Returns : Number of bytes NOT copied.
29 */
30
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000031.Lc2u_dest_not_aligned:
Linus Torvalds1da177e2005-04-16 15:20:36 -070032 rsb ip, ip, #4
33 cmp ip, #2
34 ldrb r3, [r1], #1
Catalin Marinas247055a2010-09-13 16:03:21 +010035USER( T(strb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 ldrgeb r3, [r1], #1
Catalin Marinas247055a2010-09-13 16:03:21 +010037USER( T(strgeb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 ldrgtb r3, [r1], #1
Catalin Marinas247055a2010-09-13 16:03:21 +010039USER( T(strgtb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 sub r2, r2, ip
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000041 b .Lc2u_dest_aligned
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
Russell King02fcb972006-06-21 14:44:52 +010043ENTRY(__copy_to_user)
Linus Torvalds1da177e2005-04-16 15:20:36 -070044 stmfd sp!, {r2, r4 - r7, lr}
45 cmp r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000046 blt .Lc2u_not_enough
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 ands ip, r0, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000048 bne .Lc2u_dest_not_aligned
49.Lc2u_dest_aligned:
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51 ands ip, r1, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000052 bne .Lc2u_src_not_aligned
Linus Torvalds1da177e2005-04-16 15:20:36 -070053/*
54 * Seeing as there has to be at least 8 bytes to copy, we can
55 * copy one word, and force a user-mode page fault...
56 */
57
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000058.Lc2u_0fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000060 bmi .Lc2u_0nowords
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 ldr r3, [r1], #4
Catalin Marinas247055a2010-09-13 16:03:21 +010062USER( T(str) r3, [r0], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -070063 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
64 rsb ip, ip, #0
65 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000066 beq .Lc2u_0fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -070067/*
68 * ip = max no. of bytes to copy before needing another "strt" insn
69 */
70 cmp r2, ip
71 movlt ip, r2
72 sub r2, r2, ip
73 subs ip, ip, #32
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000074 blt .Lc2u_0rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -070075
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000076.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 stmia r0!, {r3 - r6} @ Shouldnt fault
78 ldmia r1!, {r3 - r6}
79 subs ip, ip, #32
80 stmia r0!, {r3 - r6} @ Shouldnt fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000081 bpl .Lc2u_0cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000083.Lc2u_0rem8lp: cmn ip, #16
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 ldmgeia r1!, {r3 - r6}
85 stmgeia r0!, {r3 - r6} @ Shouldnt fault
86 tst ip, #8
87 ldmneia r1!, {r3 - r4}
88 stmneia r0!, {r3 - r4} @ Shouldnt fault
89 tst ip, #4
90 ldrne r3, [r1], #4
Catalin Marinas247055a2010-09-13 16:03:21 +010091 T(strne) r3, [r0], #4 @ Shouldnt fault
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +000093 beq .Lc2u_0fupi
94.Lc2u_0nowords: teq ip, #0
95 beq .Lc2u_finished
96.Lc2u_nowords: cmp ip, #2
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 ldrb r3, [r1], #1
Catalin Marinas247055a2010-09-13 16:03:21 +010098USER( T(strb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 ldrgeb r3, [r1], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100100USER( T(strgeb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 ldrgtb r3, [r1], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100102USER( T(strgtb) r3, [r0], #1) @ May fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000103 b .Lc2u_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000105.Lc2u_not_enough:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 movs ip, r2
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000107 bne .Lc2u_nowords
108.Lc2u_finished: mov r0, #0
Russell King1b93a712006-06-25 11:23:45 +0100109 ldmfd sp!, {r2, r4 - r7, pc}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000111.Lc2u_src_not_aligned:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 bic r1, r1, #3
113 ldr r7, [r1], #4
114 cmp ip, #2
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000115 bgt .Lc2u_3fupi
116 beq .Lc2u_2fupi
117.Lc2u_1fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000119 bmi .Lc2u_1nowords
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120 mov r3, r7, pull #8
121 ldr r7, [r1], #4
122 orr r3, r3, r7, push #24
Catalin Marinas247055a2010-09-13 16:03:21 +0100123USER( T(str) r3, [r0], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124 mov ip, r0, lsl #32 - PAGE_SHIFT
125 rsb ip, ip, #0
126 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000127 beq .Lc2u_1fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 cmp r2, ip
129 movlt ip, r2
130 sub r2, r2, ip
131 subs ip, ip, #16
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000132 blt .Lc2u_1rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000134.Lc2u_1cpy8lp: mov r3, r7, pull #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 ldmia r1!, {r4 - r7}
136 subs ip, ip, #16
137 orr r3, r3, r4, push #24
138 mov r4, r4, pull #8
139 orr r4, r4, r5, push #24
140 mov r5, r5, pull #8
141 orr r5, r5, r6, push #24
142 mov r6, r6, pull #8
143 orr r6, r6, r7, push #24
144 stmia r0!, {r3 - r6} @ Shouldnt fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000145 bpl .Lc2u_1cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000147.Lc2u_1rem8lp: tst ip, #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148 movne r3, r7, pull #8
149 ldmneia r1!, {r4, r7}
150 orrne r3, r3, r4, push #24
151 movne r4, r4, pull #8
152 orrne r4, r4, r7, push #24
153 stmneia r0!, {r3 - r4} @ Shouldnt fault
154 tst ip, #4
155 movne r3, r7, pull #8
156 ldrne r7, [r1], #4
157 orrne r3, r3, r7, push #24
Catalin Marinas247055a2010-09-13 16:03:21 +0100158 T(strne) r3, [r0], #4 @ Shouldnt fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000160 beq .Lc2u_1fupi
161.Lc2u_1nowords: mov r3, r7, get_byte_1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 teq ip, #0
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000163 beq .Lc2u_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 cmp ip, #2
Catalin Marinas247055a2010-09-13 16:03:21 +0100165USER( T(strb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 movge r3, r7, get_byte_2
Catalin Marinas247055a2010-09-13 16:03:21 +0100167USER( T(strgeb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 movgt r3, r7, get_byte_3
Catalin Marinas247055a2010-09-13 16:03:21 +0100169USER( T(strgtb) r3, [r0], #1) @ May fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000170 b .Lc2u_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000172.Lc2u_2fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000174 bmi .Lc2u_2nowords
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 mov r3, r7, pull #16
176 ldr r7, [r1], #4
177 orr r3, r3, r7, push #16
Catalin Marinas247055a2010-09-13 16:03:21 +0100178USER( T(str) r3, [r0], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 mov ip, r0, lsl #32 - PAGE_SHIFT
180 rsb ip, ip, #0
181 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000182 beq .Lc2u_2fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 cmp r2, ip
184 movlt ip, r2
185 sub r2, r2, ip
186 subs ip, ip, #16
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000187 blt .Lc2u_2rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000189.Lc2u_2cpy8lp: mov r3, r7, pull #16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 ldmia r1!, {r4 - r7}
191 subs ip, ip, #16
192 orr r3, r3, r4, push #16
193 mov r4, r4, pull #16
194 orr r4, r4, r5, push #16
195 mov r5, r5, pull #16
196 orr r5, r5, r6, push #16
197 mov r6, r6, pull #16
198 orr r6, r6, r7, push #16
199 stmia r0!, {r3 - r6} @ Shouldnt fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000200 bpl .Lc2u_2cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000202.Lc2u_2rem8lp: tst ip, #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203 movne r3, r7, pull #16
204 ldmneia r1!, {r4, r7}
205 orrne r3, r3, r4, push #16
206 movne r4, r4, pull #16
207 orrne r4, r4, r7, push #16
208 stmneia r0!, {r3 - r4} @ Shouldnt fault
209 tst ip, #4
210 movne r3, r7, pull #16
211 ldrne r7, [r1], #4
212 orrne r3, r3, r7, push #16
Catalin Marinas247055a2010-09-13 16:03:21 +0100213 T(strne) r3, [r0], #4 @ Shouldnt fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000215 beq .Lc2u_2fupi
216.Lc2u_2nowords: mov r3, r7, get_byte_2
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 teq ip, #0
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000218 beq .Lc2u_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 cmp ip, #2
Catalin Marinas247055a2010-09-13 16:03:21 +0100220USER( T(strb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 movge r3, r7, get_byte_3
Catalin Marinas247055a2010-09-13 16:03:21 +0100222USER( T(strgeb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 ldrgtb r3, [r1], #0
Catalin Marinas247055a2010-09-13 16:03:21 +0100224USER( T(strgtb) r3, [r0], #1) @ May fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000225 b .Lc2u_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000227.Lc2u_3fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000229 bmi .Lc2u_3nowords
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 mov r3, r7, pull #24
231 ldr r7, [r1], #4
232 orr r3, r3, r7, push #8
Catalin Marinas247055a2010-09-13 16:03:21 +0100233USER( T(str) r3, [r0], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 mov ip, r0, lsl #32 - PAGE_SHIFT
235 rsb ip, ip, #0
236 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000237 beq .Lc2u_3fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 cmp r2, ip
239 movlt ip, r2
240 sub r2, r2, ip
241 subs ip, ip, #16
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000242 blt .Lc2u_3rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000244.Lc2u_3cpy8lp: mov r3, r7, pull #24
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 ldmia r1!, {r4 - r7}
246 subs ip, ip, #16
247 orr r3, r3, r4, push #8
248 mov r4, r4, pull #24
249 orr r4, r4, r5, push #8
250 mov r5, r5, pull #24
251 orr r5, r5, r6, push #8
252 mov r6, r6, pull #24
253 orr r6, r6, r7, push #8
254 stmia r0!, {r3 - r6} @ Shouldnt fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000255 bpl .Lc2u_3cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000257.Lc2u_3rem8lp: tst ip, #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 movne r3, r7, pull #24
259 ldmneia r1!, {r4, r7}
260 orrne r3, r3, r4, push #8
261 movne r4, r4, pull #24
262 orrne r4, r4, r7, push #8
263 stmneia r0!, {r3 - r4} @ Shouldnt fault
264 tst ip, #4
265 movne r3, r7, pull #24
266 ldrne r7, [r1], #4
267 orrne r3, r3, r7, push #8
Catalin Marinas247055a2010-09-13 16:03:21 +0100268 T(strne) r3, [r0], #4 @ Shouldnt fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000270 beq .Lc2u_3fupi
271.Lc2u_3nowords: mov r3, r7, get_byte_3
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 teq ip, #0
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000273 beq .Lc2u_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 cmp ip, #2
Catalin Marinas247055a2010-09-13 16:03:21 +0100275USER( T(strb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 ldrgeb r3, [r1], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100277USER( T(strgeb) r3, [r0], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 ldrgtb r3, [r1], #0
Catalin Marinas247055a2010-09-13 16:03:21 +0100279USER( T(strgtb) r3, [r0], #1) @ May fault
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000280 b .Lc2u_finished
Catalin Marinas93ed3972008-08-28 11:22:32 +0100281ENDPROC(__copy_to_user)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282
Russell King42604152010-04-19 10:15:03 +0100283 .pushsection .fixup,"ax"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 .align 0
Russell King1b93a712006-06-25 11:23:45 +01002859001: ldmfd sp!, {r0, r4 - r7, pc}
Russell King42604152010-04-19 10:15:03 +0100286 .popsection
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287
Russell King02fcb972006-06-21 14:44:52 +0100288/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 * Purpose : copy a block from user memory to kernel memory
290 * Params : to - kernel memory
291 * : from - user memory
292 * : n - number of bytes to copy
293 * Returns : Number of bytes NOT copied.
294 */
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000295.Lcfu_dest_not_aligned:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 rsb ip, ip, #4
297 cmp ip, #2
Catalin Marinas247055a2010-09-13 16:03:21 +0100298USER( T(ldrb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 strb r3, [r0], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100300USER( T(ldrgeb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 strgeb r3, [r0], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100302USER( T(ldrgtb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 strgtb r3, [r0], #1
304 sub r2, r2, ip
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000305 b .Lcfu_dest_aligned
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
Russell King02fcb972006-06-21 14:44:52 +0100307ENTRY(__copy_from_user)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 stmfd sp!, {r0, r2, r4 - r7, lr}
309 cmp r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000310 blt .Lcfu_not_enough
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 ands ip, r0, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000312 bne .Lcfu_dest_not_aligned
313.Lcfu_dest_aligned:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 ands ip, r1, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000315 bne .Lcfu_src_not_aligned
316
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317/*
318 * Seeing as there has to be at least 8 bytes to copy, we can
319 * copy one word, and force a user-mode page fault...
320 */
321
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000322.Lcfu_0fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000324 bmi .Lcfu_0nowords
Catalin Marinas247055a2010-09-13 16:03:21 +0100325USER( T(ldr) r3, [r1], #4)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 str r3, [r0], #4
327 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
328 rsb ip, ip, #0
329 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000330 beq .Lcfu_0fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331/*
332 * ip = max no. of bytes to copy before needing another "strt" insn
333 */
334 cmp r2, ip
335 movlt ip, r2
336 sub r2, r2, ip
337 subs ip, ip, #32
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000338 blt .Lcfu_0rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000340.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 stmia r0!, {r3 - r6}
342 ldmia r1!, {r3 - r6} @ Shouldnt fault
343 subs ip, ip, #32
344 stmia r0!, {r3 - r6}
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000345 bpl .Lcfu_0cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000347.Lcfu_0rem8lp: cmn ip, #16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 ldmgeia r1!, {r3 - r6} @ Shouldnt fault
349 stmgeia r0!, {r3 - r6}
350 tst ip, #8
351 ldmneia r1!, {r3 - r4} @ Shouldnt fault
352 stmneia r0!, {r3 - r4}
353 tst ip, #4
Catalin Marinas247055a2010-09-13 16:03:21 +0100354 T(ldrne) r3, [r1], #4 @ Shouldnt fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 strne r3, [r0], #4
356 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000357 beq .Lcfu_0fupi
358.Lcfu_0nowords: teq ip, #0
359 beq .Lcfu_finished
360.Lcfu_nowords: cmp ip, #2
Catalin Marinas247055a2010-09-13 16:03:21 +0100361USER( T(ldrb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 strb r3, [r0], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100363USER( T(ldrgeb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 strgeb r3, [r0], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100365USER( T(ldrgtb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 strgtb r3, [r0], #1
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000367 b .Lcfu_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000369.Lcfu_not_enough:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 movs ip, r2
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000371 bne .Lcfu_nowords
372.Lcfu_finished: mov r0, #0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 add sp, sp, #8
Russell King1b93a712006-06-25 11:23:45 +0100374 ldmfd sp!, {r4 - r7, pc}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000376.Lcfu_src_not_aligned:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 bic r1, r1, #3
Catalin Marinas247055a2010-09-13 16:03:21 +0100378USER( T(ldr) r7, [r1], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 cmp ip, #2
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000380 bgt .Lcfu_3fupi
381 beq .Lcfu_2fupi
382.Lcfu_1fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000384 bmi .Lcfu_1nowords
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 mov r3, r7, pull #8
Catalin Marinas247055a2010-09-13 16:03:21 +0100386USER( T(ldr) r7, [r1], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 orr r3, r3, r7, push #24
388 str r3, [r0], #4
389 mov ip, r1, lsl #32 - PAGE_SHIFT
390 rsb ip, ip, #0
391 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000392 beq .Lcfu_1fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 cmp r2, ip
394 movlt ip, r2
395 sub r2, r2, ip
396 subs ip, ip, #16
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000397 blt .Lcfu_1rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000399.Lcfu_1cpy8lp: mov r3, r7, pull #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 ldmia r1!, {r4 - r7} @ Shouldnt fault
401 subs ip, ip, #16
402 orr r3, r3, r4, push #24
403 mov r4, r4, pull #8
404 orr r4, r4, r5, push #24
405 mov r5, r5, pull #8
406 orr r5, r5, r6, push #24
407 mov r6, r6, pull #8
408 orr r6, r6, r7, push #24
409 stmia r0!, {r3 - r6}
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000410 bpl .Lcfu_1cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000412.Lcfu_1rem8lp: tst ip, #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 movne r3, r7, pull #8
414 ldmneia r1!, {r4, r7} @ Shouldnt fault
415 orrne r3, r3, r4, push #24
416 movne r4, r4, pull #8
417 orrne r4, r4, r7, push #24
418 stmneia r0!, {r3 - r4}
419 tst ip, #4
420 movne r3, r7, pull #8
Catalin Marinas247055a2010-09-13 16:03:21 +0100421USER( T(ldrne) r7, [r1], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 orrne r3, r3, r7, push #24
423 strne r3, [r0], #4
424 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000425 beq .Lcfu_1fupi
426.Lcfu_1nowords: mov r3, r7, get_byte_1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 teq ip, #0
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000428 beq .Lcfu_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429 cmp ip, #2
430 strb r3, [r0], #1
431 movge r3, r7, get_byte_2
432 strgeb r3, [r0], #1
433 movgt r3, r7, get_byte_3
434 strgtb r3, [r0], #1
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000435 b .Lcfu_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000437.Lcfu_2fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000439 bmi .Lcfu_2nowords
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 mov r3, r7, pull #16
Catalin Marinas247055a2010-09-13 16:03:21 +0100441USER( T(ldr) r7, [r1], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 orr r3, r3, r7, push #16
443 str r3, [r0], #4
444 mov ip, r1, lsl #32 - PAGE_SHIFT
445 rsb ip, ip, #0
446 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000447 beq .Lcfu_2fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 cmp r2, ip
449 movlt ip, r2
450 sub r2, r2, ip
451 subs ip, ip, #16
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000452 blt .Lcfu_2rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000454
455.Lcfu_2cpy8lp: mov r3, r7, pull #16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 ldmia r1!, {r4 - r7} @ Shouldnt fault
457 subs ip, ip, #16
458 orr r3, r3, r4, push #16
459 mov r4, r4, pull #16
460 orr r4, r4, r5, push #16
461 mov r5, r5, pull #16
462 orr r5, r5, r6, push #16
463 mov r6, r6, pull #16
464 orr r6, r6, r7, push #16
465 stmia r0!, {r3 - r6}
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000466 bpl .Lcfu_2cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000468.Lcfu_2rem8lp: tst ip, #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 movne r3, r7, pull #16
470 ldmneia r1!, {r4, r7} @ Shouldnt fault
471 orrne r3, r3, r4, push #16
472 movne r4, r4, pull #16
473 orrne r4, r4, r7, push #16
474 stmneia r0!, {r3 - r4}
475 tst ip, #4
476 movne r3, r7, pull #16
Catalin Marinas247055a2010-09-13 16:03:21 +0100477USER( T(ldrne) r7, [r1], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 orrne r3, r3, r7, push #16
479 strne r3, [r0], #4
480 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000481 beq .Lcfu_2fupi
482.Lcfu_2nowords: mov r3, r7, get_byte_2
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 teq ip, #0
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000484 beq .Lcfu_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 cmp ip, #2
486 strb r3, [r0], #1
487 movge r3, r7, get_byte_3
488 strgeb r3, [r0], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100489USER( T(ldrgtb) r3, [r1], #0) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 strgtb r3, [r0], #1
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000491 b .Lcfu_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000493.Lcfu_3fupi: subs r2, r2, #4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 addmi ip, r2, #4
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000495 bmi .Lcfu_3nowords
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 mov r3, r7, pull #24
Catalin Marinas247055a2010-09-13 16:03:21 +0100497USER( T(ldr) r7, [r1], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 orr r3, r3, r7, push #8
499 str r3, [r0], #4
500 mov ip, r1, lsl #32 - PAGE_SHIFT
501 rsb ip, ip, #0
502 movs ip, ip, lsr #32 - PAGE_SHIFT
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000503 beq .Lcfu_3fupi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 cmp r2, ip
505 movlt ip, r2
506 sub r2, r2, ip
507 subs ip, ip, #16
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000508 blt .Lcfu_3rem8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000510.Lcfu_3cpy8lp: mov r3, r7, pull #24
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 ldmia r1!, {r4 - r7} @ Shouldnt fault
512 orr r3, r3, r4, push #8
513 mov r4, r4, pull #24
514 orr r4, r4, r5, push #8
515 mov r5, r5, pull #24
516 orr r5, r5, r6, push #8
517 mov r6, r6, pull #24
518 orr r6, r6, r7, push #8
519 stmia r0!, {r3 - r6}
520 subs ip, ip, #16
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000521 bpl .Lcfu_3cpy8lp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000523.Lcfu_3rem8lp: tst ip, #8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 movne r3, r7, pull #24
525 ldmneia r1!, {r4, r7} @ Shouldnt fault
526 orrne r3, r3, r4, push #8
527 movne r4, r4, pull #24
528 orrne r4, r4, r7, push #8
529 stmneia r0!, {r3 - r4}
530 tst ip, #4
531 movne r3, r7, pull #24
Catalin Marinas247055a2010-09-13 16:03:21 +0100532USER( T(ldrne) r7, [r1], #4) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 orrne r3, r3, r7, push #8
534 strne r3, [r0], #4
535 ands ip, ip, #3
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000536 beq .Lcfu_3fupi
537.Lcfu_3nowords: mov r3, r7, get_byte_3
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538 teq ip, #0
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000539 beq .Lcfu_finished
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540 cmp ip, #2
541 strb r3, [r0], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100542USER( T(ldrgeb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 strgeb r3, [r0], #1
Catalin Marinas247055a2010-09-13 16:03:21 +0100544USER( T(ldrgtb) r3, [r1], #1) @ May fault
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 strgtb r3, [r0], #1
Nicolas Pitre7ba11a92005-11-11 21:51:47 +0000546 b .Lcfu_finished
Catalin Marinas93ed3972008-08-28 11:22:32 +0100547ENDPROC(__copy_from_user)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548
Russell King42604152010-04-19 10:15:03 +0100549 .pushsection .fixup,"ax"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 .align 0
551 /*
552 * We took an exception. r0 contains a pointer to
553 * the byte not copied.
554 */
5559001: ldr r2, [sp], #4 @ void *to
556 sub r2, r0, r2 @ bytes copied
557 ldr r1, [sp], #4 @ unsigned long count
558 subs r4, r1, r2 @ bytes left to copy
559 movne r1, r4
560 blne __memzero
561 mov r0, r4
Russell King1b93a712006-06-25 11:23:45 +0100562 ldmfd sp!, {r4 - r7, pc}
Russell King42604152010-04-19 10:15:03 +0100563 .popsection
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564