blob: 4d2df328e51476cadc4cfef6f8eb636506f18d95 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* clear_page.S: UltraSparc optimized copy page.
2 *
3 * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com)
4 * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
5 */
6
7#include <asm/visasm.h>
8#include <asm/thread_info.h>
9#include <asm/page.h>
10#include <asm/pgtable.h>
11#include <asm/spitfire.h>
12#include <asm/head.h>
13
14 /* What we used to do was lock a TLB entry into a specific
15 * TLB slot, clear the page with interrupts disabled, then
16 * restore the original TLB entry. This was great for
17 * disturbing the TLB as little as possible, but it meant
18 * we had to keep interrupts disabled for a long time.
19 *
20 * Now, we simply use the normal TLB loading mechanism,
21 * and this makes the cpu choose a slot all by itself.
22 * Then we do a normal TLB flush on exit. We need only
23 * disable preemption during the clear.
24 */
25
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#define DCACHE_SIZE (PAGE_SIZE * 2)
27
David S. Millerf7fe9332008-07-17 23:43:55 -070028#if (PAGE_SHIFT == 13)
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#define PAGE_SIZE_REM 0x80
David S. Millerf7fe9332008-07-17 23:43:55 -070030#elif (PAGE_SHIFT == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#define PAGE_SIZE_REM 0x100
32#else
33#error Wrong PAGE_SHIFT specified
34#endif
35
36#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
David S. Miller6f1d8272012-06-27 01:25:23 -070037 fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \
38 fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \
39 fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \
40 fsrc2 %reg6, %f60; fsrc2 %reg7, %f62;
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
42 .text
43
44 .align 32
45 .globl copy_user_page
46 .type copy_user_page,#function
47copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
48 lduw [%g6 + TI_PRE_COUNT], %o4
49 sethi %uhi(PAGE_OFFSET), %g2
50 sethi %hi(PAGE_SIZE), %o3
51
52 sllx %g2, 32, %g2
David S. Millerc4bce902006-02-11 21:57:54 -080053 sethi %hi(PAGE_KERNEL_LOCKED), %g3
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
David S. Millerc4bce902006-02-11 21:57:54 -080055 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 sub %o0, %g2, %g1 ! dest paddr
57
58 sub %o1, %g2, %g2 ! src paddr
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
60 and %o2, %o3, %o0 ! vaddr D-cache alias bit
61 or %g1, %g3, %g1 ! dest TTE data
62
63 or %g2, %g3, %g2 ! src TTE data
64 sethi %hi(TLBTEMP_BASE), %o3
65
66 sethi %hi(DCACHE_SIZE), %o1
67 add %o0, %o3, %o0 ! dest TTE vaddr
68
69 add %o4, 1, %o2
70 add %o0, %o1, %o1 ! src TTE vaddr
71
72 /* Disable preemption. */
73 mov TLB_TAG_ACCESS, %g3
74 stw %o2, [%g6 + TI_PRE_COUNT]
75
76 /* Load TLB entries. */
77 rdpr %pstate, %o2
78 wrpr %o2, PSTATE_IE, %pstate
79 stxa %o0, [%g3] ASI_DMMU
80 stxa %g1, [%g0] ASI_DTLB_DATA_IN
81 membar #Sync
82 stxa %o1, [%g3] ASI_DMMU
83 stxa %g2, [%g0] ASI_DTLB_DATA_IN
84 membar #Sync
85 wrpr %o2, 0x0, %pstate
86
David S. Millerdbd2fdf2005-08-30 11:26:15 -070087cheetah_copy_page_insn:
Linus Torvalds1da177e2005-04-16 15:20:36 -070088 ba,pt %xcc, 9f
89 nop
90
911:
92 VISEntryHalf
93 membar #StoreLoad | #StoreStore | #LoadStore
94 sethi %hi((PAGE_SIZE/64)-2), %o2
95 mov %o0, %g1
96 prefetch [%o1 + 0x000], #one_read
97 or %o2, %lo((PAGE_SIZE/64)-2), %o2
98 prefetch [%o1 + 0x040], #one_read
99 prefetch [%o1 + 0x080], #one_read
100 prefetch [%o1 + 0x0c0], #one_read
101 ldd [%o1 + 0x000], %f0
102 prefetch [%o1 + 0x100], #one_read
103 ldd [%o1 + 0x008], %f2
104 prefetch [%o1 + 0x140], #one_read
105 ldd [%o1 + 0x010], %f4
106 prefetch [%o1 + 0x180], #one_read
David S. Miller6f1d8272012-06-27 01:25:23 -0700107 fsrc2 %f0, %f16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 ldd [%o1 + 0x018], %f6
David S. Miller6f1d8272012-06-27 01:25:23 -0700109 fsrc2 %f2, %f18
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 ldd [%o1 + 0x020], %f8
David S. Miller6f1d8272012-06-27 01:25:23 -0700111 fsrc2 %f4, %f20
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 ldd [%o1 + 0x028], %f10
David S. Miller6f1d8272012-06-27 01:25:23 -0700113 fsrc2 %f6, %f22
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 ldd [%o1 + 0x030], %f12
David S. Miller6f1d8272012-06-27 01:25:23 -0700115 fsrc2 %f8, %f24
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 ldd [%o1 + 0x038], %f14
David S. Miller6f1d8272012-06-27 01:25:23 -0700117 fsrc2 %f10, %f26
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 ldd [%o1 + 0x040], %f0
1191: ldd [%o1 + 0x048], %f2
David S. Miller6f1d8272012-06-27 01:25:23 -0700120 fsrc2 %f12, %f28
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 ldd [%o1 + 0x050], %f4
David S. Miller6f1d8272012-06-27 01:25:23 -0700122 fsrc2 %f14, %f30
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123 stda %f16, [%o0] ASI_BLK_P
124 ldd [%o1 + 0x058], %f6
David S. Miller6f1d8272012-06-27 01:25:23 -0700125 fsrc2 %f0, %f16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 ldd [%o1 + 0x060], %f8
David S. Miller6f1d8272012-06-27 01:25:23 -0700127 fsrc2 %f2, %f18
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 ldd [%o1 + 0x068], %f10
David S. Miller6f1d8272012-06-27 01:25:23 -0700129 fsrc2 %f4, %f20
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 ldd [%o1 + 0x070], %f12
David S. Miller6f1d8272012-06-27 01:25:23 -0700131 fsrc2 %f6, %f22
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 ldd [%o1 + 0x078], %f14
David S. Miller6f1d8272012-06-27 01:25:23 -0700133 fsrc2 %f8, %f24
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 ldd [%o1 + 0x080], %f0
135 prefetch [%o1 + 0x180], #one_read
David S. Miller6f1d8272012-06-27 01:25:23 -0700136 fsrc2 %f10, %f26
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 subcc %o2, 1, %o2
138 add %o0, 0x40, %o0
139 bne,pt %xcc, 1b
140 add %o1, 0x40, %o1
141
142 ldd [%o1 + 0x048], %f2
David S. Miller6f1d8272012-06-27 01:25:23 -0700143 fsrc2 %f12, %f28
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 ldd [%o1 + 0x050], %f4
David S. Miller6f1d8272012-06-27 01:25:23 -0700145 fsrc2 %f14, %f30
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 stda %f16, [%o0] ASI_BLK_P
147 ldd [%o1 + 0x058], %f6
David S. Miller6f1d8272012-06-27 01:25:23 -0700148 fsrc2 %f0, %f16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 ldd [%o1 + 0x060], %f8
David S. Miller6f1d8272012-06-27 01:25:23 -0700150 fsrc2 %f2, %f18
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 ldd [%o1 + 0x068], %f10
David S. Miller6f1d8272012-06-27 01:25:23 -0700152 fsrc2 %f4, %f20
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 ldd [%o1 + 0x070], %f12
David S. Miller6f1d8272012-06-27 01:25:23 -0700154 fsrc2 %f6, %f22
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 add %o0, 0x40, %o0
156 ldd [%o1 + 0x078], %f14
David S. Miller6f1d8272012-06-27 01:25:23 -0700157 fsrc2 %f8, %f24
158 fsrc2 %f10, %f26
159 fsrc2 %f12, %f28
160 fsrc2 %f14, %f30
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 stda %f16, [%o0] ASI_BLK_P
162 membar #Sync
163 VISExitHalf
164 ba,pt %xcc, 5f
165 nop
166
1679:
168 VISEntry
169 ldub [%g6 + TI_FAULT_CODE], %g3
170 mov %o0, %g1
171 cmp %g3, 0
172 rd %asi, %g3
173 be,a,pt %icc, 1f
174 wr %g0, ASI_BLK_P, %asi
175 wr %g0, ASI_BLK_COMMIT_P, %asi
1761: ldda [%o1] ASI_BLK_P, %f0
177 add %o1, 0x40, %o1
178 ldda [%o1] ASI_BLK_P, %f16
179 add %o1, 0x40, %o1
180 sethi %hi(PAGE_SIZE), %o2
1811: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
182 ldda [%o1] ASI_BLK_P, %f32
183 stda %f48, [%o0] %asi
184 add %o1, 0x40, %o1
185 sub %o2, 0x40, %o2
186 add %o0, 0x40, %o0
187 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
188 ldda [%o1] ASI_BLK_P, %f0
189 stda %f48, [%o0] %asi
190 add %o1, 0x40, %o1
191 sub %o2, 0x40, %o2
192 add %o0, 0x40, %o0
193 TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
194 ldda [%o1] ASI_BLK_P, %f16
195 stda %f48, [%o0] %asi
196 sub %o2, 0x40, %o2
197 add %o1, 0x40, %o1
198 cmp %o2, PAGE_SIZE_REM
199 bne,pt %xcc, 1b
200 add %o0, 0x40, %o0
David S. Millerf7fe9332008-07-17 23:43:55 -0700201#if (PAGE_SHIFT == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
203 ldda [%o1] ASI_BLK_P, %f32
204 stda %f48, [%o0] %asi
205 add %o1, 0x40, %o1
206 sub %o2, 0x40, %o2
207 add %o0, 0x40, %o0
208 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
209 ldda [%o1] ASI_BLK_P, %f0
210 stda %f48, [%o0] %asi
211 add %o1, 0x40, %o1
212 sub %o2, 0x40, %o2
213 add %o0, 0x40, %o0
214 membar #Sync
215 stda %f32, [%o0] %asi
216 add %o0, 0x40, %o0
217 stda %f0, [%o0] %asi
218#else
219 membar #Sync
220 stda %f0, [%o0] %asi
221 add %o0, 0x40, %o0
222 stda %f16, [%o0] %asi
223#endif
224 membar #Sync
225 wr %g3, 0x0, %asi
226 VISExit
227
2285:
229 stxa %g0, [%g1] ASI_DMMU_DEMAP
230 membar #Sync
231
232 sethi %hi(DCACHE_SIZE), %g2
233 stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP
234 membar #Sync
235
236 retl
237 stw %o4, [%g6 + TI_PRE_COUNT]
238
239 .size copy_user_page, .-copy_user_page
David S. Millerdbd2fdf2005-08-30 11:26:15 -0700240
241 .globl cheetah_patch_copy_page
242cheetah_patch_copy_page:
243 sethi %hi(0x01000000), %o1 ! NOP
244 sethi %hi(cheetah_copy_page_insn), %o0
245 or %o0, %lo(cheetah_copy_page_insn), %o0
246 stw %o1, [%o0]
247 membar #StoreStore
248 flush %o0
249 retl
250 nop