blob: 18a1ccd4d1348c0134ed9d755512b66d1f54779d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1998, 1999, 2000 by Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
Steven J. Hill26c5e072013-03-25 13:40:49 -05008 * Copyright (C) 2007 by Maciej W. Rozycki
9 * Copyright (C) 2011, 2012 MIPS Technologies, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 */
11#include <asm/asm.h>
Sam Ravnborg048eb582005-09-09 22:32:31 +020012#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <asm/regdef.h>
14
Atsushi Nemotoa5831582006-12-18 00:07:40 +090015#if LONGSIZE == 4
16#define LONG_S_L swl
17#define LONG_S_R swr
18#else
19#define LONG_S_L sdl
20#define LONG_S_R sdr
21#endif
22
Steven J. Hill26c5e072013-03-25 13:40:49 -050023#ifdef CONFIG_CPU_MICROMIPS
24#define STORSIZE (LONGSIZE * 2)
25#define STORMASK (STORSIZE - 1)
26#define FILL64RG t8
27#define FILLPTRG t7
28#undef LONG_S
29#define LONG_S LONG_SP
30#else
31#define STORSIZE LONGSIZE
32#define STORMASK LONGMASK
33#define FILL64RG a1
34#define FILLPTRG t0
35#endif
36
Markos Chandras6d5155c2014-01-03 09:23:16 +000037#define LEGACY_MODE 1
38#define EVA_MODE 2
39
Markos Chandrasfd9720e2014-01-03 10:11:45 +000040/*
41 * No need to protect it with EVA #ifdefery. The generated block of code
42 * will never be assembled if EVA is not enabled.
43 */
44#define __EVAFY(insn, reg, addr) __BUILD_EVA_INSN(insn##e, reg, addr)
45#define ___BUILD_EVA_INSN(insn, reg, addr) __EVAFY(insn, reg, addr)
46
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#define EX(insn,reg,addr,handler) \
Markos Chandrasfd9720e2014-01-03 10:11:45 +000048 .if \mode == LEGACY_MODE; \
499: insn reg, addr; \
50 .else; \
519: ___BUILD_EVA_INSN(insn, reg, addr); \
52 .endif; \
Ralf Baechle70342282013-01-22 12:59:30 +010053 .section __ex_table,"a"; \
54 PTR 9b, handler; \
Linus Torvalds1da177e2005-04-16 15:20:36 -070055 .previous
56
Markos Chandrasfd9720e2014-01-03 10:11:45 +000057 .macro f_fill64 dst, offset, val, fixup, mode
Steven J. Hill26c5e072013-03-25 13:40:49 -050058 EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup)
59 EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup)
60 EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup)
61 EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup)
62#if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS))
63 EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup)
64 EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup)
65 EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup)
66 EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup)
67#endif
68#if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4))
69 EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup)
70 EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup)
71 EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup)
72 EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup)
73 EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup)
74 EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup)
75 EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup)
76 EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup)
Atsushi Nemotoa5831582006-12-18 00:07:40 +090077#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 .endm
79
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 .set noreorder
81 .align 5
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Markos Chandras6d5155c2014-01-03 09:23:16 +000083 /*
84 * Macro to generate the __bzero{,_user} symbol
85 * Arguments:
86 * mode: LEGACY_MODE or EVA_MODE
87 */
88 .macro __BUILD_BZERO mode
89 /* Initialize __memset if this is the first time we call this macro */
90 .ifnotdef __memset
91 .set __memset, 1
92 .hidden __memset /* Make sure it does not leak */
93 .endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
Steven J. Hill26c5e072013-03-25 13:40:49 -050095 sltiu t0, a2, STORSIZE /* very small region? */
Markos Chandras6d5155c2014-01-03 09:23:16 +000096 bnez t0, .Lsmall_memset\@
Markos Chandras8483b142014-01-03 09:21:20 +000097 andi t0, a0, STORMASK /* aligned? */
Linus Torvalds1da177e2005-04-16 15:20:36 -070098
Steven J. Hill26c5e072013-03-25 13:40:49 -050099#ifdef CONFIG_CPU_MICROMIPS
100 move t8, a1 /* used by 'swp' instruction */
101 move t9, a1
102#endif
Maciej W. Rozycki619b6e12007-10-23 12:43:25 +0100103#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 beqz t0, 1f
Markos Chandras8483b142014-01-03 09:21:20 +0000105 PTR_SUBU t0, STORSIZE /* alignment in bytes */
Maciej W. Rozycki619b6e12007-10-23 12:43:25 +0100106#else
107 .set noat
Steven J. Hill26c5e072013-03-25 13:40:49 -0500108 li AT, STORSIZE
Maciej W. Rozycki619b6e12007-10-23 12:43:25 +0100109 beqz t0, 1f
Markos Chandras8483b142014-01-03 09:21:20 +0000110 PTR_SUBU t0, AT /* alignment in bytes */
Maciej W. Rozycki619b6e12007-10-23 12:43:25 +0100111 .set at
112#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113
Leonid Yegoshin8c562082014-11-18 09:04:34 +0000114#ifndef CONFIG_CPU_MIPSR6
Thomas Bogendoerfer930bff82007-11-25 11:47:56 +0100115 R10KCBARRIER(0(ra))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#ifdef __MIPSEB__
Markos Chandras6d5155c2014-01-03 09:23:16 +0000117 EX(LONG_S_L, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
Markos Chandrasdd2adea2014-11-19 08:58:10 +0000118#else
Markos Chandras6d5155c2014-01-03 09:23:16 +0000119 EX(LONG_S_R, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120#endif
121 PTR_SUBU a0, t0 /* long align ptr */
122 PTR_ADDU a2, t0 /* correct size */
123
Leonid Yegoshin8c562082014-11-18 09:04:34 +0000124#else /* CONFIG_CPU_MIPSR6 */
125#define STORE_BYTE(N) \
126 EX(sb, a1, N(a0), .Lbyte_fixup\@); \
127 beqz t0, 0f; \
128 PTR_ADDU t0, 1;
129
130 PTR_ADDU a2, t0 /* correct size */
131 PTR_ADDU t0, 1
132 STORE_BYTE(0)
133 STORE_BYTE(1)
134#if LONGSIZE == 4
135 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
136#else
137 STORE_BYTE(2)
138 STORE_BYTE(3)
139 STORE_BYTE(4)
140 STORE_BYTE(5)
141 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
142#endif
1430:
144 ori a0, STORMASK
145 xori a0, STORMASK
146 PTR_ADDIU a0, STORSIZE
147#endif /* CONFIG_CPU_MIPSR6 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481: ori t1, a2, 0x3f /* # of full blocks */
149 xori t1, 0x3f
Markos Chandras6d5155c2014-01-03 09:23:16 +0000150 beqz t1, .Lmemset_partial\@ /* no block to fill */
Markos Chandras8483b142014-01-03 09:21:20 +0000151 andi t0, a2, 0x40-STORSIZE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152
153 PTR_ADDU t1, a0 /* end address */
154 .set reorder
1551: PTR_ADDIU a0, 64
Thomas Bogendoerfer930bff82007-11-25 11:47:56 +0100156 R10KCBARRIER(0(ra))
Markos Chandrasfd9720e2014-01-03 10:11:45 +0000157 f_fill64 a0, -64, FILL64RG, .Lfwd_fixup\@, \mode
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 bne t1, a0, 1b
159 .set noreorder
160
Markos Chandras6d5155c2014-01-03 09:23:16 +0000161.Lmemset_partial\@:
Thomas Bogendoerfer930bff82007-11-25 11:47:56 +0100162 R10KCBARRIER(0(ra))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 PTR_LA t1, 2f /* where to start */
Steven J. Hill26c5e072013-03-25 13:40:49 -0500164#ifdef CONFIG_CPU_MICROMIPS
165 LONG_SRL t7, t0, 1
166#endif
Atsushi Nemotoa5831582006-12-18 00:07:40 +0900167#if LONGSIZE == 4
Steven J. Hill26c5e072013-03-25 13:40:49 -0500168 PTR_SUBU t1, FILLPTRG
Atsushi Nemotoa5831582006-12-18 00:07:40 +0900169#else
170 .set noat
Steven J. Hill26c5e072013-03-25 13:40:49 -0500171 LONG_SRL AT, FILLPTRG, 1
Atsushi Nemotoa5831582006-12-18 00:07:40 +0900172 PTR_SUBU t1, AT
Maciej W. Rozycki619b6e12007-10-23 12:43:25 +0100173 .set at
Atsushi Nemotoa5831582006-12-18 00:07:40 +0900174#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 jr t1
Markos Chandras8483b142014-01-03 09:21:20 +0000176 PTR_ADDU a0, t0 /* dest ptr */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177
178 .set push
179 .set noreorder
180 .set nomacro
Markos Chandras6d5155c2014-01-03 09:23:16 +0000181 /* ... but first do longs ... */
Markos Chandrasfd9720e2014-01-03 10:11:45 +0000182 f_fill64 a0, -64, FILL64RG, .Lpartial_fixup\@, \mode
Linus Torvalds1da177e2005-04-16 15:20:36 -07001832: .set pop
Steven J. Hill26c5e072013-03-25 13:40:49 -0500184 andi a2, STORMASK /* At most one long to go */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185
186 beqz a2, 1f
Leonid Yegoshin8c562082014-11-18 09:04:34 +0000187#ifndef CONFIG_CPU_MIPSR6
Markos Chandras8483b142014-01-03 09:21:20 +0000188 PTR_ADDU a0, a2 /* What's left */
Thomas Bogendoerfer930bff82007-11-25 11:47:56 +0100189 R10KCBARRIER(0(ra))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190#ifdef __MIPSEB__
Markos Chandras6d5155c2014-01-03 09:23:16 +0000191 EX(LONG_S_R, a1, -1(a0), .Llast_fixup\@)
Markos Chandrasdd2adea2014-11-19 08:58:10 +0000192#else
Markos Chandras6d5155c2014-01-03 09:23:16 +0000193 EX(LONG_S_L, a1, -1(a0), .Llast_fixup\@)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194#endif
Leonid Yegoshin8c562082014-11-18 09:04:34 +0000195#else
196 PTR_SUBU t0, $0, a2
197 PTR_ADDIU t0, 1
198 STORE_BYTE(0)
199 STORE_BYTE(1)
200#if LONGSIZE == 4
201 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
202#else
203 STORE_BYTE(2)
204 STORE_BYTE(3)
205 STORE_BYTE(4)
206 STORE_BYTE(5)
207 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
208#endif
2090:
210#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002111: jr ra
Markos Chandras8483b142014-01-03 09:21:20 +0000212 move a2, zero
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213
Markos Chandras6d5155c2014-01-03 09:23:16 +0000214.Lsmall_memset\@:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 beqz a2, 2f
Markos Chandras8483b142014-01-03 09:21:20 +0000216 PTR_ADDU t1, a0, a2
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
2181: PTR_ADDIU a0, 1 /* fill bytewise */
Thomas Bogendoerfer930bff82007-11-25 11:47:56 +0100219 R10KCBARRIER(0(ra))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 bne t1, a0, 1b
Markos Chandras8483b142014-01-03 09:21:20 +0000221 sb a1, -1(a0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
2232: jr ra /* done */
Markos Chandras8483b142014-01-03 09:21:20 +0000224 move a2, zero
Markos Chandras6d5155c2014-01-03 09:23:16 +0000225 .if __memset == 1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 END(memset)
Markos Chandras6d5155c2014-01-03 09:23:16 +0000227 .set __memset, 0
228 .hidden __memset
229 .endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
Maciej W. Rozycki8e85f2752016-02-07 11:05:58 +0000231#ifdef CONFIG_CPU_MIPSR6
Leonid Yegoshin8c562082014-11-18 09:04:34 +0000232.Lbyte_fixup\@:
233 PTR_SUBU a2, $0, t0
234 jr ra
235 PTR_ADDIU a2, 1
Maciej W. Rozycki8e85f2752016-02-07 11:05:58 +0000236#endif /* CONFIG_CPU_MIPSR6 */
Leonid Yegoshin8c562082014-11-18 09:04:34 +0000237
Markos Chandras6d5155c2014-01-03 09:23:16 +0000238.Lfirst_fixup\@:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 jr ra
Markos Chandras8483b142014-01-03 09:21:20 +0000240 nop
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241
Markos Chandras6d5155c2014-01-03 09:23:16 +0000242.Lfwd_fixup\@:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 PTR_L t0, TI_TASK($28)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 andi a2, 0x3f
Tony Wue5674ad2010-11-10 21:48:15 +0800245 LONG_L t0, THREAD_BUADDR(t0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 LONG_ADDU a2, t1
247 jr ra
Markos Chandras8483b142014-01-03 09:21:20 +0000248 LONG_SUBU a2, t0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
Markos Chandras6d5155c2014-01-03 09:23:16 +0000250.Lpartial_fixup\@:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 PTR_L t0, TI_TASK($28)
Steven J. Hill26c5e072013-03-25 13:40:49 -0500252 andi a2, STORMASK
Tony Wue5674ad2010-11-10 21:48:15 +0800253 LONG_L t0, THREAD_BUADDR(t0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 LONG_ADDU a2, t1
255 jr ra
Markos Chandras8483b142014-01-03 09:21:20 +0000256 LONG_SUBU a2, t0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257
Markos Chandras6d5155c2014-01-03 09:23:16 +0000258.Llast_fixup\@:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 jr ra
Markos Chandras8483b142014-01-03 09:21:20 +0000260 andi v1, a2, STORMASK
Markos Chandras6d5155c2014-01-03 09:23:16 +0000261
262 .endm
263
264/*
265 * memset(void *s, int c, size_t n)
266 *
267 * a0: start of area to clear
268 * a1: char to fill with
269 * a2: size of area to clear
270 */
271
272LEAF(memset)
273 beqz a1, 1f
274 move v0, a0 /* result */
275
276 andi a1, 0xff /* spread fillword */
277 LONG_SLL t1, a1, 8
278 or a1, t1
279 LONG_SLL t1, a1, 16
280#if LONGSIZE == 8
281 or a1, t1
282 LONG_SLL t1, a1, 32
283#endif
284 or a1, t1
2851:
Markos Chandrasfd9720e2014-01-03 10:11:45 +0000286#ifndef CONFIG_EVA
Markos Chandras6d5155c2014-01-03 09:23:16 +0000287FEXPORT(__bzero)
James Hogand6a428f2015-08-05 16:41:39 +0100288#else
289FEXPORT(__bzero_kernel)
Markos Chandrasfd9720e2014-01-03 10:11:45 +0000290#endif
Markos Chandras6d5155c2014-01-03 09:23:16 +0000291 __BUILD_BZERO LEGACY_MODE
Markos Chandrasfd9720e2014-01-03 10:11:45 +0000292
293#ifdef CONFIG_EVA
294LEAF(__bzero)
295 __BUILD_BZERO EVA_MODE
296END(__bzero)
297#endif