arch/cris/arch-v32/lib/usercopy.c - kernel/msm-4.9 - Gitiles

 /*
  * User address space access functions.
  * The non-inlined parts of asm-cris/uaccess.h are here.
  *
  * Copyright (C) 2000, 2003 Axis Communications AB.
  *
  * Written by Hans-Peter Nilsson.
  * Pieces used from memcpy, originally by Kenny Ranerup long time ago.
  */

 #include <asm/uaccess.h>

 /* Asm:s have been tweaked (within the domain of correctness) to give
    satisfactory results for "gcc version 3.2.1 Axis release R53/1.53-v32".

    Check regularly...

    Note that for CRISv32, the PC saved at a bus-fault is the address
    *at* the faulting instruction, with a special case for instructions
    in delay slots: then it's the address of the branch.  Note also that
    in contrast to v10, a postincrement in the instruction is *not*
    performed at a bus-fault; the register is seen having the original
    value in fault handlers.  */


 /* Copy to userspace.  This is based on the memcpy used for
    kernel-to-kernel copying; see "string.c".  */

 unsigned long
 __copy_user (void __user *pdst, const void *psrc, unsigned long pn)
 {
   /* We want the parameters put in special registers.
      Make sure the compiler is able to make something useful of this.
      As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).

      FIXME: Comment for old gcc version.  Check.
      If gcc was alright, it really would need no temporaries, and no
      stack space to save stuff on. */

   register char *dst __asm__ ("r13") = pdst;
   register const char *src __asm__ ("r11") = psrc;
   register int n __asm__ ("r12") = pn;
   register int retn __asm__ ("r10") = 0;


   /* When src is aligned but not dst, this makes a few extra needless
      cycles.  I believe it would take as many to check that the
      re-alignment was unnecessary.  */
   if (((unsigned long) dst & 3) != 0
       /* Don't align if we wouldn't copy more than a few bytes; so we
 	 don't have to check further for overflows.  */
       && n >= 3)
   {
     if ((unsigned long) dst & 1)
     {
       __asm_copy_to_user_1 (dst, src, retn);
       n--;
     }

     if ((unsigned long) dst & 2)
     {
       __asm_copy_to_user_2 (dst, src, retn);
       n -= 2;
     }
   }

   /* Movem is dirt cheap.  The overheap is low enough to always use the
      minimum possible block size as the threshold.  */
   if (n >= 44)
   {
     /* For large copies we use 'movem'.  */

     /* It is not optimal to tell the compiler about clobbering any
        registers; that will move the saving/restoring of those registers
        to the function prologue/epilogue, and make non-movem sizes
        suboptimal.  */
     __asm__ volatile ("\
         ;; Check that the register asm declaration got right.		\n\
         ;; The GCC manual explicitly says TRT will happen.		\n\
 	.ifnc %0%1%2%3,$r13$r11$r12$r10					\n\
 	.err								\n\
 	.endif								\n\
 									\n\
 	;; Save the registers we'll use in the movem process		\n\
 	;; on the stack.						\n\
 	subq	11*4,$sp						\n\
 	movem	$r10,[$sp]						\n\
 									\n\
 	;; Now we've got this:						\n\
 	;; r11 - src							\n\
 	;; r13 - dst							\n\
 	;; r12 - n							\n\
 									\n\
 	;; Update n for the first loop					\n\
 	subq	44,$r12							\n\
 0:									\n\
 	movem	[$r11+],$r10						\n\
 	subq   44,$r12							\n\
 1:	bge	0b							\n\
 	movem	$r10,[$r13+]						\n\
 3:									\n\
 	addq   44,$r12  ;; compensate for last loop underflowing n	\n\
 									\n\
 	;; Restore registers from stack					\n\
 	movem [$sp+],$r10						\n\
 2:									\n\
 	.section .fixup,\"ax\"						\n\
 4:									\n\
 ; When failing on any of the 1..44 bytes in a chunk, we adjust back the	\n\
 ; source pointer and just drop through	to the by-16 and by-4 loops to	\n\
 ; get the correct number of failing bytes.  This necessarily means a	\n\
 ; few extra exceptions, but invalid user pointers shouldn't happen in	\n\
 ; time-critical code anyway.						\n\
 	jump 3b								\n\
 	subq 44,$r11							\n\
 									\n\
 	.previous							\n\
 	.section __ex_table,\"a\"					\n\
 	.dword 1b,4b							\n\
 	.previous"

      /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
      /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn));

   }

   while (n >= 16)
   {
     __asm_copy_to_user_16 (dst, src, retn);
     n -= 16;
   }

   /* Having a separate by-four loops cuts down on cache footprint.
      FIXME:  Test with and without; increasing switch to be 0..15.  */
   while (n >= 4)
   {
     __asm_copy_to_user_4 (dst, src, retn);
     n -= 4;
   }

   switch (n)
   {
     case 0:
       break;
     case 1:
       __asm_copy_to_user_1 (dst, src, retn);
       break;
     case 2:
       __asm_copy_to_user_2 (dst, src, retn);
       break;
     case 3:
       __asm_copy_to_user_3 (dst, src, retn);
       break;
   }

   return retn;
 }

 /* Copy from user to kernel, zeroing the bytes that were inaccessible in
    userland.  The return-value is the number of bytes that were
    inaccessible.  */

 unsigned long
 __copy_user_zeroing (void __user *pdst, const void *psrc, unsigned long pn)
 {
   /* We want the parameters put in special registers.
      Make sure the compiler is able to make something useful of this.
      As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).

      FIXME: Comment for old gcc version.  Check.
      If gcc was alright, it really would need no temporaries, and no
      stack space to save stuff on.  */

   register char *dst __asm__ ("r13") = pdst;
   register const char *src __asm__ ("r11") = psrc;
   register int n __asm__ ("r12") = pn;
   register int retn __asm__ ("r10") = 0;

   /* The best reason to align src is that we then know that a read-fault
      was for aligned bytes; there's no 1..3 remaining good bytes to
      pickle.  */
   if (((unsigned long) src & 3) != 0)
   {
     if (((unsigned long) src & 1) && n != 0)
     {
       __asm_copy_from_user_1 (dst, src, retn);
       n--;
     }

     if (((unsigned long) src & 2) && n >= 2)
     {
       __asm_copy_from_user_2 (dst, src, retn);
       n -= 2;
     }

     /* We only need one check after the unalignment-adjustments, because
        if both adjustments were done, either both or neither reference
        had an exception.  */
     if (retn != 0)
       goto copy_exception_bytes;
   }

   /* Movem is dirt cheap.  The overheap is low enough to always use the
      minimum possible block size as the threshold.  */
   if (n >= 44)
   {
     /* It is not optimal to tell the compiler about clobbering any
        registers; that will move the saving/restoring of those registers
        to the function prologue/epilogue, and make non-movem sizes
        suboptimal.  */
     __asm__ volatile ("\
 	.ifnc %0%1%2%3,$r13$r11$r12$r10					\n\
 	.err								\n\
 	.endif								\n\
 									\n\
 	;; Save the registers we'll use in the movem process		\n\
 	;; on the stack.						\n\
 	subq	11*4,$sp						\n\
 	movem	$r10,[$sp]						\n\
 									\n\
 	;; Now we've got this:						\n\
 	;; r11 - src							\n\
 	;; r13 - dst							\n\
 	;; r12 - n							\n\
 									\n\
 	;; Update n for the first loop					\n\
 	subq	44,$r12							\n\
 0:									\n\
 	movem	[$r11+],$r10						\n\
 									\n\
 	subq   44,$r12							\n\
 	bge	0b							\n\
 	movem	$r10,[$r13+]						\n\
 									\n\
 4:									\n\
 	addq   44,$r12  ;; compensate for last loop underflowing n	\n\
 									\n\
 	;; Restore registers from stack					\n\
 	movem [$sp+],$r10						\n\
 	.section .fixup,\"ax\"						\n\
 									\n\
 ;; Do not jump back into the loop if we fail.  For some uses, we get a	\n\
 ;; page fault somewhere on the line.  Without checking for page limits,	\n\
 ;; we don't know where, but we need to copy accurately and keep an	\n\
 ;; accurate count; not just clear the whole line.  To do that, we fall	\n\
 ;; down in the code below, proceeding with smaller amounts.  It should	\n\
 ;; be kept in mind that we have to cater to code like what at one time	\n\
 ;; was in fs/super.c:							\n\
 ;;  i = size - copy_from_user((void *)page, data, size);		\n\
 ;; which would cause repeated faults while clearing the remainder of	\n\
 ;; the SIZE bytes at PAGE after the first fault.			\n\
 ;; A caveat here is that we must not fall through from a failing page	\n\
 ;; to a valid page.							\n\
 									\n\
 3:									\n\
 	jump	4b ;; Fall through, pretending the fault didn't happen.	\n\
 	nop								\n\
 									\n\
 	.previous							\n\
 	.section __ex_table,\"a\"					\n\
 	.dword 0b,3b							\n\
 	.previous"

      /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
      /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn));
   }

   /* Either we directly start copying here, using dword copying in a loop,
      or we copy as much as possible with 'movem' and then the last block
      (<44 bytes) is copied here.  This will work since 'movem' will have
      updated src, dst and n.  (Except with failing src.)

      Since we want to keep src accurate, we can't use
      __asm_copy_from_user_N with N != (1, 2, 4); it updates dst and
      retn, but not src (by design; it's value is ignored elsewhere).  */

   while (n >= 4)
   {
     __asm_copy_from_user_4 (dst, src, retn);
     n -= 4;

     if (retn)
       goto copy_exception_bytes;
   }

   /* If we get here, there were no memory read faults.  */
   switch (n)
   {
     /* These copies are at least "naturally aligned" (so we don't have
        to check each byte), due to the src alignment code before the
        movem loop.  The *_3 case *will* get the correct count for retn.  */
     case 0:
       /* This case deliberately left in (if you have doubts check the
 	 generated assembly code).  */
       break;
     case 1:
       __asm_copy_from_user_1 (dst, src, retn);
       break;
     case 2:
       __asm_copy_from_user_2 (dst, src, retn);
       break;
     case 3:
       __asm_copy_from_user_3 (dst, src, retn);
       break;
   }

   /* If we get here, retn correctly reflects the number of failing
      bytes.  */
   return retn;

 copy_exception_bytes:
   /* We already have "retn" bytes cleared, and need to clear the
      remaining "n" bytes.  A non-optimized simple byte-for-byte in-line
      memset is preferred here, since this isn't speed-critical code and
      we'd rather have this a leaf-function than calling memset.  */
   {
     char *endp;
     for (endp = dst + n; dst < endp; dst++)
       *dst = 0;
   }

   return retn + n;
 }

 /* Zero userspace.  */

 unsigned long
 __do_clear_user (void __user *pto, unsigned long pn)
 {
   /* We want the parameters put in special registers.
      Make sure the compiler is able to make something useful of this.
       As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).

      FIXME: Comment for old gcc version.  Check.
      If gcc was alright, it really would need no temporaries, and no
      stack space to save stuff on. */

   register char *dst __asm__ ("r13") = pto;
   register int n __asm__ ("r12") = pn;
   register int retn __asm__ ("r10") = 0;


   if (((unsigned long) dst & 3) != 0
      /* Don't align if we wouldn't copy more than a few bytes.  */
       && n >= 3)
   {
     if ((unsigned long) dst & 1)
     {
       __asm_clear_1 (dst, retn);
       n--;
     }

     if ((unsigned long) dst & 2)
     {
       __asm_clear_2 (dst, retn);
       n -= 2;
     }
   }

   /* Decide which copying method to use.
      FIXME: This number is from the "ordinary" kernel memset.  */
   if (n >= 48)
   {
     /* For large clears we use 'movem' */

     /* It is not optimal to tell the compiler about clobbering any
        call-saved registers; that will move the saving/restoring of
        those registers to the function prologue/epilogue, and make
        non-movem sizes suboptimal.

        This method is not foolproof; it assumes that the "asm reg"
        declarations at the beginning of the function really are used
        here (beware: they may be moved to temporary registers).
        This way, we do not have to save/move the registers around into
        temporaries; we can safely use them straight away.

       If you want to check that the allocation was right; then
       check the equalities in the first comment.  It should say
       something like "r13=r13, r11=r11, r12=r12". */
     __asm__ volatile ("\
 	.ifnc %0%1%2,$r13$r12$r10					\n\
 	.err								\n\
 	.endif								\n\
 									\n\
 	;; Save the registers we'll clobber in the movem process	\n\
 	;; on the stack.  Don't mention them to gcc, it will only be	\n\
 	;; upset.							\n\
 	subq	11*4,$sp						\n\
 	movem	$r10,[$sp]						\n\
 									\n\
 	clear.d $r0							\n\
 	clear.d $r1							\n\
 	clear.d $r2							\n\
 	clear.d $r3							\n\
 	clear.d $r4							\n\
 	clear.d $r5							\n\
 	clear.d $r6							\n\
 	clear.d $r7							\n\
 	clear.d $r8							\n\
 	clear.d $r9							\n\
 	clear.d $r10							\n\
 	clear.d $r11							\n\
 									\n\
 	;; Now we've got this:						\n\
 	;; r13 - dst							\n\
 	;; r12 - n							\n\
 									\n\
 	;; Update n for the first loop					\n\
 	subq	12*4,$r12						\n\
 0:									\n\
 	subq   12*4,$r12						\n\
 1:									\n\
 	bge	0b							\n\
 	movem	$r11,[$r13+]						\n\
 									\n\
 	addq   12*4,$r12 ;; compensate for last loop underflowing n	\n\
 									\n\
 	;; Restore registers from stack					\n\
 	movem [$sp+],$r10						\n\
 2:									\n\
 	.section .fixup,\"ax\"						\n\
 3:									\n\
 	movem [$sp],$r10						\n\
 	addq 12*4,$r10							\n\
 	addq 12*4,$r13							\n\
 	movem $r10,[$sp]						\n\
 	jump 0b								\n\
 	clear.d $r10							\n\
 									\n\
 	.previous							\n\
 	.section __ex_table,\"a\"					\n\
 	.dword 1b,3b							\n\
 	.previous"

      /* Outputs */ : "=r" (dst), "=r" (n), "=r" (retn)
      /* Inputs */ : "0" (dst), "1" (n), "2" (retn)
      /* Clobber */ : "r11");
   }

   while (n >= 16)
   {
     __asm_clear_16 (dst, retn);
     n -= 16;
   }

   /* Having a separate by-four loops cuts down on cache footprint.
      FIXME:  Test with and without; increasing switch to be 0..15.  */
   while (n >= 4)
   {
     __asm_clear_4 (dst, retn);
     n -= 4;
   }

   switch (n)
   {
     case 0:
       break;
     case 1:
       __asm_clear_1 (dst, retn);
       break;
     case 2:
       __asm_clear_2 (dst, retn);
       break;
     case 3:
       __asm_clear_3 (dst, retn);
       break;
   }

   return retn;
 }
	/*
	* User address space access functions.
	* The non-inlined parts of asm-cris/uaccess.h are here.
	*
	* Copyright (C) 2000, 2003 Axis Communications AB.
	*
	* Written by Hans-Peter Nilsson.
	* Pieces used from memcpy, originally by Kenny Ranerup long time ago.
	*/

	#include <asm/uaccess.h>

	/* Asm:s have been tweaked (within the domain of correctness) to give
	satisfactory results for "gcc version 3.2.1 Axis release R53/1.53-v32".

	Check regularly...

	Note that for CRISv32, the PC saved at a bus-fault is the address
	at the faulting instruction, with a special case for instructions
	in delay slots: then it's the address of the branch. Note also that
	in contrast to v10, a postincrement in the instruction is not
	performed at a bus-fault; the register is seen having the original
	value in fault handlers. */


	/* Copy to userspace. This is based on the memcpy used for
	kernel-to-kernel copying; see "string.c". */

	unsigned long
	__copy_user (void __user pdst, const void psrc, unsigned long pn)
	{
	/* We want the parameters put in special registers.
	Make sure the compiler is able to make something useful of this.
	As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).

	FIXME: Comment for old gcc version. Check.
	If gcc was alright, it really would need no temporaries, and no
	stack space to save stuff on. */

	register char *dst __asm__ ("r13") = pdst;
	register const char *src __asm__ ("r11") = psrc;
	register int n __asm__ ("r12") = pn;
	register int retn __asm__ ("r10") = 0;


	/* When src is aligned but not dst, this makes a few extra needless
	cycles. I believe it would take as many to check that the
	re-alignment was unnecessary. */
	if (((unsigned long) dst & 3) != 0
	/* Don't align if we wouldn't copy more than a few bytes; so we
	don't have to check further for overflows. */
	&& n >= 3)
	{
	if ((unsigned long) dst & 1)
	{
	__asm_copy_to_user_1 (dst, src, retn);
	n--;
	}

	if ((unsigned long) dst & 2)
	{
	__asm_copy_to_user_2 (dst, src, retn);
	n -= 2;
	}
	}

	/* Movem is dirt cheap. The overheap is low enough to always use the
	minimum possible block size as the threshold. */
	if (n >= 44)
	{
	/* For large copies we use 'movem'. */

	/* It is not optimal to tell the compiler about clobbering any
	registers; that will move the saving/restoring of those registers
	to the function prologue/epilogue, and make non-movem sizes
	suboptimal. */
	__asm__ volatile ("\
	;; Check that the register asm declaration got right. \n\
	;; The GCC manual explicitly says TRT will happen. \n\
	.ifnc %0%1%2%3,$r13$r11$r12$r10 \n\
	.err \n\
	.endif \n\
	\n\
	;; Save the registers we'll use in the movem process \n\
	;; on the stack. \n\
	subq 11*4,$sp \n\
	movem $r10,[$sp] \n\
	\n\
	;; Now we've got this: \n\
	;; r11 - src \n\
	;; r13 - dst \n\
	;; r12 - n \n\
	\n\
	;; Update n for the first loop \n\
	subq 44,$r12 \n\
	0: \n\
	movem [$r11+],$r10 \n\
	subq 44,$r12 \n\
	1: bge 0b \n\
	movem $r10,[$r13+] \n\
	3: \n\
	addq 44,$r12 ;; compensate for last loop underflowing n \n\
	\n\
	;; Restore registers from stack \n\
	movem [$sp+],$r10 \n\
	2: \n\
	.section .fixup,\"ax\" \n\
	4: \n\
	; When failing on any of the 1..44 bytes in a chunk, we adjust back the \n\
	; source pointer and just drop through to the by-16 and by-4 loops to \n\
	; get the correct number of failing bytes. This necessarily means a \n\
	; few extra exceptions, but invalid user pointers shouldn't happen in \n\
	; time-critical code anyway. \n\
	jump 3b \n\
	subq 44,$r11 \n\
	\n\
	.previous \n\
	.section __ex_table,\"a\" \n\
	.dword 1b,4b \n\
	.previous"

	/* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
	/* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn));

	}

	while (n >= 16)
	{
	__asm_copy_to_user_16 (dst, src, retn);
	n -= 16;
	}

	/* Having a separate by-four loops cuts down on cache footprint.
	FIXME: Test with and without; increasing switch to be 0..15. */
	while (n >= 4)
	{
	__asm_copy_to_user_4 (dst, src, retn);
	n -= 4;
	}

	switch (n)
	{
	case 0:
	break;
	case 1:
	__asm_copy_to_user_1 (dst, src, retn);
	break;
	case 2:
	__asm_copy_to_user_2 (dst, src, retn);
	break;
	case 3:
	__asm_copy_to_user_3 (dst, src, retn);
	break;
	}

	return retn;
	}

	/* Copy from user to kernel, zeroing the bytes that were inaccessible in
	userland. The return-value is the number of bytes that were
	inaccessible. */

	unsigned long
	__copy_user_zeroing (void __user pdst, const void psrc, unsigned long pn)
	{
	/* We want the parameters put in special registers.
	Make sure the compiler is able to make something useful of this.
	As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).

	FIXME: Comment for old gcc version. Check.
	If gcc was alright, it really would need no temporaries, and no
	stack space to save stuff on. */

	register char *dst __asm__ ("r13") = pdst;
	register const char *src __asm__ ("r11") = psrc;
	register int n __asm__ ("r12") = pn;
	register int retn __asm__ ("r10") = 0;

	/* The best reason to align src is that we then know that a read-fault
	was for aligned bytes; there's no 1..3 remaining good bytes to
	pickle. */
	if (((unsigned long) src & 3) != 0)
	{
	if (((unsigned long) src & 1) && n != 0)
	{
	__asm_copy_from_user_1 (dst, src, retn);
	n--;
	}

	if (((unsigned long) src & 2) && n >= 2)
	{
	__asm_copy_from_user_2 (dst, src, retn);
	n -= 2;
	}

	/* We only need one check after the unalignment-adjustments, because
	if both adjustments were done, either both or neither reference
	had an exception. */
	if (retn != 0)
	goto copy_exception_bytes;
	}

	/* Movem is dirt cheap. The overheap is low enough to always use the
	minimum possible block size as the threshold. */
	if (n >= 44)
	{
	/* It is not optimal to tell the compiler about clobbering any
	registers; that will move the saving/restoring of those registers
	to the function prologue/epilogue, and make non-movem sizes
	suboptimal. */
	__asm__ volatile ("\
	.ifnc %0%1%2%3,$r13$r11$r12$r10 \n\
	.err \n\
	.endif \n\
	\n\
	;; Save the registers we'll use in the movem process \n\
	;; on the stack. \n\
	subq 11*4,$sp \n\
	movem $r10,[$sp] \n\
	\n\
	;; Now we've got this: \n\
	;; r11 - src \n\
	;; r13 - dst \n\
	;; r12 - n \n\
	\n\
	;; Update n for the first loop \n\
	subq 44,$r12 \n\
	0: \n\
	movem [$r11+],$r10 \n\
	\n\
	subq 44,$r12 \n\
	bge 0b \n\
	movem $r10,[$r13+] \n\
	\n\
	4: \n\
	addq 44,$r12 ;; compensate for last loop underflowing n \n\
	\n\
	;; Restore registers from stack \n\
	movem [$sp+],$r10 \n\
	.section .fixup,\"ax\" \n\
	\n\
	;; Do not jump back into the loop if we fail. For some uses, we get a \n\
	;; page fault somewhere on the line. Without checking for page limits, \n\
	;; we don't know where, but we need to copy accurately and keep an \n\
	;; accurate count; not just clear the whole line. To do that, we fall \n\
	;; down in the code below, proceeding with smaller amounts. It should \n\
	;; be kept in mind that we have to cater to code like what at one time \n\
	;; was in fs/super.c: \n\
	;; i = size - copy_from_user((void *)page, data, size); \n\
	;; which would cause repeated faults while clearing the remainder of \n\
	;; the SIZE bytes at PAGE after the first fault. \n\
	;; A caveat here is that we must not fall through from a failing page \n\
	;; to a valid page. \n\
	\n\
	3: \n\
	jump 4b ;; Fall through, pretending the fault didn't happen. \n\
	nop \n\
	\n\
	.previous \n\
	.section __ex_table,\"a\" \n\
	.dword 0b,3b \n\
	.previous"

	/* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
	/* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn));
	}

	/* Either we directly start copying here, using dword copying in a loop,
	or we copy as much as possible with 'movem' and then the last block
	(<44 bytes) is copied here. This will work since 'movem' will have
	updated src, dst and n. (Except with failing src.)

	Since we want to keep src accurate, we can't use
	__asm_copy_from_user_N with N != (1, 2, 4); it updates dst and
	retn, but not src (by design; it's value is ignored elsewhere). */

	while (n >= 4)
	{
	__asm_copy_from_user_4 (dst, src, retn);
	n -= 4;

	if (retn)
	goto copy_exception_bytes;
	}

	/* If we get here, there were no memory read faults. */
	switch (n)
	{
	/* These copies are at least "naturally aligned" (so we don't have
	to check each byte), due to the src alignment code before the
	movem loop. The _3 case will* get the correct count for retn. */
	case 0:
	/* This case deliberately left in (if you have doubts check the
	generated assembly code). */
	break;
	case 1:
	__asm_copy_from_user_1 (dst, src, retn);
	break;
	case 2:
	__asm_copy_from_user_2 (dst, src, retn);
	break;
	case 3:
	__asm_copy_from_user_3 (dst, src, retn);
	break;
	}

	/* If we get here, retn correctly reflects the number of failing
	bytes. */
	return retn;

	copy_exception_bytes:
	/* We already have "retn" bytes cleared, and need to clear the
	remaining "n" bytes. A non-optimized simple byte-for-byte in-line
	memset is preferred here, since this isn't speed-critical code and
	we'd rather have this a leaf-function than calling memset. */
	{
	char *endp;
	for (endp = dst + n; dst < endp; dst++)
	*dst = 0;
	}

	return retn + n;
	}

	/* Zero userspace. */

	unsigned long
	__do_clear_user (void __user *pto, unsigned long pn)
	{
	/* We want the parameters put in special registers.
	Make sure the compiler is able to make something useful of this.
	As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).

	FIXME: Comment for old gcc version. Check.
	If gcc was alright, it really would need no temporaries, and no
	stack space to save stuff on. */

	register char *dst __asm__ ("r13") = pto;
	register int n __asm__ ("r12") = pn;
	register int retn __asm__ ("r10") = 0;


	if (((unsigned long) dst & 3) != 0
	/* Don't align if we wouldn't copy more than a few bytes. */
	&& n >= 3)
	{
	if ((unsigned long) dst & 1)
	{
	__asm_clear_1 (dst, retn);
	n--;
	}

	if ((unsigned long) dst & 2)
	{
	__asm_clear_2 (dst, retn);
	n -= 2;
	}
	}

	/* Decide which copying method to use.
	FIXME: This number is from the "ordinary" kernel memset. */
	if (n >= 48)
	{
	/* For large clears we use 'movem' */

	/* It is not optimal to tell the compiler about clobbering any
	call-saved registers; that will move the saving/restoring of
	those registers to the function prologue/epilogue, and make
	non-movem sizes suboptimal.

	This method is not foolproof; it assumes that the "asm reg"
	declarations at the beginning of the function really are used
	here (beware: they may be moved to temporary registers).
	This way, we do not have to save/move the registers around into
	temporaries; we can safely use them straight away.

	If you want to check that the allocation was right; then
	check the equalities in the first comment. It should say
	something like "r13=r13, r11=r11, r12=r12". */
	__asm__ volatile ("\
	.ifnc %0%1%2,$r13$r12$r10 \n\
	.err \n\
	.endif \n\
	\n\
	;; Save the registers we'll clobber in the movem process \n\
	;; on the stack. Don't mention them to gcc, it will only be \n\
	;; upset. \n\
	subq 11*4,$sp \n\
	movem $r10,[$sp] \n\
	\n\
	clear.d $r0 \n\
	clear.d $r1 \n\
	clear.d $r2 \n\
	clear.d $r3 \n\
	clear.d $r4 \n\
	clear.d $r5 \n\
	clear.d $r6 \n\
	clear.d $r7 \n\
	clear.d $r8 \n\
	clear.d $r9 \n\
	clear.d $r10 \n\
	clear.d $r11 \n\
	\n\
	;; Now we've got this: \n\
	;; r13 - dst \n\
	;; r12 - n \n\
	\n\
	;; Update n for the first loop \n\
	subq 12*4,$r12 \n\
	0: \n\
	subq 12*4,$r12 \n\
	1: \n\
	bge 0b \n\
	movem $r11,[$r13+] \n\
	\n\
	addq 12*4,$r12 ;; compensate for last loop underflowing n \n\
	\n\
	;; Restore registers from stack \n\
	movem [$sp+],$r10 \n\
	2: \n\
	.section .fixup,\"ax\" \n\
	3: \n\
	movem [$sp],$r10 \n\
	addq 12*4,$r10 \n\
	addq 12*4,$r13 \n\
	movem $r10,[$sp] \n\
	jump 0b \n\
	clear.d $r10 \n\
	\n\
	.previous \n\
	.section __ex_table,\"a\" \n\
	.dword 1b,3b \n\
	.previous"

	/* Outputs */ : "=r" (dst), "=r" (n), "=r" (retn)
	/* Inputs */ : "0" (dst), "1" (n), "2" (retn)
	/* Clobber */ : "r11");
	}

	while (n >= 16)
	{
	__asm_clear_16 (dst, retn);
	n -= 16;
	}

	/* Having a separate by-four loops cuts down on cache footprint.
	FIXME: Test with and without; increasing switch to be 0..15. */
	while (n >= 4)
	{
	__asm_clear_4 (dst, retn);
	n -= 4;
	}

	switch (n)
	{
	case 0:
	break;
	case 1:
	__asm_clear_1 (dst, retn);
	break;
	case 2:
	__asm_clear_2 (dst, retn);
	break;
	case 3:
	__asm_clear_3 (dst, retn);
	break;
	}

	return retn;
	}