blob: db24b215fc50668bcf3f9da0f6757016aab7956e [file] [log] [blame]
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001#ifndef _ASM_X86_UACCESS_64_H
2#define _ASM_X86_UACCESS_64_H
Linus Torvalds1da177e2005-04-16 15:20:36 -07003
4/*
5 * User space memory access functions
6 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <linux/compiler.h>
8#include <linux/errno.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/prefetch.h>
Nick Piggin16dbc6c2008-10-02 14:50:12 -070010#include <linux/lockdep.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <asm/page.h>
12
Linus Torvalds1da177e2005-04-16 15:20:36 -070013/*
14 * Copy To/From Userspace
15 */
16
17/* Handles exceptions in both to and from, but doesn't do access_ok */
Andi Kleen95912002006-09-26 10:52:39 +020018__must_check unsigned long
19copy_user_generic(void *to, const void *from, unsigned len);
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
Andi Kleen95912002006-09-26 10:52:39 +020021__must_check unsigned long
22copy_to_user(void __user *to, const void *from, unsigned len);
23__must_check unsigned long
24copy_from_user(void *to, const void __user *from, unsigned len);
25__must_check unsigned long
26copy_in_user(void __user *to, const void __user *from, unsigned len);
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Andi Kleen95912002006-09-26 10:52:39 +020028static __always_inline __must_check
29int __copy_from_user(void *dst, const void __user *src, unsigned size)
Joe Perchesb8963132008-03-23 01:03:49 -070030{
Andi Kleen383d0792006-09-26 10:52:40 +020031 int ret = 0;
Ingo Molnard1a76182008-10-28 16:54:49 +010032
33 might_fault();
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 if (!__builtin_constant_p(size))
Joe Perchesb8963132008-03-23 01:03:49 -070035 return copy_user_generic(dst, (__force void *)src, size);
36 switch (size) {
37 case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
38 ret, "b", "b", "=q", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -070039 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070040 case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
41 ret, "w", "w", "=r", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070043 case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
44 ret, "l", "k", "=r", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -070045 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070046 case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
47 ret, "q", "", "=r", 8);
48 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 case 10:
Joe Perchesb8963132008-03-23 01:03:49 -070050 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
Hiroshi Shimamoto20a4a232008-11-13 18:06:04 -080051 ret, "q", "", "=r", 10);
Joe Perchesb8963132008-03-23 01:03:49 -070052 if (unlikely(ret))
53 return ret;
54 __get_user_asm(*(u16 *)(8 + (char *)dst),
55 (u16 __user *)(8 + (char __user *)src),
56 ret, "w", "w", "=r", 2);
57 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 case 16:
Joe Perchesb8963132008-03-23 01:03:49 -070059 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
60 ret, "q", "", "=r", 16);
61 if (unlikely(ret))
62 return ret;
63 __get_user_asm(*(u64 *)(8 + (char *)dst),
64 (u64 __user *)(8 + (char __user *)src),
65 ret, "q", "", "=r", 8);
66 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 default:
Joe Perchesb8963132008-03-23 01:03:49 -070068 return copy_user_generic(dst, (__force void *)src, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 }
Joe Perchesb8963132008-03-23 01:03:49 -070070}
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Andi Kleen95912002006-09-26 10:52:39 +020072static __always_inline __must_check
73int __copy_to_user(void __user *dst, const void *src, unsigned size)
Joe Perchesb8963132008-03-23 01:03:49 -070074{
Andi Kleen383d0792006-09-26 10:52:40 +020075 int ret = 0;
Ingo Molnard1a76182008-10-28 16:54:49 +010076
77 might_fault();
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 if (!__builtin_constant_p(size))
Joe Perchesb8963132008-03-23 01:03:49 -070079 return copy_user_generic((__force void *)dst, src, size);
80 switch (size) {
81 case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
82 ret, "b", "b", "iq", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070084 case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
85 ret, "w", "w", "ir", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -070086 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070087 case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
88 ret, "l", "k", "ir", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 return ret;
Joe Perchesb8963132008-03-23 01:03:49 -070090 case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +020091 ret, "q", "", "er", 8);
Joe Perchesb8963132008-03-23 01:03:49 -070092 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 case 10:
Joe Perchesb8963132008-03-23 01:03:49 -070094 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +020095 ret, "q", "", "er", 10);
Joe Perchesb8963132008-03-23 01:03:49 -070096 if (unlikely(ret))
97 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 asm("":::"memory");
Joe Perchesb8963132008-03-23 01:03:49 -070099 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
100 ret, "w", "w", "ir", 2);
101 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 case 16:
Joe Perchesb8963132008-03-23 01:03:49 -0700103 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200104 ret, "q", "", "er", 16);
Joe Perchesb8963132008-03-23 01:03:49 -0700105 if (unlikely(ret))
106 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 asm("":::"memory");
Joe Perchesb8963132008-03-23 01:03:49 -0700108 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200109 ret, "q", "", "er", 8);
Joe Perchesb8963132008-03-23 01:03:49 -0700110 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 default:
Joe Perchesb8963132008-03-23 01:03:49 -0700112 return copy_user_generic((__force void *)dst, src, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 }
Joe Perchesb8963132008-03-23 01:03:49 -0700114}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
Andi Kleen95912002006-09-26 10:52:39 +0200116static __always_inline __must_check
117int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
Joe Perchesb8963132008-03-23 01:03:49 -0700118{
Andi Kleen383d0792006-09-26 10:52:40 +0200119 int ret = 0;
Ingo Molnard1a76182008-10-28 16:54:49 +0100120
121 might_fault();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 if (!__builtin_constant_p(size))
Joe Perchesb8963132008-03-23 01:03:49 -0700123 return copy_user_generic((__force void *)dst,
124 (__force void *)src, size);
125 switch (size) {
126 case 1: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 u8 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700128 __get_user_asm(tmp, (u8 __user *)src,
129 ret, "b", "b", "=q", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700131 __put_user_asm(tmp, (u8 __user *)dst,
132 ret, "b", "b", "iq", 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133 return ret;
134 }
Joe Perchesb8963132008-03-23 01:03:49 -0700135 case 2: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 u16 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700137 __get_user_asm(tmp, (u16 __user *)src,
138 ret, "w", "w", "=r", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700140 __put_user_asm(tmp, (u16 __user *)dst,
141 ret, "w", "w", "ir", 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142 return ret;
143 }
144
Joe Perchesb8963132008-03-23 01:03:49 -0700145 case 4: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 u32 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700147 __get_user_asm(tmp, (u32 __user *)src,
148 ret, "l", "k", "=r", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700150 __put_user_asm(tmp, (u32 __user *)dst,
151 ret, "l", "k", "ir", 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 return ret;
153 }
Joe Perchesb8963132008-03-23 01:03:49 -0700154 case 8: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 u64 tmp;
Joe Perchesb8963132008-03-23 01:03:49 -0700156 __get_user_asm(tmp, (u64 __user *)src,
157 ret, "q", "", "=r", 8);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 if (likely(!ret))
Joe Perchesb8963132008-03-23 01:03:49 -0700159 __put_user_asm(tmp, (u64 __user *)dst,
Uros Bizjak155b7352009-07-19 18:06:35 +0200160 ret, "q", "", "er", 8);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 return ret;
162 }
163 default:
Joe Perchesb8963132008-03-23 01:03:49 -0700164 return copy_user_generic((__force void *)dst,
165 (__force void *)src, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 }
Joe Perchesb8963132008-03-23 01:03:49 -0700167}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
Joe Perchesb8963132008-03-23 01:03:49 -0700169__must_check long
Andi Kleen95912002006-09-26 10:52:39 +0200170strncpy_from_user(char *dst, const char __user *src, long count);
Joe Perchesb8963132008-03-23 01:03:49 -0700171__must_check long
Andi Kleen95912002006-09-26 10:52:39 +0200172__strncpy_from_user(char *dst, const char __user *src, long count);
173__must_check long strnlen_user(const char __user *str, long n);
174__must_check long __strnlen_user(const char __user *str, long n);
175__must_check long strlen_user(const char __user *str);
176__must_check unsigned long clear_user(void __user *mem, unsigned long len);
177__must_check unsigned long __clear_user(void __user *mem, unsigned long len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Joe Perchesb8963132008-03-23 01:03:49 -0700179__must_check long __copy_from_user_inatomic(void *dst, const void __user *src,
180 unsigned size);
Andi Kleenb8858082006-09-30 01:47:55 +0200181
182static __must_check __always_inline int
183__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
184{
185 return copy_user_generic((__force void *)dst, src, size);
186}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187
Joe Perchesb8963132008-03-23 01:03:49 -0700188extern long __copy_user_nocache(void *dst, const void __user *src,
189 unsigned size, int zerorest);
Andi Kleen0812a572007-02-13 13:26:19 +0100190
Ingo Molnarf1800532009-03-02 11:00:57 +0100191static inline int
192__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
Andi Kleen0812a572007-02-13 13:26:19 +0100193{
194 might_sleep();
Ingo Molnarf1800532009-03-02 11:00:57 +0100195 return __copy_user_nocache(dst, src, size, 1);
Andi Kleen0812a572007-02-13 13:26:19 +0100196}
197
Ingo Molnarf1800532009-03-02 11:00:57 +0100198static inline int
199__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
200 unsigned size)
Andi Kleen0812a572007-02-13 13:26:19 +0100201{
Ingo Molnarf1800532009-03-02 11:00:57 +0100202 return __copy_user_nocache(dst, src, size, 0);
Andi Kleen0812a572007-02-13 13:26:19 +0100203}
204
Vitaly Mayatskikh11295852008-07-02 15:48:21 +0200205unsigned long
206copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest);
207
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700208#endif /* _ASM_X86_UACCESS_64_H */