Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 1 | /* binder_alloc.c |
| 2 | * |
| 3 | * Android IPC Subsystem |
| 4 | * |
| 5 | * Copyright (C) 2007-2017 Google, Inc. |
| 6 | * |
| 7 | * This software is licensed under the terms of the GNU General Public |
| 8 | * License version 2, as published by the Free Software Foundation, and |
| 9 | * may be copied, distributed, and modified under those terms. |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | */ |
| 17 | |
| 18 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 19 | |
| 20 | #include <asm/cacheflush.h> |
| 21 | #include <linux/list.h> |
| 22 | #include <linux/mm.h> |
| 23 | #include <linux/module.h> |
| 24 | #include <linux/rtmutex.h> |
| 25 | #include <linux/rbtree.h> |
| 26 | #include <linux/seq_file.h> |
| 27 | #include <linux/vmalloc.h> |
| 28 | #include <linux/slab.h> |
| 29 | #include <linux/sched.h> |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 30 | #include <linux/list_lru.h> |
Todd Kjos | d504949 | 2019-02-08 10:35:14 -0800 | [diff] [blame] | 31 | #include <linux/uaccess.h> |
| 32 | #include <linux/highmem.h> |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 33 | #include "binder_alloc.h" |
| 34 | #include "binder_trace.h" |
| 35 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 36 | struct list_lru binder_alloc_lru; |
| 37 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 38 | static DEFINE_MUTEX(binder_alloc_mmap_lock); |
| 39 | |
| 40 | enum { |
| 41 | BINDER_DEBUG_OPEN_CLOSE = 1U << 1, |
| 42 | BINDER_DEBUG_BUFFER_ALLOC = 1U << 2, |
| 43 | BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 3, |
| 44 | }; |
| 45 | static uint32_t binder_alloc_debug_mask; |
| 46 | |
| 47 | module_param_named(debug_mask, binder_alloc_debug_mask, |
| 48 | uint, 0644); |
| 49 | |
| 50 | #define binder_alloc_debug(mask, x...) \ |
| 51 | do { \ |
| 52 | if (binder_alloc_debug_mask & mask) \ |
| 53 | pr_info(x); \ |
| 54 | } while (0) |
| 55 | |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 56 | static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer) |
| 57 | { |
| 58 | return list_entry(buffer->entry.next, struct binder_buffer, entry); |
| 59 | } |
| 60 | |
| 61 | static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer) |
| 62 | { |
| 63 | return list_entry(buffer->entry.prev, struct binder_buffer, entry); |
| 64 | } |
| 65 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 66 | static size_t binder_alloc_buffer_size(struct binder_alloc *alloc, |
| 67 | struct binder_buffer *buffer) |
| 68 | { |
| 69 | if (list_is_last(&buffer->entry, &alloc->buffers)) |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 70 | return alloc->buffer + alloc->buffer_size - buffer->user_data; |
| 71 | return binder_buffer_next(buffer)->user_data - buffer->user_data; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 72 | } |
| 73 | |
| 74 | static void binder_insert_free_buffer(struct binder_alloc *alloc, |
| 75 | struct binder_buffer *new_buffer) |
| 76 | { |
| 77 | struct rb_node **p = &alloc->free_buffers.rb_node; |
| 78 | struct rb_node *parent = NULL; |
| 79 | struct binder_buffer *buffer; |
| 80 | size_t buffer_size; |
| 81 | size_t new_buffer_size; |
| 82 | |
| 83 | BUG_ON(!new_buffer->free); |
| 84 | |
| 85 | new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); |
| 86 | |
| 87 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 88 | "%d: add free buffer, size %zd, at %pK\n", |
| 89 | alloc->pid, new_buffer_size, new_buffer); |
| 90 | |
| 91 | while (*p) { |
| 92 | parent = *p; |
| 93 | buffer = rb_entry(parent, struct binder_buffer, rb_node); |
| 94 | BUG_ON(!buffer->free); |
| 95 | |
| 96 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 97 | |
| 98 | if (new_buffer_size < buffer_size) |
| 99 | p = &parent->rb_left; |
| 100 | else |
| 101 | p = &parent->rb_right; |
| 102 | } |
| 103 | rb_link_node(&new_buffer->rb_node, parent, p); |
| 104 | rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers); |
| 105 | } |
| 106 | |
| 107 | static void binder_insert_allocated_buffer_locked( |
| 108 | struct binder_alloc *alloc, struct binder_buffer *new_buffer) |
| 109 | { |
| 110 | struct rb_node **p = &alloc->allocated_buffers.rb_node; |
| 111 | struct rb_node *parent = NULL; |
| 112 | struct binder_buffer *buffer; |
| 113 | |
| 114 | BUG_ON(new_buffer->free); |
| 115 | |
| 116 | while (*p) { |
| 117 | parent = *p; |
| 118 | buffer = rb_entry(parent, struct binder_buffer, rb_node); |
| 119 | BUG_ON(buffer->free); |
| 120 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 121 | if (new_buffer->user_data < buffer->user_data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 122 | p = &parent->rb_left; |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 123 | else if (new_buffer->user_data > buffer->user_data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 124 | p = &parent->rb_right; |
| 125 | else |
| 126 | BUG(); |
| 127 | } |
| 128 | rb_link_node(&new_buffer->rb_node, parent, p); |
| 129 | rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers); |
| 130 | } |
| 131 | |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 132 | static struct binder_buffer *binder_alloc_prepare_to_free_locked( |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 133 | struct binder_alloc *alloc, |
| 134 | uintptr_t user_ptr) |
| 135 | { |
| 136 | struct rb_node *n = alloc->allocated_buffers.rb_node; |
| 137 | struct binder_buffer *buffer; |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 138 | void __user *uptr; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 139 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 140 | uptr = (void __user *)user_ptr; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 141 | |
| 142 | while (n) { |
| 143 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 144 | BUG_ON(buffer->free); |
| 145 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 146 | if (uptr < buffer->user_data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 147 | n = n->rb_left; |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 148 | else if (uptr > buffer->user_data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 149 | n = n->rb_right; |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 150 | else { |
| 151 | /* |
| 152 | * Guard against user threads attempting to |
Todd Kjos | d29b73e | 2018-11-06 15:55:32 -0800 | [diff] [blame] | 153 | * free the buffer when in use by kernel or |
| 154 | * after it's already been freed. |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 155 | */ |
Todd Kjos | d29b73e | 2018-11-06 15:55:32 -0800 | [diff] [blame] | 156 | if (!buffer->allow_user_free) |
| 157 | return ERR_PTR(-EPERM); |
| 158 | buffer->allow_user_free = 0; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 159 | return buffer; |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 160 | } |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 161 | } |
| 162 | return NULL; |
| 163 | } |
| 164 | |
| 165 | /** |
| 166 | * binder_alloc_buffer_lookup() - get buffer given user ptr |
| 167 | * @alloc: binder_alloc for this proc |
| 168 | * @user_ptr: User pointer to buffer data |
| 169 | * |
| 170 | * Validate userspace pointer to buffer data and return buffer corresponding to |
| 171 | * that user pointer. Search the rb tree for buffer that matches user data |
| 172 | * pointer. |
| 173 | * |
| 174 | * Return: Pointer to buffer or NULL |
| 175 | */ |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 176 | struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc, |
| 177 | uintptr_t user_ptr) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 178 | { |
| 179 | struct binder_buffer *buffer; |
| 180 | |
| 181 | mutex_lock(&alloc->mutex); |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 182 | buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 183 | mutex_unlock(&alloc->mutex); |
| 184 | return buffer; |
| 185 | } |
| 186 | |
| 187 | static int binder_update_page_range(struct binder_alloc *alloc, int allocate, |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 188 | void __user *start, void __user *end) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 189 | { |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 190 | void __user *page_addr; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 191 | unsigned long user_page_addr; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 192 | struct binder_lru_page *page; |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 193 | struct vm_area_struct *vma = NULL; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 194 | struct mm_struct *mm = NULL; |
| 195 | bool need_mm = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 196 | |
| 197 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 198 | "%d: %s pages %pK-%pK\n", alloc->pid, |
| 199 | allocate ? "allocate" : "free", start, end); |
| 200 | |
| 201 | if (end <= start) |
| 202 | return 0; |
| 203 | |
| 204 | trace_binder_update_page_range(alloc, allocate, start, end); |
| 205 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 206 | if (allocate == 0) |
| 207 | goto free_range; |
| 208 | |
| 209 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { |
| 210 | page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE]; |
| 211 | if (!page->page_ptr) { |
| 212 | need_mm = true; |
| 213 | break; |
| 214 | } |
| 215 | } |
| 216 | |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 217 | if (need_mm && mmget_not_zero(alloc->vma_vm_mm)) |
| 218 | mm = alloc->vma_vm_mm; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 219 | |
| 220 | if (mm) { |
Minchan Kim | 2cafd5b | 2018-05-07 23:15:37 +0900 | [diff] [blame] | 221 | down_read(&mm->mmap_sem); |
Todd Kjos | 288506e | 2019-08-12 10:11:52 -0700 | [diff] [blame] | 222 | if (!mmget_still_valid(mm)) { |
| 223 | if (allocate == 0) |
| 224 | goto free_range; |
| 225 | goto err_no_vma; |
| 226 | } |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 227 | vma = alloc->vma; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 228 | } |
| 229 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 230 | if (!vma && need_mm) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 231 | pr_err("%d: binder_alloc_buf failed to map pages in userspace, no vma\n", |
| 232 | alloc->pid); |
| 233 | goto err_no_vma; |
| 234 | } |
| 235 | |
| 236 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { |
| 237 | int ret; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 238 | bool on_lru; |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 239 | size_t index; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 240 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 241 | index = (page_addr - alloc->buffer) / PAGE_SIZE; |
| 242 | page = &alloc->pages[index]; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 243 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 244 | if (page->page_ptr) { |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 245 | trace_binder_alloc_lru_start(alloc, index); |
| 246 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 247 | on_lru = list_lru_del(&binder_alloc_lru, &page->lru); |
| 248 | WARN_ON(!on_lru); |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 249 | |
| 250 | trace_binder_alloc_lru_end(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 251 | continue; |
| 252 | } |
| 253 | |
| 254 | if (WARN_ON(!vma)) |
| 255 | goto err_page_ptr_cleared; |
| 256 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 257 | trace_binder_alloc_page_start(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 258 | page->page_ptr = alloc_page(GFP_KERNEL | |
| 259 | __GFP_HIGHMEM | |
| 260 | __GFP_ZERO); |
| 261 | if (!page->page_ptr) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 262 | pr_err("%d: binder_alloc_buf failed for page at %pK\n", |
| 263 | alloc->pid, page_addr); |
| 264 | goto err_alloc_page_failed; |
| 265 | } |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 266 | page->alloc = alloc; |
| 267 | INIT_LIST_HEAD(&page->lru); |
| 268 | |
Todd Kjos | 8d24e2a | 2019-02-08 10:35:19 -0800 | [diff] [blame] | 269 | user_page_addr = (uintptr_t)page_addr; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 270 | ret = vm_insert_page(vma, user_page_addr, page[0].page_ptr); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 271 | if (ret) { |
| 272 | pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n", |
| 273 | alloc->pid, user_page_addr); |
| 274 | goto err_vm_insert_page_failed; |
| 275 | } |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 276 | |
Martijn Coenen | c05ec29 | 2017-10-24 16:37:39 +0200 | [diff] [blame] | 277 | if (index + 1 > alloc->pages_high) |
| 278 | alloc->pages_high = index + 1; |
| 279 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 280 | trace_binder_alloc_page_end(alloc, index); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 281 | /* vm_insert_page does not seem to increment the refcount */ |
| 282 | } |
| 283 | if (mm) { |
Minchan Kim | 2cafd5b | 2018-05-07 23:15:37 +0900 | [diff] [blame] | 284 | up_read(&mm->mmap_sem); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 285 | mmput(mm); |
| 286 | } |
| 287 | return 0; |
| 288 | |
| 289 | free_range: |
| 290 | for (page_addr = end - PAGE_SIZE; page_addr >= start; |
| 291 | page_addr -= PAGE_SIZE) { |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 292 | bool ret; |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 293 | size_t index; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 294 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 295 | index = (page_addr - alloc->buffer) / PAGE_SIZE; |
| 296 | page = &alloc->pages[index]; |
| 297 | |
| 298 | trace_binder_free_lru_start(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 299 | |
| 300 | ret = list_lru_add(&binder_alloc_lru, &page->lru); |
| 301 | WARN_ON(!ret); |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 302 | |
| 303 | trace_binder_free_lru_end(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 304 | continue; |
| 305 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 306 | err_vm_insert_page_failed: |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 307 | __free_page(page->page_ptr); |
| 308 | page->page_ptr = NULL; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 309 | err_alloc_page_failed: |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 310 | err_page_ptr_cleared: |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 311 | ; |
| 312 | } |
| 313 | err_no_vma: |
| 314 | if (mm) { |
Minchan Kim | 2cafd5b | 2018-05-07 23:15:37 +0900 | [diff] [blame] | 315 | up_read(&mm->mmap_sem); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 316 | mmput(mm); |
| 317 | } |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 318 | return vma ? -ENOMEM : -ESRCH; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 319 | } |
| 320 | |
Xiongwei Song | 9e9a3e1 | 2017-12-14 12:15:42 +0800 | [diff] [blame] | 321 | static struct binder_buffer *binder_alloc_new_buf_locked( |
| 322 | struct binder_alloc *alloc, |
| 323 | size_t data_size, |
| 324 | size_t offsets_size, |
| 325 | size_t extra_buffers_size, |
| 326 | int is_async) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 327 | { |
| 328 | struct rb_node *n = alloc->free_buffers.rb_node; |
| 329 | struct binder_buffer *buffer; |
| 330 | size_t buffer_size; |
| 331 | struct rb_node *best_fit = NULL; |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 332 | void __user *has_page_addr; |
| 333 | void __user *end_page_addr; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 334 | size_t size, data_offsets_size; |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 335 | int ret; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 336 | |
| 337 | if (alloc->vma == NULL) { |
| 338 | pr_err("%d: binder_alloc_buf, no vma\n", |
| 339 | alloc->pid); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 340 | return ERR_PTR(-ESRCH); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 341 | } |
| 342 | |
| 343 | data_offsets_size = ALIGN(data_size, sizeof(void *)) + |
| 344 | ALIGN(offsets_size, sizeof(void *)); |
| 345 | |
| 346 | if (data_offsets_size < data_size || data_offsets_size < offsets_size) { |
| 347 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 348 | "%d: got transaction with invalid size %zd-%zd\n", |
| 349 | alloc->pid, data_size, offsets_size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 350 | return ERR_PTR(-EINVAL); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 351 | } |
| 352 | size = data_offsets_size + ALIGN(extra_buffers_size, sizeof(void *)); |
| 353 | if (size < data_offsets_size || size < extra_buffers_size) { |
| 354 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 355 | "%d: got transaction with invalid extra_buffers_size %zd\n", |
| 356 | alloc->pid, extra_buffers_size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 357 | return ERR_PTR(-EINVAL); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 358 | } |
| 359 | if (is_async && |
| 360 | alloc->free_async_space < size + sizeof(struct binder_buffer)) { |
| 361 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 362 | "%d: binder_alloc_buf size %zd failed, no async space left\n", |
| 363 | alloc->pid, size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 364 | return ERR_PTR(-ENOSPC); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 365 | } |
| 366 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 367 | /* Pad 0-size buffers so they get assigned unique addresses */ |
| 368 | size = max(size, sizeof(void *)); |
| 369 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 370 | while (n) { |
| 371 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 372 | BUG_ON(!buffer->free); |
| 373 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 374 | |
| 375 | if (size < buffer_size) { |
| 376 | best_fit = n; |
| 377 | n = n->rb_left; |
| 378 | } else if (size > buffer_size) |
| 379 | n = n->rb_right; |
| 380 | else { |
| 381 | best_fit = n; |
| 382 | break; |
| 383 | } |
| 384 | } |
| 385 | if (best_fit == NULL) { |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 386 | size_t allocated_buffers = 0; |
| 387 | size_t largest_alloc_size = 0; |
| 388 | size_t total_alloc_size = 0; |
| 389 | size_t free_buffers = 0; |
| 390 | size_t largest_free_size = 0; |
| 391 | size_t total_free_size = 0; |
| 392 | |
| 393 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; |
| 394 | n = rb_next(n)) { |
| 395 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 396 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 397 | allocated_buffers++; |
| 398 | total_alloc_size += buffer_size; |
| 399 | if (buffer_size > largest_alloc_size) |
| 400 | largest_alloc_size = buffer_size; |
| 401 | } |
| 402 | for (n = rb_first(&alloc->free_buffers); n != NULL; |
| 403 | n = rb_next(n)) { |
| 404 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 405 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 406 | free_buffers++; |
| 407 | total_free_size += buffer_size; |
| 408 | if (buffer_size > largest_free_size) |
| 409 | largest_free_size = buffer_size; |
| 410 | } |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 411 | pr_err("%d: binder_alloc_buf size %zd failed, no address space\n", |
| 412 | alloc->pid, size); |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 413 | pr_err("allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n", |
| 414 | total_alloc_size, allocated_buffers, largest_alloc_size, |
| 415 | total_free_size, free_buffers, largest_free_size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 416 | return ERR_PTR(-ENOSPC); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 417 | } |
| 418 | if (n == NULL) { |
| 419 | buffer = rb_entry(best_fit, struct binder_buffer, rb_node); |
| 420 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 421 | } |
| 422 | |
| 423 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 424 | "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n", |
| 425 | alloc->pid, size, buffer, buffer_size); |
| 426 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 427 | has_page_addr = (void __user *) |
| 428 | (((uintptr_t)buffer->user_data + buffer_size) & PAGE_MASK); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 429 | WARN_ON(n && buffer_size != size); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 430 | end_page_addr = |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 431 | (void __user *)PAGE_ALIGN((uintptr_t)buffer->user_data + size); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 432 | if (end_page_addr > has_page_addr) |
| 433 | end_page_addr = has_page_addr; |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 434 | ret = binder_update_page_range(alloc, 1, (void __user *) |
| 435 | PAGE_ALIGN((uintptr_t)buffer->user_data), end_page_addr); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 436 | if (ret) |
| 437 | return ERR_PTR(ret); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 438 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 439 | if (buffer_size != size) { |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 440 | struct binder_buffer *new_buffer; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 441 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 442 | new_buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); |
| 443 | if (!new_buffer) { |
| 444 | pr_err("%s: %d failed to alloc new buffer struct\n", |
| 445 | __func__, alloc->pid); |
| 446 | goto err_alloc_buf_struct_failed; |
| 447 | } |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 448 | new_buffer->user_data = (u8 __user *)buffer->user_data + size; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 449 | list_add(&new_buffer->entry, &buffer->entry); |
| 450 | new_buffer->free = 1; |
| 451 | binder_insert_free_buffer(alloc, new_buffer); |
| 452 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 453 | |
| 454 | rb_erase(best_fit, &alloc->free_buffers); |
| 455 | buffer->free = 0; |
Todd Kjos | d29b73e | 2018-11-06 15:55:32 -0800 | [diff] [blame] | 456 | buffer->allow_user_free = 0; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 457 | binder_insert_allocated_buffer_locked(alloc, buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 458 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 459 | "%d: binder_alloc_buf size %zd got %pK\n", |
| 460 | alloc->pid, size, buffer); |
| 461 | buffer->data_size = data_size; |
| 462 | buffer->offsets_size = offsets_size; |
| 463 | buffer->async_transaction = is_async; |
| 464 | buffer->extra_buffers_size = extra_buffers_size; |
| 465 | if (is_async) { |
| 466 | alloc->free_async_space -= size + sizeof(struct binder_buffer); |
| 467 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, |
| 468 | "%d: binder_alloc_buf size %zd async free %zd\n", |
| 469 | alloc->pid, size, alloc->free_async_space); |
| 470 | } |
| 471 | return buffer; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 472 | |
| 473 | err_alloc_buf_struct_failed: |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 474 | binder_update_page_range(alloc, 0, (void __user *) |
| 475 | PAGE_ALIGN((uintptr_t)buffer->user_data), |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 476 | end_page_addr); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 477 | return ERR_PTR(-ENOMEM); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 478 | } |
| 479 | |
| 480 | /** |
| 481 | * binder_alloc_new_buf() - Allocate a new binder buffer |
| 482 | * @alloc: binder_alloc for this proc |
| 483 | * @data_size: size of user data buffer |
| 484 | * @offsets_size: user specified buffer offset |
| 485 | * @extra_buffers_size: size of extra space for meta-data (eg, security context) |
| 486 | * @is_async: buffer for async transaction |
| 487 | * |
| 488 | * Allocate a new buffer given the requested sizes. Returns |
| 489 | * the kernel version of the buffer pointer. The size allocated |
| 490 | * is the sum of the three given sizes (each rounded up to |
| 491 | * pointer-sized boundary) |
| 492 | * |
| 493 | * Return: The allocated buffer or %NULL if error |
| 494 | */ |
| 495 | struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc, |
| 496 | size_t data_size, |
| 497 | size_t offsets_size, |
| 498 | size_t extra_buffers_size, |
| 499 | int is_async) |
| 500 | { |
| 501 | struct binder_buffer *buffer; |
| 502 | |
| 503 | mutex_lock(&alloc->mutex); |
| 504 | buffer = binder_alloc_new_buf_locked(alloc, data_size, offsets_size, |
| 505 | extra_buffers_size, is_async); |
| 506 | mutex_unlock(&alloc->mutex); |
| 507 | return buffer; |
| 508 | } |
| 509 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 510 | static void __user *buffer_start_page(struct binder_buffer *buffer) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 511 | { |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 512 | return (void __user *)((uintptr_t)buffer->user_data & PAGE_MASK); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 513 | } |
| 514 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 515 | static void __user *prev_buffer_end_page(struct binder_buffer *buffer) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 516 | { |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 517 | return (void __user *) |
| 518 | (((uintptr_t)(buffer->user_data) - 1) & PAGE_MASK); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 519 | } |
| 520 | |
| 521 | static void binder_delete_free_buffer(struct binder_alloc *alloc, |
| 522 | struct binder_buffer *buffer) |
| 523 | { |
| 524 | struct binder_buffer *prev, *next = NULL; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 525 | bool to_free = true; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 526 | BUG_ON(alloc->buffers.next == &buffer->entry); |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 527 | prev = binder_buffer_prev(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 528 | BUG_ON(!prev->free); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 529 | if (prev_buffer_end_page(prev) == buffer_start_page(buffer)) { |
| 530 | to_free = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 531 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 532 | "%d: merge free, buffer %pK share page with %pK\n", |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 533 | alloc->pid, buffer->user_data, |
| 534 | prev->user_data); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 535 | } |
| 536 | |
| 537 | if (!list_is_last(&buffer->entry, &alloc->buffers)) { |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 538 | next = binder_buffer_next(buffer); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 539 | if (buffer_start_page(next) == buffer_start_page(buffer)) { |
| 540 | to_free = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 541 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 542 | "%d: merge free, buffer %pK share page with %pK\n", |
| 543 | alloc->pid, |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 544 | buffer->user_data, |
| 545 | next->user_data); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 546 | } |
| 547 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 548 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 549 | if (PAGE_ALIGNED(buffer->user_data)) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 550 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 551 | "%d: merge free, buffer start %pK is page aligned\n", |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 552 | alloc->pid, buffer->user_data); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 553 | to_free = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 554 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 555 | |
| 556 | if (to_free) { |
| 557 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 558 | "%d: merge free, buffer %pK do not share page with %pK or %pK\n", |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 559 | alloc->pid, buffer->user_data, |
| 560 | prev->user_data, |
| 561 | next ? next->user_data : NULL); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 562 | binder_update_page_range(alloc, 0, buffer_start_page(buffer), |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 563 | buffer_start_page(buffer) + PAGE_SIZE); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 564 | } |
| 565 | list_del(&buffer->entry); |
| 566 | kfree(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 567 | } |
| 568 | |
| 569 | static void binder_free_buf_locked(struct binder_alloc *alloc, |
| 570 | struct binder_buffer *buffer) |
| 571 | { |
| 572 | size_t size, buffer_size; |
| 573 | |
| 574 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 575 | |
| 576 | size = ALIGN(buffer->data_size, sizeof(void *)) + |
| 577 | ALIGN(buffer->offsets_size, sizeof(void *)) + |
| 578 | ALIGN(buffer->extra_buffers_size, sizeof(void *)); |
| 579 | |
| 580 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 581 | "%d: binder_free_buf %pK size %zd buffer_size %zd\n", |
| 582 | alloc->pid, buffer, size, buffer_size); |
| 583 | |
| 584 | BUG_ON(buffer->free); |
| 585 | BUG_ON(size > buffer_size); |
| 586 | BUG_ON(buffer->transaction != NULL); |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 587 | BUG_ON(buffer->user_data < alloc->buffer); |
| 588 | BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 589 | |
| 590 | if (buffer->async_transaction) { |
| 591 | alloc->free_async_space += size + sizeof(struct binder_buffer); |
| 592 | |
| 593 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, |
| 594 | "%d: binder_free_buf size %zd async free %zd\n", |
| 595 | alloc->pid, size, alloc->free_async_space); |
| 596 | } |
| 597 | |
| 598 | binder_update_page_range(alloc, 0, |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 599 | (void __user *)PAGE_ALIGN((uintptr_t)buffer->user_data), |
| 600 | (void __user *)(((uintptr_t) |
| 601 | buffer->user_data + buffer_size) & PAGE_MASK)); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 602 | |
| 603 | rb_erase(&buffer->rb_node, &alloc->allocated_buffers); |
| 604 | buffer->free = 1; |
| 605 | if (!list_is_last(&buffer->entry, &alloc->buffers)) { |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 606 | struct binder_buffer *next = binder_buffer_next(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 607 | |
| 608 | if (next->free) { |
| 609 | rb_erase(&next->rb_node, &alloc->free_buffers); |
| 610 | binder_delete_free_buffer(alloc, next); |
| 611 | } |
| 612 | } |
| 613 | if (alloc->buffers.next != &buffer->entry) { |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 614 | struct binder_buffer *prev = binder_buffer_prev(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 615 | |
| 616 | if (prev->free) { |
| 617 | binder_delete_free_buffer(alloc, buffer); |
| 618 | rb_erase(&prev->rb_node, &alloc->free_buffers); |
| 619 | buffer = prev; |
| 620 | } |
| 621 | } |
| 622 | binder_insert_free_buffer(alloc, buffer); |
| 623 | } |
| 624 | |
| 625 | /** |
| 626 | * binder_alloc_free_buf() - free a binder buffer |
| 627 | * @alloc: binder_alloc for this proc |
| 628 | * @buffer: kernel pointer to buffer |
| 629 | * |
| 630 | * Free the buffer allocated via binder_alloc_new_buffer() |
| 631 | */ |
| 632 | void binder_alloc_free_buf(struct binder_alloc *alloc, |
| 633 | struct binder_buffer *buffer) |
| 634 | { |
| 635 | mutex_lock(&alloc->mutex); |
| 636 | binder_free_buf_locked(alloc, buffer); |
| 637 | mutex_unlock(&alloc->mutex); |
| 638 | } |
| 639 | |
| 640 | /** |
| 641 | * binder_alloc_mmap_handler() - map virtual address space for proc |
| 642 | * @alloc: alloc structure for this proc |
| 643 | * @vma: vma passed to mmap() |
| 644 | * |
| 645 | * Called by binder_mmap() to initialize the space specified in |
| 646 | * vma for allocating binder buffers |
| 647 | * |
| 648 | * Return: |
| 649 | * 0 = success |
| 650 | * -EBUSY = address space already mapped |
| 651 | * -ENOMEM = failed to map memory to given address space |
| 652 | */ |
| 653 | int binder_alloc_mmap_handler(struct binder_alloc *alloc, |
| 654 | struct vm_area_struct *vma) |
| 655 | { |
| 656 | int ret; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 657 | const char *failure_string; |
| 658 | struct binder_buffer *buffer; |
| 659 | |
| 660 | mutex_lock(&binder_alloc_mmap_lock); |
| 661 | if (alloc->buffer) { |
| 662 | ret = -EBUSY; |
| 663 | failure_string = "already mapped"; |
| 664 | goto err_already_mapped; |
| 665 | } |
| 666 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 667 | alloc->buffer = (void __user *)vma->vm_start; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 668 | mutex_unlock(&binder_alloc_mmap_lock); |
| 669 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 670 | alloc->pages = kzalloc(sizeof(alloc->pages[0]) * |
| 671 | ((vma->vm_end - vma->vm_start) / PAGE_SIZE), |
| 672 | GFP_KERNEL); |
| 673 | if (alloc->pages == NULL) { |
| 674 | ret = -ENOMEM; |
| 675 | failure_string = "alloc page array"; |
| 676 | goto err_alloc_pages_failed; |
| 677 | } |
| 678 | alloc->buffer_size = vma->vm_end - vma->vm_start; |
| 679 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 680 | buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); |
| 681 | if (!buffer) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 682 | ret = -ENOMEM; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 683 | failure_string = "alloc buffer struct"; |
| 684 | goto err_alloc_buf_struct_failed; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 685 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 686 | |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 687 | buffer->user_data = alloc->buffer; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 688 | list_add(&buffer->entry, &alloc->buffers); |
| 689 | buffer->free = 1; |
| 690 | binder_insert_free_buffer(alloc, buffer); |
| 691 | alloc->free_async_space = alloc->buffer_size / 2; |
| 692 | barrier(); |
| 693 | alloc->vma = vma; |
| 694 | alloc->vma_vm_mm = vma->vm_mm; |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 695 | /* Same as mmgrab() in later kernel versions */ |
| 696 | atomic_inc(&alloc->vma_vm_mm->mm_count); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 697 | |
| 698 | return 0; |
| 699 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 700 | err_alloc_buf_struct_failed: |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 701 | kfree(alloc->pages); |
| 702 | alloc->pages = NULL; |
| 703 | err_alloc_pages_failed: |
| 704 | mutex_lock(&binder_alloc_mmap_lock); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 705 | alloc->buffer = NULL; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 706 | err_already_mapped: |
| 707 | mutex_unlock(&binder_alloc_mmap_lock); |
| 708 | pr_err("%s: %d %lx-%lx %s failed %d\n", __func__, |
| 709 | alloc->pid, vma->vm_start, vma->vm_end, failure_string, ret); |
| 710 | return ret; |
| 711 | } |
| 712 | |
| 713 | |
| 714 | void binder_alloc_deferred_release(struct binder_alloc *alloc) |
| 715 | { |
| 716 | struct rb_node *n; |
| 717 | int buffers, page_count; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 718 | struct binder_buffer *buffer; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 719 | |
| 720 | BUG_ON(alloc->vma); |
| 721 | |
| 722 | buffers = 0; |
| 723 | mutex_lock(&alloc->mutex); |
| 724 | while ((n = rb_first(&alloc->allocated_buffers))) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 725 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 726 | |
| 727 | /* Transaction should already have been freed */ |
| 728 | BUG_ON(buffer->transaction); |
| 729 | |
| 730 | binder_free_buf_locked(alloc, buffer); |
| 731 | buffers++; |
| 732 | } |
| 733 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 734 | while (!list_empty(&alloc->buffers)) { |
| 735 | buffer = list_first_entry(&alloc->buffers, |
| 736 | struct binder_buffer, entry); |
| 737 | WARN_ON(!buffer->free); |
| 738 | |
| 739 | list_del(&buffer->entry); |
| 740 | WARN_ON_ONCE(!list_empty(&alloc->buffers)); |
| 741 | kfree(buffer); |
| 742 | } |
| 743 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 744 | page_count = 0; |
| 745 | if (alloc->pages) { |
| 746 | int i; |
| 747 | |
| 748 | for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 749 | void __user *page_addr; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 750 | bool on_lru; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 751 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 752 | if (!alloc->pages[i].page_ptr) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 753 | continue; |
| 754 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 755 | on_lru = list_lru_del(&binder_alloc_lru, |
| 756 | &alloc->pages[i].lru); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 757 | page_addr = alloc->buffer + i * PAGE_SIZE; |
| 758 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 759 | "%s: %d: page %d at %pK %s\n", |
| 760 | __func__, alloc->pid, i, page_addr, |
| 761 | on_lru ? "on lru" : "active"); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 762 | __free_page(alloc->pages[i].page_ptr); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 763 | page_count++; |
| 764 | } |
| 765 | kfree(alloc->pages); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 766 | } |
| 767 | mutex_unlock(&alloc->mutex); |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 768 | if (alloc->vma_vm_mm) |
| 769 | mmdrop(alloc->vma_vm_mm); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 770 | |
| 771 | binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE, |
| 772 | "%s: %d buffers %d, pages %d\n", |
| 773 | __func__, alloc->pid, buffers, page_count); |
| 774 | } |
| 775 | |
| 776 | static void print_binder_buffer(struct seq_file *m, const char *prefix, |
| 777 | struct binder_buffer *buffer) |
| 778 | { |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 779 | seq_printf(m, "%s %d: %pK size %zd:%zd:%zd %s\n", |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 780 | prefix, buffer->debug_id, buffer->user_data, |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 781 | buffer->data_size, buffer->offsets_size, |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 782 | buffer->extra_buffers_size, |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 783 | buffer->transaction ? "active" : "delivered"); |
| 784 | } |
| 785 | |
| 786 | /** |
| 787 | * binder_alloc_print_allocated() - print buffer info |
| 788 | * @m: seq_file for output via seq_printf() |
| 789 | * @alloc: binder_alloc for this proc |
| 790 | * |
| 791 | * Prints information about every buffer associated with |
| 792 | * the binder_alloc state to the given seq_file |
| 793 | */ |
| 794 | void binder_alloc_print_allocated(struct seq_file *m, |
| 795 | struct binder_alloc *alloc) |
| 796 | { |
| 797 | struct rb_node *n; |
| 798 | |
| 799 | mutex_lock(&alloc->mutex); |
| 800 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) |
| 801 | print_binder_buffer(m, " buffer", |
| 802 | rb_entry(n, struct binder_buffer, rb_node)); |
| 803 | mutex_unlock(&alloc->mutex); |
| 804 | } |
| 805 | |
| 806 | /** |
Sherry Yang | 9100442 | 2017-08-22 17:26:57 -0700 | [diff] [blame] | 807 | * binder_alloc_print_pages() - print page usage |
| 808 | * @m: seq_file for output via seq_printf() |
| 809 | * @alloc: binder_alloc for this proc |
| 810 | */ |
| 811 | void binder_alloc_print_pages(struct seq_file *m, |
| 812 | struct binder_alloc *alloc) |
| 813 | { |
| 814 | struct binder_lru_page *page; |
| 815 | int i; |
| 816 | int active = 0; |
| 817 | int lru = 0; |
| 818 | int free = 0; |
| 819 | |
| 820 | mutex_lock(&alloc->mutex); |
| 821 | for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { |
| 822 | page = &alloc->pages[i]; |
| 823 | if (!page->page_ptr) |
| 824 | free++; |
| 825 | else if (list_empty(&page->lru)) |
| 826 | active++; |
| 827 | else |
| 828 | lru++; |
| 829 | } |
| 830 | mutex_unlock(&alloc->mutex); |
| 831 | seq_printf(m, " pages: %d:%d:%d\n", active, lru, free); |
Martijn Coenen | c05ec29 | 2017-10-24 16:37:39 +0200 | [diff] [blame] | 832 | seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high); |
Sherry Yang | 9100442 | 2017-08-22 17:26:57 -0700 | [diff] [blame] | 833 | } |
| 834 | |
| 835 | /** |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 836 | * binder_alloc_get_allocated_count() - return count of buffers |
| 837 | * @alloc: binder_alloc for this proc |
| 838 | * |
| 839 | * Return: count of allocated buffers |
| 840 | */ |
| 841 | int binder_alloc_get_allocated_count(struct binder_alloc *alloc) |
| 842 | { |
| 843 | struct rb_node *n; |
| 844 | int count = 0; |
| 845 | |
| 846 | mutex_lock(&alloc->mutex); |
| 847 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) |
| 848 | count++; |
| 849 | mutex_unlock(&alloc->mutex); |
| 850 | return count; |
| 851 | } |
| 852 | |
| 853 | |
| 854 | /** |
| 855 | * binder_alloc_vma_close() - invalidate address space |
| 856 | * @alloc: binder_alloc for this proc |
| 857 | * |
| 858 | * Called from binder_vma_close() when releasing address space. |
| 859 | * Clears alloc->vma to prevent new incoming transactions from |
| 860 | * allocating more buffers. |
| 861 | */ |
| 862 | void binder_alloc_vma_close(struct binder_alloc *alloc) |
| 863 | { |
| 864 | WRITE_ONCE(alloc->vma, NULL); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 865 | } |
| 866 | |
| 867 | /** |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 868 | * binder_alloc_free_page() - shrinker callback to free pages |
| 869 | * @item: item to free |
| 870 | * @lock: lock protecting the item |
| 871 | * @cb_arg: callback argument |
| 872 | * |
| 873 | * Called from list_lru_walk() in binder_shrink_scan() to free |
| 874 | * up pages when the system is under memory pressure. |
| 875 | */ |
| 876 | enum lru_status binder_alloc_free_page(struct list_head *item, |
| 877 | struct list_lru_one *lru, |
| 878 | spinlock_t *lock, |
| 879 | void *cb_arg) |
| 880 | { |
| 881 | struct mm_struct *mm = NULL; |
| 882 | struct binder_lru_page *page = container_of(item, |
| 883 | struct binder_lru_page, |
| 884 | lru); |
| 885 | struct binder_alloc *alloc; |
| 886 | uintptr_t page_addr; |
| 887 | size_t index; |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 888 | struct vm_area_struct *vma; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 889 | |
| 890 | alloc = page->alloc; |
| 891 | if (!mutex_trylock(&alloc->mutex)) |
| 892 | goto err_get_alloc_mutex_failed; |
| 893 | |
| 894 | if (!page->page_ptr) |
| 895 | goto err_page_already_freed; |
| 896 | |
| 897 | index = page - alloc->pages; |
| 898 | page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE; |
Todd Kjos | e53fcdd | 2019-06-05 09:37:46 -0700 | [diff] [blame] | 899 | |
| 900 | mm = alloc->vma_vm_mm; |
| 901 | if (!mmget_not_zero(mm)) |
| 902 | goto err_mmget; |
| 903 | if (!down_write_trylock(&mm->mmap_sem)) |
| 904 | goto err_down_write_mmap_sem_failed; |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 905 | vma = alloc->vma; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 906 | |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 907 | list_lru_isolate(lru, item); |
| 908 | spin_unlock(lock); |
| 909 | |
| 910 | if (vma) { |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 911 | trace_binder_unmap_user_start(alloc, index); |
| 912 | |
Todd Kjos | 8d24e2a | 2019-02-08 10:35:19 -0800 | [diff] [blame] | 913 | zap_page_range(vma, page_addr, PAGE_SIZE, NULL); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 914 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 915 | trace_binder_unmap_user_end(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 916 | } |
Todd Kjos | e53fcdd | 2019-06-05 09:37:46 -0700 | [diff] [blame] | 917 | up_write(&mm->mmap_sem); |
| 918 | mmput(mm); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 919 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 920 | trace_binder_unmap_kernel_start(alloc, index); |
| 921 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 922 | __free_page(page->page_ptr); |
| 923 | page->page_ptr = NULL; |
| 924 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 925 | trace_binder_unmap_kernel_end(alloc, index); |
| 926 | |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 927 | spin_lock(lock); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 928 | mutex_unlock(&alloc->mutex); |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 929 | return LRU_REMOVED_RETRY; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 930 | |
| 931 | err_down_write_mmap_sem_failed: |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 932 | mmput_async(mm); |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 933 | err_mmget: |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 934 | err_page_already_freed: |
| 935 | mutex_unlock(&alloc->mutex); |
| 936 | err_get_alloc_mutex_failed: |
| 937 | return LRU_SKIP; |
| 938 | } |
| 939 | |
| 940 | static unsigned long |
| 941 | binder_shrink_count(struct shrinker *shrink, struct shrink_control *sc) |
| 942 | { |
| 943 | unsigned long ret = list_lru_count(&binder_alloc_lru); |
| 944 | return ret; |
| 945 | } |
| 946 | |
| 947 | static unsigned long |
| 948 | binder_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) |
| 949 | { |
| 950 | unsigned long ret; |
| 951 | |
| 952 | ret = list_lru_walk(&binder_alloc_lru, binder_alloc_free_page, |
| 953 | NULL, sc->nr_to_scan); |
| 954 | return ret; |
| 955 | } |
| 956 | |
Sherry Yang | 0b33853 | 2017-10-05 17:00:57 -0400 | [diff] [blame] | 957 | static struct shrinker binder_shrinker = { |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 958 | .count_objects = binder_shrink_count, |
| 959 | .scan_objects = binder_shrink_scan, |
| 960 | .seeks = DEFAULT_SEEKS, |
| 961 | }; |
| 962 | |
| 963 | /** |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 964 | * binder_alloc_init() - called by binder_open() for per-proc initialization |
| 965 | * @alloc: binder_alloc for this proc |
| 966 | * |
| 967 | * Called from binder_open() to initialize binder_alloc fields for |
| 968 | * new binder proc |
| 969 | */ |
| 970 | void binder_alloc_init(struct binder_alloc *alloc) |
| 971 | { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 972 | alloc->pid = current->group_leader->pid; |
| 973 | mutex_init(&alloc->mutex); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 974 | INIT_LIST_HEAD(&alloc->buffers); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 975 | } |
| 976 | |
Tetsuo Handa | f8cb822 | 2017-11-29 22:29:47 +0900 | [diff] [blame] | 977 | int binder_alloc_shrinker_init(void) |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 978 | { |
Tetsuo Handa | f8cb822 | 2017-11-29 22:29:47 +0900 | [diff] [blame] | 979 | int ret = list_lru_init(&binder_alloc_lru); |
| 980 | |
| 981 | if (ret == 0) { |
| 982 | ret = register_shrinker(&binder_shrinker); |
| 983 | if (ret) |
| 984 | list_lru_destroy(&binder_alloc_lru); |
| 985 | } |
| 986 | return ret; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 987 | } |
Todd Kjos | d504949 | 2019-02-08 10:35:14 -0800 | [diff] [blame] | 988 | |
| 989 | /** |
| 990 | * check_buffer() - verify that buffer/offset is safe to access |
| 991 | * @alloc: binder_alloc for this proc |
| 992 | * @buffer: binder buffer to be accessed |
| 993 | * @offset: offset into @buffer data |
| 994 | * @bytes: bytes to access from offset |
| 995 | * |
| 996 | * Check that the @offset/@bytes are within the size of the given |
| 997 | * @buffer and that the buffer is currently active and not freeable. |
| 998 | * Offsets must also be multiples of sizeof(u32). The kernel is |
| 999 | * allowed to touch the buffer in two cases: |
| 1000 | * |
| 1001 | * 1) when the buffer is being created: |
| 1002 | * (buffer->free == 0 && buffer->allow_user_free == 0) |
| 1003 | * 2) when the buffer is being torn down: |
| 1004 | * (buffer->free == 0 && buffer->transaction == NULL). |
| 1005 | * |
| 1006 | * Return: true if the buffer is safe to access |
| 1007 | */ |
| 1008 | static inline bool check_buffer(struct binder_alloc *alloc, |
| 1009 | struct binder_buffer *buffer, |
| 1010 | binder_size_t offset, size_t bytes) |
| 1011 | { |
| 1012 | size_t buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 1013 | |
| 1014 | return buffer_size >= bytes && |
| 1015 | offset <= buffer_size - bytes && |
| 1016 | IS_ALIGNED(offset, sizeof(u32)) && |
| 1017 | !buffer->free && |
| 1018 | (!buffer->allow_user_free || !buffer->transaction); |
| 1019 | } |
| 1020 | |
| 1021 | /** |
| 1022 | * binder_alloc_get_page() - get kernel pointer for given buffer offset |
| 1023 | * @alloc: binder_alloc for this proc |
| 1024 | * @buffer: binder buffer to be accessed |
| 1025 | * @buffer_offset: offset into @buffer data |
| 1026 | * @pgoffp: address to copy final page offset to |
| 1027 | * |
| 1028 | * Lookup the struct page corresponding to the address |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 1029 | * at @buffer_offset into @buffer->user_data. If @pgoffp is not |
Todd Kjos | d504949 | 2019-02-08 10:35:14 -0800 | [diff] [blame] | 1030 | * NULL, the byte-offset into the page is written there. |
| 1031 | * |
| 1032 | * The caller is responsible to ensure that the offset points |
| 1033 | * to a valid address within the @buffer and that @buffer is |
| 1034 | * not freeable by the user. Since it can't be freed, we are |
| 1035 | * guaranteed that the corresponding elements of @alloc->pages[] |
| 1036 | * cannot change. |
| 1037 | * |
| 1038 | * Return: struct page |
| 1039 | */ |
| 1040 | static struct page *binder_alloc_get_page(struct binder_alloc *alloc, |
| 1041 | struct binder_buffer *buffer, |
| 1042 | binder_size_t buffer_offset, |
| 1043 | pgoff_t *pgoffp) |
| 1044 | { |
| 1045 | binder_size_t buffer_space_offset = buffer_offset + |
Todd Kjos | 8539b1e | 2019-02-08 10:35:20 -0800 | [diff] [blame] | 1046 | (buffer->user_data - alloc->buffer); |
Todd Kjos | d504949 | 2019-02-08 10:35:14 -0800 | [diff] [blame] | 1047 | pgoff_t pgoff = buffer_space_offset & ~PAGE_MASK; |
| 1048 | size_t index = buffer_space_offset >> PAGE_SHIFT; |
| 1049 | struct binder_lru_page *lru_page; |
| 1050 | |
| 1051 | lru_page = &alloc->pages[index]; |
| 1052 | *pgoffp = pgoff; |
| 1053 | return lru_page->page_ptr; |
| 1054 | } |
| 1055 | |
| 1056 | /** |
| 1057 | * binder_alloc_copy_user_to_buffer() - copy src user to tgt user |
| 1058 | * @alloc: binder_alloc for this proc |
| 1059 | * @buffer: binder buffer to be accessed |
| 1060 | * @buffer_offset: offset into @buffer data |
| 1061 | * @from: userspace pointer to source buffer |
| 1062 | * @bytes: bytes to copy |
| 1063 | * |
| 1064 | * Copy bytes from source userspace to target buffer. |
| 1065 | * |
| 1066 | * Return: bytes remaining to be copied |
| 1067 | */ |
| 1068 | unsigned long |
| 1069 | binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc, |
| 1070 | struct binder_buffer *buffer, |
| 1071 | binder_size_t buffer_offset, |
| 1072 | const void __user *from, |
| 1073 | size_t bytes) |
| 1074 | { |
| 1075 | if (!check_buffer(alloc, buffer, buffer_offset, bytes)) |
| 1076 | return bytes; |
| 1077 | |
| 1078 | while (bytes) { |
| 1079 | unsigned long size; |
| 1080 | unsigned long ret; |
| 1081 | struct page *page; |
| 1082 | pgoff_t pgoff; |
| 1083 | void *kptr; |
| 1084 | |
| 1085 | page = binder_alloc_get_page(alloc, buffer, |
| 1086 | buffer_offset, &pgoff); |
| 1087 | size = min_t(size_t, bytes, PAGE_SIZE - pgoff); |
| 1088 | kptr = kmap(page) + pgoff; |
| 1089 | ret = copy_from_user(kptr, from, size); |
| 1090 | kunmap(page); |
| 1091 | if (ret) |
| 1092 | return bytes - size + ret; |
| 1093 | bytes -= size; |
| 1094 | from += size; |
| 1095 | buffer_offset += size; |
| 1096 | } |
| 1097 | return 0; |
| 1098 | } |
Todd Kjos | 90a570c | 2019-02-08 10:35:15 -0800 | [diff] [blame] | 1099 | |
| 1100 | static void binder_alloc_do_buffer_copy(struct binder_alloc *alloc, |
| 1101 | bool to_buffer, |
| 1102 | struct binder_buffer *buffer, |
| 1103 | binder_size_t buffer_offset, |
| 1104 | void *ptr, |
| 1105 | size_t bytes) |
| 1106 | { |
| 1107 | /* All copies must be 32-bit aligned and 32-bit size */ |
| 1108 | BUG_ON(!check_buffer(alloc, buffer, buffer_offset, bytes)); |
| 1109 | |
| 1110 | while (bytes) { |
| 1111 | unsigned long size; |
| 1112 | struct page *page; |
| 1113 | pgoff_t pgoff; |
| 1114 | void *tmpptr; |
| 1115 | void *base_ptr; |
| 1116 | |
| 1117 | page = binder_alloc_get_page(alloc, buffer, |
| 1118 | buffer_offset, &pgoff); |
| 1119 | size = min_t(size_t, bytes, PAGE_SIZE - pgoff); |
| 1120 | base_ptr = kmap_atomic(page); |
| 1121 | tmpptr = base_ptr + pgoff; |
| 1122 | if (to_buffer) |
| 1123 | memcpy(tmpptr, ptr, size); |
| 1124 | else |
| 1125 | memcpy(ptr, tmpptr, size); |
| 1126 | /* |
| 1127 | * kunmap_atomic() takes care of flushing the cache |
| 1128 | * if this device has VIVT cache arch |
| 1129 | */ |
| 1130 | kunmap_atomic(base_ptr); |
| 1131 | bytes -= size; |
| 1132 | pgoff = 0; |
| 1133 | ptr = ptr + size; |
| 1134 | buffer_offset += size; |
| 1135 | } |
| 1136 | } |
| 1137 | |
| 1138 | void binder_alloc_copy_to_buffer(struct binder_alloc *alloc, |
| 1139 | struct binder_buffer *buffer, |
| 1140 | binder_size_t buffer_offset, |
| 1141 | void *src, |
| 1142 | size_t bytes) |
| 1143 | { |
| 1144 | binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset, |
| 1145 | src, bytes); |
| 1146 | } |
| 1147 | |
| 1148 | void binder_alloc_copy_from_buffer(struct binder_alloc *alloc, |
| 1149 | void *dest, |
| 1150 | struct binder_buffer *buffer, |
| 1151 | binder_size_t buffer_offset, |
| 1152 | size_t bytes) |
| 1153 | { |
| 1154 | binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset, |
| 1155 | dest, bytes); |
| 1156 | } |
| 1157 | |