Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 1 | /* binder_alloc.c |
| 2 | * |
| 3 | * Android IPC Subsystem |
| 4 | * |
| 5 | * Copyright (C) 2007-2017 Google, Inc. |
| 6 | * |
| 7 | * This software is licensed under the terms of the GNU General Public |
| 8 | * License version 2, as published by the Free Software Foundation, and |
| 9 | * may be copied, distributed, and modified under those terms. |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | */ |
| 17 | |
| 18 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 19 | |
| 20 | #include <asm/cacheflush.h> |
| 21 | #include <linux/list.h> |
| 22 | #include <linux/mm.h> |
| 23 | #include <linux/module.h> |
| 24 | #include <linux/rtmutex.h> |
| 25 | #include <linux/rbtree.h> |
| 26 | #include <linux/seq_file.h> |
| 27 | #include <linux/vmalloc.h> |
| 28 | #include <linux/slab.h> |
| 29 | #include <linux/sched.h> |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 30 | #include <linux/list_lru.h> |
Todd Kjos | d504949 | 2019-02-08 10:35:14 -0800 | [diff] [blame] | 31 | #include <linux/uaccess.h> |
| 32 | #include <linux/highmem.h> |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 33 | #include "binder_alloc.h" |
| 34 | #include "binder_trace.h" |
| 35 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 36 | struct list_lru binder_alloc_lru; |
| 37 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 38 | static DEFINE_MUTEX(binder_alloc_mmap_lock); |
| 39 | |
| 40 | enum { |
| 41 | BINDER_DEBUG_OPEN_CLOSE = 1U << 1, |
| 42 | BINDER_DEBUG_BUFFER_ALLOC = 1U << 2, |
| 43 | BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 3, |
| 44 | }; |
| 45 | static uint32_t binder_alloc_debug_mask; |
| 46 | |
| 47 | module_param_named(debug_mask, binder_alloc_debug_mask, |
| 48 | uint, 0644); |
| 49 | |
| 50 | #define binder_alloc_debug(mask, x...) \ |
| 51 | do { \ |
| 52 | if (binder_alloc_debug_mask & mask) \ |
| 53 | pr_info(x); \ |
| 54 | } while (0) |
| 55 | |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 56 | static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer) |
| 57 | { |
| 58 | return list_entry(buffer->entry.next, struct binder_buffer, entry); |
| 59 | } |
| 60 | |
| 61 | static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer) |
| 62 | { |
| 63 | return list_entry(buffer->entry.prev, struct binder_buffer, entry); |
| 64 | } |
| 65 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 66 | static size_t binder_alloc_buffer_size(struct binder_alloc *alloc, |
| 67 | struct binder_buffer *buffer) |
| 68 | { |
| 69 | if (list_is_last(&buffer->entry, &alloc->buffers)) |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 70 | return (u8 *)alloc->buffer + |
| 71 | alloc->buffer_size - (u8 *)buffer->data; |
| 72 | return (u8 *)binder_buffer_next(buffer)->data - (u8 *)buffer->data; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 73 | } |
| 74 | |
| 75 | static void binder_insert_free_buffer(struct binder_alloc *alloc, |
| 76 | struct binder_buffer *new_buffer) |
| 77 | { |
| 78 | struct rb_node **p = &alloc->free_buffers.rb_node; |
| 79 | struct rb_node *parent = NULL; |
| 80 | struct binder_buffer *buffer; |
| 81 | size_t buffer_size; |
| 82 | size_t new_buffer_size; |
| 83 | |
| 84 | BUG_ON(!new_buffer->free); |
| 85 | |
| 86 | new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); |
| 87 | |
| 88 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 89 | "%d: add free buffer, size %zd, at %pK\n", |
| 90 | alloc->pid, new_buffer_size, new_buffer); |
| 91 | |
| 92 | while (*p) { |
| 93 | parent = *p; |
| 94 | buffer = rb_entry(parent, struct binder_buffer, rb_node); |
| 95 | BUG_ON(!buffer->free); |
| 96 | |
| 97 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 98 | |
| 99 | if (new_buffer_size < buffer_size) |
| 100 | p = &parent->rb_left; |
| 101 | else |
| 102 | p = &parent->rb_right; |
| 103 | } |
| 104 | rb_link_node(&new_buffer->rb_node, parent, p); |
| 105 | rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers); |
| 106 | } |
| 107 | |
| 108 | static void binder_insert_allocated_buffer_locked( |
| 109 | struct binder_alloc *alloc, struct binder_buffer *new_buffer) |
| 110 | { |
| 111 | struct rb_node **p = &alloc->allocated_buffers.rb_node; |
| 112 | struct rb_node *parent = NULL; |
| 113 | struct binder_buffer *buffer; |
| 114 | |
| 115 | BUG_ON(new_buffer->free); |
| 116 | |
| 117 | while (*p) { |
| 118 | parent = *p; |
| 119 | buffer = rb_entry(parent, struct binder_buffer, rb_node); |
| 120 | BUG_ON(buffer->free); |
| 121 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 122 | if (new_buffer->data < buffer->data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 123 | p = &parent->rb_left; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 124 | else if (new_buffer->data > buffer->data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 125 | p = &parent->rb_right; |
| 126 | else |
| 127 | BUG(); |
| 128 | } |
| 129 | rb_link_node(&new_buffer->rb_node, parent, p); |
| 130 | rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers); |
| 131 | } |
| 132 | |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 133 | static struct binder_buffer *binder_alloc_prepare_to_free_locked( |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 134 | struct binder_alloc *alloc, |
| 135 | uintptr_t user_ptr) |
| 136 | { |
| 137 | struct rb_node *n = alloc->allocated_buffers.rb_node; |
| 138 | struct binder_buffer *buffer; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 139 | void *kern_ptr; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 140 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 141 | kern_ptr = (void *)(user_ptr - alloc->user_buffer_offset); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 142 | |
| 143 | while (n) { |
| 144 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 145 | BUG_ON(buffer->free); |
| 146 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 147 | if (kern_ptr < buffer->data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 148 | n = n->rb_left; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 149 | else if (kern_ptr > buffer->data) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 150 | n = n->rb_right; |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 151 | else { |
| 152 | /* |
| 153 | * Guard against user threads attempting to |
Todd Kjos | d29b73e | 2018-11-06 15:55:32 -0800 | [diff] [blame] | 154 | * free the buffer when in use by kernel or |
| 155 | * after it's already been freed. |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 156 | */ |
Todd Kjos | d29b73e | 2018-11-06 15:55:32 -0800 | [diff] [blame] | 157 | if (!buffer->allow_user_free) |
| 158 | return ERR_PTR(-EPERM); |
| 159 | buffer->allow_user_free = 0; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 160 | return buffer; |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 161 | } |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 162 | } |
| 163 | return NULL; |
| 164 | } |
| 165 | |
| 166 | /** |
| 167 | * binder_alloc_buffer_lookup() - get buffer given user ptr |
| 168 | * @alloc: binder_alloc for this proc |
| 169 | * @user_ptr: User pointer to buffer data |
| 170 | * |
| 171 | * Validate userspace pointer to buffer data and return buffer corresponding to |
| 172 | * that user pointer. Search the rb tree for buffer that matches user data |
| 173 | * pointer. |
| 174 | * |
| 175 | * Return: Pointer to buffer or NULL |
| 176 | */ |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 177 | struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc, |
| 178 | uintptr_t user_ptr) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 179 | { |
| 180 | struct binder_buffer *buffer; |
| 181 | |
| 182 | mutex_lock(&alloc->mutex); |
Todd Kjos | 076072a | 2017-04-21 14:32:11 -0700 | [diff] [blame] | 183 | buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 184 | mutex_unlock(&alloc->mutex); |
| 185 | return buffer; |
| 186 | } |
| 187 | |
| 188 | static int binder_update_page_range(struct binder_alloc *alloc, int allocate, |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 189 | void *start, void *end) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 190 | { |
| 191 | void *page_addr; |
| 192 | unsigned long user_page_addr; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 193 | struct binder_lru_page *page; |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 194 | struct vm_area_struct *vma = NULL; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 195 | struct mm_struct *mm = NULL; |
| 196 | bool need_mm = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 197 | |
| 198 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 199 | "%d: %s pages %pK-%pK\n", alloc->pid, |
| 200 | allocate ? "allocate" : "free", start, end); |
| 201 | |
| 202 | if (end <= start) |
| 203 | return 0; |
| 204 | |
| 205 | trace_binder_update_page_range(alloc, allocate, start, end); |
| 206 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 207 | if (allocate == 0) |
| 208 | goto free_range; |
| 209 | |
| 210 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { |
| 211 | page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE]; |
| 212 | if (!page->page_ptr) { |
| 213 | need_mm = true; |
| 214 | break; |
| 215 | } |
| 216 | } |
| 217 | |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 218 | if (need_mm && mmget_not_zero(alloc->vma_vm_mm)) |
| 219 | mm = alloc->vma_vm_mm; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 220 | |
| 221 | if (mm) { |
Minchan Kim | 2cafd5b | 2018-05-07 23:15:37 +0900 | [diff] [blame] | 222 | down_read(&mm->mmap_sem); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 223 | vma = alloc->vma; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 224 | } |
| 225 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 226 | if (!vma && need_mm) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 227 | pr_err("%d: binder_alloc_buf failed to map pages in userspace, no vma\n", |
| 228 | alloc->pid); |
| 229 | goto err_no_vma; |
| 230 | } |
| 231 | |
| 232 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { |
| 233 | int ret; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 234 | bool on_lru; |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 235 | size_t index; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 236 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 237 | index = (page_addr - alloc->buffer) / PAGE_SIZE; |
| 238 | page = &alloc->pages[index]; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 239 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 240 | if (page->page_ptr) { |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 241 | trace_binder_alloc_lru_start(alloc, index); |
| 242 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 243 | on_lru = list_lru_del(&binder_alloc_lru, &page->lru); |
| 244 | WARN_ON(!on_lru); |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 245 | |
| 246 | trace_binder_alloc_lru_end(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 247 | continue; |
| 248 | } |
| 249 | |
| 250 | if (WARN_ON(!vma)) |
| 251 | goto err_page_ptr_cleared; |
| 252 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 253 | trace_binder_alloc_page_start(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 254 | page->page_ptr = alloc_page(GFP_KERNEL | |
| 255 | __GFP_HIGHMEM | |
| 256 | __GFP_ZERO); |
| 257 | if (!page->page_ptr) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 258 | pr_err("%d: binder_alloc_buf failed for page at %pK\n", |
| 259 | alloc->pid, page_addr); |
| 260 | goto err_alloc_page_failed; |
| 261 | } |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 262 | page->alloc = alloc; |
| 263 | INIT_LIST_HEAD(&page->lru); |
| 264 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 265 | user_page_addr = |
| 266 | (uintptr_t)page_addr + alloc->user_buffer_offset; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 267 | ret = vm_insert_page(vma, user_page_addr, page[0].page_ptr); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 268 | if (ret) { |
| 269 | pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n", |
| 270 | alloc->pid, user_page_addr); |
| 271 | goto err_vm_insert_page_failed; |
| 272 | } |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 273 | |
Martijn Coenen | c05ec29 | 2017-10-24 16:37:39 +0200 | [diff] [blame] | 274 | if (index + 1 > alloc->pages_high) |
| 275 | alloc->pages_high = index + 1; |
| 276 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 277 | trace_binder_alloc_page_end(alloc, index); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 278 | /* vm_insert_page does not seem to increment the refcount */ |
| 279 | } |
| 280 | if (mm) { |
Minchan Kim | 2cafd5b | 2018-05-07 23:15:37 +0900 | [diff] [blame] | 281 | up_read(&mm->mmap_sem); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 282 | mmput(mm); |
| 283 | } |
| 284 | return 0; |
| 285 | |
| 286 | free_range: |
| 287 | for (page_addr = end - PAGE_SIZE; page_addr >= start; |
| 288 | page_addr -= PAGE_SIZE) { |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 289 | bool ret; |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 290 | size_t index; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 291 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 292 | index = (page_addr - alloc->buffer) / PAGE_SIZE; |
| 293 | page = &alloc->pages[index]; |
| 294 | |
| 295 | trace_binder_free_lru_start(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 296 | |
| 297 | ret = list_lru_add(&binder_alloc_lru, &page->lru); |
| 298 | WARN_ON(!ret); |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 299 | |
| 300 | trace_binder_free_lru_end(alloc, index); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 301 | continue; |
| 302 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 303 | err_vm_insert_page_failed: |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 304 | __free_page(page->page_ptr); |
| 305 | page->page_ptr = NULL; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 306 | err_alloc_page_failed: |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 307 | err_page_ptr_cleared: |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 308 | ; |
| 309 | } |
| 310 | err_no_vma: |
| 311 | if (mm) { |
Minchan Kim | 2cafd5b | 2018-05-07 23:15:37 +0900 | [diff] [blame] | 312 | up_read(&mm->mmap_sem); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 313 | mmput(mm); |
| 314 | } |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 315 | return vma ? -ENOMEM : -ESRCH; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 316 | } |
| 317 | |
Xiongwei Song | 9e9a3e1 | 2017-12-14 12:15:42 +0800 | [diff] [blame] | 318 | static struct binder_buffer *binder_alloc_new_buf_locked( |
| 319 | struct binder_alloc *alloc, |
| 320 | size_t data_size, |
| 321 | size_t offsets_size, |
| 322 | size_t extra_buffers_size, |
| 323 | int is_async) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 324 | { |
| 325 | struct rb_node *n = alloc->free_buffers.rb_node; |
| 326 | struct binder_buffer *buffer; |
| 327 | size_t buffer_size; |
| 328 | struct rb_node *best_fit = NULL; |
| 329 | void *has_page_addr; |
| 330 | void *end_page_addr; |
| 331 | size_t size, data_offsets_size; |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 332 | int ret; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 333 | |
| 334 | if (alloc->vma == NULL) { |
| 335 | pr_err("%d: binder_alloc_buf, no vma\n", |
| 336 | alloc->pid); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 337 | return ERR_PTR(-ESRCH); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 338 | } |
| 339 | |
| 340 | data_offsets_size = ALIGN(data_size, sizeof(void *)) + |
| 341 | ALIGN(offsets_size, sizeof(void *)); |
| 342 | |
| 343 | if (data_offsets_size < data_size || data_offsets_size < offsets_size) { |
| 344 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 345 | "%d: got transaction with invalid size %zd-%zd\n", |
| 346 | alloc->pid, data_size, offsets_size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 347 | return ERR_PTR(-EINVAL); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 348 | } |
| 349 | size = data_offsets_size + ALIGN(extra_buffers_size, sizeof(void *)); |
| 350 | if (size < data_offsets_size || size < extra_buffers_size) { |
| 351 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 352 | "%d: got transaction with invalid extra_buffers_size %zd\n", |
| 353 | alloc->pid, extra_buffers_size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 354 | return ERR_PTR(-EINVAL); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 355 | } |
| 356 | if (is_async && |
| 357 | alloc->free_async_space < size + sizeof(struct binder_buffer)) { |
| 358 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 359 | "%d: binder_alloc_buf size %zd failed, no async space left\n", |
| 360 | alloc->pid, size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 361 | return ERR_PTR(-ENOSPC); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 362 | } |
| 363 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 364 | /* Pad 0-size buffers so they get assigned unique addresses */ |
| 365 | size = max(size, sizeof(void *)); |
| 366 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 367 | while (n) { |
| 368 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 369 | BUG_ON(!buffer->free); |
| 370 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 371 | |
| 372 | if (size < buffer_size) { |
| 373 | best_fit = n; |
| 374 | n = n->rb_left; |
| 375 | } else if (size > buffer_size) |
| 376 | n = n->rb_right; |
| 377 | else { |
| 378 | best_fit = n; |
| 379 | break; |
| 380 | } |
| 381 | } |
| 382 | if (best_fit == NULL) { |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 383 | size_t allocated_buffers = 0; |
| 384 | size_t largest_alloc_size = 0; |
| 385 | size_t total_alloc_size = 0; |
| 386 | size_t free_buffers = 0; |
| 387 | size_t largest_free_size = 0; |
| 388 | size_t total_free_size = 0; |
| 389 | |
| 390 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; |
| 391 | n = rb_next(n)) { |
| 392 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 393 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 394 | allocated_buffers++; |
| 395 | total_alloc_size += buffer_size; |
| 396 | if (buffer_size > largest_alloc_size) |
| 397 | largest_alloc_size = buffer_size; |
| 398 | } |
| 399 | for (n = rb_first(&alloc->free_buffers); n != NULL; |
| 400 | n = rb_next(n)) { |
| 401 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 402 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 403 | free_buffers++; |
| 404 | total_free_size += buffer_size; |
| 405 | if (buffer_size > largest_free_size) |
| 406 | largest_free_size = buffer_size; |
| 407 | } |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 408 | pr_err("%d: binder_alloc_buf size %zd failed, no address space\n", |
| 409 | alloc->pid, size); |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 410 | pr_err("allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n", |
| 411 | total_alloc_size, allocated_buffers, largest_alloc_size, |
| 412 | total_free_size, free_buffers, largest_free_size); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 413 | return ERR_PTR(-ENOSPC); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 414 | } |
| 415 | if (n == NULL) { |
| 416 | buffer = rb_entry(best_fit, struct binder_buffer, rb_node); |
| 417 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 418 | } |
| 419 | |
| 420 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 421 | "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n", |
| 422 | alloc->pid, size, buffer, buffer_size); |
| 423 | |
| 424 | has_page_addr = |
| 425 | (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 426 | WARN_ON(n && buffer_size != size); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 427 | end_page_addr = |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 428 | (void *)PAGE_ALIGN((uintptr_t)buffer->data + size); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 429 | if (end_page_addr > has_page_addr) |
| 430 | end_page_addr = has_page_addr; |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 431 | ret = binder_update_page_range(alloc, 1, |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 432 | (void *)PAGE_ALIGN((uintptr_t)buffer->data), end_page_addr); |
Todd Kjos | e598d17 | 2017-03-22 17:19:52 -0700 | [diff] [blame] | 433 | if (ret) |
| 434 | return ERR_PTR(ret); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 435 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 436 | if (buffer_size != size) { |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 437 | struct binder_buffer *new_buffer; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 438 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 439 | new_buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); |
| 440 | if (!new_buffer) { |
| 441 | pr_err("%s: %d failed to alloc new buffer struct\n", |
| 442 | __func__, alloc->pid); |
| 443 | goto err_alloc_buf_struct_failed; |
| 444 | } |
| 445 | new_buffer->data = (u8 *)buffer->data + size; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 446 | list_add(&new_buffer->entry, &buffer->entry); |
| 447 | new_buffer->free = 1; |
| 448 | binder_insert_free_buffer(alloc, new_buffer); |
| 449 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 450 | |
| 451 | rb_erase(best_fit, &alloc->free_buffers); |
| 452 | buffer->free = 0; |
Todd Kjos | d29b73e | 2018-11-06 15:55:32 -0800 | [diff] [blame] | 453 | buffer->allow_user_free = 0; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 454 | binder_insert_allocated_buffer_locked(alloc, buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 455 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 456 | "%d: binder_alloc_buf size %zd got %pK\n", |
| 457 | alloc->pid, size, buffer); |
| 458 | buffer->data_size = data_size; |
| 459 | buffer->offsets_size = offsets_size; |
| 460 | buffer->async_transaction = is_async; |
| 461 | buffer->extra_buffers_size = extra_buffers_size; |
| 462 | if (is_async) { |
| 463 | alloc->free_async_space -= size + sizeof(struct binder_buffer); |
| 464 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, |
| 465 | "%d: binder_alloc_buf size %zd async free %zd\n", |
| 466 | alloc->pid, size, alloc->free_async_space); |
| 467 | } |
| 468 | return buffer; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 469 | |
| 470 | err_alloc_buf_struct_failed: |
| 471 | binder_update_page_range(alloc, 0, |
| 472 | (void *)PAGE_ALIGN((uintptr_t)buffer->data), |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 473 | end_page_addr); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 474 | return ERR_PTR(-ENOMEM); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 475 | } |
| 476 | |
| 477 | /** |
| 478 | * binder_alloc_new_buf() - Allocate a new binder buffer |
| 479 | * @alloc: binder_alloc for this proc |
| 480 | * @data_size: size of user data buffer |
| 481 | * @offsets_size: user specified buffer offset |
| 482 | * @extra_buffers_size: size of extra space for meta-data (eg, security context) |
| 483 | * @is_async: buffer for async transaction |
| 484 | * |
| 485 | * Allocate a new buffer given the requested sizes. Returns |
| 486 | * the kernel version of the buffer pointer. The size allocated |
| 487 | * is the sum of the three given sizes (each rounded up to |
| 488 | * pointer-sized boundary) |
| 489 | * |
| 490 | * Return: The allocated buffer or %NULL if error |
| 491 | */ |
| 492 | struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc, |
| 493 | size_t data_size, |
| 494 | size_t offsets_size, |
| 495 | size_t extra_buffers_size, |
| 496 | int is_async) |
| 497 | { |
| 498 | struct binder_buffer *buffer; |
| 499 | |
| 500 | mutex_lock(&alloc->mutex); |
| 501 | buffer = binder_alloc_new_buf_locked(alloc, data_size, offsets_size, |
| 502 | extra_buffers_size, is_async); |
| 503 | mutex_unlock(&alloc->mutex); |
| 504 | return buffer; |
| 505 | } |
| 506 | |
| 507 | static void *buffer_start_page(struct binder_buffer *buffer) |
| 508 | { |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 509 | return (void *)((uintptr_t)buffer->data & PAGE_MASK); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 510 | } |
| 511 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 512 | static void *prev_buffer_end_page(struct binder_buffer *buffer) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 513 | { |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 514 | return (void *)(((uintptr_t)(buffer->data) - 1) & PAGE_MASK); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 515 | } |
| 516 | |
| 517 | static void binder_delete_free_buffer(struct binder_alloc *alloc, |
| 518 | struct binder_buffer *buffer) |
| 519 | { |
| 520 | struct binder_buffer *prev, *next = NULL; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 521 | bool to_free = true; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 522 | BUG_ON(alloc->buffers.next == &buffer->entry); |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 523 | prev = binder_buffer_prev(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 524 | BUG_ON(!prev->free); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 525 | if (prev_buffer_end_page(prev) == buffer_start_page(buffer)) { |
| 526 | to_free = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 527 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 528 | "%d: merge free, buffer %pK share page with %pK\n", |
| 529 | alloc->pid, buffer->data, prev->data); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 530 | } |
| 531 | |
| 532 | if (!list_is_last(&buffer->entry, &alloc->buffers)) { |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 533 | next = binder_buffer_next(buffer); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 534 | if (buffer_start_page(next) == buffer_start_page(buffer)) { |
| 535 | to_free = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 536 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 537 | "%d: merge free, buffer %pK share page with %pK\n", |
| 538 | alloc->pid, |
| 539 | buffer->data, |
| 540 | next->data); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 541 | } |
| 542 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 543 | |
| 544 | if (PAGE_ALIGNED(buffer->data)) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 545 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 546 | "%d: merge free, buffer start %pK is page aligned\n", |
| 547 | alloc->pid, buffer->data); |
| 548 | to_free = false; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 549 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 550 | |
| 551 | if (to_free) { |
| 552 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 553 | "%d: merge free, buffer %pK do not share page with %pK or %pK\n", |
| 554 | alloc->pid, buffer->data, |
Sherry Yang | 27155df | 2017-10-05 17:13:47 -0400 | [diff] [blame] | 555 | prev->data, next ? next->data : NULL); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 556 | binder_update_page_range(alloc, 0, buffer_start_page(buffer), |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 557 | buffer_start_page(buffer) + PAGE_SIZE); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 558 | } |
| 559 | list_del(&buffer->entry); |
| 560 | kfree(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 561 | } |
| 562 | |
| 563 | static void binder_free_buf_locked(struct binder_alloc *alloc, |
| 564 | struct binder_buffer *buffer) |
| 565 | { |
| 566 | size_t size, buffer_size; |
| 567 | |
| 568 | buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 569 | |
| 570 | size = ALIGN(buffer->data_size, sizeof(void *)) + |
| 571 | ALIGN(buffer->offsets_size, sizeof(void *)) + |
| 572 | ALIGN(buffer->extra_buffers_size, sizeof(void *)); |
| 573 | |
| 574 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
| 575 | "%d: binder_free_buf %pK size %zd buffer_size %zd\n", |
| 576 | alloc->pid, buffer, size, buffer_size); |
| 577 | |
| 578 | BUG_ON(buffer->free); |
| 579 | BUG_ON(size > buffer_size); |
| 580 | BUG_ON(buffer->transaction != NULL); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 581 | BUG_ON(buffer->data < alloc->buffer); |
| 582 | BUG_ON(buffer->data > alloc->buffer + alloc->buffer_size); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 583 | |
| 584 | if (buffer->async_transaction) { |
| 585 | alloc->free_async_space += size + sizeof(struct binder_buffer); |
| 586 | |
| 587 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, |
| 588 | "%d: binder_free_buf size %zd async free %zd\n", |
| 589 | alloc->pid, size, alloc->free_async_space); |
| 590 | } |
| 591 | |
| 592 | binder_update_page_range(alloc, 0, |
| 593 | (void *)PAGE_ALIGN((uintptr_t)buffer->data), |
Sherry Yang | edd2131 | 2017-09-15 20:40:03 -0400 | [diff] [blame] | 594 | (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK)); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 595 | |
| 596 | rb_erase(&buffer->rb_node, &alloc->allocated_buffers); |
| 597 | buffer->free = 1; |
| 598 | if (!list_is_last(&buffer->entry, &alloc->buffers)) { |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 599 | struct binder_buffer *next = binder_buffer_next(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 600 | |
| 601 | if (next->free) { |
| 602 | rb_erase(&next->rb_node, &alloc->free_buffers); |
| 603 | binder_delete_free_buffer(alloc, next); |
| 604 | } |
| 605 | } |
| 606 | if (alloc->buffers.next != &buffer->entry) { |
Sherry Yang | 7aed47a | 2017-06-30 10:22:23 -0700 | [diff] [blame] | 607 | struct binder_buffer *prev = binder_buffer_prev(buffer); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 608 | |
| 609 | if (prev->free) { |
| 610 | binder_delete_free_buffer(alloc, buffer); |
| 611 | rb_erase(&prev->rb_node, &alloc->free_buffers); |
| 612 | buffer = prev; |
| 613 | } |
| 614 | } |
| 615 | binder_insert_free_buffer(alloc, buffer); |
| 616 | } |
| 617 | |
| 618 | /** |
| 619 | * binder_alloc_free_buf() - free a binder buffer |
| 620 | * @alloc: binder_alloc for this proc |
| 621 | * @buffer: kernel pointer to buffer |
| 622 | * |
| 623 | * Free the buffer allocated via binder_alloc_new_buffer() |
| 624 | */ |
| 625 | void binder_alloc_free_buf(struct binder_alloc *alloc, |
| 626 | struct binder_buffer *buffer) |
| 627 | { |
| 628 | mutex_lock(&alloc->mutex); |
| 629 | binder_free_buf_locked(alloc, buffer); |
| 630 | mutex_unlock(&alloc->mutex); |
| 631 | } |
| 632 | |
| 633 | /** |
| 634 | * binder_alloc_mmap_handler() - map virtual address space for proc |
| 635 | * @alloc: alloc structure for this proc |
| 636 | * @vma: vma passed to mmap() |
| 637 | * |
| 638 | * Called by binder_mmap() to initialize the space specified in |
| 639 | * vma for allocating binder buffers |
| 640 | * |
| 641 | * Return: |
| 642 | * 0 = success |
| 643 | * -EBUSY = address space already mapped |
| 644 | * -ENOMEM = failed to map memory to given address space |
| 645 | */ |
| 646 | int binder_alloc_mmap_handler(struct binder_alloc *alloc, |
| 647 | struct vm_area_struct *vma) |
| 648 | { |
| 649 | int ret; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 650 | const char *failure_string; |
| 651 | struct binder_buffer *buffer; |
| 652 | |
| 653 | mutex_lock(&binder_alloc_mmap_lock); |
| 654 | if (alloc->buffer) { |
| 655 | ret = -EBUSY; |
| 656 | failure_string = "already mapped"; |
| 657 | goto err_already_mapped; |
| 658 | } |
| 659 | |
Todd Kjos | 46dc639 | 2019-02-08 10:35:18 -0800 | [diff] [blame^] | 660 | alloc->buffer = (void *)vma->vm_start; |
| 661 | alloc->user_buffer_offset = 0; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 662 | mutex_unlock(&binder_alloc_mmap_lock); |
| 663 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 664 | alloc->pages = kzalloc(sizeof(alloc->pages[0]) * |
| 665 | ((vma->vm_end - vma->vm_start) / PAGE_SIZE), |
| 666 | GFP_KERNEL); |
| 667 | if (alloc->pages == NULL) { |
| 668 | ret = -ENOMEM; |
| 669 | failure_string = "alloc page array"; |
| 670 | goto err_alloc_pages_failed; |
| 671 | } |
| 672 | alloc->buffer_size = vma->vm_end - vma->vm_start; |
| 673 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 674 | buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); |
| 675 | if (!buffer) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 676 | ret = -ENOMEM; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 677 | failure_string = "alloc buffer struct"; |
| 678 | goto err_alloc_buf_struct_failed; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 679 | } |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 680 | |
| 681 | buffer->data = alloc->buffer; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 682 | list_add(&buffer->entry, &alloc->buffers); |
| 683 | buffer->free = 1; |
| 684 | binder_insert_free_buffer(alloc, buffer); |
| 685 | alloc->free_async_space = alloc->buffer_size / 2; |
| 686 | barrier(); |
| 687 | alloc->vma = vma; |
| 688 | alloc->vma_vm_mm = vma->vm_mm; |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 689 | /* Same as mmgrab() in later kernel versions */ |
| 690 | atomic_inc(&alloc->vma_vm_mm->mm_count); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 691 | |
| 692 | return 0; |
| 693 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 694 | err_alloc_buf_struct_failed: |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 695 | kfree(alloc->pages); |
| 696 | alloc->pages = NULL; |
| 697 | err_alloc_pages_failed: |
| 698 | mutex_lock(&binder_alloc_mmap_lock); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 699 | alloc->buffer = NULL; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 700 | err_already_mapped: |
| 701 | mutex_unlock(&binder_alloc_mmap_lock); |
| 702 | pr_err("%s: %d %lx-%lx %s failed %d\n", __func__, |
| 703 | alloc->pid, vma->vm_start, vma->vm_end, failure_string, ret); |
| 704 | return ret; |
| 705 | } |
| 706 | |
| 707 | |
| 708 | void binder_alloc_deferred_release(struct binder_alloc *alloc) |
| 709 | { |
| 710 | struct rb_node *n; |
| 711 | int buffers, page_count; |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 712 | struct binder_buffer *buffer; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 713 | |
| 714 | BUG_ON(alloc->vma); |
| 715 | |
| 716 | buffers = 0; |
| 717 | mutex_lock(&alloc->mutex); |
| 718 | while ((n = rb_first(&alloc->allocated_buffers))) { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 719 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
| 720 | |
| 721 | /* Transaction should already have been freed */ |
| 722 | BUG_ON(buffer->transaction); |
| 723 | |
| 724 | binder_free_buf_locked(alloc, buffer); |
| 725 | buffers++; |
| 726 | } |
| 727 | |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 728 | while (!list_empty(&alloc->buffers)) { |
| 729 | buffer = list_first_entry(&alloc->buffers, |
| 730 | struct binder_buffer, entry); |
| 731 | WARN_ON(!buffer->free); |
| 732 | |
| 733 | list_del(&buffer->entry); |
| 734 | WARN_ON_ONCE(!list_empty(&alloc->buffers)); |
| 735 | kfree(buffer); |
| 736 | } |
| 737 | |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 738 | page_count = 0; |
| 739 | if (alloc->pages) { |
| 740 | int i; |
| 741 | |
| 742 | for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { |
| 743 | void *page_addr; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 744 | bool on_lru; |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 745 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 746 | if (!alloc->pages[i].page_ptr) |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 747 | continue; |
| 748 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 749 | on_lru = list_lru_del(&binder_alloc_lru, |
| 750 | &alloc->pages[i].lru); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 751 | page_addr = alloc->buffer + i * PAGE_SIZE; |
| 752 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 753 | "%s: %d: page %d at %pK %s\n", |
| 754 | __func__, alloc->pid, i, page_addr, |
| 755 | on_lru ? "on lru" : "active"); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 756 | __free_page(alloc->pages[i].page_ptr); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 757 | page_count++; |
| 758 | } |
| 759 | kfree(alloc->pages); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 760 | } |
| 761 | mutex_unlock(&alloc->mutex); |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 762 | if (alloc->vma_vm_mm) |
| 763 | mmdrop(alloc->vma_vm_mm); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 764 | |
| 765 | binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE, |
| 766 | "%s: %d buffers %d, pages %d\n", |
| 767 | __func__, alloc->pid, buffers, page_count); |
| 768 | } |
| 769 | |
| 770 | static void print_binder_buffer(struct seq_file *m, const char *prefix, |
| 771 | struct binder_buffer *buffer) |
| 772 | { |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 773 | seq_printf(m, "%s %d: %pK size %zd:%zd:%zd %s\n", |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 774 | prefix, buffer->debug_id, buffer->data, |
| 775 | buffer->data_size, buffer->offsets_size, |
Martijn Coenen | 970df8b | 2017-03-15 18:22:52 +0100 | [diff] [blame] | 776 | buffer->extra_buffers_size, |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 777 | buffer->transaction ? "active" : "delivered"); |
| 778 | } |
| 779 | |
| 780 | /** |
| 781 | * binder_alloc_print_allocated() - print buffer info |
| 782 | * @m: seq_file for output via seq_printf() |
| 783 | * @alloc: binder_alloc for this proc |
| 784 | * |
| 785 | * Prints information about every buffer associated with |
| 786 | * the binder_alloc state to the given seq_file |
| 787 | */ |
| 788 | void binder_alloc_print_allocated(struct seq_file *m, |
| 789 | struct binder_alloc *alloc) |
| 790 | { |
| 791 | struct rb_node *n; |
| 792 | |
| 793 | mutex_lock(&alloc->mutex); |
| 794 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) |
| 795 | print_binder_buffer(m, " buffer", |
| 796 | rb_entry(n, struct binder_buffer, rb_node)); |
| 797 | mutex_unlock(&alloc->mutex); |
| 798 | } |
| 799 | |
| 800 | /** |
Sherry Yang | 9100442 | 2017-08-22 17:26:57 -0700 | [diff] [blame] | 801 | * binder_alloc_print_pages() - print page usage |
| 802 | * @m: seq_file for output via seq_printf() |
| 803 | * @alloc: binder_alloc for this proc |
| 804 | */ |
| 805 | void binder_alloc_print_pages(struct seq_file *m, |
| 806 | struct binder_alloc *alloc) |
| 807 | { |
| 808 | struct binder_lru_page *page; |
| 809 | int i; |
| 810 | int active = 0; |
| 811 | int lru = 0; |
| 812 | int free = 0; |
| 813 | |
| 814 | mutex_lock(&alloc->mutex); |
| 815 | for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { |
| 816 | page = &alloc->pages[i]; |
| 817 | if (!page->page_ptr) |
| 818 | free++; |
| 819 | else if (list_empty(&page->lru)) |
| 820 | active++; |
| 821 | else |
| 822 | lru++; |
| 823 | } |
| 824 | mutex_unlock(&alloc->mutex); |
| 825 | seq_printf(m, " pages: %d:%d:%d\n", active, lru, free); |
Martijn Coenen | c05ec29 | 2017-10-24 16:37:39 +0200 | [diff] [blame] | 826 | seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high); |
Sherry Yang | 9100442 | 2017-08-22 17:26:57 -0700 | [diff] [blame] | 827 | } |
| 828 | |
| 829 | /** |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 830 | * binder_alloc_get_allocated_count() - return count of buffers |
| 831 | * @alloc: binder_alloc for this proc |
| 832 | * |
| 833 | * Return: count of allocated buffers |
| 834 | */ |
| 835 | int binder_alloc_get_allocated_count(struct binder_alloc *alloc) |
| 836 | { |
| 837 | struct rb_node *n; |
| 838 | int count = 0; |
| 839 | |
| 840 | mutex_lock(&alloc->mutex); |
| 841 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) |
| 842 | count++; |
| 843 | mutex_unlock(&alloc->mutex); |
| 844 | return count; |
| 845 | } |
| 846 | |
| 847 | |
| 848 | /** |
| 849 | * binder_alloc_vma_close() - invalidate address space |
| 850 | * @alloc: binder_alloc for this proc |
| 851 | * |
| 852 | * Called from binder_vma_close() when releasing address space. |
| 853 | * Clears alloc->vma to prevent new incoming transactions from |
| 854 | * allocating more buffers. |
| 855 | */ |
| 856 | void binder_alloc_vma_close(struct binder_alloc *alloc) |
| 857 | { |
| 858 | WRITE_ONCE(alloc->vma, NULL); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 859 | } |
| 860 | |
| 861 | /** |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 862 | * binder_alloc_free_page() - shrinker callback to free pages |
| 863 | * @item: item to free |
| 864 | * @lock: lock protecting the item |
| 865 | * @cb_arg: callback argument |
| 866 | * |
| 867 | * Called from list_lru_walk() in binder_shrink_scan() to free |
| 868 | * up pages when the system is under memory pressure. |
| 869 | */ |
| 870 | enum lru_status binder_alloc_free_page(struct list_head *item, |
| 871 | struct list_lru_one *lru, |
| 872 | spinlock_t *lock, |
| 873 | void *cb_arg) |
| 874 | { |
| 875 | struct mm_struct *mm = NULL; |
| 876 | struct binder_lru_page *page = container_of(item, |
| 877 | struct binder_lru_page, |
| 878 | lru); |
| 879 | struct binder_alloc *alloc; |
| 880 | uintptr_t page_addr; |
| 881 | size_t index; |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 882 | struct vm_area_struct *vma; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 883 | |
| 884 | alloc = page->alloc; |
| 885 | if (!mutex_trylock(&alloc->mutex)) |
| 886 | goto err_get_alloc_mutex_failed; |
| 887 | |
| 888 | if (!page->page_ptr) |
| 889 | goto err_page_already_freed; |
| 890 | |
| 891 | index = page - alloc->pages; |
| 892 | page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE; |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 893 | vma = alloc->vma; |
| 894 | if (vma) { |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 895 | if (!mmget_not_zero(alloc->vma_vm_mm)) |
| 896 | goto err_mmget; |
| 897 | mm = alloc->vma_vm_mm; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 898 | if (!down_write_trylock(&mm->mmap_sem)) |
| 899 | goto err_down_write_mmap_sem_failed; |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 900 | } |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 901 | |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 902 | list_lru_isolate(lru, item); |
| 903 | spin_unlock(lock); |
| 904 | |
| 905 | if (vma) { |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 906 | trace_binder_unmap_user_start(alloc, index); |
| 907 | |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 908 | zap_page_range(vma, |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 909 | page_addr + |
| 910 | alloc->user_buffer_offset, |
| 911 | PAGE_SIZE, NULL); |
| 912 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 913 | trace_binder_unmap_user_end(alloc, index); |
| 914 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 915 | up_write(&mm->mmap_sem); |
| 916 | mmput(mm); |
| 917 | } |
| 918 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 919 | trace_binder_unmap_kernel_start(alloc, index); |
| 920 | |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 921 | __free_page(page->page_ptr); |
| 922 | page->page_ptr = NULL; |
| 923 | |
Sherry Yang | 4a61ba6 | 2017-08-02 14:02:37 -0700 | [diff] [blame] | 924 | trace_binder_unmap_kernel_end(alloc, index); |
| 925 | |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 926 | spin_lock(lock); |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 927 | mutex_unlock(&alloc->mutex); |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 928 | return LRU_REMOVED_RETRY; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 929 | |
| 930 | err_down_write_mmap_sem_failed: |
Sherry Yang | e3368fd | 2017-09-08 02:09:26 -0400 | [diff] [blame] | 931 | mmput_async(mm); |
Sherry Yang | 5e34c9b | 2017-09-15 21:12:15 -0400 | [diff] [blame] | 932 | err_mmget: |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 933 | err_page_already_freed: |
| 934 | mutex_unlock(&alloc->mutex); |
| 935 | err_get_alloc_mutex_failed: |
| 936 | return LRU_SKIP; |
| 937 | } |
| 938 | |
| 939 | static unsigned long |
| 940 | binder_shrink_count(struct shrinker *shrink, struct shrink_control *sc) |
| 941 | { |
| 942 | unsigned long ret = list_lru_count(&binder_alloc_lru); |
| 943 | return ret; |
| 944 | } |
| 945 | |
| 946 | static unsigned long |
| 947 | binder_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) |
| 948 | { |
| 949 | unsigned long ret; |
| 950 | |
| 951 | ret = list_lru_walk(&binder_alloc_lru, binder_alloc_free_page, |
| 952 | NULL, sc->nr_to_scan); |
| 953 | return ret; |
| 954 | } |
| 955 | |
Sherry Yang | 0b33853 | 2017-10-05 17:00:57 -0400 | [diff] [blame] | 956 | static struct shrinker binder_shrinker = { |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 957 | .count_objects = binder_shrink_count, |
| 958 | .scan_objects = binder_shrink_scan, |
| 959 | .seeks = DEFAULT_SEEKS, |
| 960 | }; |
| 961 | |
| 962 | /** |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 963 | * binder_alloc_init() - called by binder_open() for per-proc initialization |
| 964 | * @alloc: binder_alloc for this proc |
| 965 | * |
| 966 | * Called from binder_open() to initialize binder_alloc fields for |
| 967 | * new binder proc |
| 968 | */ |
| 969 | void binder_alloc_init(struct binder_alloc *alloc) |
| 970 | { |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 971 | alloc->pid = current->group_leader->pid; |
| 972 | mutex_init(&alloc->mutex); |
Sherry Yang | 7ffd494 | 2017-08-03 11:33:53 -0700 | [diff] [blame] | 973 | INIT_LIST_HEAD(&alloc->buffers); |
Todd Kjos | b934102 | 2016-10-10 10:40:53 -0700 | [diff] [blame] | 974 | } |
| 975 | |
Tetsuo Handa | f8cb822 | 2017-11-29 22:29:47 +0900 | [diff] [blame] | 976 | int binder_alloc_shrinker_init(void) |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 977 | { |
Tetsuo Handa | f8cb822 | 2017-11-29 22:29:47 +0900 | [diff] [blame] | 978 | int ret = list_lru_init(&binder_alloc_lru); |
| 979 | |
| 980 | if (ret == 0) { |
| 981 | ret = register_shrinker(&binder_shrinker); |
| 982 | if (ret) |
| 983 | list_lru_destroy(&binder_alloc_lru); |
| 984 | } |
| 985 | return ret; |
Sherry Yang | 5828d70 | 2017-07-29 13:24:11 -0700 | [diff] [blame] | 986 | } |
Todd Kjos | d504949 | 2019-02-08 10:35:14 -0800 | [diff] [blame] | 987 | |
| 988 | /** |
| 989 | * check_buffer() - verify that buffer/offset is safe to access |
| 990 | * @alloc: binder_alloc for this proc |
| 991 | * @buffer: binder buffer to be accessed |
| 992 | * @offset: offset into @buffer data |
| 993 | * @bytes: bytes to access from offset |
| 994 | * |
| 995 | * Check that the @offset/@bytes are within the size of the given |
| 996 | * @buffer and that the buffer is currently active and not freeable. |
| 997 | * Offsets must also be multiples of sizeof(u32). The kernel is |
| 998 | * allowed to touch the buffer in two cases: |
| 999 | * |
| 1000 | * 1) when the buffer is being created: |
| 1001 | * (buffer->free == 0 && buffer->allow_user_free == 0) |
| 1002 | * 2) when the buffer is being torn down: |
| 1003 | * (buffer->free == 0 && buffer->transaction == NULL). |
| 1004 | * |
| 1005 | * Return: true if the buffer is safe to access |
| 1006 | */ |
| 1007 | static inline bool check_buffer(struct binder_alloc *alloc, |
| 1008 | struct binder_buffer *buffer, |
| 1009 | binder_size_t offset, size_t bytes) |
| 1010 | { |
| 1011 | size_t buffer_size = binder_alloc_buffer_size(alloc, buffer); |
| 1012 | |
| 1013 | return buffer_size >= bytes && |
| 1014 | offset <= buffer_size - bytes && |
| 1015 | IS_ALIGNED(offset, sizeof(u32)) && |
| 1016 | !buffer->free && |
| 1017 | (!buffer->allow_user_free || !buffer->transaction); |
| 1018 | } |
| 1019 | |
| 1020 | /** |
| 1021 | * binder_alloc_get_page() - get kernel pointer for given buffer offset |
| 1022 | * @alloc: binder_alloc for this proc |
| 1023 | * @buffer: binder buffer to be accessed |
| 1024 | * @buffer_offset: offset into @buffer data |
| 1025 | * @pgoffp: address to copy final page offset to |
| 1026 | * |
| 1027 | * Lookup the struct page corresponding to the address |
| 1028 | * at @buffer_offset into @buffer->data. If @pgoffp is not |
| 1029 | * NULL, the byte-offset into the page is written there. |
| 1030 | * |
| 1031 | * The caller is responsible to ensure that the offset points |
| 1032 | * to a valid address within the @buffer and that @buffer is |
| 1033 | * not freeable by the user. Since it can't be freed, we are |
| 1034 | * guaranteed that the corresponding elements of @alloc->pages[] |
| 1035 | * cannot change. |
| 1036 | * |
| 1037 | * Return: struct page |
| 1038 | */ |
| 1039 | static struct page *binder_alloc_get_page(struct binder_alloc *alloc, |
| 1040 | struct binder_buffer *buffer, |
| 1041 | binder_size_t buffer_offset, |
| 1042 | pgoff_t *pgoffp) |
| 1043 | { |
| 1044 | binder_size_t buffer_space_offset = buffer_offset + |
| 1045 | (buffer->data - alloc->buffer); |
| 1046 | pgoff_t pgoff = buffer_space_offset & ~PAGE_MASK; |
| 1047 | size_t index = buffer_space_offset >> PAGE_SHIFT; |
| 1048 | struct binder_lru_page *lru_page; |
| 1049 | |
| 1050 | lru_page = &alloc->pages[index]; |
| 1051 | *pgoffp = pgoff; |
| 1052 | return lru_page->page_ptr; |
| 1053 | } |
| 1054 | |
| 1055 | /** |
| 1056 | * binder_alloc_copy_user_to_buffer() - copy src user to tgt user |
| 1057 | * @alloc: binder_alloc for this proc |
| 1058 | * @buffer: binder buffer to be accessed |
| 1059 | * @buffer_offset: offset into @buffer data |
| 1060 | * @from: userspace pointer to source buffer |
| 1061 | * @bytes: bytes to copy |
| 1062 | * |
| 1063 | * Copy bytes from source userspace to target buffer. |
| 1064 | * |
| 1065 | * Return: bytes remaining to be copied |
| 1066 | */ |
| 1067 | unsigned long |
| 1068 | binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc, |
| 1069 | struct binder_buffer *buffer, |
| 1070 | binder_size_t buffer_offset, |
| 1071 | const void __user *from, |
| 1072 | size_t bytes) |
| 1073 | { |
| 1074 | if (!check_buffer(alloc, buffer, buffer_offset, bytes)) |
| 1075 | return bytes; |
| 1076 | |
| 1077 | while (bytes) { |
| 1078 | unsigned long size; |
| 1079 | unsigned long ret; |
| 1080 | struct page *page; |
| 1081 | pgoff_t pgoff; |
| 1082 | void *kptr; |
| 1083 | |
| 1084 | page = binder_alloc_get_page(alloc, buffer, |
| 1085 | buffer_offset, &pgoff); |
| 1086 | size = min_t(size_t, bytes, PAGE_SIZE - pgoff); |
| 1087 | kptr = kmap(page) + pgoff; |
| 1088 | ret = copy_from_user(kptr, from, size); |
| 1089 | kunmap(page); |
| 1090 | if (ret) |
| 1091 | return bytes - size + ret; |
| 1092 | bytes -= size; |
| 1093 | from += size; |
| 1094 | buffer_offset += size; |
| 1095 | } |
| 1096 | return 0; |
| 1097 | } |
Todd Kjos | 90a570c | 2019-02-08 10:35:15 -0800 | [diff] [blame] | 1098 | |
| 1099 | static void binder_alloc_do_buffer_copy(struct binder_alloc *alloc, |
| 1100 | bool to_buffer, |
| 1101 | struct binder_buffer *buffer, |
| 1102 | binder_size_t buffer_offset, |
| 1103 | void *ptr, |
| 1104 | size_t bytes) |
| 1105 | { |
| 1106 | /* All copies must be 32-bit aligned and 32-bit size */ |
| 1107 | BUG_ON(!check_buffer(alloc, buffer, buffer_offset, bytes)); |
| 1108 | |
| 1109 | while (bytes) { |
| 1110 | unsigned long size; |
| 1111 | struct page *page; |
| 1112 | pgoff_t pgoff; |
| 1113 | void *tmpptr; |
| 1114 | void *base_ptr; |
| 1115 | |
| 1116 | page = binder_alloc_get_page(alloc, buffer, |
| 1117 | buffer_offset, &pgoff); |
| 1118 | size = min_t(size_t, bytes, PAGE_SIZE - pgoff); |
| 1119 | base_ptr = kmap_atomic(page); |
| 1120 | tmpptr = base_ptr + pgoff; |
| 1121 | if (to_buffer) |
| 1122 | memcpy(tmpptr, ptr, size); |
| 1123 | else |
| 1124 | memcpy(ptr, tmpptr, size); |
| 1125 | /* |
| 1126 | * kunmap_atomic() takes care of flushing the cache |
| 1127 | * if this device has VIVT cache arch |
| 1128 | */ |
| 1129 | kunmap_atomic(base_ptr); |
| 1130 | bytes -= size; |
| 1131 | pgoff = 0; |
| 1132 | ptr = ptr + size; |
| 1133 | buffer_offset += size; |
| 1134 | } |
| 1135 | } |
| 1136 | |
| 1137 | void binder_alloc_copy_to_buffer(struct binder_alloc *alloc, |
| 1138 | struct binder_buffer *buffer, |
| 1139 | binder_size_t buffer_offset, |
| 1140 | void *src, |
| 1141 | size_t bytes) |
| 1142 | { |
| 1143 | binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset, |
| 1144 | src, bytes); |
| 1145 | } |
| 1146 | |
| 1147 | void binder_alloc_copy_from_buffer(struct binder_alloc *alloc, |
| 1148 | void *dest, |
| 1149 | struct binder_buffer *buffer, |
| 1150 | binder_size_t buffer_offset, |
| 1151 | size_t bytes) |
| 1152 | { |
| 1153 | binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset, |
| 1154 | dest, bytes); |
| 1155 | } |
| 1156 | |