| /* |
| * |
| * sep_main.c - Security Processor Driver main group of functions |
| * |
| * Copyright(c) 2009-2011 Intel Corporation. All rights reserved. |
| * Contributions(c) 2009-2011 Discretix. All rights reserved. |
| * |
| * This program is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License as published by the Free |
| * Software Foundation; version 2 of the License. |
| * |
| * This program is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| * more details. |
| * |
| * You should have received a copy of the GNU General Public License along with |
| * this program; if not, write to the Free Software Foundation, Inc., 59 |
| * Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| * |
| * CONTACTS: |
| * |
| * Mark Allyn mark.a.allyn@intel.com |
| * Jayant Mangalampalli jayant.mangalampalli@intel.com |
| * |
| * CHANGES: |
| * |
| * 2009.06.26 Initial publish |
| * 2010.09.14 Upgrade to Medfield |
| * 2011.01.21 Move to sep_main.c to allow for sep_crypto.c |
| * 2011.02.22 Enable kernel crypto operation |
| * |
| * Please note that this driver is based on information in the Discretix |
| * CryptoCell 5.2 Driver Implementation Guide; the Discretix CryptoCell 5.2 |
| * Integration Intel Medfield appendix; the Discretix CryptoCell 5.2 |
| * Linux Driver Integration Guide; and the Discretix CryptoCell 5.2 System |
| * Overview and Integration Guide. |
| */ |
| /* #define DEBUG */ |
| /* #define SEP_PERF_DEBUG */ |
| |
| #include <linux/init.h> |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/miscdevice.h> |
| #include <linux/fs.h> |
| #include <linux/cdev.h> |
| #include <linux/kdev_t.h> |
| #include <linux/mutex.h> |
| #include <linux/sched.h> |
| #include <linux/mm.h> |
| #include <linux/poll.h> |
| #include <linux/wait.h> |
| #include <linux/pci.h> |
| #include <linux/pm_runtime.h> |
| #include <linux/slab.h> |
| #include <linux/ioctl.h> |
| #include <asm/current.h> |
| #include <linux/ioport.h> |
| #include <linux/io.h> |
| #include <linux/interrupt.h> |
| #include <linux/pagemap.h> |
| #include <asm/cacheflush.h> |
| #include <linux/delay.h> |
| #include <linux/jiffies.h> |
| #include <linux/async.h> |
| #include <linux/crypto.h> |
| #include <crypto/internal/hash.h> |
| #include <crypto/scatterwalk.h> |
| #include <crypto/sha.h> |
| #include <crypto/md5.h> |
| #include <crypto/aes.h> |
| #include <crypto/des.h> |
| #include <crypto/hash.h> |
| |
| #include "sep_driver_hw_defs.h" |
| #include "sep_driver_config.h" |
| #include "sep_driver_api.h" |
| #include "sep_dev.h" |
| #include "sep_crypto.h" |
| |
| #define CREATE_TRACE_POINTS |
| #include "sep_trace_events.h" |
| |
| /* |
| * Let's not spend cycles iterating over message |
| * area contents if debugging not enabled |
| */ |
| #ifdef DEBUG |
| #define sep_dump_message(sep) _sep_dump_message(sep) |
| #else |
| #define sep_dump_message(sep) |
| #endif |
| |
| /** |
| * Currently, there is only one SEP device per platform; |
| * In event platforms in the future have more than one SEP |
| * device, this will be a linked list |
| */ |
| |
| struct sep_device *sep_dev; |
| |
| /** |
| * sep_queue_status_remove - Removes transaction from status queue |
| * @sep: SEP device |
| * @sep_queue_info: pointer to status queue |
| * |
| * This function will remove information about transaction from the queue. |
| */ |
| void sep_queue_status_remove(struct sep_device *sep, |
| struct sep_queue_info **queue_elem) |
| { |
| unsigned long lck_flags; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_queue_status_remove\n", |
| current->pid); |
| |
| if (!queue_elem || !(*queue_elem)) { |
| dev_dbg(&sep->pdev->dev, "PID%d %s null\n", |
| current->pid, __func__); |
| return; |
| } |
| |
| spin_lock_irqsave(&sep->sep_queue_lock, lck_flags); |
| list_del(&(*queue_elem)->list); |
| sep->sep_queue_num--; |
| spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags); |
| |
| kfree(*queue_elem); |
| *queue_elem = NULL; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_queue_status_remove return\n", |
| current->pid); |
| return; |
| } |
| |
| /** |
| * sep_queue_status_add - Adds transaction to status queue |
| * @sep: SEP device |
| * @opcode: transaction opcode |
| * @size: input data size |
| * @pid: pid of current process |
| * @name: current process name |
| * @name_len: length of name (current process) |
| * |
| * This function adds information about about transaction started to the status |
| * queue. |
| */ |
| struct sep_queue_info *sep_queue_status_add( |
| struct sep_device *sep, |
| u32 opcode, |
| u32 size, |
| u32 pid, |
| u8 *name, size_t name_len) |
| { |
| unsigned long lck_flags; |
| struct sep_queue_info *my_elem = NULL; |
| |
| my_elem = kzalloc(sizeof(struct sep_queue_info), GFP_KERNEL); |
| |
| if (!my_elem) |
| return NULL; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] kzalloc ok\n", current->pid); |
| |
| my_elem->data.opcode = opcode; |
| my_elem->data.size = size; |
| my_elem->data.pid = pid; |
| |
| if (name_len > TASK_COMM_LEN) |
| name_len = TASK_COMM_LEN; |
| |
| memcpy(&my_elem->data.name, name, name_len); |
| |
| spin_lock_irqsave(&sep->sep_queue_lock, lck_flags); |
| |
| list_add_tail(&my_elem->list, &sep->sep_queue_status); |
| sep->sep_queue_num++; |
| |
| spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags); |
| |
| return my_elem; |
| } |
| |
| /** |
| * sep_allocate_dmatables_region - Allocates buf for the MLLI/DMA tables |
| * @sep: SEP device |
| * @dmatables_region: Destination pointer for the buffer |
| * @dma_ctx: DMA context for the transaction |
| * @table_count: Number of MLLI/DMA tables to create |
| * The buffer created will not work as-is for DMA operations, |
| * it needs to be copied over to the appropriate place in the |
| * shared area. |
| */ |
| static int sep_allocate_dmatables_region(struct sep_device *sep, |
| void **dmatables_region, |
| struct sep_dma_context *dma_ctx, |
| const u32 table_count) |
| { |
| const size_t new_len = |
| SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES - 1; |
| |
| void *tmp_region = NULL; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] dma_ctx = 0x%p\n", |
| current->pid, dma_ctx); |
| dev_dbg(&sep->pdev->dev, "[PID%d] dmatables_region = 0x%p\n", |
| current->pid, dmatables_region); |
| |
| if (!dma_ctx || !dmatables_region) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] dma context/region uninitialized\n", |
| current->pid); |
| return -EINVAL; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] newlen = 0x%08zX\n", |
| current->pid, new_len); |
| dev_dbg(&sep->pdev->dev, "[PID%d] oldlen = 0x%08X\n", current->pid, |
| dma_ctx->dmatables_len); |
| tmp_region = kzalloc(new_len + dma_ctx->dmatables_len, GFP_KERNEL); |
| if (!tmp_region) |
| return -ENOMEM; |
| |
| /* Were there any previous tables that need to be preserved ? */ |
| if (*dmatables_region) { |
| memcpy(tmp_region, *dmatables_region, dma_ctx->dmatables_len); |
| kfree(*dmatables_region); |
| *dmatables_region = NULL; |
| } |
| |
| *dmatables_region = tmp_region; |
| |
| dma_ctx->dmatables_len += new_len; |
| |
| return 0; |
| } |
| |
| /** |
| * sep_wait_transaction - Used for synchronizing transactions |
| * @sep: SEP device |
| */ |
| int sep_wait_transaction(struct sep_device *sep) |
| { |
| int error = 0; |
| DEFINE_WAIT(wait); |
| |
| if (0 == test_and_set_bit(SEP_TRANSACTION_STARTED_LOCK_BIT, |
| &sep->in_use_flags)) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] no transactions, returning\n", |
| current->pid); |
| goto end_function_setpid; |
| } |
| |
| /* |
| * Looping needed even for exclusive waitq entries |
| * due to process wakeup latencies, previous process |
| * might have already created another transaction. |
| */ |
| for (;;) { |
| /* |
| * Exclusive waitq entry, so that only one process is |
| * woken up from the queue at a time. |
| */ |
| prepare_to_wait_exclusive(&sep->event_transactions, |
| &wait, |
| TASK_INTERRUPTIBLE); |
| if (0 == test_and_set_bit(SEP_TRANSACTION_STARTED_LOCK_BIT, |
| &sep->in_use_flags)) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] no transactions, breaking\n", |
| current->pid); |
| break; |
| } |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] transactions ongoing, sleeping\n", |
| current->pid); |
| schedule(); |
| dev_dbg(&sep->pdev->dev, "[PID%d] woken up\n", current->pid); |
| |
| if (signal_pending(current)) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] received signal\n", |
| current->pid); |
| error = -EINTR; |
| goto end_function; |
| } |
| } |
| end_function_setpid: |
| /* |
| * The pid_doing_transaction indicates that this process |
| * now owns the facilities to perform a transaction with |
| * the SEP. While this process is performing a transaction, |
| * no other process who has the SEP device open can perform |
| * any transactions. This method allows more than one process |
| * to have the device open at any given time, which provides |
| * finer granularity for device utilization by multiple |
| * processes. |
| */ |
| /* Only one process is able to progress here at a time */ |
| sep->pid_doing_transaction = current->pid; |
| |
| end_function: |
| finish_wait(&sep->event_transactions, &wait); |
| |
| return error; |
| } |
| |
| /** |
| * sep_check_transaction_owner - Checks if current process owns transaction |
| * @sep: SEP device |
| */ |
| static inline int sep_check_transaction_owner(struct sep_device *sep) |
| { |
| dev_dbg(&sep->pdev->dev, "[PID%d] transaction pid = %d\n", |
| current->pid, |
| sep->pid_doing_transaction); |
| |
| if ((sep->pid_doing_transaction == 0) || |
| (current->pid != sep->pid_doing_transaction)) { |
| return -EACCES; |
| } |
| |
| /* We own the transaction */ |
| return 0; |
| } |
| |
| #ifdef DEBUG |
| |
| /** |
| * sep_dump_message - dump the message that is pending |
| * @sep: SEP device |
| * This will only print dump if DEBUG is set; it does |
| * follow kernel debug print enabling |
| */ |
| static void _sep_dump_message(struct sep_device *sep) |
| { |
| int count; |
| |
| u32 *p = sep->shared_addr; |
| |
| for (count = 0; count < 10 * 4; count += 4) |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] Word %d of the message is %x\n", |
| current->pid, count/4, *p++); |
| } |
| |
| #endif |
| |
| /** |
| * sep_map_and_alloc_shared_area -allocate shared block |
| * @sep: security processor |
| * @size: size of shared area |
| */ |
| static int sep_map_and_alloc_shared_area(struct sep_device *sep) |
| { |
| sep->shared_addr = dma_alloc_coherent(&sep->pdev->dev, |
| sep->shared_size, |
| &sep->shared_bus, GFP_KERNEL); |
| |
| if (!sep->shared_addr) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] shared memory dma_alloc_coherent failed\n", |
| current->pid); |
| return -ENOMEM; |
| } |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] shared_addr %zx bytes @%p (bus %llx)\n", |
| current->pid, |
| sep->shared_size, sep->shared_addr, |
| (unsigned long long)sep->shared_bus); |
| return 0; |
| } |
| |
| /** |
| * sep_unmap_and_free_shared_area - free shared block |
| * @sep: security processor |
| */ |
| static void sep_unmap_and_free_shared_area(struct sep_device *sep) |
| { |
| dma_free_coherent(&sep->pdev->dev, sep->shared_size, |
| sep->shared_addr, sep->shared_bus); |
| } |
| |
| #ifdef DEBUG |
| |
| /** |
| * sep_shared_bus_to_virt - convert bus/virt addresses |
| * @sep: pointer to struct sep_device |
| * @bus_address: address to convert |
| * |
| * Returns virtual address inside the shared area according |
| * to the bus address. |
| */ |
| static void *sep_shared_bus_to_virt(struct sep_device *sep, |
| dma_addr_t bus_address) |
| { |
| return sep->shared_addr + (bus_address - sep->shared_bus); |
| } |
| |
| #endif |
| |
| /** |
| * sep_open - device open method |
| * @inode: inode of SEP device |
| * @filp: file handle to SEP device |
| * |
| * Open method for the SEP device. Called when userspace opens |
| * the SEP device node. |
| * |
| * Returns zero on success otherwise an error code. |
| */ |
| static int sep_open(struct inode *inode, struct file *filp) |
| { |
| struct sep_device *sep; |
| struct sep_private_data *priv; |
| |
| dev_dbg(&sep_dev->pdev->dev, "[PID%d] open\n", current->pid); |
| |
| if (filp->f_flags & O_NONBLOCK) |
| return -ENOTSUPP; |
| |
| /* |
| * Get the SEP device structure and use it for the |
| * private_data field in filp for other methods |
| */ |
| |
| priv = kzalloc(sizeof(*priv), GFP_KERNEL); |
| if (!priv) |
| return -ENOMEM; |
| |
| sep = sep_dev; |
| priv->device = sep; |
| filp->private_data = priv; |
| |
| dev_dbg(&sep_dev->pdev->dev, "[PID%d] priv is 0x%p\n", |
| current->pid, priv); |
| |
| /* Anyone can open; locking takes place at transaction level */ |
| return 0; |
| } |
| |
| /** |
| * sep_free_dma_table_data_handler - free DMA table |
| * @sep: pointer to struct sep_device |
| * @dma_ctx: dma context |
| * |
| * Handles the request to free DMA table for synchronic actions |
| */ |
| int sep_free_dma_table_data_handler(struct sep_device *sep, |
| struct sep_dma_context **dma_ctx) |
| { |
| int count; |
| int dcb_counter; |
| /* Pointer to the current dma_resource struct */ |
| struct sep_dma_resource *dma; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] sep_free_dma_table_data_handler\n", |
| current->pid); |
| |
| if (!dma_ctx || !(*dma_ctx)) { |
| /* No context or context already freed */ |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] no DMA context or context already freed\n", |
| current->pid); |
| |
| return 0; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] (*dma_ctx)->nr_dcb_creat 0x%x\n", |
| current->pid, |
| (*dma_ctx)->nr_dcb_creat); |
| |
| for (dcb_counter = 0; |
| dcb_counter < (*dma_ctx)->nr_dcb_creat; dcb_counter++) { |
| dma = &(*dma_ctx)->dma_res_arr[dcb_counter]; |
| |
| /* Unmap and free input map array */ |
| if (dma->in_map_array) { |
| for (count = 0; count < dma->in_num_pages; count++) { |
| dma_unmap_page(&sep->pdev->dev, |
| dma->in_map_array[count].dma_addr, |
| dma->in_map_array[count].size, |
| DMA_TO_DEVICE); |
| } |
| kfree(dma->in_map_array); |
| } |
| |
| /** |
| * Output is handled different. If |
| * this was a secure dma into restricted memory, |
| * then we skip this step altogether as restricted |
| * memory is not available to the o/s at all. |
| */ |
| if (((*dma_ctx)->secure_dma == false) && |
| (dma->out_map_array)) { |
| |
| for (count = 0; count < dma->out_num_pages; count++) { |
| dma_unmap_page(&sep->pdev->dev, |
| dma->out_map_array[count].dma_addr, |
| dma->out_map_array[count].size, |
| DMA_FROM_DEVICE); |
| } |
| kfree(dma->out_map_array); |
| } |
| |
| /* Free page cache for output */ |
| if (dma->in_page_array) { |
| for (count = 0; count < dma->in_num_pages; count++) { |
| flush_dcache_page(dma->in_page_array[count]); |
| page_cache_release(dma->in_page_array[count]); |
| } |
| kfree(dma->in_page_array); |
| } |
| |
| /* Again, we do this only for non secure dma */ |
| if (((*dma_ctx)->secure_dma == false) && |
| (dma->out_page_array)) { |
| |
| for (count = 0; count < dma->out_num_pages; count++) { |
| if (!PageReserved(dma->out_page_array[count])) |
| |
| SetPageDirty(dma-> |
| out_page_array[count]); |
| |
| flush_dcache_page(dma->out_page_array[count]); |
| page_cache_release(dma->out_page_array[count]); |
| } |
| kfree(dma->out_page_array); |
| } |
| |
| /** |
| * Note that here we use in_map_num_entries because we |
| * don't have a page array; the page array is generated |
| * only in the lock_user_pages, which is not called |
| * for kernel crypto, which is what the sg (scatter gather |
| * is used for exclusively) |
| */ |
| if (dma->src_sg) { |
| dma_unmap_sg(&sep->pdev->dev, dma->src_sg, |
| dma->in_map_num_entries, DMA_TO_DEVICE); |
| dma->src_sg = NULL; |
| } |
| |
| if (dma->dst_sg) { |
| dma_unmap_sg(&sep->pdev->dev, dma->dst_sg, |
| dma->in_map_num_entries, DMA_FROM_DEVICE); |
| dma->dst_sg = NULL; |
| } |
| |
| /* Reset all the values */ |
| dma->in_page_array = NULL; |
| dma->out_page_array = NULL; |
| dma->in_num_pages = 0; |
| dma->out_num_pages = 0; |
| dma->in_map_array = NULL; |
| dma->out_map_array = NULL; |
| dma->in_map_num_entries = 0; |
| dma->out_map_num_entries = 0; |
| } |
| |
| (*dma_ctx)->nr_dcb_creat = 0; |
| (*dma_ctx)->num_lli_tables_created = 0; |
| |
| kfree(*dma_ctx); |
| *dma_ctx = NULL; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] sep_free_dma_table_data_handler end\n", |
| current->pid); |
| |
| return 0; |
| } |
| |
| /** |
| * sep_end_transaction_handler - end transaction |
| * @sep: pointer to struct sep_device |
| * @dma_ctx: DMA context |
| * @call_status: Call status |
| * |
| * This API handles the end transaction request. |
| */ |
| static int sep_end_transaction_handler(struct sep_device *sep, |
| struct sep_dma_context **dma_ctx, |
| struct sep_call_status *call_status, |
| struct sep_queue_info **my_queue_elem) |
| { |
| dev_dbg(&sep->pdev->dev, "[PID%d] ending transaction\n", current->pid); |
| |
| /* |
| * Extraneous transaction clearing would mess up PM |
| * device usage counters and SEP would get suspended |
| * just before we send a command to SEP in the next |
| * transaction |
| * */ |
| if (sep_check_transaction_owner(sep)) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] not transaction owner\n", |
| current->pid); |
| return 0; |
| } |
| |
| /* Update queue status */ |
| sep_queue_status_remove(sep, my_queue_elem); |
| |
| /* Check that all the DMA resources were freed */ |
| if (dma_ctx) |
| sep_free_dma_table_data_handler(sep, dma_ctx); |
| |
| /* Reset call status for next transaction */ |
| if (call_status) |
| call_status->status = 0; |
| |
| /* Clear the message area to avoid next transaction reading |
| * sensitive results from previous transaction */ |
| memset(sep->shared_addr, 0, |
| SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES); |
| |
| /* start suspend delay */ |
| #ifdef SEP_ENABLE_RUNTIME_PM |
| if (sep->in_use) { |
| sep->in_use = 0; |
| pm_runtime_mark_last_busy(&sep->pdev->dev); |
| pm_runtime_put_autosuspend(&sep->pdev->dev); |
| } |
| #endif |
| |
| clear_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags); |
| sep->pid_doing_transaction = 0; |
| |
| /* Now it's safe for next process to proceed */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] waking up next transaction\n", |
| current->pid); |
| clear_bit(SEP_TRANSACTION_STARTED_LOCK_BIT, &sep->in_use_flags); |
| wake_up(&sep->event_transactions); |
| |
| return 0; |
| } |
| |
| |
| /** |
| * sep_release - close a SEP device |
| * @inode: inode of SEP device |
| * @filp: file handle being closed |
| * |
| * Called on the final close of a SEP device. |
| */ |
| static int sep_release(struct inode *inode, struct file *filp) |
| { |
| struct sep_private_data * const private_data = filp->private_data; |
| struct sep_call_status *call_status = &private_data->call_status; |
| struct sep_device *sep = private_data->device; |
| struct sep_dma_context **dma_ctx = &private_data->dma_ctx; |
| struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] release\n", current->pid); |
| |
| sep_end_transaction_handler(sep, dma_ctx, call_status, |
| my_queue_elem); |
| |
| kfree(filp->private_data); |
| |
| return 0; |
| } |
| |
| /** |
| * sep_mmap - maps the shared area to user space |
| * @filp: pointer to struct file |
| * @vma: pointer to vm_area_struct |
| * |
| * Called on an mmap of our space via the normal SEP device |
| */ |
| static int sep_mmap(struct file *filp, struct vm_area_struct *vma) |
| { |
| struct sep_private_data * const private_data = filp->private_data; |
| struct sep_call_status *call_status = &private_data->call_status; |
| struct sep_device *sep = private_data->device; |
| struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem; |
| dma_addr_t bus_addr; |
| unsigned long error = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_mmap\n", current->pid); |
| |
| /* Set the transaction busy (own the device) */ |
| /* |
| * Problem for multithreaded applications is that here we're |
| * possibly going to sleep while holding a write lock on |
| * current->mm->mmap_sem, which will cause deadlock for ongoing |
| * transaction trying to create DMA tables |
| */ |
| error = sep_wait_transaction(sep); |
| if (error) |
| /* Interrupted by signal, don't clear transaction */ |
| goto end_function; |
| |
| /* Clear the message area to avoid next transaction reading |
| * sensitive results from previous transaction */ |
| memset(sep->shared_addr, 0, |
| SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES); |
| |
| /* |
| * Check that the size of the mapped range is as the size of the message |
| * shared area |
| */ |
| if ((vma->vm_end - vma->vm_start) > SEP_DRIVER_MMMAP_AREA_SIZE) { |
| error = -EINVAL; |
| goto end_function_with_error; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] shared_addr is %p\n", |
| current->pid, sep->shared_addr); |
| |
| /* Get bus address */ |
| bus_addr = sep->shared_bus; |
| |
| if (remap_pfn_range(vma, vma->vm_start, bus_addr >> PAGE_SHIFT, |
| vma->vm_end - vma->vm_start, vma->vm_page_prot)) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] remap_pfn_range failed\n", |
| current->pid); |
| error = -EAGAIN; |
| goto end_function_with_error; |
| } |
| |
| /* Update call status */ |
| set_bit(SEP_LEGACY_MMAP_DONE_OFFSET, &call_status->status); |
| |
| goto end_function; |
| |
| end_function_with_error: |
| /* Clear our transaction */ |
| sep_end_transaction_handler(sep, NULL, call_status, |
| my_queue_elem); |
| |
| end_function: |
| return error; |
| } |
| |
| /** |
| * sep_poll - poll handler |
| * @filp: pointer to struct file |
| * @wait: pointer to poll_table |
| * |
| * Called by the OS when the kernel is asked to do a poll on |
| * a SEP file handle. |
| */ |
| static unsigned int sep_poll(struct file *filp, poll_table *wait) |
| { |
| struct sep_private_data * const private_data = filp->private_data; |
| struct sep_call_status *call_status = &private_data->call_status; |
| struct sep_device *sep = private_data->device; |
| u32 mask = 0; |
| u32 retval = 0; |
| u32 retval2 = 0; |
| unsigned long lock_irq_flag; |
| |
| /* Am I the process that owns the transaction? */ |
| if (sep_check_transaction_owner(sep)) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] poll pid not owner\n", |
| current->pid); |
| mask = POLLERR; |
| goto end_function; |
| } |
| |
| /* Check if send command or send_reply were activated previously */ |
| if (0 == test_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET, |
| &call_status->status)) { |
| dev_warn(&sep->pdev->dev, "[PID%d] sendmsg not called\n", |
| current->pid); |
| mask = POLLERR; |
| goto end_function; |
| } |
| |
| |
| /* Add the event to the polling wait table */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] poll: calling wait sep_event\n", |
| current->pid); |
| |
| poll_wait(filp, &sep->event_interrupt, wait); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] poll: send_ct is %lx reply ct is %lx\n", |
| current->pid, sep->send_ct, sep->reply_ct); |
| |
| /* Check if error occurred during poll */ |
| retval2 = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR3_REG_ADDR); |
| if ((retval2 != 0x0) && (retval2 != 0x8)) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] poll; poll error %x\n", |
| current->pid, retval2); |
| mask |= POLLERR; |
| goto end_function; |
| } |
| |
| spin_lock_irqsave(&sep->snd_rply_lck, lock_irq_flag); |
| |
| if (sep->send_ct == sep->reply_ct) { |
| spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag); |
| retval = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] poll: data ready check (GPR2) %x\n", |
| current->pid, retval); |
| |
| /* Check if printf request */ |
| if ((retval >> 30) & 0x1) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] poll: SEP printf request\n", |
| current->pid); |
| goto end_function; |
| } |
| |
| /* Check if the this is SEP reply or request */ |
| if (retval >> 31) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] poll: SEP request\n", |
| current->pid); |
| } else { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] poll: normal return\n", |
| current->pid); |
| sep_dump_message(sep); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] poll; SEP reply POLLIN|POLLRDNORM\n", |
| current->pid); |
| mask |= POLLIN | POLLRDNORM; |
| } |
| set_bit(SEP_LEGACY_POLL_DONE_OFFSET, &call_status->status); |
| } else { |
| spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] poll; no reply; returning mask of 0\n", |
| current->pid); |
| mask = 0; |
| } |
| |
| end_function: |
| return mask; |
| } |
| |
| /** |
| * sep_time_address - address in SEP memory of time |
| * @sep: SEP device we want the address from |
| * |
| * Return the address of the two dwords in memory used for time |
| * setting. |
| */ |
| static u32 *sep_time_address(struct sep_device *sep) |
| { |
| return sep->shared_addr + |
| SEP_DRIVER_SYSTEM_TIME_MEMORY_OFFSET_IN_BYTES; |
| } |
| |
| /** |
| * sep_set_time - set the SEP time |
| * @sep: the SEP we are setting the time for |
| * |
| * Calculates time and sets it at the predefined address. |
| * Called with the SEP mutex held. |
| */ |
| static unsigned long sep_set_time(struct sep_device *sep) |
| { |
| struct timeval time; |
| u32 *time_addr; /* Address of time as seen by the kernel */ |
| |
| |
| do_gettimeofday(&time); |
| |
| /* Set value in the SYSTEM MEMORY offset */ |
| time_addr = sep_time_address(sep); |
| |
| time_addr[0] = SEP_TIME_VAL_TOKEN; |
| time_addr[1] = time.tv_sec; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] time.tv_sec is %lu\n", |
| current->pid, time.tv_sec); |
| dev_dbg(&sep->pdev->dev, "[PID%d] time_addr is %p\n", |
| current->pid, time_addr); |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep->shared_addr is %p\n", |
| current->pid, sep->shared_addr); |
| |
| return time.tv_sec; |
| } |
| |
| /** |
| * sep_send_command_handler - kick off a command |
| * @sep: SEP being signalled |
| * |
| * This function raises interrupt to SEP that signals that is has a new |
| * command from the host |
| * |
| * Note that this function does fall under the ioctl lock |
| */ |
| int sep_send_command_handler(struct sep_device *sep) |
| { |
| unsigned long lock_irq_flag; |
| u32 *msg_pool; |
| int error = 0; |
| |
| /* Basic sanity check; set msg pool to start of shared area */ |
| msg_pool = (u32 *)sep->shared_addr; |
| msg_pool += 2; |
| |
| /* Look for start msg token */ |
| if (*msg_pool != SEP_START_MSG_TOKEN) { |
| dev_warn(&sep->pdev->dev, "start message token not present\n"); |
| error = -EPROTO; |
| goto end_function; |
| } |
| |
| /* Do we have a reasonable size? */ |
| msg_pool += 1; |
| if ((*msg_pool < 2) || |
| (*msg_pool > SEP_DRIVER_MAX_MESSAGE_SIZE_IN_BYTES)) { |
| |
| dev_warn(&sep->pdev->dev, "invalid message size\n"); |
| error = -EPROTO; |
| goto end_function; |
| } |
| |
| /* Does the command look reasonable? */ |
| msg_pool += 1; |
| if (*msg_pool < 2) { |
| dev_warn(&sep->pdev->dev, "invalid message opcode\n"); |
| error = -EPROTO; |
| goto end_function; |
| } |
| |
| #if defined(CONFIG_PM_RUNTIME) && defined(SEP_ENABLE_RUNTIME_PM) |
| dev_dbg(&sep->pdev->dev, "[PID%d] before pm sync status 0x%X\n", |
| current->pid, |
| sep->pdev->dev.power.runtime_status); |
| sep->in_use = 1; /* device is about to be used */ |
| pm_runtime_get_sync(&sep->pdev->dev); |
| #endif |
| |
| if (test_and_set_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags)) { |
| error = -EPROTO; |
| goto end_function; |
| } |
| sep->in_use = 1; /* device is about to be used */ |
| sep_set_time(sep); |
| |
| sep_dump_message(sep); |
| |
| /* Update counter */ |
| spin_lock_irqsave(&sep->snd_rply_lck, lock_irq_flag); |
| sep->send_ct++; |
| spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] sep_send_command_handler send_ct %lx reply_ct %lx\n", |
| current->pid, sep->send_ct, sep->reply_ct); |
| |
| /* Send interrupt to SEP */ |
| sep_write_reg(sep, HW_HOST_HOST_SEP_GPR0_REG_ADDR, 0x2); |
| |
| end_function: |
| return error; |
| } |
| |
| /** |
| * sep_crypto_dma - |
| * @sep: pointer to struct sep_device |
| * @sg: pointer to struct scatterlist |
| * @direction: |
| * @dma_maps: pointer to place a pointer to array of dma maps |
| * This is filled in; anything previous there will be lost |
| * The structure for dma maps is sep_dma_map |
| * @returns number of dma maps on success; negative on error |
| * |
| * This creates the dma table from the scatterlist |
| * It is used only for kernel crypto as it works with scatterlists |
| * representation of data buffers |
| * |
| */ |
| static int sep_crypto_dma( |
| struct sep_device *sep, |
| struct scatterlist *sg, |
| struct sep_dma_map **dma_maps, |
| enum dma_data_direction direction) |
| { |
| struct scatterlist *temp_sg; |
| |
| u32 count_segment; |
| u32 count_mapped; |
| struct sep_dma_map *sep_dma; |
| int ct1; |
| |
| if (sg->length == 0) |
| return 0; |
| |
| /* Count the segments */ |
| temp_sg = sg; |
| count_segment = 0; |
| while (temp_sg) { |
| count_segment += 1; |
| temp_sg = scatterwalk_sg_next(temp_sg); |
| } |
| dev_dbg(&sep->pdev->dev, |
| "There are (hex) %x segments in sg\n", count_segment); |
| |
| /* DMA map segments */ |
| count_mapped = dma_map_sg(&sep->pdev->dev, sg, |
| count_segment, direction); |
| |
| dev_dbg(&sep->pdev->dev, |
| "There are (hex) %x maps in sg\n", count_mapped); |
| |
| if (count_mapped == 0) { |
| dev_dbg(&sep->pdev->dev, "Cannot dma_map_sg\n"); |
| return -ENOMEM; |
| } |
| |
| sep_dma = kmalloc(sizeof(struct sep_dma_map) * |
| count_mapped, GFP_ATOMIC); |
| |
| if (sep_dma == NULL) { |
| dev_dbg(&sep->pdev->dev, "Cannot allocate dma_maps\n"); |
| return -ENOMEM; |
| } |
| |
| for_each_sg(sg, temp_sg, count_mapped, ct1) { |
| sep_dma[ct1].dma_addr = sg_dma_address(temp_sg); |
| sep_dma[ct1].size = sg_dma_len(temp_sg); |
| dev_dbg(&sep->pdev->dev, "(all hex) map %x dma %lx len %lx\n", |
| ct1, (unsigned long)sep_dma[ct1].dma_addr, |
| (unsigned long)sep_dma[ct1].size); |
| } |
| |
| *dma_maps = sep_dma; |
| return count_mapped; |
| |
| } |
| |
| /** |
| * sep_crypto_lli - |
| * @sep: pointer to struct sep_device |
| * @sg: pointer to struct scatterlist |
| * @data_size: total data size |
| * @direction: |
| * @dma_maps: pointer to place a pointer to array of dma maps |
| * This is filled in; anything previous there will be lost |
| * The structure for dma maps is sep_dma_map |
| * @lli_maps: pointer to place a pointer to array of lli maps |
| * This is filled in; anything previous there will be lost |
| * The structure for dma maps is sep_dma_map |
| * @returns number of dma maps on success; negative on error |
| * |
| * This creates the LLI table from the scatterlist |
| * It is only used for kernel crypto as it works exclusively |
| * with scatterlists (struct scatterlist) representation of |
| * data buffers |
| */ |
| static int sep_crypto_lli( |
| struct sep_device *sep, |
| struct scatterlist *sg, |
| struct sep_dma_map **maps, |
| struct sep_lli_entry **llis, |
| u32 data_size, |
| enum dma_data_direction direction) |
| { |
| |
| int ct1; |
| struct sep_lli_entry *sep_lli; |
| struct sep_dma_map *sep_map; |
| |
| int nbr_ents; |
| |
| nbr_ents = sep_crypto_dma(sep, sg, maps, direction); |
| if (nbr_ents <= 0) { |
| dev_dbg(&sep->pdev->dev, "crypto_dma failed %x\n", |
| nbr_ents); |
| return nbr_ents; |
| } |
| |
| sep_map = *maps; |
| |
| sep_lli = kmalloc(sizeof(struct sep_lli_entry) * nbr_ents, GFP_ATOMIC); |
| |
| if (sep_lli == NULL) { |
| dev_dbg(&sep->pdev->dev, "Cannot allocate lli_maps\n"); |
| |
| kfree(*maps); |
| *maps = NULL; |
| return -ENOMEM; |
| } |
| |
| for (ct1 = 0; ct1 < nbr_ents; ct1 += 1) { |
| sep_lli[ct1].bus_address = (u32)sep_map[ct1].dma_addr; |
| |
| /* Maximum for page is total data size */ |
| if (sep_map[ct1].size > data_size) |
| sep_map[ct1].size = data_size; |
| |
| sep_lli[ct1].block_size = (u32)sep_map[ct1].size; |
| } |
| |
| *llis = sep_lli; |
| return nbr_ents; |
| } |
| |
| /** |
| * sep_lock_kernel_pages - map kernel pages for DMA |
| * @sep: pointer to struct sep_device |
| * @kernel_virt_addr: address of data buffer in kernel |
| * @data_size: size of data |
| * @lli_array_ptr: lli array |
| * @in_out_flag: input into device or output from device |
| * |
| * This function locks all the physical pages of the kernel virtual buffer |
| * and construct a basic lli array, where each entry holds the physical |
| * page address and the size that application data holds in this page |
| * This function is used only during kernel crypto mod calls from within |
| * the kernel (when ioctl is not used) |
| * |
| * This is used only for kernel crypto. Kernel pages |
| * are handled differently as they are done via |
| * scatter gather lists (struct scatterlist) |
| */ |
| static int sep_lock_kernel_pages(struct sep_device *sep, |
| unsigned long kernel_virt_addr, |
| u32 data_size, |
| struct sep_lli_entry **lli_array_ptr, |
| int in_out_flag, |
| struct sep_dma_context *dma_ctx) |
| |
| { |
| u32 num_pages; |
| struct scatterlist *sg; |
| |
| /* Array of lli */ |
| struct sep_lli_entry *lli_array; |
| /* Map array */ |
| struct sep_dma_map *map_array; |
| |
| enum dma_data_direction direction; |
| |
| lli_array = NULL; |
| map_array = NULL; |
| |
| if (in_out_flag == SEP_DRIVER_IN_FLAG) { |
| direction = DMA_TO_DEVICE; |
| sg = dma_ctx->src_sg; |
| } else { |
| direction = DMA_FROM_DEVICE; |
| sg = dma_ctx->dst_sg; |
| } |
| |
| num_pages = sep_crypto_lli(sep, sg, &map_array, &lli_array, |
| data_size, direction); |
| |
| if (num_pages <= 0) { |
| dev_dbg(&sep->pdev->dev, "sep_crypto_lli returned error %x\n", |
| num_pages); |
| return -ENOMEM; |
| } |
| |
| /* Put mapped kernel sg into kernel resource array */ |
| |
| /* Set output params according to the in_out flag */ |
| if (in_out_flag == SEP_DRIVER_IN_FLAG) { |
| *lli_array_ptr = lli_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages = |
| num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = |
| NULL; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array = |
| map_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_num_entries = |
| num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].src_sg = |
| dma_ctx->src_sg; |
| } else { |
| *lli_array_ptr = lli_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages = |
| num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = |
| NULL; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array = |
| map_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat]. |
| out_map_num_entries = num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].dst_sg = |
| dma_ctx->dst_sg; |
| } |
| |
| return 0; |
| } |
| |
| /** |
| * sep_lock_user_pages - lock and map user pages for DMA |
| * @sep: pointer to struct sep_device |
| * @app_virt_addr: user memory data buffer |
| * @data_size: size of data buffer |
| * @lli_array_ptr: lli array |
| * @in_out_flag: input or output to device |
| * |
| * This function locks all the physical pages of the application |
| * virtual buffer and construct a basic lli array, where each entry |
| * holds the physical page address and the size that application |
| * data holds in this physical pages |
| */ |
| static int sep_lock_user_pages(struct sep_device *sep, |
| u32 app_virt_addr, |
| u32 data_size, |
| struct sep_lli_entry **lli_array_ptr, |
| int in_out_flag, |
| struct sep_dma_context *dma_ctx) |
| |
| { |
| int error = 0; |
| u32 count; |
| int result; |
| /* The the page of the end address of the user space buffer */ |
| u32 end_page; |
| /* The page of the start address of the user space buffer */ |
| u32 start_page; |
| /* The range in pages */ |
| u32 num_pages; |
| /* Array of pointers to page */ |
| struct page **page_array; |
| /* Array of lli */ |
| struct sep_lli_entry *lli_array; |
| /* Map array */ |
| struct sep_dma_map *map_array; |
| |
| /* Set start and end pages and num pages */ |
| end_page = (app_virt_addr + data_size - 1) >> PAGE_SHIFT; |
| start_page = app_virt_addr >> PAGE_SHIFT; |
| num_pages = end_page - start_page + 1; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lock user pages app_virt_addr is %x\n", |
| current->pid, app_virt_addr); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] data_size is (hex) %x\n", |
| current->pid, data_size); |
| dev_dbg(&sep->pdev->dev, "[PID%d] start_page is (hex) %x\n", |
| current->pid, start_page); |
| dev_dbg(&sep->pdev->dev, "[PID%d] end_page is (hex) %x\n", |
| current->pid, end_page); |
| dev_dbg(&sep->pdev->dev, "[PID%d] num_pages is (hex) %x\n", |
| current->pid, num_pages); |
| |
| /* Allocate array of pages structure pointers */ |
| page_array = kmalloc_array(num_pages, sizeof(struct page *), |
| GFP_ATOMIC); |
| if (!page_array) { |
| error = -ENOMEM; |
| goto end_function; |
| } |
| |
| map_array = kmalloc_array(num_pages, sizeof(struct sep_dma_map), |
| GFP_ATOMIC); |
| if (!map_array) { |
| error = -ENOMEM; |
| goto end_function_with_error1; |
| } |
| |
| lli_array = kmalloc_array(num_pages, sizeof(struct sep_lli_entry), |
| GFP_ATOMIC); |
| if (!lli_array) { |
| error = -ENOMEM; |
| goto end_function_with_error2; |
| } |
| |
| /* Convert the application virtual address into a set of physical */ |
| down_read(¤t->mm->mmap_sem); |
| result = get_user_pages(current, current->mm, app_virt_addr, |
| num_pages, |
| ((in_out_flag == SEP_DRIVER_IN_FLAG) ? 0 : 1), |
| 0, page_array, NULL); |
| |
| up_read(¤t->mm->mmap_sem); |
| |
| /* Check the number of pages locked - if not all then exit with error */ |
| if (result != num_pages) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] not all pages locked by get_user_pages, " |
| "result 0x%X, num_pages 0x%X\n", |
| current->pid, result, num_pages); |
| error = -ENOMEM; |
| goto end_function_with_error3; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] get_user_pages succeeded\n", |
| current->pid); |
| |
| /* |
| * Fill the array using page array data and |
| * map the pages - this action will also flush the cache as needed |
| */ |
| for (count = 0; count < num_pages; count++) { |
| /* Fill the map array */ |
| map_array[count].dma_addr = |
| dma_map_page(&sep->pdev->dev, page_array[count], |
| 0, PAGE_SIZE, DMA_BIDIRECTIONAL); |
| |
| map_array[count].size = PAGE_SIZE; |
| |
| /* Fill the lli array entry */ |
| lli_array[count].bus_address = (u32)map_array[count].dma_addr; |
| lli_array[count].block_size = PAGE_SIZE; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_array[%x].bus_address is %08lx, " |
| "lli_array[%x].block_size is (hex) %x\n", current->pid, |
| count, (unsigned long)lli_array[count].bus_address, |
| count, lli_array[count].block_size); |
| } |
| |
| /* Check the offset for the first page */ |
| lli_array[0].bus_address = |
| lli_array[0].bus_address + (app_virt_addr & (~PAGE_MASK)); |
| |
| /* Check that not all the data is in the first page only */ |
| if ((PAGE_SIZE - (app_virt_addr & (~PAGE_MASK))) >= data_size) |
| lli_array[0].block_size = data_size; |
| else |
| lli_array[0].block_size = |
| PAGE_SIZE - (app_virt_addr & (~PAGE_MASK)); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] After check if page 0 has all data\n", |
| current->pid); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_array[0].bus_address is (hex) %08lx, " |
| "lli_array[0].block_size is (hex) %x\n", |
| current->pid, |
| (unsigned long)lli_array[0].bus_address, |
| lli_array[0].block_size); |
| |
| |
| /* Check the size of the last page */ |
| if (num_pages > 1) { |
| lli_array[num_pages - 1].block_size = |
| (app_virt_addr + data_size) & (~PAGE_MASK); |
| if (lli_array[num_pages - 1].block_size == 0) |
| lli_array[num_pages - 1].block_size = PAGE_SIZE; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] After last page size adjustment\n", |
| current->pid); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_array[%x].bus_address is (hex) %08lx, " |
| "lli_array[%x].block_size is (hex) %x\n", |
| current->pid, |
| num_pages - 1, |
| (unsigned long)lli_array[num_pages - 1].bus_address, |
| num_pages - 1, |
| lli_array[num_pages - 1].block_size); |
| } |
| |
| /* Set output params according to the in_out flag */ |
| if (in_out_flag == SEP_DRIVER_IN_FLAG) { |
| *lli_array_ptr = lli_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages = |
| num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = |
| page_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array = |
| map_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_num_entries = |
| num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].src_sg = NULL; |
| } else { |
| *lli_array_ptr = lli_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages = |
| num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = |
| page_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array = |
| map_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat]. |
| out_map_num_entries = num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].dst_sg = NULL; |
| } |
| goto end_function; |
| |
| end_function_with_error3: |
| /* Free lli array */ |
| kfree(lli_array); |
| |
| end_function_with_error2: |
| kfree(map_array); |
| |
| end_function_with_error1: |
| /* Free page array */ |
| kfree(page_array); |
| |
| end_function: |
| return error; |
| } |
| |
| /** |
| * sep_lli_table_secure_dma - get lli array for IMR addresses |
| * @sep: pointer to struct sep_device |
| * @app_virt_addr: user memory data buffer |
| * @data_size: size of data buffer |
| * @lli_array_ptr: lli array |
| * @in_out_flag: not used |
| * @dma_ctx: pointer to struct sep_dma_context |
| * |
| * This function creates lli tables for outputting data to |
| * IMR memory, which is memory that cannot be accessed by the |
| * the x86 processor. |
| */ |
| static int sep_lli_table_secure_dma(struct sep_device *sep, |
| u32 app_virt_addr, |
| u32 data_size, |
| struct sep_lli_entry **lli_array_ptr, |
| int in_out_flag, |
| struct sep_dma_context *dma_ctx) |
| |
| { |
| int error = 0; |
| u32 count; |
| /* The the page of the end address of the user space buffer */ |
| u32 end_page; |
| /* The page of the start address of the user space buffer */ |
| u32 start_page; |
| /* The range in pages */ |
| u32 num_pages; |
| /* Array of lli */ |
| struct sep_lli_entry *lli_array; |
| |
| /* Set start and end pages and num pages */ |
| end_page = (app_virt_addr + data_size - 1) >> PAGE_SHIFT; |
| start_page = app_virt_addr >> PAGE_SHIFT; |
| num_pages = end_page - start_page + 1; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lock user pages app_virt_addr is %x\n", |
| current->pid, app_virt_addr); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] data_size is (hex) %x\n", |
| current->pid, data_size); |
| dev_dbg(&sep->pdev->dev, "[PID%d] start_page is (hex) %x\n", |
| current->pid, start_page); |
| dev_dbg(&sep->pdev->dev, "[PID%d] end_page is (hex) %x\n", |
| current->pid, end_page); |
| dev_dbg(&sep->pdev->dev, "[PID%d] num_pages is (hex) %x\n", |
| current->pid, num_pages); |
| |
| lli_array = kmalloc_array(num_pages, sizeof(struct sep_lli_entry), |
| GFP_ATOMIC); |
| if (!lli_array) |
| return -ENOMEM; |
| |
| /* |
| * Fill the lli_array |
| */ |
| start_page = start_page << PAGE_SHIFT; |
| for (count = 0; count < num_pages; count++) { |
| /* Fill the lli array entry */ |
| lli_array[count].bus_address = start_page; |
| lli_array[count].block_size = PAGE_SIZE; |
| |
| start_page += PAGE_SIZE; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_array[%x].bus_address is %08lx, " |
| "lli_array[%x].block_size is (hex) %x\n", |
| current->pid, |
| count, (unsigned long)lli_array[count].bus_address, |
| count, lli_array[count].block_size); |
| } |
| |
| /* Check the offset for the first page */ |
| lli_array[0].bus_address = |
| lli_array[0].bus_address + (app_virt_addr & (~PAGE_MASK)); |
| |
| /* Check that not all the data is in the first page only */ |
| if ((PAGE_SIZE - (app_virt_addr & (~PAGE_MASK))) >= data_size) |
| lli_array[0].block_size = data_size; |
| else |
| lli_array[0].block_size = |
| PAGE_SIZE - (app_virt_addr & (~PAGE_MASK)); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] After check if page 0 has all data\n" |
| "lli_array[0].bus_address is (hex) %08lx, " |
| "lli_array[0].block_size is (hex) %x\n", |
| current->pid, |
| (unsigned long)lli_array[0].bus_address, |
| lli_array[0].block_size); |
| |
| /* Check the size of the last page */ |
| if (num_pages > 1) { |
| lli_array[num_pages - 1].block_size = |
| (app_virt_addr + data_size) & (~PAGE_MASK); |
| if (lli_array[num_pages - 1].block_size == 0) |
| lli_array[num_pages - 1].block_size = PAGE_SIZE; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] After last page size adjustment\n" |
| "lli_array[%x].bus_address is (hex) %08lx, " |
| "lli_array[%x].block_size is (hex) %x\n", |
| current->pid, num_pages - 1, |
| (unsigned long)lli_array[num_pages - 1].bus_address, |
| num_pages - 1, |
| lli_array[num_pages - 1].block_size); |
| } |
| *lli_array_ptr = lli_array; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages = num_pages; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = NULL; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array = NULL; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_num_entries = 0; |
| |
| return error; |
| } |
| |
| /** |
| * sep_calculate_lli_table_max_size - size the LLI table |
| * @sep: pointer to struct sep_device |
| * @lli_in_array_ptr |
| * @num_array_entries |
| * @last_table_flag |
| * |
| * This function calculates the size of data that can be inserted into |
| * the lli table from this array, such that either the table is full |
| * (all entries are entered), or there are no more entries in the |
| * lli array |
| */ |
| static u32 sep_calculate_lli_table_max_size(struct sep_device *sep, |
| struct sep_lli_entry *lli_in_array_ptr, |
| u32 num_array_entries, |
| u32 *last_table_flag) |
| { |
| u32 counter; |
| /* Table data size */ |
| u32 table_data_size = 0; |
| /* Data size for the next table */ |
| u32 next_table_data_size; |
| |
| *last_table_flag = 0; |
| |
| /* |
| * Calculate the data in the out lli table till we fill the whole |
| * table or till the data has ended |
| */ |
| for (counter = 0; |
| (counter < (SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP - 1)) && |
| (counter < num_array_entries); counter++) |
| table_data_size += lli_in_array_ptr[counter].block_size; |
| |
| /* |
| * Check if we reached the last entry, |
| * meaning this ia the last table to build, |
| * and no need to check the block alignment |
| */ |
| if (counter == num_array_entries) { |
| /* Set the last table flag */ |
| *last_table_flag = 1; |
| goto end_function; |
| } |
| |
| /* |
| * Calculate the data size of the next table. |
| * Stop if no entries left or if data size is more the DMA restriction |
| */ |
| next_table_data_size = 0; |
| for (; counter < num_array_entries; counter++) { |
| next_table_data_size += lli_in_array_ptr[counter].block_size; |
| if (next_table_data_size >= SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE) |
| break; |
| } |
| |
| /* |
| * Check if the next table data size is less then DMA rstriction. |
| * if it is - recalculate the current table size, so that the next |
| * table data size will be adaquete for DMA |
| */ |
| if (next_table_data_size && |
| next_table_data_size < SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE) |
| |
| table_data_size -= (SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE - |
| next_table_data_size); |
| |
| end_function: |
| return table_data_size; |
| } |
| |
| /** |
| * sep_build_lli_table - build an lli array for the given table |
| * @sep: pointer to struct sep_device |
| * @lli_array_ptr: pointer to lli array |
| * @lli_table_ptr: pointer to lli table |
| * @num_processed_entries_ptr: pointer to number of entries |
| * @num_table_entries_ptr: pointer to number of tables |
| * @table_data_size: total data size |
| * |
| * Builds an lli table from the lli_array according to |
| * the given size of data |
| */ |
| static void sep_build_lli_table(struct sep_device *sep, |
| struct sep_lli_entry *lli_array_ptr, |
| struct sep_lli_entry *lli_table_ptr, |
| u32 *num_processed_entries_ptr, |
| u32 *num_table_entries_ptr, |
| u32 table_data_size) |
| { |
| /* Current table data size */ |
| u32 curr_table_data_size; |
| /* Counter of lli array entry */ |
| u32 array_counter; |
| |
| /* Init current table data size and lli array entry counter */ |
| curr_table_data_size = 0; |
| array_counter = 0; |
| *num_table_entries_ptr = 1; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] build lli table table_data_size: (hex) %x\n", |
| current->pid, table_data_size); |
| |
| /* Fill the table till table size reaches the needed amount */ |
| while (curr_table_data_size < table_data_size) { |
| /* Update the number of entries in table */ |
| (*num_table_entries_ptr)++; |
| |
| lli_table_ptr->bus_address = |
| cpu_to_le32(lli_array_ptr[array_counter].bus_address); |
| |
| lli_table_ptr->block_size = |
| cpu_to_le32(lli_array_ptr[array_counter].block_size); |
| |
| curr_table_data_size += lli_array_ptr[array_counter].block_size; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_table_ptr is %p\n", |
| current->pid, lli_table_ptr); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_table_ptr->bus_address: %08lx\n", |
| current->pid, |
| (unsigned long)lli_table_ptr->bus_address); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_table_ptr->block_size is (hex) %x\n", |
| current->pid, lli_table_ptr->block_size); |
| |
| /* Check for overflow of the table data */ |
| if (curr_table_data_size > table_data_size) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] curr_table_data_size too large\n", |
| current->pid); |
| |
| /* Update the size of block in the table */ |
| lli_table_ptr->block_size = |
| cpu_to_le32(lli_table_ptr->block_size) - |
| (curr_table_data_size - table_data_size); |
| |
| /* Update the physical address in the lli array */ |
| lli_array_ptr[array_counter].bus_address += |
| cpu_to_le32(lli_table_ptr->block_size); |
| |
| /* Update the block size left in the lli array */ |
| lli_array_ptr[array_counter].block_size = |
| (curr_table_data_size - table_data_size); |
| } else |
| /* Advance to the next entry in the lli_array */ |
| array_counter++; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_table_ptr->bus_address is %08lx\n", |
| current->pid, |
| (unsigned long)lli_table_ptr->bus_address); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_table_ptr->block_size is (hex) %x\n", |
| current->pid, |
| lli_table_ptr->block_size); |
| |
| /* Move to the next entry in table */ |
| lli_table_ptr++; |
| } |
| |
| /* Set the info entry to default */ |
| lli_table_ptr->bus_address = 0xffffffff; |
| lli_table_ptr->block_size = 0; |
| |
| /* Set the output parameter */ |
| *num_processed_entries_ptr += array_counter; |
| |
| } |
| |
| /** |
| * sep_shared_area_virt_to_bus - map shared area to bus address |
| * @sep: pointer to struct sep_device |
| * @virt_address: virtual address to convert |
| * |
| * This functions returns the physical address inside shared area according |
| * to the virtual address. It can be either on the external RAM device |
| * (ioremapped), or on the system RAM |
| * This implementation is for the external RAM |
| */ |
| static dma_addr_t sep_shared_area_virt_to_bus(struct sep_device *sep, |
| void *virt_address) |
| { |
| dev_dbg(&sep->pdev->dev, "[PID%d] sh virt to phys v %p\n", |
| current->pid, virt_address); |
| dev_dbg(&sep->pdev->dev, "[PID%d] sh virt to phys p %08lx\n", |
| current->pid, |
| (unsigned long) |
| sep->shared_bus + (virt_address - sep->shared_addr)); |
| |
| return sep->shared_bus + (size_t)(virt_address - sep->shared_addr); |
| } |
| |
| /** |
| * sep_shared_area_bus_to_virt - map shared area bus address to kernel |
| * @sep: pointer to struct sep_device |
| * @bus_address: bus address to convert |
| * |
| * This functions returns the virtual address inside shared area |
| * according to the physical address. It can be either on the |
| * external RAM device (ioremapped), or on the system RAM |
| * This implementation is for the external RAM |
| */ |
| static void *sep_shared_area_bus_to_virt(struct sep_device *sep, |
| dma_addr_t bus_address) |
| { |
| dev_dbg(&sep->pdev->dev, "[PID%d] shared bus to virt b=%lx v=%lx\n", |
| current->pid, |
| (unsigned long)bus_address, (unsigned long)(sep->shared_addr + |
| (size_t)(bus_address - sep->shared_bus))); |
| |
| return sep->shared_addr + (size_t)(bus_address - sep->shared_bus); |
| } |
| |
| /** |
| * sep_debug_print_lli_tables - dump LLI table |
| * @sep: pointer to struct sep_device |
| * @lli_table_ptr: pointer to sep_lli_entry |
| * @num_table_entries: number of entries |
| * @table_data_size: total data size |
| * |
| * Walk the the list of the print created tables and print all the data |
| */ |
| static void sep_debug_print_lli_tables(struct sep_device *sep, |
| struct sep_lli_entry *lli_table_ptr, |
| unsigned long num_table_entries, |
| unsigned long table_data_size) |
| { |
| #ifdef DEBUG |
| unsigned long table_count = 1; |
| unsigned long entries_count = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_debug_print_lli_tables start\n", |
| current->pid); |
| if (num_table_entries == 0) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] no table to print\n", |
| current->pid); |
| return; |
| } |
| |
| while ((unsigned long) lli_table_ptr->bus_address != 0xffffffff) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli table %08lx, " |
| "table_data_size is (hex) %lx\n", |
| current->pid, table_count, table_data_size); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] num_table_entries is (hex) %lx\n", |
| current->pid, num_table_entries); |
| |
| /* Print entries of the table (without info entry) */ |
| for (entries_count = 0; entries_count < num_table_entries; |
| entries_count++, lli_table_ptr++) { |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] lli_table_ptr address is %08lx\n", |
| current->pid, |
| (unsigned long) lli_table_ptr); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] phys address is %08lx " |
| "block size is (hex) %x\n", current->pid, |
| (unsigned long)lli_table_ptr->bus_address, |
| lli_table_ptr->block_size); |
| } |
| |
| /* Point to the info entry */ |
| lli_table_ptr--; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] phys lli_table_ptr->block_size " |
| "is (hex) %x\n", |
| current->pid, |
| lli_table_ptr->block_size); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] phys lli_table_ptr->physical_address " |
| "is %08lx\n", |
| current->pid, |
| (unsigned long)lli_table_ptr->bus_address); |
| |
| |
| table_data_size = lli_table_ptr->block_size & 0xffffff; |
| num_table_entries = (lli_table_ptr->block_size >> 24) & 0xff; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] phys table_data_size is " |
| "(hex) %lx num_table_entries is" |
| " %lx bus_address is%lx\n", |
| current->pid, |
| table_data_size, |
| num_table_entries, |
| (unsigned long)lli_table_ptr->bus_address); |
| |
| if ((unsigned long)lli_table_ptr->bus_address != 0xffffffff) |
| lli_table_ptr = (struct sep_lli_entry *) |
| sep_shared_bus_to_virt(sep, |
| (unsigned long)lli_table_ptr->bus_address); |
| |
| table_count++; |
| } |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_debug_print_lli_tables end\n", |
| current->pid); |
| #endif |
| } |
| |
| |
| /** |
| * sep_prepare_empty_lli_table - create a blank LLI table |
| * @sep: pointer to struct sep_device |
| * @lli_table_addr_ptr: pointer to lli table |
| * @num_entries_ptr: pointer to number of entries |
| * @table_data_size_ptr: point to table data size |
| * @dmatables_region: Optional buffer for DMA tables |
| * @dma_ctx: DMA context |
| * |
| * This function creates empty lli tables when there is no data |
| */ |
| static void sep_prepare_empty_lli_table(struct sep_device *sep, |
| dma_addr_t *lli_table_addr_ptr, |
| u32 *num_entries_ptr, |
| u32 *table_data_size_ptr, |
| void **dmatables_region, |
| struct sep_dma_context *dma_ctx) |
| { |
| struct sep_lli_entry *lli_table_ptr; |
| |
| /* Find the area for new table */ |
| lli_table_ptr = |
| (struct sep_lli_entry *)(sep->shared_addr + |
| SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES + |
| dma_ctx->num_lli_tables_created * sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP); |
| |
| if (dmatables_region && *dmatables_region) |
| lli_table_ptr = *dmatables_region; |
| |
| lli_table_ptr->bus_address = 0; |
| lli_table_ptr->block_size = 0; |
| |
| lli_table_ptr++; |
| lli_table_ptr->bus_address = 0xFFFFFFFF; |
| lli_table_ptr->block_size = 0; |
| |
| /* Set the output parameter value */ |
| *lli_table_addr_ptr = sep->shared_bus + |
| SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES + |
| dma_ctx->num_lli_tables_created * |
| sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP; |
| |
| /* Set the num of entries and table data size for empty table */ |
| *num_entries_ptr = 2; |
| *table_data_size_ptr = 0; |
| |
| /* Update the number of created tables */ |
| dma_ctx->num_lli_tables_created++; |
| } |
| |
| /** |
| * sep_prepare_input_dma_table - prepare input DMA mappings |
| * @sep: pointer to struct sep_device |
| * @data_size: |
| * @block_size: |
| * @lli_table_ptr: |
| * @num_entries_ptr: |
| * @table_data_size_ptr: |
| * @is_kva: set for kernel data (kernel crypt io call) |
| * |
| * This function prepares only input DMA table for synchronic symmetric |
| * operations (HASH) |
| * Note that all bus addresses that are passed to the SEP |
| * are in 32 bit format; the SEP is a 32 bit device |
| */ |
| static int sep_prepare_input_dma_table(struct sep_device *sep, |
| unsigned long app_virt_addr, |
| u32 data_size, |
| u32 block_size, |
| dma_addr_t *lli_table_ptr, |
| u32 *num_entries_ptr, |
| u32 *table_data_size_ptr, |
| bool is_kva, |
| void **dmatables_region, |
| struct sep_dma_context *dma_ctx |
| ) |
| { |
| int error = 0; |
| /* Pointer to the info entry of the table - the last entry */ |
| struct sep_lli_entry *info_entry_ptr; |
| /* Array of pointers to page */ |
| struct sep_lli_entry *lli_array_ptr; |
| /* Points to the first entry to be processed in the lli_in_array */ |
| u32 current_entry = 0; |
| /* Num entries in the virtual buffer */ |
| u32 sep_lli_entries = 0; |
| /* Lli table pointer */ |
| struct sep_lli_entry *in_lli_table_ptr; |
| /* The total data in one table */ |
| u32 table_data_size = 0; |
| /* Flag for last table */ |
| u32 last_table_flag = 0; |
| /* Number of entries in lli table */ |
| u32 num_entries_in_table = 0; |
| /* Next table address */ |
| void *lli_table_alloc_addr = NULL; |
| void *dma_lli_table_alloc_addr = NULL; |
| void *dma_in_lli_table_ptr = NULL; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] prepare intput dma tbl data size: (hex) %x\n", |
| current->pid, data_size); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] block_size is (hex) %x\n", |
| current->pid, block_size); |
| |
| /* Initialize the pages pointers */ |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages = 0; |
| |
| /* Set the kernel address for first table to be allocated */ |
| lli_table_alloc_addr = (void *)(sep->shared_addr + |
| SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES + |
| dma_ctx->num_lli_tables_created * sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP); |
| |
| if (data_size == 0) { |
| if (dmatables_region) { |
| error = sep_allocate_dmatables_region(sep, |
| dmatables_region, |
| dma_ctx, |
| 1); |
| if (error) |
| return error; |
| } |
| /* Special case - create meptu table - 2 entries, zero data */ |
| sep_prepare_empty_lli_table(sep, lli_table_ptr, |
| num_entries_ptr, table_data_size_ptr, |
| dmatables_region, dma_ctx); |
| goto update_dcb_counter; |
| } |
| |
| /* Check if the pages are in Kernel Virtual Address layout */ |
| if (is_kva == true) |
| error = sep_lock_kernel_pages(sep, app_virt_addr, |
| data_size, &lli_array_ptr, SEP_DRIVER_IN_FLAG, |
| dma_ctx); |
| else |
| /* |
| * Lock the pages of the user buffer |
| * and translate them to pages |
| */ |
| error = sep_lock_user_pages(sep, app_virt_addr, |
| data_size, &lli_array_ptr, SEP_DRIVER_IN_FLAG, |
| dma_ctx); |
| |
| if (error) |
| goto end_function; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output sep_in_num_pages is (hex) %x\n", |
| current->pid, |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages); |
| |
| current_entry = 0; |
| info_entry_ptr = NULL; |
| |
| sep_lli_entries = |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages; |
| |
| dma_lli_table_alloc_addr = lli_table_alloc_addr; |
| if (dmatables_region) { |
| error = sep_allocate_dmatables_region(sep, |
| dmatables_region, |
| dma_ctx, |
| sep_lli_entries); |
| if (error) |
| goto end_function_error; |
| lli_table_alloc_addr = *dmatables_region; |
| } |
| |
| /* Loop till all the entries in in array are processed */ |
| while (current_entry < sep_lli_entries) { |
| |
| /* Set the new input and output tables */ |
| in_lli_table_ptr = |
| (struct sep_lli_entry *)lli_table_alloc_addr; |
| dma_in_lli_table_ptr = |
| (struct sep_lli_entry *)dma_lli_table_alloc_addr; |
| |
| lli_table_alloc_addr += sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP; |
| dma_lli_table_alloc_addr += sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP; |
| |
| if (dma_lli_table_alloc_addr > |
| ((void *)sep->shared_addr + |
| SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES + |
| SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES)) { |
| |
| error = -ENOMEM; |
| goto end_function_error; |
| |
| } |
| |
| /* Update the number of created tables */ |
| dma_ctx->num_lli_tables_created++; |
| |
| /* Calculate the maximum size of data for input table */ |
| table_data_size = sep_calculate_lli_table_max_size(sep, |
| &lli_array_ptr[current_entry], |
| (sep_lli_entries - current_entry), |
| &last_table_flag); |
| |
| /* |
| * If this is not the last table - |
| * then align it to the block size |
| */ |
| if (!last_table_flag) |
| table_data_size = |
| (table_data_size / block_size) * block_size; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output table_data_size is (hex) %x\n", |
| current->pid, |
| table_data_size); |
| |
| /* Construct input lli table */ |
| sep_build_lli_table(sep, &lli_array_ptr[current_entry], |
| in_lli_table_ptr, |
| ¤t_entry, &num_entries_in_table, table_data_size); |
| |
| if (info_entry_ptr == NULL) { |
| |
| /* Set the output parameters to physical addresses */ |
| *lli_table_ptr = sep_shared_area_virt_to_bus(sep, |
| dma_in_lli_table_ptr); |
| *num_entries_ptr = num_entries_in_table; |
| *table_data_size_ptr = table_data_size; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output lli_table_in_ptr is %08lx\n", |
| current->pid, |
| (unsigned long)*lli_table_ptr); |
| |
| } else { |
| /* Update the info entry of the previous in table */ |
| info_entry_ptr->bus_address = |
| sep_shared_area_virt_to_bus(sep, |
| dma_in_lli_table_ptr); |
| info_entry_ptr->block_size = |
| ((num_entries_in_table) << 24) | |
| (table_data_size); |
| } |
| /* Save the pointer to the info entry of the current tables */ |
| info_entry_ptr = in_lli_table_ptr + num_entries_in_table - 1; |
| } |
| /* Print input tables */ |
| if (!dmatables_region) { |
| sep_debug_print_lli_tables(sep, (struct sep_lli_entry *) |
| sep_shared_area_bus_to_virt(sep, *lli_table_ptr), |
| *num_entries_ptr, *table_data_size_ptr); |
| } |
| |
| /* The array of the pages */ |
| kfree(lli_array_ptr); |
| |
| update_dcb_counter: |
| /* Update DCB counter */ |
| dma_ctx->nr_dcb_creat++; |
| goto end_function; |
| |
| end_function_error: |
| /* Free all the allocated resources */ |
| kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array); |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array = NULL; |
| kfree(lli_array_ptr); |
| kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array); |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL; |
| |
| end_function: |
| return error; |
| |
| } |
| |
| /** |
| * sep_construct_dma_tables_from_lli - prepare AES/DES mappings |
| * @sep: pointer to struct sep_device |
| * @lli_in_array: |
| * @sep_in_lli_entries: |
| * @lli_out_array: |
| * @sep_out_lli_entries |
| * @block_size |
| * @lli_table_in_ptr |
| * @lli_table_out_ptr |
| * @in_num_entries_ptr |
| * @out_num_entries_ptr |
| * @table_data_size_ptr |
| * |
| * This function creates the input and output DMA tables for |
| * symmetric operations (AES/DES) according to the block |
| * size from LLI arays |
| * Note that all bus addresses that are passed to the SEP |
| * are in 32 bit format; the SEP is a 32 bit device |
| */ |
| static int sep_construct_dma_tables_from_lli( |
| struct sep_device *sep, |
| struct sep_lli_entry *lli_in_array, |
| u32 sep_in_lli_entries, |
| struct sep_lli_entry *lli_out_array, |
| u32 sep_out_lli_entries, |
| u32 block_size, |
| dma_addr_t *lli_table_in_ptr, |
| dma_addr_t *lli_table_out_ptr, |
| u32 *in_num_entries_ptr, |
| u32 *out_num_entries_ptr, |
| u32 *table_data_size_ptr, |
| void **dmatables_region, |
| struct sep_dma_context *dma_ctx) |
| { |
| /* Points to the area where next lli table can be allocated */ |
| void *lli_table_alloc_addr = NULL; |
| /* |
| * Points to the area in shared region where next lli table |
| * can be allocated |
| */ |
| void *dma_lli_table_alloc_addr = NULL; |
| /* Input lli table in dmatables_region or shared region */ |
| struct sep_lli_entry *in_lli_table_ptr = NULL; |
| /* Input lli table location in the shared region */ |
| struct sep_lli_entry *dma_in_lli_table_ptr = NULL; |
| /* Output lli table in dmatables_region or shared region */ |
| struct sep_lli_entry *out_lli_table_ptr = NULL; |
| /* Output lli table location in the shared region */ |
| struct sep_lli_entry *dma_out_lli_table_ptr = NULL; |
| /* Pointer to the info entry of the table - the last entry */ |
| struct sep_lli_entry *info_in_entry_ptr = NULL; |
| /* Pointer to the info entry of the table - the last entry */ |
| struct sep_lli_entry *info_out_entry_ptr = NULL; |
| /* Points to the first entry to be processed in the lli_in_array */ |
| u32 current_in_entry = 0; |
| /* Points to the first entry to be processed in the lli_out_array */ |
| u32 current_out_entry = 0; |
| /* Max size of the input table */ |
| u32 in_table_data_size = 0; |
| /* Max size of the output table */ |
| u32 out_table_data_size = 0; |
| /* Flag te signifies if this is the last tables build */ |
| u32 last_table_flag = 0; |
| /* The data size that should be in table */ |
| u32 table_data_size = 0; |
| /* Number of entries in the input table */ |
| u32 num_entries_in_table = 0; |
| /* Number of entries in the output table */ |
| u32 num_entries_out_table = 0; |
| |
| if (!dma_ctx) { |
| dev_warn(&sep->pdev->dev, "DMA context uninitialized\n"); |
| return -EINVAL; |
| } |
| |
| /* Initiate to point after the message area */ |
| lli_table_alloc_addr = (void *)(sep->shared_addr + |
| SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES + |
| (dma_ctx->num_lli_tables_created * |
| (sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP))); |
| dma_lli_table_alloc_addr = lli_table_alloc_addr; |
| |
| if (dmatables_region) { |
| /* 2 for both in+out table */ |
| if (sep_allocate_dmatables_region(sep, |
| dmatables_region, |
| dma_ctx, |
| 2*sep_in_lli_entries)) |
| return -ENOMEM; |
| lli_table_alloc_addr = *dmatables_region; |
| } |
| |
| /* Loop till all the entries in in array are not processed */ |
| while (current_in_entry < sep_in_lli_entries) { |
| /* Set the new input and output tables */ |
| in_lli_table_ptr = |
| (struct sep_lli_entry *)lli_table_alloc_addr; |
| dma_in_lli_table_ptr = |
| (struct sep_lli_entry *)dma_lli_table_alloc_addr; |
| |
| lli_table_alloc_addr += sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP; |
| dma_lli_table_alloc_addr += sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP; |
| |
| /* Set the first output tables */ |
| out_lli_table_ptr = |
| (struct sep_lli_entry *)lli_table_alloc_addr; |
| dma_out_lli_table_ptr = |
| (struct sep_lli_entry *)dma_lli_table_alloc_addr; |
| |
| /* Check if the DMA table area limit was overrun */ |
| if ((dma_lli_table_alloc_addr + sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP) > |
| ((void *)sep->shared_addr + |
| SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES + |
| SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES)) { |
| |
| dev_warn(&sep->pdev->dev, "dma table limit overrun\n"); |
| return -ENOMEM; |
| } |
| |
| /* Update the number of the lli tables created */ |
| dma_ctx->num_lli_tables_created += 2; |
| |
| lli_table_alloc_addr += sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP; |
| dma_lli_table_alloc_addr += sizeof(struct sep_lli_entry) * |
| SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP; |
| |
| /* Calculate the maximum size of data for input table */ |
| in_table_data_size = |
| sep_calculate_lli_table_max_size(sep, |
| &lli_in_array[current_in_entry], |
| (sep_in_lli_entries - current_in_entry), |
| &last_table_flag); |
| |
| /* Calculate the maximum size of data for output table */ |
| out_table_data_size = |
| sep_calculate_lli_table_max_size(sep, |
| &lli_out_array[current_out_entry], |
| (sep_out_lli_entries - current_out_entry), |
| &last_table_flag); |
| |
| if (!last_table_flag) { |
| in_table_data_size = (in_table_data_size / |
| block_size) * block_size; |
| out_table_data_size = (out_table_data_size / |
| block_size) * block_size; |
| } |
| |
| table_data_size = in_table_data_size; |
| if (table_data_size > out_table_data_size) |
| table_data_size = out_table_data_size; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] construct tables from lli" |
| " in_table_data_size is (hex) %x\n", current->pid, |
| in_table_data_size); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] construct tables from lli" |
| "out_table_data_size is (hex) %x\n", current->pid, |
| out_table_data_size); |
| |
| /* Construct input lli table */ |
| sep_build_lli_table(sep, &lli_in_array[current_in_entry], |
| in_lli_table_ptr, |
| ¤t_in_entry, |
| &num_entries_in_table, |
| table_data_size); |
| |
| /* Construct output lli table */ |
| sep_build_lli_table(sep, &lli_out_array[current_out_entry], |
| out_lli_table_ptr, |
| ¤t_out_entry, |
| &num_entries_out_table, |
| table_data_size); |
| |
| /* If info entry is null - this is the first table built */ |
| if (info_in_entry_ptr == NULL || info_out_entry_ptr == NULL) { |
| /* Set the output parameters to physical addresses */ |
| *lli_table_in_ptr = |
| sep_shared_area_virt_to_bus(sep, dma_in_lli_table_ptr); |
| |
| *in_num_entries_ptr = num_entries_in_table; |
| |
| *lli_table_out_ptr = |
| sep_shared_area_virt_to_bus(sep, |
| dma_out_lli_table_ptr); |
| |
| *out_num_entries_ptr = num_entries_out_table; |
| *table_data_size_ptr = table_data_size; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output lli_table_in_ptr is %08lx\n", |
| current->pid, |
| (unsigned long)*lli_table_in_ptr); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output lli_table_out_ptr is %08lx\n", |
| current->pid, |
| (unsigned long)*lli_table_out_ptr); |
| } else { |
| /* Update the info entry of the previous in table */ |
| info_in_entry_ptr->bus_address = |
| sep_shared_area_virt_to_bus(sep, |
| dma_in_lli_table_ptr); |
| |
| info_in_entry_ptr->block_size = |
| ((num_entries_in_table) << 24) | |
| (table_data_size); |
| |
| /* Update the info entry of the previous in table */ |
| info_out_entry_ptr->bus_address = |
| sep_shared_area_virt_to_bus(sep, |
| dma_out_lli_table_ptr); |
| |
| info_out_entry_ptr->block_size = |
| ((num_entries_out_table) << 24) | |
| (table_data_size); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output lli_table_in_ptr:%08lx %08x\n", |
| current->pid, |
| (unsigned long)info_in_entry_ptr->bus_address, |
| info_in_entry_ptr->block_size); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output lli_table_out_ptr:" |
| "%08lx %08x\n", |
| current->pid, |
| (unsigned long)info_out_entry_ptr->bus_address, |
| info_out_entry_ptr->block_size); |
| } |
| |
| /* Save the pointer to the info entry of the current tables */ |
| info_in_entry_ptr = in_lli_table_ptr + |
| num_entries_in_table - 1; |
| info_out_entry_ptr = out_lli_table_ptr + |
| num_entries_out_table - 1; |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output num_entries_out_table is %x\n", |
| current->pid, |
| (u32)num_entries_out_table); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output info_in_entry_ptr is %lx\n", |
| current->pid, |
| (unsigned long)info_in_entry_ptr); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] output info_out_entry_ptr is %lx\n", |
| current->pid, |
| (unsigned long)info_out_entry_ptr); |
| } |
| |
| /* Print input tables */ |
| if (!dmatables_region) { |
| sep_debug_print_lli_tables( |
| sep, |
| (struct sep_lli_entry *) |
| sep_shared_area_bus_to_virt(sep, *lli_table_in_ptr), |
| *in_num_entries_ptr, |
| *table_data_size_ptr); |
| } |
| |
| /* Print output tables */ |
| if (!dmatables_region) { |
| sep_debug_print_lli_tables( |
| sep, |
| (struct sep_lli_entry *) |
| sep_shared_area_bus_to_virt(sep, *lli_table_out_ptr), |
| *out_num_entries_ptr, |
| *table_data_size_ptr); |
| } |
| |
| return 0; |
| } |
| |
| /** |
| * sep_prepare_input_output_dma_table - prepare DMA I/O table |
| * @app_virt_in_addr: |
| * @app_virt_out_addr: |
| * @data_size: |
| * @block_size: |
| * @lli_table_in_ptr: |
| * @lli_table_out_ptr: |
| * @in_num_entries_ptr: |
| * @out_num_entries_ptr: |
| * @table_data_size_ptr: |
| * @is_kva: set for kernel data; used only for kernel crypto module |
| * |
| * This function builds input and output DMA tables for synchronic |
| * symmetric operations (AES, DES, HASH). It also checks that each table |
| * is of the modular block size |
| * Note that all bus addresses that are passed to the SEP |
| * are in 32 bit format; the SEP is a 32 bit device |
| */ |
| static int sep_prepare_input_output_dma_table(struct sep_device *sep, |
| unsigned long app_virt_in_addr, |
| unsigned long app_virt_out_addr, |
| u32 data_size, |
| u32 block_size, |
| dma_addr_t *lli_table_in_ptr, |
| dma_addr_t *lli_table_out_ptr, |
| u32 *in_num_entries_ptr, |
| u32 *out_num_entries_ptr, |
| u32 *table_data_size_ptr, |
| bool is_kva, |
| void **dmatables_region, |
| struct sep_dma_context *dma_ctx) |
| |
| { |
| int error = 0; |
| /* Array of pointers of page */ |
| struct sep_lli_entry *lli_in_array; |
| /* Array of pointers of page */ |
| struct sep_lli_entry *lli_out_array; |
| |
| if (!dma_ctx) { |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| if (data_size == 0) { |
| /* Prepare empty table for input and output */ |
| if (dmatables_region) { |
| error = sep_allocate_dmatables_region( |
| sep, |
| dmatables_region, |
| dma_ctx, |
| 2); |
| if (error) |
| goto end_function; |
| } |
| sep_prepare_empty_lli_table(sep, lli_table_in_ptr, |
| in_num_entries_ptr, table_data_size_ptr, |
| dmatables_region, dma_ctx); |
| |
| sep_prepare_empty_lli_table(sep, lli_table_out_ptr, |
| out_num_entries_ptr, table_data_size_ptr, |
| dmatables_region, dma_ctx); |
| |
| goto update_dcb_counter; |
| } |
| |
| /* Initialize the pages pointers */ |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL; |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = NULL; |
| |
| /* Lock the pages of the buffer and translate them to pages */ |
| if (is_kva == true) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] Locking kernel input pages\n", |
| current->pid); |
| error = sep_lock_kernel_pages(sep, app_virt_in_addr, |
| data_size, &lli_in_array, SEP_DRIVER_IN_FLAG, |
| dma_ctx); |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] sep_lock_kernel_pages for input " |
| "virtual buffer failed\n", current->pid); |
| |
| goto end_function; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] Locking kernel output pages\n", |
| current->pid); |
| error = sep_lock_kernel_pages(sep, app_virt_out_addr, |
| data_size, &lli_out_array, SEP_DRIVER_OUT_FLAG, |
| dma_ctx); |
| |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] sep_lock_kernel_pages for output " |
| "virtual buffer failed\n", current->pid); |
| |
| goto end_function_free_lli_in; |
| } |
| |
| } |
| |
| else { |
| dev_dbg(&sep->pdev->dev, "[PID%d] Locking user input pages\n", |
| current->pid); |
| error = sep_lock_user_pages(sep, app_virt_in_addr, |
| data_size, &lli_in_array, SEP_DRIVER_IN_FLAG, |
| dma_ctx); |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] sep_lock_user_pages for input " |
| "virtual buffer failed\n", current->pid); |
| |
| goto end_function; |
| } |
| |
| if (dma_ctx->secure_dma == true) { |
| /* secure_dma requires use of non accessible memory */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] in secure_dma\n", |
| current->pid); |
| error = sep_lli_table_secure_dma(sep, |
| app_virt_out_addr, data_size, &lli_out_array, |
| SEP_DRIVER_OUT_FLAG, dma_ctx); |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] secure dma table setup " |
| " for output virtual buffer failed\n", |
| current->pid); |
| |
| goto end_function_free_lli_in; |
| } |
| } else { |
| /* For normal, non-secure dma */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] not in secure_dma\n", |
| current->pid); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] Locking user output pages\n", |
| current->pid); |
| |
| error = sep_lock_user_pages(sep, app_virt_out_addr, |
| data_size, &lli_out_array, SEP_DRIVER_OUT_FLAG, |
| dma_ctx); |
| |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] sep_lock_user_pages" |
| " for output virtual buffer failed\n", |
| current->pid); |
| |
| goto end_function_free_lli_in; |
| } |
| } |
| } |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] After lock; prep input output dma table sep_in_num_pages is (hex) %x\n", |
| current->pid, |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_out_num_pages is (hex) %x\n", |
| current->pid, |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP is (hex) %x\n", |
| current->pid, SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP); |
| |
| /* Call the function that creates table from the lli arrays */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] calling create table from lli\n", |
| current->pid); |
| error = sep_construct_dma_tables_from_lli( |
| sep, lli_in_array, |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat]. |
| in_num_pages, |
| lli_out_array, |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat]. |
| out_num_pages, |
| block_size, lli_table_in_ptr, lli_table_out_ptr, |
| in_num_entries_ptr, out_num_entries_ptr, |
| table_data_size_ptr, dmatables_region, dma_ctx); |
| |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] sep_construct_dma_tables_from_lli failed\n", |
| current->pid); |
| goto end_function_with_error; |
| } |
| |
| kfree(lli_out_array); |
| kfree(lli_in_array); |
| |
| update_dcb_counter: |
| /* Update DCB counter */ |
| dma_ctx->nr_dcb_creat++; |
| |
| goto end_function; |
| |
| end_function_with_error: |
| kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array); |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array = NULL; |
| kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array); |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = NULL; |
| kfree(lli_out_array); |
| |
| |
| end_function_free_lli_in: |
| kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array); |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array = NULL; |
| kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array); |
| dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL; |
| kfree(lli_in_array); |
| |
| end_function: |
| |
| return error; |
| |
| } |
| |
| /** |
| * sep_prepare_input_output_dma_table_in_dcb - prepare control blocks |
| * @app_in_address: unsigned long; for data buffer in (user space) |
| * @app_out_address: unsigned long; for data buffer out (user space) |
| * @data_in_size: u32; for size of data |
| * @block_size: u32; for block size |
| * @tail_block_size: u32; for size of tail block |
| * @isapplet: bool; to indicate external app |
| * @is_kva: bool; kernel buffer; only used for kernel crypto module |
| * @secure_dma; indicates whether this is secure_dma using IMR |
| * |
| * This function prepares the linked DMA tables and puts the |
| * address for the linked list of tables inta a DCB (data control |
| * block) the address of which is known by the SEP hardware |
| * Note that all bus addresses that are passed to the SEP |
| * are in 32 bit format; the SEP is a 32 bit device |
| */ |
| int sep_prepare_input_output_dma_table_in_dcb(struct sep_device *sep, |
| unsigned long app_in_address, |
| unsigned long app_out_address, |
| u32 data_in_size, |
| u32 block_size, |
| u32 tail_block_size, |
| bool isapplet, |
| bool is_kva, |
| bool secure_dma, |
| struct sep_dcblock *dcb_region, |
| void **dmatables_region, |
| struct sep_dma_context **dma_ctx, |
| struct scatterlist *src_sg, |
| struct scatterlist *dst_sg) |
| { |
| int error = 0; |
| /* Size of tail */ |
| u32 tail_size = 0; |
| /* Address of the created DCB table */ |
| struct sep_dcblock *dcb_table_ptr = NULL; |
| /* The physical address of the first input DMA table */ |
| dma_addr_t in_first_mlli_address = 0; |
| /* Number of entries in the first input DMA table */ |
| u32 in_first_num_entries = 0; |
| /* The physical address of the first output DMA table */ |
| dma_addr_t out_first_mlli_address = 0; |
| /* Number of entries in the first output DMA table */ |
| u32 out_first_num_entries = 0; |
| /* Data in the first input/output table */ |
| u32 first_data_size = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] app_in_address %lx\n", |
| current->pid, app_in_address); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] app_out_address %lx\n", |
| current->pid, app_out_address); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] data_in_size %x\n", |
| current->pid, data_in_size); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] block_size %x\n", |
| current->pid, block_size); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] tail_block_size %x\n", |
| current->pid, tail_block_size); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] isapplet %x\n", |
| current->pid, isapplet); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] is_kva %x\n", |
| current->pid, is_kva); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] src_sg %p\n", |
| current->pid, src_sg); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] dst_sg %p\n", |
| current->pid, dst_sg); |
| |
| if (!dma_ctx) { |
| dev_warn(&sep->pdev->dev, "[PID%d] no DMA context pointer\n", |
| current->pid); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| if (*dma_ctx) { |
| /* In case there are multiple DCBs for this transaction */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] DMA context already set\n", |
| current->pid); |
| } else { |
| *dma_ctx = kzalloc(sizeof(**dma_ctx), GFP_KERNEL); |
| if (!(*dma_ctx)) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] Not enough memory for DMA context\n", |
| current->pid); |
| error = -ENOMEM; |
| goto end_function; |
| } |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] Created DMA context addr at 0x%p\n", |
| current->pid, *dma_ctx); |
| } |
| |
| (*dma_ctx)->secure_dma = secure_dma; |
| |
| /* these are for kernel crypto only */ |
| (*dma_ctx)->src_sg = src_sg; |
| (*dma_ctx)->dst_sg = dst_sg; |
| |
| if ((*dma_ctx)->nr_dcb_creat == SEP_MAX_NUM_SYNC_DMA_OPS) { |
| /* No more DCBs to allocate */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] no more DCBs available\n", |
| current->pid); |
| error = -ENOSPC; |
| goto end_function_error; |
| } |
| |
| /* Allocate new DCB */ |
| if (dcb_region) { |
| dcb_table_ptr = dcb_region; |
| } else { |
| dcb_table_ptr = (struct sep_dcblock *)(sep->shared_addr + |
| SEP_DRIVER_SYSTEM_DCB_MEMORY_OFFSET_IN_BYTES + |
| ((*dma_ctx)->nr_dcb_creat * |
| sizeof(struct sep_dcblock))); |
| } |
| |
| /* Set the default values in the DCB */ |
| dcb_table_ptr->input_mlli_address = 0; |
| dcb_table_ptr->input_mlli_num_entries = 0; |
| dcb_table_ptr->input_mlli_data_size = 0; |
| dcb_table_ptr->output_mlli_address = 0; |
| dcb_table_ptr->output_mlli_num_entries = 0; |
| dcb_table_ptr->output_mlli_data_size = 0; |
| dcb_table_ptr->tail_data_size = 0; |
| dcb_table_ptr->out_vr_tail_pt = 0; |
| |
| if (isapplet == true) { |
| |
| /* Check if there is enough data for DMA operation */ |
| if (data_in_size < SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE) { |
| if (is_kva == true) { |
| error = -ENODEV; |
| goto end_function_error; |
| } else { |
| if (copy_from_user(dcb_table_ptr->tail_data, |
| (void __user *)app_in_address, |
| data_in_size)) { |
| error = -EFAULT; |
| goto end_function_error; |
| } |
| } |
| |
| dcb_table_ptr->tail_data_size = data_in_size; |
| |
| /* Set the output user-space address for mem2mem op */ |
| if (app_out_address) |
| dcb_table_ptr->out_vr_tail_pt = |
| (aligned_u64)app_out_address; |
| |
| /* |
| * Update both data length parameters in order to avoid |
| * second data copy and allow building of empty mlli |
| * tables |
| */ |
| tail_size = 0x0; |
| data_in_size = 0x0; |
| |
| } else { |
| if (!app_out_address) { |
| tail_size = data_in_size % block_size; |
| if (!tail_size) { |
| if (tail_block_size == block_size) |
| tail_size = block_size; |
| } |
| } else { |
| tail_size = 0; |
| } |
| } |
| if (tail_size) { |
| if (tail_size > sizeof(dcb_table_ptr->tail_data)) |
| return -EINVAL; |
| if (is_kva == true) { |
| error = -ENODEV; |
| goto end_function_error; |
| } else { |
| /* We have tail data - copy it to DCB */ |
| if (copy_from_user(dcb_table_ptr->tail_data, |
| (void __user *)(app_in_address + |
| data_in_size - tail_size), tail_size)) { |
| error = -EFAULT; |
| goto end_function_error; |
| } |
| } |
| if (app_out_address) |
| /* |
| * Calculate the output address |
| * according to tail data size |
| */ |
| dcb_table_ptr->out_vr_tail_pt = |
| (aligned_u64)app_out_address + |
| data_in_size - tail_size; |
| |
| /* Save the real tail data size */ |
| dcb_table_ptr->tail_data_size = tail_size; |
| /* |
| * Update the data size without the tail |
| * data size AKA data for the dma |
| */ |
| data_in_size = (data_in_size - tail_size); |
| } |
| } |
| /* Check if we need to build only input table or input/output */ |
| if (app_out_address) { |
| /* Prepare input/output tables */ |
| error = sep_prepare_input_output_dma_table(sep, |
| app_in_address, |
| app_out_address, |
| data_in_size, |
| block_size, |
| &in_first_mlli_address, |
| &out_first_mlli_address, |
| &in_first_num_entries, |
| &out_first_num_entries, |
| &first_data_size, |
| is_kva, |
| dmatables_region, |
| *dma_ctx); |
| } else { |
| /* Prepare input tables */ |
| error = sep_prepare_input_dma_table(sep, |
| app_in_address, |
| data_in_size, |
| block_size, |
| &in_first_mlli_address, |
| &in_first_num_entries, |
| &first_data_size, |
| is_kva, |
| dmatables_region, |
| *dma_ctx); |
| } |
| |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "prepare DMA table call failed " |
| "from prepare DCB call\n"); |
| goto end_function_error; |
| } |
| |
| /* Set the DCB values */ |
| dcb_table_ptr->input_mlli_address = in_first_mlli_address; |
| dcb_table_ptr->input_mlli_num_entries = in_first_num_entries; |
| dcb_table_ptr->input_mlli_data_size = first_data_size; |
| dcb_table_ptr->output_mlli_address = out_first_mlli_address; |
| dcb_table_ptr->output_mlli_num_entries = out_first_num_entries; |
| dcb_table_ptr->output_mlli_data_size = first_data_size; |
| |
| goto end_function; |
| |
| end_function_error: |
| kfree(*dma_ctx); |
| *dma_ctx = NULL; |
| |
| end_function: |
| return error; |
| |
| } |
| |
| |
| /** |
| * sep_free_dma_tables_and_dcb - free DMA tables and DCBs |
| * @sep: pointer to struct sep_device |
| * @isapplet: indicates external application (used for kernel access) |
| * @is_kva: indicates kernel addresses (only used for kernel crypto) |
| * |
| * This function frees the DMA tables and DCB |
| */ |
| static int sep_free_dma_tables_and_dcb(struct sep_device *sep, bool isapplet, |
| bool is_kva, struct sep_dma_context **dma_ctx) |
| { |
| struct sep_dcblock *dcb_table_ptr; |
| unsigned long pt_hold; |
| void *tail_pt; |
| |
| int i = 0; |
| int error = 0; |
| int error_temp = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_free_dma_tables_and_dcb\n", |
| current->pid); |
| if (!dma_ctx || !*dma_ctx) /* nothing to be done here*/ |
| return 0; |
| |
| if (((*dma_ctx)->secure_dma == false) && (isapplet == true)) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] handling applet\n", |
| current->pid); |
| |
| /* Tail stuff is only for non secure_dma */ |
| /* Set pointer to first DCB table */ |
| dcb_table_ptr = (struct sep_dcblock *) |
| (sep->shared_addr + |
| SEP_DRIVER_SYSTEM_DCB_MEMORY_OFFSET_IN_BYTES); |
| |
| /** |
| * Go over each DCB and see if |
| * tail pointer must be updated |
| */ |
| for (i = 0; i < (*dma_ctx)->nr_dcb_creat; i++, dcb_table_ptr++) { |
| if (dcb_table_ptr->out_vr_tail_pt) { |
| pt_hold = (unsigned long)dcb_table_ptr-> |
| out_vr_tail_pt; |
| tail_pt = (void *)pt_hold; |
| if (is_kva == true) { |
| error = -ENODEV; |
| break; |
| } else { |
| error_temp = copy_to_user( |
| (void __user *)tail_pt, |
| dcb_table_ptr->tail_data, |
| dcb_table_ptr->tail_data_size); |
| } |
| if (error_temp) { |
| /* Release the DMA resource */ |
| error = -EFAULT; |
| break; |
| } |
| } |
| } |
| } |
| |
| /* Free the output pages, if any */ |
| sep_free_dma_table_data_handler(sep, dma_ctx); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep_free_dma_tables_and_dcb end\n", |
| current->pid); |
| |
| return error; |
| } |
| |
| /** |
| * sep_prepare_dcb_handler - prepare a control block |
| * @sep: pointer to struct sep_device |
| * @arg: pointer to user parameters |
| * @secure_dma: indicate whether we are using secure_dma on IMR |
| * |
| * This function will retrieve the RAR buffer physical addresses, type |
| * & size corresponding to the RAR handles provided in the buffers vector. |
| */ |
| static int sep_prepare_dcb_handler(struct sep_device *sep, unsigned long arg, |
| bool secure_dma, |
| struct sep_dma_context **dma_ctx) |
| { |
| int error; |
| /* Command arguments */ |
| static struct build_dcb_struct command_args; |
| |
| /* Get the command arguments */ |
| if (copy_from_user(&command_args, (void __user *)arg, |
| sizeof(struct build_dcb_struct))) { |
| error = -EFAULT; |
| goto end_function; |
| } |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] prep dcb handler app_in_address is %08llx\n", |
| current->pid, command_args.app_in_address); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] app_out_address is %08llx\n", |
| current->pid, command_args.app_out_address); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] data_size is %x\n", |
| current->pid, command_args.data_in_size); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] block_size is %x\n", |
| current->pid, command_args.block_size); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] tail block_size is %x\n", |
| current->pid, command_args.tail_block_size); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] is_applet is %x\n", |
| current->pid, command_args.is_applet); |
| |
| if (!command_args.app_in_address) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] null app_in_address\n", current->pid); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| error = sep_prepare_input_output_dma_table_in_dcb(sep, |
| (unsigned long)command_args.app_in_address, |
| (unsigned long)command_args.app_out_address, |
| command_args.data_in_size, command_args.block_size, |
| command_args.tail_block_size, |
| command_args.is_applet, false, |
| secure_dma, NULL, NULL, dma_ctx, NULL, NULL); |
| |
| end_function: |
| return error; |
| |
| } |
| |
| /** |
| * sep_free_dcb_handler - free control block resources |
| * @sep: pointer to struct sep_device |
| * |
| * This function frees the DCB resources and updates the needed |
| * user-space buffers. |
| */ |
| static int sep_free_dcb_handler(struct sep_device *sep, |
| struct sep_dma_context **dma_ctx) |
| { |
| if (!dma_ctx || !(*dma_ctx)) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] no dma context defined, nothing to free\n", |
| current->pid); |
| return -EINVAL; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] free dcbs num of DCBs %x\n", |
| current->pid, |
| (*dma_ctx)->nr_dcb_creat); |
| |
| return sep_free_dma_tables_and_dcb(sep, false, false, dma_ctx); |
| } |
| |
| /** |
| * sep_ioctl - ioctl handler for sep device |
| * @filp: pointer to struct file |
| * @cmd: command |
| * @arg: pointer to argument structure |
| * |
| * Implement the ioctl methods available on the SEP device. |
| */ |
| static long sep_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) |
| { |
| struct sep_private_data * const private_data = filp->private_data; |
| struct sep_call_status *call_status = &private_data->call_status; |
| struct sep_device *sep = private_data->device; |
| struct sep_dma_context **dma_ctx = &private_data->dma_ctx; |
| struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem; |
| int error = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] ioctl cmd 0x%x\n", |
| current->pid, cmd); |
| dev_dbg(&sep->pdev->dev, "[PID%d] dma context addr 0x%p\n", |
| current->pid, *dma_ctx); |
| |
| /* Make sure we own this device */ |
| error = sep_check_transaction_owner(sep); |
| if (error) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] ioctl pid is not owner\n", |
| current->pid); |
| goto end_function; |
| } |
| |
| /* Check that sep_mmap has been called before */ |
| if (0 == test_bit(SEP_LEGACY_MMAP_DONE_OFFSET, |
| &call_status->status)) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] mmap not called\n", current->pid); |
| error = -EPROTO; |
| goto end_function; |
| } |
| |
| /* Check that the command is for SEP device */ |
| if (_IOC_TYPE(cmd) != SEP_IOC_MAGIC_NUMBER) { |
| error = -ENOTTY; |
| goto end_function; |
| } |
| |
| switch (cmd) { |
| case SEP_IOCSENDSEPCOMMAND: |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCSENDSEPCOMMAND start\n", |
| current->pid); |
| if (1 == test_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET, |
| &call_status->status)) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] send msg already done\n", |
| current->pid); |
| error = -EPROTO; |
| goto end_function; |
| } |
| /* Send command to SEP */ |
| error = sep_send_command_handler(sep); |
| if (!error) |
| set_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET, |
| &call_status->status); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCSENDSEPCOMMAND end\n", |
| current->pid); |
| break; |
| case SEP_IOCENDTRANSACTION: |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCENDTRANSACTION start\n", |
| current->pid); |
| error = sep_end_transaction_handler(sep, dma_ctx, call_status, |
| my_queue_elem); |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCENDTRANSACTION end\n", |
| current->pid); |
| break; |
| case SEP_IOCPREPAREDCB: |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCPREPAREDCB start\n", |
| current->pid); |
| case SEP_IOCPREPAREDCB_SECURE_DMA: |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCPREPAREDCB_SECURE_DMA start\n", |
| current->pid); |
| if (1 == test_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET, |
| &call_status->status)) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] dcb prep needed before send msg\n", |
| current->pid); |
| error = -EPROTO; |
| goto end_function; |
| } |
| |
| if (!arg) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] dcb null arg\n", current->pid); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| if (cmd == SEP_IOCPREPAREDCB) { |
| /* No secure dma */ |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCPREPAREDCB (no secure_dma)\n", |
| current->pid); |
| |
| error = sep_prepare_dcb_handler(sep, arg, false, |
| dma_ctx); |
| } else { |
| /* Secure dma */ |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOC_POC (with secure_dma)\n", |
| current->pid); |
| |
| error = sep_prepare_dcb_handler(sep, arg, true, |
| dma_ctx); |
| } |
| dev_dbg(&sep->pdev->dev, "[PID%d] dcb's end\n", |
| current->pid); |
| break; |
| case SEP_IOCFREEDCB: |
| dev_dbg(&sep->pdev->dev, "[PID%d] SEP_IOCFREEDCB start\n", |
| current->pid); |
| case SEP_IOCFREEDCB_SECURE_DMA: |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] SEP_IOCFREEDCB_SECURE_DMA start\n", |
| current->pid); |
| error = sep_free_dcb_handler(sep, dma_ctx); |
| dev_dbg(&sep->pdev->dev, "[PID%d] SEP_IOCFREEDCB end\n", |
| current->pid); |
| break; |
| default: |
| error = -ENOTTY; |
| dev_dbg(&sep->pdev->dev, "[PID%d] default end\n", |
| current->pid); |
| break; |
| } |
| |
| end_function: |
| dev_dbg(&sep->pdev->dev, "[PID%d] ioctl end\n", current->pid); |
| |
| return error; |
| } |
| |
| /** |
| * sep_inthandler - interrupt handler for sep device |
| * @irq: interrupt |
| * @dev_id: device id |
| */ |
| static irqreturn_t sep_inthandler(int irq, void *dev_id) |
| { |
| unsigned long lock_irq_flag; |
| u32 reg_val, reg_val2 = 0; |
| struct sep_device *sep = dev_id; |
| irqreturn_t int_error = IRQ_HANDLED; |
| |
| /* Are we in power save? */ |
| #if defined(CONFIG_PM_RUNTIME) && defined(SEP_ENABLE_RUNTIME_PM) |
| if (sep->pdev->dev.power.runtime_status != RPM_ACTIVE) { |
| dev_dbg(&sep->pdev->dev, "interrupt during pwr save\n"); |
| return IRQ_NONE; |
| } |
| #endif |
| |
| if (test_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags) == 0) { |
| dev_dbg(&sep->pdev->dev, "interrupt while nobody using sep\n"); |
| return IRQ_NONE; |
| } |
| |
| /* Read the IRR register to check if this is SEP interrupt */ |
| reg_val = sep_read_reg(sep, HW_HOST_IRR_REG_ADDR); |
| |
| dev_dbg(&sep->pdev->dev, "sep int: IRR REG val: %x\n", reg_val); |
| |
| if (reg_val & (0x1 << 13)) { |
| |
| /* Lock and update the counter of reply messages */ |
| spin_lock_irqsave(&sep->snd_rply_lck, lock_irq_flag); |
| sep->reply_ct++; |
| spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag); |
| |
| dev_dbg(&sep->pdev->dev, "sep int: send_ct %lx reply_ct %lx\n", |
| sep->send_ct, sep->reply_ct); |
| |
| /* Is this a kernel client request */ |
| if (sep->in_kernel) { |
| tasklet_schedule(&sep->finish_tasklet); |
| goto finished_interrupt; |
| } |
| |
| /* Is this printf or daemon request? */ |
| reg_val2 = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR); |
| dev_dbg(&sep->pdev->dev, |
| "SEP Interrupt - GPR2 is %08x\n", reg_val2); |
| |
| clear_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags); |
| |
| if ((reg_val2 >> 30) & 0x1) { |
| dev_dbg(&sep->pdev->dev, "int: printf request\n"); |
| } else if (reg_val2 >> 31) { |
| dev_dbg(&sep->pdev->dev, "int: daemon request\n"); |
| } else { |
| dev_dbg(&sep->pdev->dev, "int: SEP reply\n"); |
| wake_up(&sep->event_interrupt); |
| } |
| } else { |
| dev_dbg(&sep->pdev->dev, "int: not SEP interrupt\n"); |
| int_error = IRQ_NONE; |
| } |
| |
| finished_interrupt: |
| |
| if (int_error == IRQ_HANDLED) |
| sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, reg_val); |
| |
| return int_error; |
| } |
| |
| /** |
| * sep_reconfig_shared_area - reconfigure shared area |
| * @sep: pointer to struct sep_device |
| * |
| * Reconfig the shared area between HOST and SEP - needed in case |
| * the DX_CC_Init function was called before OS loading. |
| */ |
| static int sep_reconfig_shared_area(struct sep_device *sep) |
| { |
| int ret_val; |
| |
| /* use to limit waiting for SEP */ |
| unsigned long end_time; |
| |
| /* Send the new SHARED MESSAGE AREA to the SEP */ |
| dev_dbg(&sep->pdev->dev, "reconfig shared; sending %08llx to sep\n", |
| (unsigned long long)sep->shared_bus); |
| |
| sep_write_reg(sep, HW_HOST_HOST_SEP_GPR1_REG_ADDR, sep->shared_bus); |
| |
| /* Poll for SEP response */ |
| ret_val = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR1_REG_ADDR); |
| |
| end_time = jiffies + (WAIT_TIME * HZ); |
| |
| while ((time_before(jiffies, end_time)) && (ret_val != 0xffffffff) && |
| (ret_val != sep->shared_bus)) |
| ret_val = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR1_REG_ADDR); |
| |
| /* Check the return value (register) */ |
| if (ret_val != sep->shared_bus) { |
| dev_warn(&sep->pdev->dev, "could not reconfig shared area\n"); |
| dev_warn(&sep->pdev->dev, "result was %x\n", ret_val); |
| ret_val = -ENOMEM; |
| } else |
| ret_val = 0; |
| |
| dev_dbg(&sep->pdev->dev, "reconfig shared area end\n"); |
| |
| return ret_val; |
| } |
| |
| /** |
| * sep_activate_dcb_dmatables_context - Takes DCB & DMA tables |
| * contexts into use |
| * @sep: SEP device |
| * @dcb_region: DCB region copy |
| * @dmatables_region: MLLI/DMA tables copy |
| * @dma_ctx: DMA context for current transaction |
| */ |
| ssize_t sep_activate_dcb_dmatables_context(struct sep_device *sep, |
| struct sep_dcblock **dcb_region, |
| void **dmatables_region, |
| struct sep_dma_context *dma_ctx) |
| { |
| void *dmaregion_free_start = NULL; |
| void *dmaregion_free_end = NULL; |
| void *dcbregion_free_start = NULL; |
| void *dcbregion_free_end = NULL; |
| ssize_t error = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] activating dcb/dma region\n", |
| current->pid); |
| |
| if (1 > dma_ctx->nr_dcb_creat) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] invalid number of dcbs to activate 0x%08X\n", |
| current->pid, dma_ctx->nr_dcb_creat); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| dmaregion_free_start = sep->shared_addr |
| + SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES; |
| dmaregion_free_end = dmaregion_free_start |
| + SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES - 1; |
| |
| if (dmaregion_free_start |
| + dma_ctx->dmatables_len > dmaregion_free_end) { |
| error = -ENOMEM; |
| goto end_function; |
| } |
| memcpy(dmaregion_free_start, |
| *dmatables_region, |
| dma_ctx->dmatables_len); |
| /* Free MLLI table copy */ |
| kfree(*dmatables_region); |
| *dmatables_region = NULL; |
| |
| /* Copy thread's DCB table copy to DCB table region */ |
| dcbregion_free_start = sep->shared_addr + |
| SEP_DRIVER_SYSTEM_DCB_MEMORY_OFFSET_IN_BYTES; |
| dcbregion_free_end = dcbregion_free_start + |
| (SEP_MAX_NUM_SYNC_DMA_OPS * |
| sizeof(struct sep_dcblock)) - 1; |
| |
| if (dcbregion_free_start |
| + (dma_ctx->nr_dcb_creat * sizeof(struct sep_dcblock)) |
| > dcbregion_free_end) { |
| error = -ENOMEM; |
| goto end_function; |
| } |
| |
| memcpy(dcbregion_free_start, |
| *dcb_region, |
| dma_ctx->nr_dcb_creat * sizeof(struct sep_dcblock)); |
| |
| /* Print the tables */ |
| dev_dbg(&sep->pdev->dev, "activate: input table\n"); |
| sep_debug_print_lli_tables(sep, |
| (struct sep_lli_entry *)sep_shared_area_bus_to_virt(sep, |
| (*dcb_region)->input_mlli_address), |
| (*dcb_region)->input_mlli_num_entries, |
| (*dcb_region)->input_mlli_data_size); |
| |
| dev_dbg(&sep->pdev->dev, "activate: output table\n"); |
| sep_debug_print_lli_tables(sep, |
| (struct sep_lli_entry *)sep_shared_area_bus_to_virt(sep, |
| (*dcb_region)->output_mlli_address), |
| (*dcb_region)->output_mlli_num_entries, |
| (*dcb_region)->output_mlli_data_size); |
| |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] printing activated tables\n", current->pid); |
| |
| end_function: |
| kfree(*dmatables_region); |
| *dmatables_region = NULL; |
| |
| kfree(*dcb_region); |
| *dcb_region = NULL; |
| |
| return error; |
| } |
| |
| /** |
| * sep_create_dcb_dmatables_context - Creates DCB & MLLI/DMA table context |
| * @sep: SEP device |
| * @dcb_region: DCB region buf to create for current transaction |
| * @dmatables_region: MLLI/DMA tables buf to create for current transaction |
| * @dma_ctx: DMA context buf to create for current transaction |
| * @user_dcb_args: User arguments for DCB/MLLI creation |
| * @num_dcbs: Number of DCBs to create |
| * @secure_dma: Indicate use of IMR restricted memory secure dma |
| */ |
| static ssize_t sep_create_dcb_dmatables_context(struct sep_device *sep, |
| struct sep_dcblock **dcb_region, |
| void **dmatables_region, |
| struct sep_dma_context **dma_ctx, |
| const struct build_dcb_struct __user *user_dcb_args, |
| const u32 num_dcbs, bool secure_dma) |
| { |
| int error = 0; |
| int i = 0; |
| struct build_dcb_struct *dcb_args = NULL; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] creating dcb/dma region\n", |
| current->pid); |
| |
| if (!dcb_region || !dma_ctx || !dmatables_region || !user_dcb_args) { |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| if (SEP_MAX_NUM_SYNC_DMA_OPS < num_dcbs) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] invalid number of dcbs 0x%08X\n", |
| current->pid, num_dcbs); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| dcb_args = kcalloc(num_dcbs, sizeof(struct build_dcb_struct), |
| GFP_KERNEL); |
| if (!dcb_args) { |
| error = -ENOMEM; |
| goto end_function; |
| } |
| |
| if (copy_from_user(dcb_args, |
| user_dcb_args, |
| num_dcbs * sizeof(struct build_dcb_struct))) { |
| error = -EFAULT; |
| goto end_function; |
| } |
| |
| /* Allocate thread-specific memory for DCB */ |
| *dcb_region = kzalloc(num_dcbs * sizeof(struct sep_dcblock), |
| GFP_KERNEL); |
| if (!(*dcb_region)) { |
| error = -ENOMEM; |
| goto end_function; |
| } |
| |
| /* Prepare DCB and MLLI table into the allocated regions */ |
| for (i = 0; i < num_dcbs; i++) { |
| error = sep_prepare_input_output_dma_table_in_dcb(sep, |
| (unsigned long)dcb_args[i].app_in_address, |
| (unsigned long)dcb_args[i].app_out_address, |
| dcb_args[i].data_in_size, |
| dcb_args[i].block_size, |
| dcb_args[i].tail_block_size, |
| dcb_args[i].is_applet, |
| false, secure_dma, |
| *dcb_region, dmatables_region, |
| dma_ctx, |
| NULL, |
| NULL); |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] dma table creation failed\n", |
| current->pid); |
| goto end_function; |
| } |
| |
| if (dcb_args[i].app_in_address != 0) |
| (*dma_ctx)->input_data_len += dcb_args[i].data_in_size; |
| } |
| |
| end_function: |
| kfree(dcb_args); |
| return error; |
| |
| } |
| |
| /** |
| * sep_create_dcb_dmatables_context_kernel - Creates DCB & MLLI/DMA table context |
| * for kernel crypto |
| * @sep: SEP device |
| * @dcb_region: DCB region buf to create for current transaction |
| * @dmatables_region: MLLI/DMA tables buf to create for current transaction |
| * @dma_ctx: DMA context buf to create for current transaction |
| * @user_dcb_args: User arguments for DCB/MLLI creation |
| * @num_dcbs: Number of DCBs to create |
| * This does that same thing as sep_create_dcb_dmatables_context |
| * except that it is used only for the kernel crypto operation. It is |
| * separate because there is no user data involved; the dcb data structure |
| * is specific for kernel crypto (build_dcb_struct_kernel) |
| */ |
| int sep_create_dcb_dmatables_context_kernel(struct sep_device *sep, |
| struct sep_dcblock **dcb_region, |
| void **dmatables_region, |
| struct sep_dma_context **dma_ctx, |
| const struct build_dcb_struct_kernel *dcb_data, |
| const u32 num_dcbs) |
| { |
| int error = 0; |
| int i = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] creating dcb/dma region\n", |
| current->pid); |
| |
| if (!dcb_region || !dma_ctx || !dmatables_region || !dcb_data) { |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| if (SEP_MAX_NUM_SYNC_DMA_OPS < num_dcbs) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] invalid number of dcbs 0x%08X\n", |
| current->pid, num_dcbs); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] num_dcbs is %d\n", |
| current->pid, num_dcbs); |
| |
| /* Allocate thread-specific memory for DCB */ |
| *dcb_region = kzalloc(num_dcbs * sizeof(struct sep_dcblock), |
| GFP_KERNEL); |
| if (!(*dcb_region)) { |
| error = -ENOMEM; |
| goto end_function; |
| } |
| |
| /* Prepare DCB and MLLI table into the allocated regions */ |
| for (i = 0; i < num_dcbs; i++) { |
| error = sep_prepare_input_output_dma_table_in_dcb(sep, |
| (unsigned long)dcb_data->app_in_address, |
| (unsigned long)dcb_data->app_out_address, |
| dcb_data->data_in_size, |
| dcb_data->block_size, |
| dcb_data->tail_block_size, |
| dcb_data->is_applet, |
| true, |
| false, |
| *dcb_region, dmatables_region, |
| dma_ctx, |
| dcb_data->src_sg, |
| dcb_data->dst_sg); |
| if (error) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] dma table creation failed\n", |
| current->pid); |
| goto end_function; |
| } |
| } |
| |
| end_function: |
| return error; |
| |
| } |
| |
| /** |
| * sep_activate_msgarea_context - Takes the message area context into use |
| * @sep: SEP device |
| * @msg_region: Message area context buf |
| * @msg_len: Message area context buffer size |
| */ |
| static ssize_t sep_activate_msgarea_context(struct sep_device *sep, |
| void **msg_region, |
| const size_t msg_len) |
| { |
| dev_dbg(&sep->pdev->dev, "[PID%d] activating msg region\n", |
| current->pid); |
| |
| if (!msg_region || !(*msg_region) || |
| SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES < msg_len) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] invalid act msgarea len 0x%08zX\n", |
| current->pid, msg_len); |
| return -EINVAL; |
| } |
| |
| memcpy(sep->shared_addr, *msg_region, msg_len); |
| |
| return 0; |
| } |
| |
| /** |
| * sep_create_msgarea_context - Creates message area context |
| * @sep: SEP device |
| * @msg_region: Msg area region buf to create for current transaction |
| * @msg_user: Content for msg area region from user |
| * @msg_len: Message area size |
| */ |
| static ssize_t sep_create_msgarea_context(struct sep_device *sep, |
| void **msg_region, |
| const void __user *msg_user, |
| const size_t msg_len) |
| { |
| int error = 0; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] creating msg region\n", |
| current->pid); |
| |
| if (!msg_region || |
| !msg_user || |
| SEP_DRIVER_MAX_MESSAGE_SIZE_IN_BYTES < msg_len || |
| SEP_DRIVER_MIN_MESSAGE_SIZE_IN_BYTES > msg_len) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] invalid creat msgarea len 0x%08zX\n", |
| current->pid, msg_len); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| /* Allocate thread-specific memory for message buffer */ |
| *msg_region = kzalloc(msg_len, GFP_KERNEL); |
| if (!(*msg_region)) { |
| error = -ENOMEM; |
| goto end_function; |
| } |
| |
| /* Copy input data to write() to allocated message buffer */ |
| if (copy_from_user(*msg_region, msg_user, msg_len)) { |
| error = -EFAULT; |
| goto end_function; |
| } |
| |
| end_function: |
| if (error && msg_region) { |
| kfree(*msg_region); |
| *msg_region = NULL; |
| } |
| |
| return error; |
| } |
| |
| |
| /** |
| * sep_read - Returns results of an operation for fastcall interface |
| * @filp: File pointer |
| * @buf_user: User buffer for storing results |
| * @count_user: User buffer size |
| * @offset: File offset, not supported |
| * |
| * The implementation does not support reading in chunks, all data must be |
| * consumed during a single read system call. |
| */ |
| static ssize_t sep_read(struct file *filp, |
| char __user *buf_user, size_t count_user, |
| loff_t *offset) |
| { |
| struct sep_private_data * const private_data = filp->private_data; |
| struct sep_call_status *call_status = &private_data->call_status; |
| struct sep_device *sep = private_data->device; |
| struct sep_dma_context **dma_ctx = &private_data->dma_ctx; |
| struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem; |
| ssize_t error = 0, error_tmp = 0; |
| |
| /* Am I the process that owns the transaction? */ |
| error = sep_check_transaction_owner(sep); |
| if (error) { |
| dev_dbg(&sep->pdev->dev, "[PID%d] read pid is not owner\n", |
| current->pid); |
| goto end_function; |
| } |
| |
| /* Checks that user has called necessary apis */ |
| if (0 == test_bit(SEP_FASTCALL_WRITE_DONE_OFFSET, |
| &call_status->status)) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] fastcall write not called\n", |
| current->pid); |
| error = -EPROTO; |
| goto end_function_error; |
| } |
| |
| if (!buf_user) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] null user buffer\n", |
| current->pid); |
| error = -EINVAL; |
| goto end_function_error; |
| } |
| |
| |
| /* Wait for SEP to finish */ |
| wait_event(sep->event_interrupt, |
| test_bit(SEP_WORKING_LOCK_BIT, |
| &sep->in_use_flags) == 0); |
| |
| sep_dump_message(sep); |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] count_user = 0x%08zX\n", |
| current->pid, count_user); |
| |
| /* In case user has allocated bigger buffer */ |
| if (count_user > SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES) |
| count_user = SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES; |
| |
| if (copy_to_user(buf_user, sep->shared_addr, count_user)) { |
| error = -EFAULT; |
| goto end_function_error; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] read succeeded\n", current->pid); |
| error = count_user; |
| |
| end_function_error: |
| /* Copy possible tail data to user and free DCB and MLLIs */ |
| error_tmp = sep_free_dcb_handler(sep, dma_ctx); |
| if (error_tmp) |
| dev_warn(&sep->pdev->dev, "[PID%d] dcb free failed\n", |
| current->pid); |
| |
| /* End the transaction, wakeup pending ones */ |
| error_tmp = sep_end_transaction_handler(sep, dma_ctx, call_status, |
| my_queue_elem); |
| if (error_tmp) |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] ending transaction failed\n", |
| current->pid); |
| |
| end_function: |
| return error; |
| } |
| |
| /** |
| * sep_fastcall_args_get - Gets fastcall params from user |
| * sep: SEP device |
| * @args: Parameters buffer |
| * @buf_user: User buffer for operation parameters |
| * @count_user: User buffer size |
| */ |
| static inline ssize_t sep_fastcall_args_get(struct sep_device *sep, |
| struct sep_fastcall_hdr *args, |
| const char __user *buf_user, |
| const size_t count_user) |
| { |
| ssize_t error = 0; |
| size_t actual_count = 0; |
| |
| if (!buf_user) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] null user buffer\n", |
| current->pid); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| if (count_user < sizeof(struct sep_fastcall_hdr)) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] too small message size 0x%08zX\n", |
| current->pid, count_user); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| |
| if (copy_from_user(args, buf_user, sizeof(struct sep_fastcall_hdr))) { |
| error = -EFAULT; |
| goto end_function; |
| } |
| |
| if (SEP_FC_MAGIC != args->magic) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] invalid fastcall magic 0x%08X\n", |
| current->pid, args->magic); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] fastcall hdr num of DCBs 0x%08X\n", |
| current->pid, args->num_dcbs); |
| dev_dbg(&sep->pdev->dev, "[PID%d] fastcall hdr msg len 0x%08X\n", |
| current->pid, args->msg_len); |
| |
| if (SEP_DRIVER_MAX_MESSAGE_SIZE_IN_BYTES < args->msg_len || |
| SEP_DRIVER_MIN_MESSAGE_SIZE_IN_BYTES > args->msg_len) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] invalid message length\n", |
| current->pid); |
| error = -EINVAL; |
| goto end_function; |
| } |
| |
| actual_count = sizeof(struct sep_fastcall_hdr) |
| + args->msg_len |
| + (args->num_dcbs * sizeof(struct build_dcb_struct)); |
| |
| if (actual_count != count_user) { |
| dev_warn(&sep->pdev->dev, |
| "[PID%d] inconsistent message " |
| "sizes 0x%08zX vs 0x%08zX\n", |
| current->pid, actual_count, count_user); |
| error = -EMSGSIZE; |
| goto end_function; |
| } |
| |
| end_function: |
| return error; |
| } |
| |
| /** |
| * sep_write - Starts an operation for fastcall interface |
| * @filp: File pointer |
| * @buf_user: User buffer for operation parameters |
| * @count_user: User buffer size |
| * @offset: File offset, not supported |
| * |
| * The implementation does not support writing in chunks, |
| * all data must be given during a single write system call. |
| */ |
| static ssize_t sep_write(struct file *filp, |
| const char __user *buf_user, size_t count_user, |
| loff_t *offset) |
| { |
| struct sep_private_data * const private_data = filp->private_data; |
| struct sep_call_status *call_status = &private_data->call_status; |
| struct sep_device *sep = private_data->device; |
| struct sep_dma_context *dma_ctx = NULL; |
| struct sep_fastcall_hdr call_hdr = {0}; |
| void *msg_region = NULL; |
| void *dmatables_region = NULL; |
| struct sep_dcblock *dcb_region = NULL; |
| ssize_t error = 0; |
| struct sep_queue_info *my_queue_elem = NULL; |
| bool my_secure_dma; /* are we using secure_dma (IMR)? */ |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] sep dev is 0x%p\n", |
| current->pid, sep); |
| dev_dbg(&sep->pdev->dev, "[PID%d] private_data is 0x%p\n", |
| current->pid, private_data); |
| |
| error = sep_fastcall_args_get(sep, &call_hdr, buf_user, count_user); |
| if (error) |
| goto end_function; |
| |
| buf_user += sizeof(struct sep_fastcall_hdr); |
| |
| if (call_hdr.secure_dma == 0) |
| my_secure_dma = false; |
| else |
| my_secure_dma = true; |
| |
| /* |
| * Controlling driver memory usage by limiting amount of |
| * buffers created. Only SEP_DOUBLEBUF_USERS_LIMIT number |
| * of threads can progress further at a time |
| */ |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] waiting for double buffering region access\n", |
| current->pid); |
| error = down_interruptible(&sep->sep_doublebuf); |
| dev_dbg(&sep->pdev->dev, "[PID%d] double buffering region start\n", |
| current->pid); |
| if (error) { |
| /* Signal received */ |
| goto end_function_error; |
| } |
| |
| |
| /* |
| * Prepare contents of the shared area regions for |
| * the operation into temporary buffers |
| */ |
| if (0 < call_hdr.num_dcbs) { |
| error = sep_create_dcb_dmatables_context(sep, |
| &dcb_region, |
| &dmatables_region, |
| &dma_ctx, |
| (const struct build_dcb_struct __user *) |
| buf_user, |
| call_hdr.num_dcbs, my_secure_dma); |
| if (error) |
| goto end_function_error_doublebuf; |
| |
| buf_user += call_hdr.num_dcbs * sizeof(struct build_dcb_struct); |
| } |
| |
| error = sep_create_msgarea_context(sep, |
| &msg_region, |
| buf_user, |
| call_hdr.msg_len); |
| if (error) |
| goto end_function_error_doublebuf; |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] updating queue status\n", |
| current->pid); |
| my_queue_elem = sep_queue_status_add(sep, |
| ((struct sep_msgarea_hdr *)msg_region)->opcode, |
| (dma_ctx) ? dma_ctx->input_data_len : 0, |
| current->pid, |
| current->comm, sizeof(current->comm)); |
| |
| if (!my_queue_elem) { |
| dev_dbg(&sep->pdev->dev, |
| "[PID%d] updating queue status error\n", current->pid); |
| error = -ENOMEM; |
| goto end_function_error_doublebuf; |
| } |
| |
| /* Wait until current process gets the transaction */ |
| error = sep_wait_transaction(sep); |
| |
| if (error) { |
| /* Interrupted by signal, don't clear transaction */ |
| dev_dbg(&sep->pdev->dev, "[PID%d] interrupted by signal\n", |
| current->pid); |
| sep_queue_status_remove(sep, &my_queue_elem); |
| goto end_function_error_doublebuf; |
| } |
| |
| dev_dbg(&sep->pdev->dev, "[PID%d] saving queue element\n", |
| current->pid); |
| private_data->my_queue_elem = my_queue_elem; |
| |
| /* Activate shared area regions for the transaction */ |
| error = sep_activate_msgarea_context(sep, &msg_region, |
| call_hdr.msg_len); |
| if (error) |
| goto end_function_error_clear_transact; |
| |
| sep_dump_message(sep); |
| |
| if (0 < call_hdr.num_dcbs) { |
| error = sep_activate_dcb_dmatables_context(sep, |
| &dcb_region, |
| &dmatables_region, |
| dma_ctx); |
| if (error) |
| goto end_function_error_clear_transact; |
| } |
| |
| /* Send command to SEP */ |
| error = sep_send_command_handler(sep); |
| if (error) |
| goto end_function_error_clear_transact; |
| |
| /* Store DMA context for the transaction */ |
| private_data->dma_ctx = dma_ctx; |
| /* Update call status */ |
| set_bit(SEP_FASTCALL_WRITE_DONE_OFFSET, &call_status->status); |
| error = count_user; |
| |
| up(&sep->sep_doublebuf); |
| dev_dbg(&sep->pdev->dev, "[PID%d] double buffering region end\n", |
| current->pid); |
| |
| goto end_function; |
| |
| end_function_error_clear_transact: |
| sep_end_transaction_handler(sep, &dma_ctx, call_status, |
| &private_data->my_queue_elem); |
| |
| end_function_error_doublebuf: |
| up(&sep->sep_doublebuf); |
| dev_dbg(&sep->pdev->dev, "[PID%d] double buffering region end\n", |
| current->pid); |
| |
| end_function_error: |
| if (dma_ctx) |
| sep_free_dma_table_data_handler(sep, &dma_ctx); |
| |
| end_function: |
| kfree(dcb_region); |
| kfree(dmatables_region); |
| kfree(msg_region); |
| |
| return error; |
| } |
| /** |
| * sep_seek - Handler for seek system call |
| * @filp: File pointer |
| * @offset: File offset |
| * @origin: Options for offset |
| * |
| * Fastcall interface does not support seeking, all reads |
| * and writes are from/to offset zero |
| */ |
| static loff_t sep_seek(struct file *filp, loff_t offset, int origin) |
| { |
| return -ENOSYS; |
| } |
| |
| |
| |
| /** |
| * sep_file_operations - file operation on sep device |
| * @sep_ioctl: ioctl handler from user space call |
| * @sep_poll: poll handler |
| * @sep_open: handles sep device open request |
| * @sep_release:handles sep device release request |
| * @sep_mmap: handles memory mapping requests |
| * @sep_read: handles read request on sep device |
| * @sep_write: handles write request on sep device |
| * @sep_seek: handles seek request on sep device |
| */ |
| static const struct file_operations sep_file_operations = { |
| .owner = THIS_MODULE, |
| .unlocked_ioctl = sep_ioctl, |
| .poll = sep_poll, |
| .open = sep_open, |
| .release = sep_release, |
| .mmap = sep_mmap, |
| .read = sep_read, |
| .write = sep_write, |
| .llseek = sep_seek, |
| }; |
| |
| /** |
| * sep_sysfs_read - read sysfs entry per gives arguments |
| * @filp: file pointer |
| * @kobj: kobject pointer |
| * @attr: binary file attributes |
| * @buf: read to this buffer |
| * @pos: offset to read |
| * @count: amount of data to read |
| * |
| * This function is to read sysfs entries for sep driver per given arguments. |
| */ |
| static ssize_t |
| sep_sysfs_read(struct file *filp, struct kobject *kobj, |
| struct bin_attribute *attr, |
| char *buf, loff_t pos, size_t count) |
| { |
| unsigned long lck_flags; |
| size_t nleft = count; |
| struct sep_device *sep = sep_dev; |
| struct sep_queue_info *queue_elem = NULL; |
| u32 queue_num = 0; |
| u32 i = 1; |
| |
| spin_lock_irqsave(&sep->sep_queue_lock, lck_flags); |
| |
| queue_num = sep->sep_queue_num; |
| if (queue_num > SEP_DOUBLEBUF_USERS_LIMIT) |
| queue_num = SEP_DOUBLEBUF_USERS_LIMIT; |
| |
| |
| if (count < sizeof(queue_num) |
| + (queue_num * sizeof(struct sep_queue_data))) { |
| spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags); |
| return -EINVAL; |
| } |
| |
| memcpy(buf, &queue_num, sizeof(queue_num)); |
| buf += sizeof(queue_num); |
| nleft -= sizeof(queue_num); |
| |
| list_for_each_entry(queue_elem, &sep->sep_queue_status, list) { |
| if (i++ > queue_num) |
| break; |
| |
| memcpy(buf, &queue_elem->data, sizeof(queue_elem->data)); |
| nleft -= sizeof(queue_elem->data); |
| buf += sizeof(queue_elem->data); |
| } |
| spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags); |
| |
| return count - nleft; |
| } |
| |
| /** |
| * bin_attributes - defines attributes for queue_status |
| * @attr: attributes (name & permissions) |
| * @read: function pointer to read this file |
| * @size: maxinum size of binary attribute |
| */ |
| static const struct bin_attribute queue_status = { |
| .attr = {.name = "queue_status", .mode = 0444}, |
| .read = sep_sysfs_read, |
| .size = sizeof(u32) |
| + (SEP_DOUBLEBUF_USERS_LIMIT * sizeof(struct sep_queue_data)), |
| }; |
| |
| /** |
| * sep_register_driver_with_fs - register misc devices |
| * @sep: pointer to struct sep_device |
| * |
| * This function registers the driver with the file system |
| */ |
| static int sep_register_driver_with_fs(struct sep_device *sep) |
| { |
| int ret_val; |
| |
| sep->miscdev_sep.minor = MISC_DYNAMIC_MINOR; |
| sep->miscdev_sep.name = SEP_DEV_NAME; |
| sep->miscdev_sep.fops = &sep_file_operations; |
| |
| ret_val = misc_register(&sep->miscdev_sep); |
| if (ret_val) { |
| dev_warn(&sep->pdev->dev, "misc reg fails for SEP %x\n", |
| ret_val); |
| return ret_val; |
| } |
| |
| ret_val = device_create_bin_file(sep->miscdev_sep.this_device, |
| &queue_status); |
| if (ret_val) { |
| dev_warn(&sep->pdev->dev, "sysfs attribute1 fails for SEP %x\n", |
| ret_val); |
| return ret_val; |
| } |
| |
| return ret_val; |
| } |
| |
| |
| /** |
| *sep_probe - probe a matching PCI device |
| *@pdev: pci_device |
| *@ent: pci_device_id |
| * |
| *Attempt to set up and configure a SEP device that has been |
| *discovered by the PCI layer. Allocates all required resources. |
| */ |
| static int sep_probe(struct pci_dev *pdev, |
| const struct pci_device_id *ent) |
| { |
| int error = 0; |
| struct sep_device *sep = NULL; |
| |
| if (sep_dev != NULL) { |
| dev_dbg(&pdev->dev, "only one SEP supported.\n"); |
| return -EBUSY; |
| } |
| |
| /* Enable the device */ |
| error = pci_enable_device(pdev); |
| if (error) { |
| dev_warn(&pdev->dev, "error enabling pci device\n"); |
| goto end_function; |
| } |
| |
| /* Allocate the sep_device structure for this device */ |
| sep_dev = kzalloc(sizeof(struct sep_device), GFP_ATOMIC); |
| if (sep_dev == NULL) { |
| error = -ENOMEM; |
| goto end_function_disable_device; |
| } |
| |
| /* |
| * We're going to use another variable for actually |
| * working with the device; this way, if we have |
| * multiple devices in the future, it would be easier |
| * to make appropriate changes |
| */ |
| sep = sep_dev; |
| |
| sep->pdev = pci_dev_get(pdev); |
| |
| init_waitqueue_head(&sep->event_transactions); |
| init_waitqueue_head(&sep->event_interrupt); |
| spin_lock_init(&sep->snd_rply_lck); |
| spin_lock_init(&sep->sep_queue_lock); |
| sema_init(&sep->sep_doublebuf, SEP_DOUBLEBUF_USERS_LIMIT); |
| |
| INIT_LIST_HEAD(&sep->sep_queue_status); |
| |
| dev_dbg(&sep->pdev->dev, |
| "sep probe: PCI obtained, device being prepared\n"); |
| |
| /* Set up our register area */ |
| sep->reg_physical_addr = pci_resource_start(sep->pdev, 0); |
| if (!sep->reg_physical_addr) { |
| dev_warn(&sep->pdev->dev, "Error getting register start\n"); |
| error = -ENODEV; |
| goto end_function_free_sep_dev; |
| } |
| |
| sep->reg_physical_end = pci_resource_end(sep->pdev, 0); |
| if (!sep->reg_physical_end) { |
| dev_warn(&sep->pdev->dev, "Error getting register end\n"); |
| error = -ENODEV; |
| goto end_function_free_sep_dev; |
| } |
| |
| sep->reg_addr = ioremap_nocache(sep->reg_physical_addr, |
| (size_t)(sep->reg_physical_end - sep->reg_physical_addr + 1)); |
| if (!sep->reg_addr) { |
| dev_warn(&sep->pdev->dev, "Error getting register virtual\n"); |
| error = -ENODEV; |
| goto end_function_free_sep_dev; |
| } |
| |
| dev_dbg(&sep->pdev->dev, |
| "Register area start %llx end %llx virtual %p\n", |
| (unsigned long long)sep->reg_physical_addr, |
| (unsigned long long)sep->reg_physical_end, |
| sep->reg_addr); |
| |
| /* Allocate the shared area */ |
| sep->shared_size = SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES + |
| SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES + |
| SEP_DRIVER_DATA_POOL_SHARED_AREA_SIZE_IN_BYTES + |
| SEP_DRIVER_STATIC_AREA_SIZE_IN_BYTES + |
| SEP_DRIVER_SYSTEM_DATA_MEMORY_SIZE_IN_BYTES; |
| |
| if (sep_map_and_alloc_shared_area(sep)) { |
| error = -ENOMEM; |
| /* Allocation failed */ |
| goto end_function_error; |
| } |
| |
| /* Clear ICR register */ |
| sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF); |
| |
| /* Set the IMR register - open only GPR 2 */ |
| sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, (~(0x1 << 13))); |
| |
| /* Read send/receive counters from SEP */ |
| sep->reply_ct = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR); |
| sep->reply_ct &= 0x3FFFFFFF; |
| sep->send_ct = sep->reply_ct; |
| |
| /* Get the interrupt line */ |
| error = request_irq(pdev->irq, sep_inthandler, IRQF_SHARED, |
| "sep_driver", sep); |
| |
| if (error) |
| goto end_function_deallocate_sep_shared_area; |
| |
| /* The new chip requires a shared area reconfigure */ |
| error = sep_reconfig_shared_area(sep); |
| if (error) |
| goto end_function_free_irq; |
| |
| sep->in_use = 1; |
| |
| /* Finally magic up the device nodes */ |
| /* Register driver with the fs */ |
| error = sep_register_driver_with_fs(sep); |
| |
| if (error) { |
| dev_err(&sep->pdev->dev, "error registering dev file\n"); |
| goto end_function_free_irq; |
| } |
| |
| sep->in_use = 0; /* through touching the device */ |
| #ifdef SEP_ENABLE_RUNTIME_PM |
| pm_runtime_put_noidle(&sep->pdev->dev); |
| pm_runtime_allow(&sep->pdev->dev); |
| pm_runtime_set_autosuspend_delay(&sep->pdev->dev, |
| SUSPEND_DELAY); |
| pm_runtime_use_autosuspend(&sep->pdev->dev); |
| pm_runtime_mark_last_busy(&sep->pdev->dev); |
| sep->power_save_setup = 1; |
| #endif |
| /* register kernel crypto driver */ |
| #if defined(CONFIG_CRYPTO) || defined(CONFIG_CRYPTO_MODULE) |
| error = sep_crypto_setup(); |
| if (error) { |
| dev_err(&sep->pdev->dev, "crypto setup failed\n"); |
| goto end_function_free_irq; |
| } |
| #endif |
| goto end_function; |
| |
| end_function_free_irq: |
| free_irq(pdev->irq, sep); |
| |
| end_function_deallocate_sep_shared_area: |
| /* De-allocate shared area */ |
| sep_unmap_and_free_shared_area(sep); |
| |
| end_function_error: |
| iounmap(sep->reg_addr); |
| |
| end_function_free_sep_dev: |
| pci_dev_put(sep_dev->pdev); |
| kfree(sep_dev); |
| sep_dev = NULL; |
| |
| end_function_disable_device: |
| pci_disable_device(pdev); |
| |
| end_function: |
| return error; |
| } |
| |
| /** |
| * sep_remove - handles removing device from pci subsystem |
| * @pdev: pointer to pci device |
| * |
| * This function will handle removing our sep device from pci subsystem on exit |
| * or unloading this module. It should free up all used resources, and unmap if |
| * any memory regions mapped. |
| */ |
| static void sep_remove(struct pci_dev *pdev) |
| { |
| struct sep_device *sep = sep_dev; |
| |
| /* Unregister from fs */ |
| misc_deregister(&sep->miscdev_sep); |
| |
| /* Unregister from kernel crypto */ |
| #if defined(CONFIG_CRYPTO) || defined(CONFIG_CRYPTO_MODULE) |
| sep_crypto_takedown(); |
| #endif |
| /* Free the irq */ |
| free_irq(sep->pdev->irq, sep); |
| |
| /* Free the shared area */ |
| sep_unmap_and_free_shared_area(sep_dev); |
| iounmap(sep_dev->reg_addr); |
| |
| #ifdef SEP_ENABLE_RUNTIME_PM |
| if (sep->in_use) { |
| sep->in_use = 0; |
| pm_runtime_forbid(&sep->pdev->dev); |
| pm_runtime_get_noresume(&sep->pdev->dev); |
| } |
| #endif |
| pci_dev_put(sep_dev->pdev); |
| kfree(sep_dev); |
| sep_dev = NULL; |
| } |
| |
| /* Initialize struct pci_device_id for our driver */ |
| static DEFINE_PCI_DEVICE_TABLE(sep_pci_id_tbl) = { |
| {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0826)}, |
| {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08e9)}, |
| {0} |
| }; |
| |
| /* Export our pci_device_id structure to user space */ |
| MODULE_DEVICE_TABLE(pci, sep_pci_id_tbl); |
| |
| #ifdef SEP_ENABLE_RUNTIME_PM |
| |
| /** |
| * sep_pm_resume - rsume routine while waking up from S3 state |
| * @dev: pointer to sep device |
| * |
| * This function is to be used to wake up sep driver while system awakes from S3 |
| * state i.e. suspend to ram. The RAM in intact. |
| * Notes - revisit with more understanding of pm, ICR/IMR & counters. |
| */ |
| static int sep_pci_resume(struct device *dev) |
| { |
| struct sep_device *sep = sep_dev; |
| |
| dev_dbg(&sep->pdev->dev, "pci resume called\n"); |
| |
| if (sep->power_state == SEP_DRIVER_POWERON) |
| return 0; |
| |
| /* Clear ICR register */ |
| sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF); |
| |
| /* Set the IMR register - open only GPR 2 */ |
| sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, (~(0x1 << 13))); |
| |
| /* Read send/receive counters from SEP */ |
| sep->reply_ct = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR); |
| sep->reply_ct &= 0x3FFFFFFF; |
| sep->send_ct = sep->reply_ct; |
| |
| sep->power_state = SEP_DRIVER_POWERON; |
| |
| return 0; |
| } |
| |
| /** |
| * sep_pm_suspend - suspend routine while going to S3 state |
| * @dev: pointer to sep device |
| * |
| * This function is to be used to suspend sep driver while system goes to S3 |
| * state i.e. suspend to ram. The RAM in intact and ON during this suspend. |
| * Notes - revisit with more understanding of pm, ICR/IMR |
| */ |
| static int sep_pci_suspend(struct device *dev) |
| { |
| struct sep_device *sep = sep_dev; |
| |
| dev_dbg(&sep->pdev->dev, "pci suspend called\n"); |
| if (sep->in_use == 1) |
| return -EAGAIN; |
| |
| sep->power_state = SEP_DRIVER_POWEROFF; |
| |
| /* Clear ICR register */ |
| sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF); |
| |
| /* Set the IMR to block all */ |
| sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, 0xFFFFFFFF); |
| |
| return 0; |
| } |
| |
| /** |
| * sep_pm_runtime_resume - runtime resume routine |
| * @dev: pointer to sep device |
| * |
| * Notes - revisit with more understanding of pm, ICR/IMR & counters |
| */ |
| static int sep_pm_runtime_resume(struct device *dev) |
| { |
| |
| u32 retval2; |
| u32 delay_count; |
| struct sep_device *sep = sep_dev; |
| |
| dev_dbg(&sep->pdev->dev, "pm runtime resume called\n"); |
| |
| /** |
| * Wait until the SCU boot is ready |
| * This is done by iterating SCU_DELAY_ITERATION (10 |
| * microseconds each) up to SCU_DELAY_MAX (50) times. |
| * This bit can be set in a random time that is less |
| * than 500 microseconds after each power resume |
| */ |
| retval2 = 0; |
| delay_count = 0; |
| while ((!retval2) && (delay_count < SCU_DELAY_MAX)) { |
| retval2 = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR3_REG_ADDR); |
| retval2 &= 0x00000008; |
| if (!retval2) { |
| udelay(SCU_DELAY_ITERATION); |
| delay_count += 1; |
| } |
| } |
| |
| if (!retval2) { |
| dev_warn(&sep->pdev->dev, "scu boot bit not set at resume\n"); |
| return -EINVAL; |
| } |
| |
| /* Clear ICR register */ |
| sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF); |
| |
| /* Set the IMR register - open only GPR 2 */ |
| sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, (~(0x1 << 13))); |
| |
| /* Read send/receive counters from SEP */ |
| sep->reply_ct = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR); |
| sep->reply_ct &= 0x3FFFFFFF; |
| sep->send_ct = sep->reply_ct; |
| |
| return 0; |
| } |
| |
| /** |
| * sep_pm_runtime_suspend - runtime suspend routine |
| * @dev: pointer to sep device |
| * |
| * Notes - revisit with more understanding of pm |
| */ |
| static int sep_pm_runtime_suspend(struct device *dev) |
| { |
| struct sep_device *sep = sep_dev; |
| |
| dev_dbg(&sep->pdev->dev, "pm runtime suspend called\n"); |
| |
| /* Clear ICR register */ |
| sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF); |
| return 0; |
| } |
| |
| /** |
| * sep_pm - power management for sep driver |
| * @sep_pm_runtime_resume: resume- no communication with cpu & main memory |
| * @sep_pm_runtime_suspend: suspend- no communication with cpu & main memory |
| * @sep_pci_suspend: suspend - main memory is still ON |
| * @sep_pci_resume: resume - main memory is still ON |
| */ |
| static const struct dev_pm_ops sep_pm = { |
| .runtime_resume = sep_pm_runtime_resume, |
| .runtime_suspend = sep_pm_runtime_suspend, |
| .resume = sep_pci_resume, |
| .suspend = sep_pci_suspend, |
| }; |
| #endif /* SEP_ENABLE_RUNTIME_PM */ |
| |
| /** |
| * sep_pci_driver - registers this device with pci subsystem |
| * @name: name identifier for this driver |
| * @sep_pci_id_tbl: pointer to struct pci_device_id table |
| * @sep_probe: pointer to probe function in PCI driver |
| * @sep_remove: pointer to remove function in PCI driver |
| */ |
| static struct pci_driver sep_pci_driver = { |
| #ifdef SEP_ENABLE_RUNTIME_PM |
| .driver = { |
| .pm = &sep_pm, |
| }, |
| #endif |
| .name = "sep_sec_driver", |
| .id_table = sep_pci_id_tbl, |
| .probe = sep_probe, |
| .remove = sep_remove |
| }; |
| |
| module_pci_driver(sep_pci_driver); |
| MODULE_LICENSE("GPL"); |