| /* |
| * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved. |
| * |
| * This program is free software; you can redistribute it and/or modify |
| * it under the terms of the GNU General Public License version 2 and |
| * only version 2 as published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| */ |
| |
| #define pr_fmt(fmt) "iommu-debug: %s: " fmt, __func__ |
| |
| #include <linux/debugfs.h> |
| #include <linux/device.h> |
| #include <linux/iommu.h> |
| #include <linux/of.h> |
| #include <linux/platform_device.h> |
| #include <linux/slab.h> |
| #include <linux/module.h> |
| #include <linux/uaccess.h> |
| #include <linux/dma-contiguous.h> |
| #include <soc/qcom/secure_buffer.h> |
| #include <linux/dma-mapping.h> |
| #include <asm/cacheflush.h> |
| #include <asm/dma-iommu.h> |
| |
| #if defined(CONFIG_IOMMU_TESTS) |
| |
| static const char *iommu_debug_attr_to_string(enum iommu_attr attr) |
| { |
| switch (attr) { |
| case DOMAIN_ATTR_GEOMETRY: |
| return "DOMAIN_ATTR_GEOMETRY"; |
| case DOMAIN_ATTR_PAGING: |
| return "DOMAIN_ATTR_PAGING"; |
| case DOMAIN_ATTR_WINDOWS: |
| return "DOMAIN_ATTR_WINDOWS"; |
| case DOMAIN_ATTR_FSL_PAMU_STASH: |
| return "DOMAIN_ATTR_FSL_PAMU_STASH"; |
| case DOMAIN_ATTR_FSL_PAMU_ENABLE: |
| return "DOMAIN_ATTR_FSL_PAMU_ENABLE"; |
| case DOMAIN_ATTR_FSL_PAMUV1: |
| return "DOMAIN_ATTR_FSL_PAMUV1"; |
| case DOMAIN_ATTR_NESTING: |
| return "DOMAIN_ATTR_NESTING"; |
| case DOMAIN_ATTR_PT_BASE_ADDR: |
| return "DOMAIN_ATTR_PT_BASE_ADDR"; |
| case DOMAIN_ATTR_SECURE_VMID: |
| return "DOMAIN_ATTR_SECURE_VMID"; |
| case DOMAIN_ATTR_ATOMIC: |
| return "DOMAIN_ATTR_ATOMIC"; |
| case DOMAIN_ATTR_CONTEXT_BANK: |
| return "DOMAIN_ATTR_CONTEXT_BANK"; |
| case DOMAIN_ATTR_TTBR0: |
| return "DOMAIN_ATTR_TTBR0"; |
| case DOMAIN_ATTR_CONTEXTIDR: |
| return "DOMAIN_ATTR_CONTEXTIDR"; |
| case DOMAIN_ATTR_PROCID: |
| return "DOMAIN_ATTR_PROCID"; |
| case DOMAIN_ATTR_DYNAMIC: |
| return "DOMAIN_ATTR_DYNAMIC"; |
| case DOMAIN_ATTR_NON_FATAL_FAULTS: |
| return "DOMAIN_ATTR_NON_FATAL_FAULTS"; |
| case DOMAIN_ATTR_S1_BYPASS: |
| return "DOMAIN_ATTR_S1_BYPASS"; |
| case DOMAIN_ATTR_FAST: |
| return "DOMAIN_ATTR_FAST"; |
| case DOMAIN_ATTR_EARLY_MAP: |
| return "DOMAIN_ATTR_EARLY_MAP"; |
| case DOMAIN_ATTR_CB_STALL_DISABLE: |
| return "DOMAIN_ATTR_CB_STALL_DISABLE"; |
| default: |
| return "Unknown attr!"; |
| } |
| } |
| #endif |
| |
| #ifdef CONFIG_IOMMU_DEBUG_TRACKING |
| |
| static DEFINE_MUTEX(iommu_debug_attachments_lock); |
| static LIST_HEAD(iommu_debug_attachments); |
| |
| /* |
| * Each group may have more than one domain; but each domain may |
| * only have one group. |
| * Used by debug tools to display the name of the device(s) associated |
| * with a particular domain. |
| */ |
| struct iommu_debug_attachment { |
| struct iommu_domain *domain; |
| struct iommu_group *group; |
| struct list_head list; |
| }; |
| |
| void iommu_debug_attach_device(struct iommu_domain *domain, |
| struct device *dev) |
| { |
| struct iommu_debug_attachment *attach; |
| struct iommu_group *group; |
| |
| group = dev->iommu_group; |
| if (!group) |
| return; |
| |
| mutex_lock(&iommu_debug_attachments_lock); |
| list_for_each_entry(attach, &iommu_debug_attachments, list) |
| if ((attach->domain == domain) && (attach->group == group)) |
| goto out; |
| |
| attach = kzalloc(sizeof(*attach), GFP_KERNEL); |
| if (!attach) |
| goto out; |
| |
| attach->domain = domain; |
| attach->group = group; |
| INIT_LIST_HEAD(&attach->list); |
| |
| list_add(&attach->list, &iommu_debug_attachments); |
| out: |
| mutex_unlock(&iommu_debug_attachments_lock); |
| } |
| |
| void iommu_debug_domain_remove(struct iommu_domain *domain) |
| { |
| struct iommu_debug_attachment *it, *tmp; |
| |
| mutex_lock(&iommu_debug_attachments_lock); |
| list_for_each_entry_safe(it, tmp, &iommu_debug_attachments, list) { |
| if (it->domain != domain) |
| continue; |
| list_del(&it->list); |
| kfree(it); |
| } |
| |
| mutex_unlock(&iommu_debug_attachments_lock); |
| } |
| |
| #endif |
| |
| #ifdef CONFIG_IOMMU_TESTS |
| |
| #ifdef CONFIG_64BIT |
| |
| #define kstrtoux kstrtou64 |
| #define kstrtox_from_user kstrtoull_from_user |
| #define kstrtosize_t kstrtoul |
| |
| #else |
| |
| #define kstrtoux kstrtou32 |
| #define kstrtox_from_user kstrtouint_from_user |
| #define kstrtosize_t kstrtouint |
| |
| #endif |
| |
| static LIST_HEAD(iommu_debug_devices); |
| static struct dentry *debugfs_tests_dir; |
| static u32 iters_per_op = 1; |
| static void *test_virt_addr; |
| |
| struct iommu_debug_device { |
| struct device *dev; |
| struct iommu_domain *domain; |
| struct dma_iommu_mapping *mapping; |
| u64 iova; |
| u64 phys; |
| size_t len; |
| struct list_head list; |
| struct mutex clk_lock; |
| unsigned int clk_count; |
| }; |
| |
| static int iommu_debug_build_phoney_sg_table(struct device *dev, |
| struct sg_table *table, |
| unsigned long total_size, |
| unsigned long chunk_size) |
| { |
| unsigned long nents = total_size / chunk_size; |
| struct scatterlist *sg; |
| int i; |
| struct page *page; |
| |
| if (!IS_ALIGNED(total_size, PAGE_SIZE)) |
| return -EINVAL; |
| if (!IS_ALIGNED(total_size, chunk_size)) |
| return -EINVAL; |
| if (sg_alloc_table(table, nents, GFP_KERNEL)) |
| return -EINVAL; |
| page = alloc_pages(GFP_KERNEL, get_order(chunk_size)); |
| if (!page) |
| goto free_table; |
| |
| /* all the same page... why not. */ |
| for_each_sg(table->sgl, sg, table->nents, i) |
| sg_set_page(sg, page, chunk_size, 0); |
| |
| return 0; |
| |
| free_table: |
| sg_free_table(table); |
| return -ENOMEM; |
| } |
| |
| static void iommu_debug_destroy_phoney_sg_table(struct device *dev, |
| struct sg_table *table, |
| unsigned long chunk_size) |
| { |
| __free_pages(sg_page(table->sgl), get_order(chunk_size)); |
| sg_free_table(table); |
| } |
| |
| static const char * const _size_to_string(unsigned long size) |
| { |
| switch (size) { |
| case SZ_4K: |
| return "4K"; |
| case SZ_8K: |
| return "8K"; |
| case SZ_16K: |
| return "16K"; |
| case SZ_64K: |
| return "64K"; |
| case SZ_2M: |
| return "2M"; |
| case SZ_1M * 12: |
| return "12M"; |
| case SZ_1M * 20: |
| return "20M"; |
| } |
| return "unknown size, please add to _size_to_string"; |
| } |
| |
| static int nr_iters_set(void *data, u64 val) |
| { |
| if (!val) |
| val = 1; |
| if (val > 10000) |
| val = 10000; |
| *(u32 *)data = val; |
| return 0; |
| } |
| |
| static int nr_iters_get(void *data, u64 *val) |
| { |
| *val = *(u32 *)data; |
| return 0; |
| } |
| |
| DEFINE_SIMPLE_ATTRIBUTE(iommu_debug_nr_iters_ops, |
| nr_iters_get, nr_iters_set, "%llu\n"); |
| |
| static void iommu_debug_device_profiling(struct seq_file *s, struct device *dev, |
| enum iommu_attr attrs[], |
| void *attr_values[], int nattrs, |
| const size_t sizes[]) |
| { |
| int i; |
| const size_t *sz; |
| struct iommu_domain *domain; |
| unsigned long iova = 0x10000; |
| phys_addr_t paddr = 0xa000; |
| |
| domain = iommu_domain_alloc(&platform_bus_type); |
| if (!domain) { |
| seq_puts(s, "Couldn't allocate domain\n"); |
| return; |
| } |
| |
| seq_puts(s, "Domain attributes: [ "); |
| for (i = 0; i < nattrs; ++i) { |
| /* not all attrs are ints, but this will get us by for now */ |
| seq_printf(s, "%s=%d%s", iommu_debug_attr_to_string(attrs[i]), |
| *((int *)attr_values[i]), |
| i < nattrs ? " " : ""); |
| } |
| seq_puts(s, "]\n"); |
| for (i = 0; i < nattrs; ++i) { |
| if (iommu_domain_set_attr(domain, attrs[i], attr_values[i])) { |
| seq_printf(s, "Couldn't set %d to the value at %p\n", |
| attrs[i], attr_values[i]); |
| goto out_domain_free; |
| } |
| } |
| |
| if (iommu_attach_group(domain, dev->iommu_group)) { |
| seq_puts(s, |
| "Couldn't attach new domain to device. Is it already attached?\n"); |
| goto out_domain_free; |
| } |
| |
| seq_printf(s, "(average over %d iterations)\n", iters_per_op); |
| seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map", "iommu_unmap"); |
| for (sz = sizes; *sz; ++sz) { |
| size_t size = *sz; |
| size_t unmapped; |
| u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; |
| u64 map_elapsed_us = 0, unmap_elapsed_us = 0; |
| u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; |
| struct timespec tbefore, tafter, diff; |
| int i; |
| |
| for (i = 0; i < iters_per_op; ++i) { |
| getnstimeofday(&tbefore); |
| if (iommu_map(domain, iova, paddr, size, |
| IOMMU_READ | IOMMU_WRITE)) { |
| seq_puts(s, "Failed to map\n"); |
| continue; |
| } |
| getnstimeofday(&tafter); |
| diff = timespec_sub(tafter, tbefore); |
| map_elapsed_ns += timespec_to_ns(&diff); |
| |
| getnstimeofday(&tbefore); |
| unmapped = iommu_unmap(domain, iova, size); |
| if (unmapped != size) { |
| seq_printf(s, |
| "Only unmapped %zx instead of %zx\n", |
| unmapped, size); |
| continue; |
| } |
| getnstimeofday(&tafter); |
| diff = timespec_sub(tafter, tbefore); |
| unmap_elapsed_ns += timespec_to_ns(&diff); |
| } |
| |
| map_elapsed_ns = div_u64_rem(map_elapsed_ns, iters_per_op, |
| &map_elapsed_rem); |
| unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, iters_per_op, |
| &unmap_elapsed_rem); |
| |
| map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, |
| &map_elapsed_rem); |
| unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, |
| &unmap_elapsed_rem); |
| |
| seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", |
| _size_to_string(size), |
| map_elapsed_us, map_elapsed_rem, |
| unmap_elapsed_us, unmap_elapsed_rem); |
| } |
| |
| seq_putc(s, '\n'); |
| seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map_sg", "iommu_unmap"); |
| for (sz = sizes; *sz; ++sz) { |
| size_t size = *sz; |
| size_t unmapped; |
| u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; |
| u64 map_elapsed_us = 0, unmap_elapsed_us = 0; |
| u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; |
| struct timespec tbefore, tafter, diff; |
| struct sg_table table; |
| unsigned long chunk_size = SZ_4K; |
| int i; |
| |
| if (iommu_debug_build_phoney_sg_table(dev, &table, size, |
| chunk_size)) { |
| seq_puts(s, |
| "couldn't build phoney sg table! bailing...\n"); |
| goto out_detach; |
| } |
| |
| for (i = 0; i < iters_per_op; ++i) { |
| getnstimeofday(&tbefore); |
| if (iommu_map_sg(domain, iova, table.sgl, table.nents, |
| IOMMU_READ | IOMMU_WRITE) != size) { |
| seq_puts(s, "Failed to map_sg\n"); |
| goto next; |
| } |
| getnstimeofday(&tafter); |
| diff = timespec_sub(tafter, tbefore); |
| map_elapsed_ns += timespec_to_ns(&diff); |
| |
| getnstimeofday(&tbefore); |
| unmapped = iommu_unmap(domain, iova, size); |
| if (unmapped != size) { |
| seq_printf(s, |
| "Only unmapped %zx instead of %zx\n", |
| unmapped, size); |
| goto next; |
| } |
| getnstimeofday(&tafter); |
| diff = timespec_sub(tafter, tbefore); |
| unmap_elapsed_ns += timespec_to_ns(&diff); |
| } |
| |
| map_elapsed_ns = div_u64_rem(map_elapsed_ns, iters_per_op, |
| &map_elapsed_rem); |
| unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, iters_per_op, |
| &unmap_elapsed_rem); |
| |
| map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, |
| &map_elapsed_rem); |
| unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, |
| &unmap_elapsed_rem); |
| |
| seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", |
| _size_to_string(size), |
| map_elapsed_us, map_elapsed_rem, |
| unmap_elapsed_us, unmap_elapsed_rem); |
| |
| next: |
| iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size); |
| } |
| |
| out_detach: |
| iommu_detach_group(domain, dev->iommu_group); |
| out_domain_free: |
| iommu_domain_free(domain); |
| } |
| |
| static int iommu_debug_profiling_show(struct seq_file *s, void *ignored) |
| { |
| struct iommu_debug_device *ddev = s->private; |
| const size_t sizes[] = { SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, |
| SZ_1M * 20, 0 }; |
| enum iommu_attr attrs[] = { |
| DOMAIN_ATTR_ATOMIC, |
| }; |
| int htw_disable = 1, atomic = 1; |
| void *attr_values[] = { &htw_disable, &atomic }; |
| |
| iommu_debug_device_profiling(s, ddev->dev, attrs, attr_values, |
| ARRAY_SIZE(attrs), sizes); |
| |
| return 0; |
| } |
| |
| static int iommu_debug_profiling_open(struct inode *inode, struct file *file) |
| { |
| return single_open(file, iommu_debug_profiling_show, inode->i_private); |
| } |
| |
| static const struct file_operations iommu_debug_profiling_fops = { |
| .open = iommu_debug_profiling_open, |
| .read = seq_read, |
| .llseek = seq_lseek, |
| .release = single_release, |
| }; |
| |
| static int iommu_debug_secure_profiling_show(struct seq_file *s, void *ignored) |
| { |
| struct iommu_debug_device *ddev = s->private; |
| const size_t sizes[] = { SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, |
| SZ_1M * 20, 0 }; |
| |
| enum iommu_attr attrs[] = { |
| DOMAIN_ATTR_ATOMIC, |
| DOMAIN_ATTR_SECURE_VMID, |
| }; |
| int one = 1, secure_vmid = VMID_CP_PIXEL; |
| void *attr_values[] = { &one, &secure_vmid }; |
| |
| iommu_debug_device_profiling(s, ddev->dev, attrs, attr_values, |
| ARRAY_SIZE(attrs), sizes); |
| |
| return 0; |
| } |
| |
| static int iommu_debug_secure_profiling_open(struct inode *inode, |
| struct file *file) |
| { |
| return single_open(file, iommu_debug_secure_profiling_show, |
| inode->i_private); |
| } |
| |
| static const struct file_operations iommu_debug_secure_profiling_fops = { |
| .open = iommu_debug_secure_profiling_open, |
| .read = seq_read, |
| .llseek = seq_lseek, |
| .release = single_release, |
| }; |
| |
| static int iommu_debug_profiling_fast_show(struct seq_file *s, void *ignored) |
| { |
| struct iommu_debug_device *ddev = s->private; |
| size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0}; |
| enum iommu_attr attrs[] = { |
| DOMAIN_ATTR_FAST, |
| DOMAIN_ATTR_ATOMIC, |
| }; |
| int one = 1; |
| void *attr_values[] = { &one, &one }; |
| |
| iommu_debug_device_profiling(s, ddev->dev, attrs, attr_values, |
| ARRAY_SIZE(attrs), sizes); |
| |
| return 0; |
| } |
| |
| static int iommu_debug_profiling_fast_open(struct inode *inode, |
| struct file *file) |
| { |
| return single_open(file, iommu_debug_profiling_fast_show, |
| inode->i_private); |
| } |
| |
| static const struct file_operations iommu_debug_profiling_fast_fops = { |
| .open = iommu_debug_profiling_fast_open, |
| .read = seq_read, |
| .llseek = seq_lseek, |
| .release = single_release, |
| }; |
| |
| static int iommu_debug_profiling_fast_dma_api_show(struct seq_file *s, |
| void *ignored) |
| { |
| int i, experiment; |
| struct iommu_debug_device *ddev = s->private; |
| struct device *dev = ddev->dev; |
| u64 map_elapsed_ns[10], unmap_elapsed_ns[10]; |
| struct dma_iommu_mapping *mapping; |
| dma_addr_t dma_addr; |
| void *virt; |
| int fast = 1; |
| const char * const extra_labels[] = { |
| "not coherent", |
| "coherent", |
| }; |
| unsigned long extra_attrs[] = { |
| 0, |
| DMA_ATTR_SKIP_CPU_SYNC, |
| }; |
| |
| virt = kmalloc(1518, GFP_KERNEL); |
| if (!virt) |
| goto out; |
| |
| mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4ULL); |
| if (!mapping) { |
| seq_puts(s, "fast_smmu_create_mapping failed\n"); |
| goto out_kfree; |
| } |
| |
| if (iommu_domain_set_attr(mapping->domain, DOMAIN_ATTR_FAST, &fast)) { |
| seq_puts(s, "iommu_domain_set_attr failed\n"); |
| goto out_release_mapping; |
| } |
| |
| if (arm_iommu_attach_device(dev, mapping)) { |
| seq_puts(s, "fast_smmu_attach_device failed\n"); |
| goto out_release_mapping; |
| } |
| |
| if (iommu_enable_config_clocks(mapping->domain)) { |
| seq_puts(s, "Couldn't enable clocks\n"); |
| goto out_detach; |
| } |
| for (experiment = 0; experiment < 2; ++experiment) { |
| size_t map_avg = 0, unmap_avg = 0; |
| |
| for (i = 0; i < 10; ++i) { |
| struct timespec tbefore, tafter, diff; |
| u64 ns; |
| |
| getnstimeofday(&tbefore); |
| dma_addr = dma_map_single_attrs( |
| dev, virt, SZ_4K, DMA_TO_DEVICE, |
| extra_attrs[experiment]); |
| getnstimeofday(&tafter); |
| diff = timespec_sub(tafter, tbefore); |
| ns = timespec_to_ns(&diff); |
| if (dma_mapping_error(dev, dma_addr)) { |
| seq_puts(s, "dma_map_single failed\n"); |
| goto out_disable_config_clocks; |
| } |
| map_elapsed_ns[i] = ns; |
| |
| getnstimeofday(&tbefore); |
| dma_unmap_single_attrs( |
| dev, dma_addr, SZ_4K, DMA_TO_DEVICE, |
| extra_attrs[experiment]); |
| getnstimeofday(&tafter); |
| diff = timespec_sub(tafter, tbefore); |
| ns = timespec_to_ns(&diff); |
| unmap_elapsed_ns[i] = ns; |
| } |
| |
| seq_printf(s, "%13s %24s (ns): [", extra_labels[experiment], |
| "dma_map_single_attrs"); |
| for (i = 0; i < 10; ++i) { |
| map_avg += map_elapsed_ns[i]; |
| seq_printf(s, "%5llu%s", map_elapsed_ns[i], |
| i < 9 ? ", " : ""); |
| } |
| map_avg /= 10; |
| seq_printf(s, "] (avg: %zu)\n", map_avg); |
| |
| seq_printf(s, "%13s %24s (ns): [", extra_labels[experiment], |
| "dma_unmap_single_attrs"); |
| for (i = 0; i < 10; ++i) { |
| unmap_avg += unmap_elapsed_ns[i]; |
| seq_printf(s, "%5llu%s", unmap_elapsed_ns[i], |
| i < 9 ? ", " : ""); |
| } |
| unmap_avg /= 10; |
| seq_printf(s, "] (avg: %zu)\n", unmap_avg); |
| } |
| |
| out_disable_config_clocks: |
| iommu_disable_config_clocks(mapping->domain); |
| out_detach: |
| arm_iommu_detach_device(dev); |
| out_release_mapping: |
| arm_iommu_release_mapping(mapping); |
| out_kfree: |
| kfree(virt); |
| out: |
| return 0; |
| } |
| |
| static int iommu_debug_profiling_fast_dma_api_open(struct inode *inode, |
| struct file *file) |
| { |
| return single_open(file, iommu_debug_profiling_fast_dma_api_show, |
| inode->i_private); |
| } |
| |
| static const struct file_operations iommu_debug_profiling_fast_dma_api_fops = { |
| .open = iommu_debug_profiling_fast_dma_api_open, |
| .read = seq_read, |
| .llseek = seq_lseek, |
| .release = single_release, |
| }; |
| |
| static int __tlb_stress_sweep(struct device *dev, struct seq_file *s) |
| { |
| int i, ret = 0; |
| u64 iova; |
| const u64 max = SZ_1G * 4ULL - 1; |
| void *virt; |
| phys_addr_t phys; |
| dma_addr_t dma_addr; |
| |
| /* |
| * we'll be doing 4K and 8K mappings. Need to own an entire 8K |
| * chunk that we can work with. |
| */ |
| virt = (void *)__get_free_pages(GFP_KERNEL, get_order(SZ_8K)); |
| phys = virt_to_phys(virt); |
| |
| /* fill the whole 4GB space */ |
| for (iova = 0, i = 0; iova < max; iova += SZ_8K, ++i) { |
| dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); |
| if (dma_addr == DMA_ERROR_CODE) { |
| dev_err(dev, "Failed map on iter %d\n", i); |
| ret = -EINVAL; |
| goto out; |
| } |
| } |
| |
| if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_ERROR_CODE) { |
| dev_err(dev, |
| "dma_map_single unexpectedly (VA should have been exhausted)\n"); |
| ret = -EINVAL; |
| goto out; |
| } |
| |
| /* |
| * free up 4K at the very beginning, then leave one 4K mapping, |
| * then free up 8K. This will result in the next 8K map to skip |
| * over the 4K hole and take the 8K one. |
| */ |
| dma_unmap_single(dev, 0, SZ_4K, DMA_TO_DEVICE); |
| dma_unmap_single(dev, SZ_8K, SZ_4K, DMA_TO_DEVICE); |
| dma_unmap_single(dev, SZ_8K + SZ_4K, SZ_4K, DMA_TO_DEVICE); |
| |
| /* remap 8K */ |
| dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); |
| if (dma_addr != SZ_8K) { |
| dma_addr_t expected = SZ_8K; |
| |
| dev_err(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", |
| &dma_addr, &expected); |
| ret = -EINVAL; |
| goto out; |
| } |
| |
| /* |
| * now remap 4K. We should get the first 4K chunk that was skipped |
| * over during the previous 8K map. If we missed a TLB invalidate |
| * at that point this should explode. |
| */ |
| dma_addr = dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE); |
| if (dma_addr != 0) { |
| dma_addr_t expected = 0; |
| |
| dev_err(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", |
| &dma_addr, &expected); |
| ret = -EINVAL; |
| goto out; |
| } |
| |
| if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_ERROR_CODE) { |
| dev_err(dev, |
| "dma_map_single unexpectedly after remaps (VA should have been exhausted)\n"); |
| ret = -EINVAL; |
| goto out; |
| } |
| |
| /* we're all full again. unmap everything. */ |
| for (iova = 0; iova < max; iova += SZ_8K) |
| dma_unmap_single(dev, (dma_addr_t)iova, SZ_8K, DMA_TO_DEVICE); |
| |
| out: |
| free_pages((unsigned long)virt, get_order(SZ_8K)); |
| return ret; |
| } |
| |
| struct fib_state { |
| unsigned long cur; |
| unsigned long prev; |
| }; |
| |
| static void fib_init(struct fib_state *f) |
| { |
| f->cur = f->prev = 1; |
| } |
| |
| static unsigned long get_next_fib(struct fib_state *f) |
| { |
| int next = f->cur + f->prev; |
| |
| f->prev = f->cur; |
| f->cur = next; |
| return next; |
| } |
| |
| /* |
| * Not actually random. Just testing the fibs (and max - the fibs). |
| */ |
| static int __rand_va_sweep(struct device *dev, struct seq_file *s, |
| const size_t size) |
| { |
| u64 iova; |
| const u64 max = SZ_1G * 4ULL - 1; |
| int i, remapped, unmapped, ret = 0; |
| void *virt; |
| dma_addr_t dma_addr, dma_addr2; |
| struct fib_state fib; |
| |
| virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); |
| if (!virt) { |
| if (size > SZ_8K) { |
| dev_err(dev, |
| "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", |
| _size_to_string(size)); |
| return 0; |
| } |
| return -ENOMEM; |
| } |
| |
| /* fill the whole 4GB space */ |
| for (iova = 0, i = 0; iova < max; iova += size, ++i) { |
| dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); |
| if (dma_addr == DMA_ERROR_CODE) { |
| dev_err(dev, "Failed map on iter %d\n", i); |
| ret = -EINVAL; |
| goto out; |
| } |
| } |
| |
| /* now unmap "random" iovas */ |
| unmapped = 0; |
| fib_init(&fib); |
| for (iova = get_next_fib(&fib) * size; |
| iova < max - size; |
| iova = (u64)get_next_fib(&fib) * size) { |
| dma_addr = (dma_addr_t)(iova); |
| dma_addr2 = (dma_addr_t)((max + 1) - size - iova); |
| if (dma_addr == dma_addr2) { |
| WARN(1, |
| "%s test needs update! The random number sequence is folding in on itself and should be changed.\n", |
| __func__); |
| return -EINVAL; |
| } |
| dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE); |
| dma_unmap_single(dev, dma_addr2, size, DMA_TO_DEVICE); |
| unmapped += 2; |
| } |
| |
| /* and map until everything fills back up */ |
| for (remapped = 0; ; ++remapped) { |
| dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); |
| if (dma_addr == DMA_ERROR_CODE) |
| break; |
| } |
| |
| if (unmapped != remapped) { |
| dev_err(dev, |
| "Unexpected random remap count! Unmapped %d but remapped %d\n", |
| unmapped, remapped); |
| ret = -EINVAL; |
| } |
| |
| for (iova = 0; iova < max; iova += size) |
| dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); |
| |
| out: |
| free_pages((unsigned long)virt, get_order(size)); |
| return ret; |
| } |
| |
| static int __check_mapping(struct device *dev, struct iommu_domain *domain, |
| dma_addr_t iova, phys_addr_t expected) |
| { |
| phys_addr_t res = iommu_iova_to_phys_hard(domain, iova); |
| phys_addr_t res2 = iommu_iova_to_phys(domain, iova); |
| |
| WARN(res != res2, "hard/soft iova_to_phys fns don't agree..."); |
| |
| if (res != expected) { |
| dev_err_ratelimited(dev, |
| "Bad translation for %pa! Expected: %pa Got: %pa\n", |
| &iova, &expected, &res); |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| static int __full_va_sweep(struct device *dev, struct seq_file *s, |
| const size_t size, struct iommu_domain *domain) |
| { |
| u64 iova; |
| dma_addr_t dma_addr; |
| void *virt; |
| phys_addr_t phys; |
| const u64 max = SZ_1G * 4ULL - 1; |
| int ret = 0, i; |
| |
| virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); |
| if (!virt) { |
| if (size > SZ_8K) { |
| dev_err(dev, |
| "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", |
| _size_to_string(size)); |
| return 0; |
| } |
| return -ENOMEM; |
| } |
| phys = virt_to_phys(virt); |
| |
| for (iova = 0, i = 0; iova < max; iova += size, ++i) { |
| unsigned long expected = iova; |
| |
| dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); |
| if (dma_addr != expected) { |
| dev_err_ratelimited(dev, |
| "Unexpected iova on iter %d (expected: 0x%lx got: 0x%lx)\n", |
| i, expected, |
| (unsigned long)dma_addr); |
| ret = -EINVAL; |
| goto out; |
| } |
| } |
| |
| if (domain) { |
| /* check every mapping from 0..6M */ |
| for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { |
| phys_addr_t expected = phys; |
| |
| if (__check_mapping(dev, domain, iova, expected)) { |
| dev_err(dev, "iter: %d\n", i); |
| ret = -EINVAL; |
| goto out; |
| } |
| } |
| /* and from 4G..4G-6M */ |
| for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { |
| phys_addr_t expected = phys; |
| unsigned long theiova = ((SZ_1G * 4ULL) - size) - iova; |
| |
| if (__check_mapping(dev, domain, theiova, expected)) { |
| dev_err(dev, "iter: %d\n", i); |
| ret = -EINVAL; |
| goto out; |
| } |
| } |
| } |
| |
| /* at this point, our VA space should be full */ |
| dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); |
| if (dma_addr != DMA_ERROR_CODE) { |
| dev_err_ratelimited(dev, |
| "dma_map_single succeeded when it should have failed. Got iova: 0x%lx\n", |
| (unsigned long)dma_addr); |
| ret = -EINVAL; |
| } |
| |
| out: |
| for (iova = 0; iova < max; iova += size) |
| dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); |
| |
| free_pages((unsigned long)virt, get_order(size)); |
| return ret; |
| } |
| |
| #define ds_printf(d, s, fmt, ...) ({ \ |
| dev_err(d, fmt, ##__VA_ARGS__); \ |
| seq_printf(s, fmt, ##__VA_ARGS__); \ |
| }) |
| |
| static int __functional_dma_api_va_test(struct device *dev, struct seq_file *s, |
| struct iommu_domain *domain, void *priv) |
| { |
| int i, j, ret = 0; |
| size_t *sz, *sizes = priv; |
| |
| for (j = 0; j < 1; ++j) { |
| for (sz = sizes; *sz; ++sz) { |
| for (i = 0; i < 2; ++i) { |
| ds_printf(dev, s, "Full VA sweep @%s %d", |
| _size_to_string(*sz), i); |
| if (__full_va_sweep(dev, s, *sz, domain)) { |
| ds_printf(dev, s, " -> FAILED\n"); |
| ret = -EINVAL; |
| } else { |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| } |
| } |
| } |
| } |
| |
| ds_printf(dev, s, "bonus map:"); |
| if (__full_va_sweep(dev, s, SZ_4K, domain)) { |
| ds_printf(dev, s, " -> FAILED\n"); |
| ret = -EINVAL; |
| } else { |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| } |
| |
| for (sz = sizes; *sz; ++sz) { |
| for (i = 0; i < 2; ++i) { |
| ds_printf(dev, s, "Rand VA sweep @%s %d", |
| _size_to_string(*sz), i); |
| if (__rand_va_sweep(dev, s, *sz)) { |
| ds_printf(dev, s, " -> FAILED\n"); |
| ret = -EINVAL; |
| } else { |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| } |
| } |
| } |
| |
| ds_printf(dev, s, "TLB stress sweep"); |
| if (__tlb_stress_sweep(dev, s)) { |
| ds_printf(dev, s, " -> FAILED\n"); |
| ret = -EINVAL; |
| } else { |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| } |
| |
| ds_printf(dev, s, "second bonus map:"); |
| if (__full_va_sweep(dev, s, SZ_4K, domain)) { |
| ds_printf(dev, s, " -> FAILED\n"); |
| ret = -EINVAL; |
| } else { |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| } |
| |
| return ret; |
| } |
| |
| static int __functional_dma_api_alloc_test(struct device *dev, |
| struct seq_file *s, |
| struct iommu_domain *domain, |
| void *ignored) |
| { |
| size_t size = SZ_1K * 742; |
| int ret = 0; |
| u8 *data; |
| dma_addr_t iova; |
| |
| /* Make sure we can allocate and use a buffer */ |
| ds_printf(dev, s, "Allocating coherent buffer"); |
| data = dma_alloc_coherent(dev, size, &iova, GFP_KERNEL); |
| if (!data) { |
| ds_printf(dev, s, " -> FAILED\n"); |
| ret = -EINVAL; |
| } else { |
| int i; |
| |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| ds_printf(dev, s, "Using coherent buffer"); |
| for (i = 0; i < 742; ++i) { |
| int ind = SZ_1K * i; |
| u8 *p = data + ind; |
| u8 val = i % 255; |
| |
| memset(data, 0xa5, size); |
| *p = val; |
| (*p)++; |
| if ((*p) != val + 1) { |
| ds_printf(dev, s, |
| " -> FAILED on iter %d since %d != %d\n", |
| i, *p, val + 1); |
| ret = -EINVAL; |
| } |
| } |
| if (!ret) |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| dma_free_coherent(dev, size, data, iova); |
| } |
| |
| return ret; |
| } |
| |
| static int __functional_dma_api_basic_test(struct device *dev, |
| struct seq_file *s, |
| struct iommu_domain *domain, |
| void *ignored) |
| { |
| size_t size = 1518; |
| int i, j, ret = 0; |
| u8 *data; |
| dma_addr_t iova; |
| phys_addr_t pa, pa2; |
| |
| ds_printf(dev, s, "Basic DMA API test"); |
| /* Make sure we can allocate and use a buffer */ |
| for (i = 0; i < 1000; ++i) { |
| data = kmalloc(size, GFP_KERNEL); |
| if (!data) { |
| ds_printf(dev, s, " -> FAILED\n"); |
| ret = -EINVAL; |
| goto out; |
| } |
| memset(data, 0xa5, size); |
| iova = dma_map_single(dev, data, size, DMA_TO_DEVICE); |
| pa = iommu_iova_to_phys(domain, iova); |
| pa2 = iommu_iova_to_phys_hard(domain, iova); |
| if (pa != pa2) { |
| dev_err(dev, |
| "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n", |
| &pa, &pa2); |
| ret = -EINVAL; |
| goto out; |
| } |
| pa2 = virt_to_phys(data); |
| if (pa != pa2) { |
| dev_err(dev, |
| "iova_to_phys doesn't match virt_to_phys: %pa != %pa\n", |
| &pa, &pa2); |
| ret = -EINVAL; |
| goto out; |
| } |
| dma_unmap_single(dev, iova, size, DMA_TO_DEVICE); |
| for (j = 0; j < size; ++j) { |
| if (data[j] != 0xa5) { |
| dev_err(dev, "data[%d] != 0xa5\n", data[j]); |
| ret = -EINVAL; |
| goto out; |
| } |
| } |
| kfree(data); |
| } |
| |
| out: |
| if (ret) |
| ds_printf(dev, s, " -> FAILED\n"); |
| else |
| ds_printf(dev, s, " -> SUCCEEDED\n"); |
| |
| return ret; |
| } |
| |
| /* Creates a fresh fast mapping and applies @fn to it */ |
| static int __apply_to_new_mapping(struct seq_file *s, |
| int (*fn)(struct device *dev, |
| struct seq_file *s, |
| struct iommu_domain *domain, |
| void *priv), |
| void *priv) |
| { |
| struct dma_iommu_mapping *mapping; |
| struct iommu_debug_device *ddev = s->private; |
| struct device *dev = ddev->dev; |
| int ret = -EINVAL, fast = 1; |
| phys_addr_t pt_phys; |
| |
| mapping = arm_iommu_create_mapping(&platform_bus_type, 0, |
| (SZ_1G * 4ULL)); |
| if (!mapping) |
| goto out; |
| |
| if (iommu_domain_set_attr(mapping->domain, DOMAIN_ATTR_FAST, &fast)) { |
| seq_puts(s, "iommu_domain_set_attr failed\n"); |
| goto out_release_mapping; |
| } |
| |
| if (arm_iommu_attach_device(dev, mapping)) |
| goto out_release_mapping; |
| |
| if (iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_PT_BASE_ADDR, |
| &pt_phys)) { |
| ds_printf(dev, s, "Couldn't get page table base address\n"); |
| goto out_release_mapping; |
| } |
| |
| dev_err(dev, "testing with pgtables at %pa\n", &pt_phys); |
| if (iommu_enable_config_clocks(mapping->domain)) { |
| ds_printf(dev, s, "Couldn't enable clocks\n"); |
| goto out_release_mapping; |
| } |
| ret = fn(dev, s, mapping->domain, priv); |
| iommu_disable_config_clocks(mapping->domain); |
| |
| arm_iommu_detach_device(dev); |
| out_release_mapping: |
| arm_iommu_release_mapping(mapping); |
| out: |
| seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS"); |
| return 0; |
| } |
| |
| static int iommu_debug_functional_fast_dma_api_show(struct seq_file *s, |
| void *ignored) |
| { |
| size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0}; |
| int ret = 0; |
| |
| ret |= __apply_to_new_mapping(s, __functional_dma_api_alloc_test, NULL); |
| ret |= __apply_to_new_mapping(s, __functional_dma_api_basic_test, NULL); |
| ret |= __apply_to_new_mapping(s, __functional_dma_api_va_test, sizes); |
| return ret; |
| } |
| |
| static int iommu_debug_functional_fast_dma_api_open(struct inode *inode, |
| struct file *file) |
| { |
| return single_open(file, iommu_debug_functional_fast_dma_api_show, |
| inode->i_private); |
| } |
| |
| static const struct file_operations iommu_debug_functional_fast_dma_api_fops = { |
| .open = iommu_debug_functional_fast_dma_api_open, |
| .read = seq_read, |
| .llseek = seq_lseek, |
| .release = single_release, |
| }; |
| |
| static int iommu_debug_functional_arm_dma_api_show(struct seq_file *s, |
| void *ignored) |
| { |
| struct dma_iommu_mapping *mapping; |
| struct iommu_debug_device *ddev = s->private; |
| struct device *dev = ddev->dev; |
| size_t sizes[] = {SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, 0}; |
| int ret = -EINVAL; |
| |
| /* Make the size equal to MAX_ULONG */ |
| mapping = arm_iommu_create_mapping(&platform_bus_type, 0, |
| (SZ_1G * 4ULL - 1)); |
| if (!mapping) |
| goto out; |
| |
| if (arm_iommu_attach_device(dev, mapping)) |
| goto out_release_mapping; |
| |
| ret = __functional_dma_api_alloc_test(dev, s, mapping->domain, sizes); |
| ret |= __functional_dma_api_basic_test(dev, s, mapping->domain, sizes); |
| |
| arm_iommu_detach_device(dev); |
| out_release_mapping: |
| arm_iommu_release_mapping(mapping); |
| out: |
| seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS"); |
| return 0; |
| } |
| |
| static int iommu_debug_functional_arm_dma_api_open(struct inode *inode, |
| struct file *file) |
| { |
| return single_open(file, iommu_debug_functional_arm_dma_api_show, |
| inode->i_private); |
| } |
| |
| static const struct file_operations iommu_debug_functional_arm_dma_api_fops = { |
| .open = iommu_debug_functional_arm_dma_api_open, |
| .read = seq_read, |
| .llseek = seq_lseek, |
| .release = single_release, |
| }; |
| |
| static int iommu_debug_attach_do_attach(struct iommu_debug_device *ddev, |
| int val, bool is_secure) |
| { |
| struct iommu_group *group = ddev->dev->iommu_group; |
| |
| ddev->domain = iommu_domain_alloc(&platform_bus_type); |
| if (!ddev->domain) { |
| pr_err("Couldn't allocate domain\n"); |
| return -ENOMEM; |
| } |
| |
| val = VMID_CP_CAMERA; |
| if (is_secure && iommu_domain_set_attr(ddev->domain, |
| DOMAIN_ATTR_SECURE_VMID, |
| &val)) { |
| pr_err("Couldn't set secure vmid to %d\n", val); |
| goto out_domain_free; |
| } |
| |
| if (iommu_attach_group(ddev->domain, group)) { |
| dev_err(ddev->dev, "Couldn't attach new domain to device\n"); |
| goto out_domain_free; |
| } |
| |
| return 0; |
| |
| out_domain_free: |
| iommu_domain_free(ddev->domain); |
| ddev->domain = NULL; |
| return -EIO; |
| } |
| |
| static ssize_t __iommu_debug_dma_attach_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| struct dma_iommu_mapping *dma_mapping; |
| ssize_t retval = -EINVAL; |
| int val; |
| |
| if (kstrtoint_from_user(ubuf, count, 0, &val)) { |
| pr_err("Invalid format. Expected a hex or decimal integer"); |
| retval = -EFAULT; |
| goto out; |
| } |
| |
| if (val) { |
| if (dev->archdata.mapping) |
| if (dev->archdata.mapping->domain) { |
| pr_err("Already attached.\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| if (WARN(dev->archdata.iommu, |
| "Attachment tracking out of sync with device\n")) { |
| retval = -EINVAL; |
| goto out; |
| } |
| |
| dma_mapping = arm_iommu_create_mapping(&platform_bus_type, 0, |
| (SZ_1G * 4ULL)); |
| |
| if (!dma_mapping) |
| goto out; |
| |
| if (arm_iommu_attach_device(dev, dma_mapping)) |
| goto out_release_mapping; |
| |
| ddev->mapping = dma_mapping; |
| pr_err("Attached\n"); |
| } else { |
| if (!dev->archdata.mapping) { |
| pr_err("No mapping. Did you already attach?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| if (!dev->archdata.mapping->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| arm_iommu_detach_device(dev); |
| arm_iommu_release_mapping(ddev->mapping); |
| pr_err("Detached\n"); |
| } |
| retval = count; |
| return retval; |
| |
| out_release_mapping: |
| arm_iommu_release_mapping(dma_mapping); |
| out: |
| return retval; |
| } |
| |
| static ssize_t __iommu_debug_attach_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset, |
| bool is_secure) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| struct iommu_domain *domain; |
| ssize_t retval; |
| int val; |
| |
| if (kstrtoint_from_user(ubuf, count, 0, &val)) { |
| pr_err("Invalid format. Expected a hex or decimal integer"); |
| retval = -EFAULT; |
| goto out; |
| } |
| |
| if (val) { |
| if (ddev->domain) { |
| pr_err("Iommu-Debug is already attached?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| |
| domain = iommu_get_domain_for_dev(dev); |
| if (domain) { |
| pr_err("Another driver is using this device's iommu\n" |
| "Iommu-Debug cannot be used concurrently\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| if (iommu_debug_attach_do_attach(ddev, val, is_secure)) { |
| retval = -EIO; |
| goto out; |
| } |
| pr_err("Attached\n"); |
| } else { |
| if (!ddev->domain) { |
| pr_err("Iommu-Debug is not attached?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| iommu_detach_group(ddev->domain, dev->iommu_group); |
| iommu_domain_free(ddev->domain); |
| ddev->domain = NULL; |
| pr_err("Detached\n"); |
| } |
| |
| retval = count; |
| out: |
| return retval; |
| } |
| |
| static ssize_t iommu_debug_dma_attach_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| return __iommu_debug_dma_attach_write(file, ubuf, count, offset); |
| |
| } |
| |
| static ssize_t iommu_debug_dma_attach_read(struct file *file, char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| char c[2]; |
| size_t buflen = sizeof(c); |
| |
| if (*offset) |
| return 0; |
| |
| if (!dev->archdata.mapping) |
| c[0] = '0'; |
| else |
| c[0] = dev->archdata.mapping->domain ? '1' : '0'; |
| |
| c[1] = '\n'; |
| buflen = min(count, buflen); |
| if (copy_to_user(ubuf, &c, buflen)) { |
| pr_err("copy_to_user failed\n"); |
| return -EFAULT; |
| } |
| *offset = 1; /* non-zero means we're done */ |
| |
| return buflen; |
| } |
| |
| static const struct file_operations iommu_debug_dma_attach_fops = { |
| .open = simple_open, |
| .write = iommu_debug_dma_attach_write, |
| .read = iommu_debug_dma_attach_read, |
| }; |
| |
| static ssize_t iommu_debug_test_virt_addr_read(struct file *file, |
| char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| char buf[100]; |
| ssize_t retval; |
| size_t buflen; |
| int buf_len = sizeof(buf); |
| |
| if (*offset) |
| return 0; |
| |
| memset(buf, 0, buf_len); |
| |
| if (!test_virt_addr) |
| strlcpy(buf, "FAIL\n", buf_len); |
| else |
| snprintf(buf, buf_len, "0x%pK\n", test_virt_addr); |
| |
| buflen = min(count, strlen(buf)); |
| if (copy_to_user(ubuf, buf, buflen)) { |
| pr_err("Couldn't copy_to_user\n"); |
| retval = -EFAULT; |
| } else { |
| *offset = 1; /* non-zero means we're done */ |
| retval = buflen; |
| } |
| |
| return retval; |
| } |
| |
| static const struct file_operations iommu_debug_test_virt_addr_fops = { |
| .open = simple_open, |
| .read = iommu_debug_test_virt_addr_read, |
| }; |
| |
| static ssize_t iommu_debug_attach_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| return __iommu_debug_attach_write(file, ubuf, count, offset, |
| false); |
| |
| } |
| |
| static ssize_t iommu_debug_attach_read(struct file *file, char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| char c[2]; |
| size_t buflen = sizeof(c); |
| |
| if (*offset) |
| return 0; |
| |
| c[0] = ddev->domain ? '1' : '0'; |
| c[1] = '\n'; |
| buflen = min(count, buflen); |
| if (copy_to_user(ubuf, &c, buflen)) { |
| pr_err("copy_to_user failed\n"); |
| return -EFAULT; |
| } |
| *offset = 1; /* non-zero means we're done */ |
| |
| return buflen; |
| } |
| |
| static const struct file_operations iommu_debug_attach_fops = { |
| .open = simple_open, |
| .write = iommu_debug_attach_write, |
| .read = iommu_debug_attach_read, |
| }; |
| |
| static ssize_t iommu_debug_attach_write_secure(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| return __iommu_debug_attach_write(file, ubuf, count, offset, |
| true); |
| } |
| |
| static const struct file_operations iommu_debug_secure_attach_fops = { |
| .open = simple_open, |
| .write = iommu_debug_attach_write_secure, |
| .read = iommu_debug_attach_read, |
| }; |
| |
| static ssize_t iommu_debug_pte_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| dma_addr_t iova; |
| |
| if (kstrtox_from_user(ubuf, count, 0, &iova)) { |
| pr_err("Invalid format for iova\n"); |
| ddev->iova = 0; |
| return -EINVAL; |
| } |
| |
| ddev->iova = iova; |
| pr_err("Saved iova=%pa for future PTE commands\n", &iova); |
| return count; |
| } |
| |
| |
| static ssize_t iommu_debug_pte_read(struct file *file, char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| uint64_t pte; |
| char buf[100]; |
| ssize_t retval; |
| size_t buflen; |
| |
| if (kptr_restrict != 0) { |
| pr_err("kptr_restrict needs to be disabled.\n"); |
| return -EPERM; |
| } |
| if (!dev->archdata.mapping) { |
| pr_err("No mapping. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| if (!dev->archdata.mapping->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| if (*offset) |
| return 0; |
| |
| memset(buf, 0, sizeof(buf)); |
| |
| pte = iommu_iova_to_pte(dev->archdata.mapping->domain, |
| ddev->iova); |
| |
| if (!pte) |
| strlcpy(buf, "FAIL\n", sizeof(buf)); |
| else |
| snprintf(buf, sizeof(buf), "pte=%016llx\n", pte); |
| |
| buflen = min(count, strlen(buf)); |
| if (copy_to_user(ubuf, buf, buflen)) { |
| pr_err("Couldn't copy_to_user\n"); |
| retval = -EFAULT; |
| } else { |
| *offset = 1; /* non-zero means we're done */ |
| retval = buflen; |
| } |
| |
| return retval; |
| } |
| |
| static const struct file_operations iommu_debug_pte_fops = { |
| .open = simple_open, |
| .write = iommu_debug_pte_write, |
| .read = iommu_debug_pte_read, |
| }; |
| |
| static ssize_t iommu_debug_atos_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| dma_addr_t iova; |
| |
| if (kstrtox_from_user(ubuf, count, 0, &iova)) { |
| pr_err("Invalid format for iova\n"); |
| ddev->iova = 0; |
| return -EINVAL; |
| } |
| |
| ddev->iova = iova; |
| pr_err("Saved iova=%pa for future ATOS commands\n", &iova); |
| return count; |
| } |
| |
| static ssize_t iommu_debug_atos_read(struct file *file, char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| phys_addr_t phys; |
| char buf[100]; |
| ssize_t retval; |
| size_t buflen; |
| |
| if (kptr_restrict != 0) { |
| pr_err("kptr_restrict needs to be disabled.\n"); |
| return -EPERM; |
| } |
| if (!ddev->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| if (*offset) |
| return 0; |
| |
| memset(buf, 0, 100); |
| |
| phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova); |
| if (!phys) { |
| strlcpy(buf, "FAIL\n", 100); |
| phys = iommu_iova_to_phys(ddev->domain, ddev->iova); |
| dev_err(ddev->dev, "ATOS for %pa failed. Software walk returned: %pa\n", |
| &ddev->iova, &phys); |
| } else { |
| snprintf(buf, 100, "%pa\n", &phys); |
| } |
| |
| buflen = min(count, strlen(buf)); |
| if (copy_to_user(ubuf, buf, buflen)) { |
| pr_err("Couldn't copy_to_user\n"); |
| retval = -EFAULT; |
| } else { |
| *offset = 1; /* non-zero means we're done */ |
| retval = buflen; |
| } |
| |
| return retval; |
| } |
| |
| static const struct file_operations iommu_debug_atos_fops = { |
| .open = simple_open, |
| .write = iommu_debug_atos_write, |
| .read = iommu_debug_atos_read, |
| }; |
| |
| static ssize_t iommu_debug_dma_atos_read(struct file *file, char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| phys_addr_t phys; |
| char buf[100]; |
| ssize_t retval; |
| size_t buflen; |
| |
| if (kptr_restrict != 0) { |
| pr_err("kptr_restrict needs to be disabled.\n"); |
| return -EPERM; |
| } |
| if (!dev->archdata.mapping) { |
| pr_err("No mapping. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| if (!dev->archdata.mapping->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| if (*offset) |
| return 0; |
| |
| memset(buf, 0, sizeof(buf)); |
| |
| phys = iommu_iova_to_phys_hard(dev->archdata.mapping->domain, |
| ddev->iova); |
| if (!phys) |
| strlcpy(buf, "FAIL\n", sizeof(buf)); |
| else |
| snprintf(buf, sizeof(buf), "%pa\n", &phys); |
| |
| buflen = min(count, strlen(buf)); |
| if (copy_to_user(ubuf, buf, buflen)) { |
| pr_err("Couldn't copy_to_user\n"); |
| retval = -EFAULT; |
| } else { |
| *offset = 1; /* non-zero means we're done */ |
| retval = buflen; |
| } |
| |
| return retval; |
| } |
| |
| static const struct file_operations iommu_debug_dma_atos_fops = { |
| .open = simple_open, |
| .write = iommu_debug_atos_write, |
| .read = iommu_debug_dma_atos_read, |
| }; |
| |
| static ssize_t iommu_debug_map_write(struct file *file, const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| ssize_t retval = -EINVAL; |
| int ret; |
| char *comma1, *comma2, *comma3; |
| char buf[100]; |
| dma_addr_t iova; |
| phys_addr_t phys; |
| size_t size; |
| int prot; |
| struct iommu_debug_device *ddev = file->private_data; |
| |
| if (count >= 100) { |
| pr_err("Value too large\n"); |
| return -EINVAL; |
| } |
| |
| if (!ddev->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| memset(buf, 0, 100); |
| |
| if (copy_from_user(buf, ubuf, count)) { |
| pr_err("Couldn't copy from user\n"); |
| retval = -EFAULT; |
| } |
| |
| comma1 = strnchr(buf, count, ','); |
| if (!comma1) |
| goto invalid_format; |
| |
| comma2 = strnchr(comma1 + 1, count, ','); |
| if (!comma2) |
| goto invalid_format; |
| |
| comma3 = strnchr(comma2 + 1, count, ','); |
| if (!comma3) |
| goto invalid_format; |
| |
| /* split up the words */ |
| *comma1 = *comma2 = *comma3 = '\0'; |
| |
| if (kstrtoux(buf, 0, &iova)) |
| goto invalid_format; |
| |
| if (kstrtoux(comma1 + 1, 0, &phys)) |
| goto invalid_format; |
| |
| if (kstrtosize_t(comma2 + 1, 0, &size)) |
| goto invalid_format; |
| |
| if (kstrtoint(comma3 + 1, 0, &prot)) |
| goto invalid_format; |
| |
| ret = iommu_map(ddev->domain, iova, phys, size, prot); |
| if (ret) { |
| pr_err("iommu_map failed with %d\n", ret); |
| retval = -EIO; |
| goto out; |
| } |
| |
| retval = count; |
| pr_err("Mapped %pa to %pa (len=0x%zx, prot=0x%x)\n", |
| &iova, &phys, size, prot); |
| out: |
| return retval; |
| |
| invalid_format: |
| pr_err("Invalid format. Expected: iova,phys,len,prot where `prot' is the bitwise OR of IOMMU_READ, IOMMU_WRITE, etc.\n"); |
| return -EINVAL; |
| } |
| |
| static const struct file_operations iommu_debug_map_fops = { |
| .open = simple_open, |
| .write = iommu_debug_map_write, |
| }; |
| |
| /* |
| * Performs DMA mapping of a given virtual address and size to an iova address. |
| * User input format: (addr,len,dma attr) where dma attr is: |
| * 0: normal mapping |
| * 1: force coherent mapping |
| * 2: force non-cohernet mapping |
| * 3: use system cache |
| */ |
| static ssize_t iommu_debug_dma_map_write(struct file *file, |
| const char __user *ubuf, size_t count, loff_t *offset) |
| { |
| ssize_t retval = -EINVAL; |
| int ret; |
| char *comma1, *comma2; |
| char buf[100]; |
| unsigned long addr; |
| void *v_addr; |
| dma_addr_t iova; |
| size_t size; |
| unsigned int attr; |
| unsigned long dma_attrs; |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| |
| if (count >= sizeof(buf)) { |
| pr_err("Value too large\n"); |
| return -EINVAL; |
| } |
| |
| if (!dev->archdata.mapping) { |
| pr_err("No mapping. Did you already attach?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| if (!dev->archdata.mapping->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| |
| memset(buf, 0, sizeof(buf)); |
| |
| if (copy_from_user(buf, ubuf, count)) { |
| pr_err("Couldn't copy from user\n"); |
| retval = -EFAULT; |
| goto out; |
| } |
| |
| comma1 = strnchr(buf, count, ','); |
| if (!comma1) |
| goto invalid_format; |
| |
| comma2 = strnchr(comma1 + 1, count, ','); |
| if (!comma2) |
| goto invalid_format; |
| |
| *comma1 = *comma2 = '\0'; |
| |
| if (kstrtoul(buf, 0, &addr)) |
| goto invalid_format; |
| v_addr = (void *)addr; |
| |
| if (kstrtosize_t(comma1 + 1, 0, &size)) |
| goto invalid_format; |
| |
| if (kstrtouint(comma2 + 1, 0, &attr)) |
| goto invalid_format; |
| |
| if (v_addr < test_virt_addr || v_addr > (test_virt_addr + SZ_1M - 1)) |
| goto invalid_addr; |
| |
| if (attr == 0) |
| dma_attrs = 0; |
| else if (attr == 1) |
| dma_attrs = DMA_ATTR_FORCE_COHERENT; |
| else if (attr == 2) |
| dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; |
| else if (attr == 3) |
| dma_attrs = DMA_ATTR_IOMMU_USE_UPSTREAM_HINT; |
| else |
| goto invalid_format; |
| |
| iova = dma_map_single_attrs(dev, v_addr, size, |
| DMA_TO_DEVICE, dma_attrs); |
| |
| if (dma_mapping_error(dev, iova)) { |
| pr_err("Failed to perform dma_map_single\n"); |
| ret = -EINVAL; |
| goto out; |
| } |
| |
| retval = count; |
| pr_err("Mapped 0x%p to %pa (len=0x%zx)\n", |
| v_addr, &iova, size); |
| ddev->iova = iova; |
| pr_err("Saved iova=%pa for future PTE commands\n", &iova); |
| out: |
| return retval; |
| |
| invalid_format: |
| pr_err("Invalid format. Expected: addr,len,dma attr where 'dma attr' is\n0: normal mapping\n1: force coherent\n2: force non-cohernet\n3: use system cache\n"); |
| return retval; |
| |
| invalid_addr: |
| pr_err("Invalid addr given! Address should be within 1MB size from start addr returned by doing 'cat test_virt_addr'.\n"); |
| return retval; |
| } |
| |
| static ssize_t iommu_debug_dma_map_read(struct file *file, char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| char buf[100]; |
| ssize_t retval; |
| size_t buflen; |
| dma_addr_t iova; |
| |
| if (!dev->archdata.mapping) { |
| pr_err("No mapping. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| if (!dev->archdata.mapping->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| if (*offset) |
| return 0; |
| |
| memset(buf, 0, sizeof(buf)); |
| |
| iova = ddev->iova; |
| snprintf(buf, sizeof(buf), "%pa\n", &iova); |
| |
| buflen = min(count, strlen(buf)); |
| if (copy_to_user(ubuf, buf, buflen)) { |
| pr_err("Couldn't copy_to_user\n"); |
| retval = -EFAULT; |
| } else { |
| *offset = 1; /* non-zero means we're done */ |
| retval = buflen; |
| } |
| |
| return retval; |
| } |
| |
| static const struct file_operations iommu_debug_dma_map_fops = { |
| .open = simple_open, |
| .write = iommu_debug_dma_map_write, |
| .read = iommu_debug_dma_map_read, |
| }; |
| |
| static ssize_t iommu_debug_unmap_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| ssize_t retval = 0; |
| char *comma1; |
| char buf[100]; |
| dma_addr_t iova; |
| size_t size; |
| size_t unmapped; |
| struct iommu_debug_device *ddev = file->private_data; |
| |
| if (count >= 100) { |
| pr_err("Value too large\n"); |
| return -EINVAL; |
| } |
| |
| if (!ddev->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| memset(buf, 0, 100); |
| |
| if (copy_from_user(buf, ubuf, count)) { |
| pr_err("Couldn't copy from user\n"); |
| retval = -EFAULT; |
| goto out; |
| } |
| |
| comma1 = strnchr(buf, count, ','); |
| if (!comma1) |
| goto invalid_format; |
| |
| /* split up the words */ |
| *comma1 = '\0'; |
| |
| if (kstrtoux(buf, 0, &iova)) |
| goto invalid_format; |
| |
| if (kstrtosize_t(comma1 + 1, 0, &size)) |
| goto invalid_format; |
| |
| unmapped = iommu_unmap(ddev->domain, iova, size); |
| if (unmapped != size) { |
| pr_err("iommu_unmap failed. Expected to unmap: 0x%zx, unmapped: 0x%zx", |
| size, unmapped); |
| return -EIO; |
| } |
| |
| retval = count; |
| pr_err("Unmapped %pa (len=0x%zx)\n", &iova, size); |
| out: |
| return retval; |
| |
| invalid_format: |
| pr_err("Invalid format. Expected: iova,len\n"); |
| return -EINVAL; |
| } |
| |
| static const struct file_operations iommu_debug_unmap_fops = { |
| .open = simple_open, |
| .write = iommu_debug_unmap_write, |
| }; |
| |
| static ssize_t iommu_debug_dma_unmap_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| ssize_t retval = 0; |
| char *comma1, *comma2; |
| char buf[100]; |
| size_t size; |
| unsigned int attr; |
| dma_addr_t iova; |
| unsigned long dma_attrs; |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| |
| if (count >= sizeof(buf)) { |
| pr_err("Value too large\n"); |
| return -EINVAL; |
| } |
| |
| if (!dev->archdata.mapping) { |
| pr_err("No mapping. Did you already attach?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| if (!dev->archdata.mapping->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| retval = -EINVAL; |
| goto out; |
| } |
| |
| memset(buf, 0, sizeof(buf)); |
| |
| if (copy_from_user(buf, ubuf, count)) { |
| pr_err("Couldn't copy from user\n"); |
| retval = -EFAULT; |
| goto out; |
| } |
| |
| comma1 = strnchr(buf, count, ','); |
| if (!comma1) |
| goto invalid_format; |
| |
| comma2 = strnchr(comma1 + 1, count, ','); |
| if (!comma2) |
| goto invalid_format; |
| |
| *comma1 = *comma2 = '\0'; |
| |
| if (kstrtoux(buf, 0, &iova)) |
| goto invalid_format; |
| |
| if (kstrtosize_t(comma1 + 1, 0, &size)) |
| goto invalid_format; |
| |
| if (kstrtouint(comma2 + 1, 0, &attr)) |
| goto invalid_format; |
| |
| if (attr == 0) |
| dma_attrs = 0; |
| else if (attr == 1) |
| dma_attrs = DMA_ATTR_FORCE_COHERENT; |
| else if (attr == 2) |
| dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; |
| else if (attr == 3) |
| dma_attrs = DMA_ATTR_IOMMU_USE_UPSTREAM_HINT; |
| else |
| goto invalid_format; |
| |
| dma_unmap_single_attrs(dev, iova, size, DMA_TO_DEVICE, dma_attrs); |
| |
| retval = count; |
| pr_err("Unmapped %pa (len=0x%zx)\n", &iova, size); |
| out: |
| return retval; |
| |
| invalid_format: |
| pr_err("Invalid format. Expected: iova,len, dma attr\n"); |
| return retval; |
| } |
| |
| static const struct file_operations iommu_debug_dma_unmap_fops = { |
| .open = simple_open, |
| .write = iommu_debug_dma_unmap_write, |
| }; |
| |
| static ssize_t iommu_debug_config_clocks_write(struct file *file, |
| const char __user *ubuf, |
| size_t count, loff_t *offset) |
| { |
| char buf; |
| struct iommu_debug_device *ddev = file->private_data; |
| struct device *dev = ddev->dev; |
| |
| /* we're expecting a single character plus (optionally) a newline */ |
| if (count > 2) { |
| dev_err(dev, "Invalid value\n"); |
| return -EINVAL; |
| } |
| |
| if (!ddev->domain) { |
| dev_err(dev, "No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| if (copy_from_user(&buf, ubuf, 1)) { |
| dev_err(dev, "Couldn't copy from user\n"); |
| return -EFAULT; |
| } |
| |
| mutex_lock(&ddev->clk_lock); |
| switch (buf) { |
| case '0': |
| if (ddev->clk_count == 0) { |
| dev_err(dev, "Config clocks already disabled\n"); |
| break; |
| } |
| |
| if (--ddev->clk_count > 0) |
| break; |
| |
| dev_err(dev, "Disabling config clocks\n"); |
| iommu_disable_config_clocks(ddev->domain); |
| break; |
| case '1': |
| if (ddev->clk_count++ > 0) |
| break; |
| |
| dev_err(dev, "Enabling config clocks\n"); |
| if (iommu_enable_config_clocks(ddev->domain)) |
| dev_err(dev, "Failed!\n"); |
| break; |
| default: |
| dev_err(dev, "Invalid value. Should be 0 or 1.\n"); |
| mutex_unlock(&ddev->clk_lock); |
| return -EINVAL; |
| } |
| mutex_unlock(&ddev->clk_lock); |
| |
| return count; |
| } |
| |
| static const struct file_operations iommu_debug_config_clocks_fops = { |
| .open = simple_open, |
| .write = iommu_debug_config_clocks_write, |
| }; |
| |
| static ssize_t iommu_debug_trigger_fault_write( |
| struct file *file, const char __user *ubuf, size_t count, |
| loff_t *offset) |
| { |
| struct iommu_debug_device *ddev = file->private_data; |
| unsigned long flags; |
| |
| if (!ddev->domain) { |
| pr_err("No domain. Did you already attach?\n"); |
| return -EINVAL; |
| } |
| |
| if (kstrtoul_from_user(ubuf, count, 0, &flags)) { |
| pr_err("Invalid flags format\n"); |
| return -EFAULT; |
| } |
| |
| iommu_trigger_fault(ddev->domain, flags); |
| |
| return count; |
| } |
| |
| static const struct file_operations iommu_debug_trigger_fault_fops = { |
| .open = simple_open, |
| .write = iommu_debug_trigger_fault_write, |
| }; |
| |
| /* |
| * The following will only work for drivers that implement the generic |
| * device tree bindings described in |
| * Documentation/devicetree/bindings/iommu/iommu.txt |
| */ |
| static int snarf_iommu_devices(struct device *dev, void *ignored) |
| { |
| struct iommu_debug_device *ddev; |
| struct dentry *dir; |
| |
| if (!of_find_property(dev->of_node, "iommus", NULL)) |
| return 0; |
| |
| if (!of_device_is_compatible(dev->of_node, "iommu-debug-test")) |
| return 0; |
| |
| /* Hold a reference count */ |
| if (!iommu_group_get(dev)) |
| return 0; |
| |
| ddev = kzalloc(sizeof(*ddev), GFP_KERNEL); |
| if (!ddev) |
| return -ENODEV; |
| mutex_init(&ddev->clk_lock); |
| ddev->dev = dev; |
| dir = debugfs_create_dir(dev_name(dev), debugfs_tests_dir); |
| if (!dir) { |
| pr_err("Couldn't create iommu/devices/%s debugfs dir\n", |
| dev_name(dev)); |
| goto err; |
| } |
| |
| if (!debugfs_create_file("nr_iters", S_IRUSR, dir, &iters_per_op, |
| &iommu_debug_nr_iters_ops)) { |
| pr_err("Couldn't create iommu/devices/%s/nr_iters debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("test_virt_addr", 0400, dir, ddev, |
| &iommu_debug_test_virt_addr_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/test_virt_addr debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("profiling", S_IRUSR, dir, ddev, |
| &iommu_debug_profiling_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/profiling debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("secure_profiling", S_IRUSR, dir, ddev, |
| &iommu_debug_secure_profiling_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/secure_profiling debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("profiling_fast", S_IRUSR, dir, ddev, |
| &iommu_debug_profiling_fast_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/profiling_fast debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("profiling_fast_dma_api", S_IRUSR, dir, ddev, |
| &iommu_debug_profiling_fast_dma_api_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/profiling_fast_dma_api debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("functional_fast_dma_api", S_IRUSR, dir, ddev, |
| &iommu_debug_functional_fast_dma_api_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/functional_fast_dma_api debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("functional_arm_dma_api", S_IRUSR, dir, ddev, |
| &iommu_debug_functional_arm_dma_api_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/functional_arm_dma_api debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("dma_attach", 0600, dir, ddev, |
| &iommu_debug_dma_attach_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/dma_attach debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("attach", S_IRUSR, dir, ddev, |
| &iommu_debug_attach_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/attach debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("secure_attach", S_IRUSR, dir, ddev, |
| &iommu_debug_secure_attach_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/secure_attach debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("atos", S_IWUSR, dir, ddev, |
| &iommu_debug_atos_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/atos debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("dma_atos", 0600, dir, ddev, |
| &iommu_debug_dma_atos_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/dma_atos debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("map", S_IWUSR, dir, ddev, |
| &iommu_debug_map_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/map debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("dma_map", 0600, dir, ddev, |
| &iommu_debug_dma_map_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/dma_map debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("unmap", S_IWUSR, dir, ddev, |
| &iommu_debug_unmap_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/unmap debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("dma_unmap", 0200, dir, ddev, |
| &iommu_debug_dma_unmap_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/dma_unmap debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("pte", 0600, dir, ddev, |
| &iommu_debug_pte_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/pte debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("config_clocks", S_IWUSR, dir, ddev, |
| &iommu_debug_config_clocks_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/config_clocks debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| if (!debugfs_create_file("trigger-fault", 0200, dir, ddev, |
| &iommu_debug_trigger_fault_fops)) { |
| pr_err("Couldn't create iommu/devices/%s/trigger-fault debugfs file\n", |
| dev_name(dev)); |
| goto err_rmdir; |
| } |
| |
| list_add(&ddev->list, &iommu_debug_devices); |
| return 0; |
| |
| err_rmdir: |
| debugfs_remove_recursive(dir); |
| err: |
| kfree(ddev); |
| return 0; |
| } |
| |
| static int iommu_debug_init_tests(void) |
| { |
| debugfs_tests_dir = debugfs_create_dir("tests", |
| iommu_debugfs_top); |
| if (!debugfs_tests_dir) { |
| pr_err("Couldn't create iommu/tests debugfs directory\n"); |
| return -ENODEV; |
| } |
| |
| test_virt_addr = kzalloc(SZ_1M, GFP_KERNEL); |
| |
| if (!test_virt_addr) |
| return -ENOMEM; |
| |
| return bus_for_each_dev(&platform_bus_type, NULL, NULL, |
| snarf_iommu_devices); |
| } |
| |
| static void iommu_debug_destroy_tests(void) |
| { |
| debugfs_remove_recursive(debugfs_tests_dir); |
| } |
| #else |
| static inline int iommu_debug_init_tests(void) { return 0; } |
| static inline void iommu_debug_destroy_tests(void) { } |
| #endif |
| |
| /* |
| * This isn't really a "driver", we just need something in the device tree |
| * so that our tests can run without any client drivers, and our tests rely |
| * on parsing the device tree for nodes with the `iommus' property. |
| */ |
| static int iommu_debug_pass(struct platform_device *pdev) |
| { |
| return 0; |
| } |
| |
| static const struct of_device_id iommu_debug_of_match[] = { |
| { .compatible = "iommu-debug-test" }, |
| { }, |
| }; |
| |
| static struct platform_driver iommu_debug_driver = { |
| .probe = iommu_debug_pass, |
| .remove = iommu_debug_pass, |
| .driver = { |
| .name = "iommu-debug", |
| .of_match_table = iommu_debug_of_match, |
| }, |
| }; |
| |
| static int iommu_debug_init(void) |
| { |
| if (iommu_debug_init_tests()) |
| return -ENODEV; |
| |
| return platform_driver_register(&iommu_debug_driver); |
| } |
| |
| static void iommu_debug_exit(void) |
| { |
| platform_driver_unregister(&iommu_debug_driver); |
| iommu_debug_destroy_tests(); |
| } |
| |
| module_init(iommu_debug_init); |
| module_exit(iommu_debug_exit); |