drivers/ntb/test/ntb_perf.c - kernel/msm-5.4 - Gitiles

 /*
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  *   redistributing this file, you may do so under either license.
  *
  *   GPL LICENSE SUMMARY
  *
  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
  *   published by the Free Software Foundation.
  *
  *   BSD LICENSE
  *
  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
  *   are met:
  *
  *     * Redistributions of source code must retain the above copyright
  *       notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above copy
  *       notice, this list of conditions and the following disclaimer in
  *       the documentation and/or other materials provided with the
  *       distribution.
  *     * Neither the name of Intel Corporation nor the names of its
  *       contributors may be used to endorse or promote products derived
  *       from this software without specific prior written permission.
  *
  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  *   PCIe NTB Perf Linux driver
  */

 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/kthread.h>
 #include <linux/time.h>
 #include <linux/timer.h>
 #include <linux/dma-mapping.h>
 #include <linux/pci.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/debugfs.h>
 #include <linux/dmaengine.h>
 #include <linux/delay.h>
 #include <linux/sizes.h>
 #include <linux/ntb.h>
 #include <linux/mutex.h>

 #define DRIVER_NAME		"ntb_perf"
 #define DRIVER_DESCRIPTION	"PCIe NTB Performance Measurement Tool"

 #define DRIVER_LICENSE		"Dual BSD/GPL"
 #define DRIVER_VERSION		"1.0"
 #define DRIVER_AUTHOR		"Dave Jiang <dave.jiang@intel.com>"

 #define PERF_LINK_DOWN_TIMEOUT	10
 #define PERF_VERSION		0xffff0001
 #define MAX_THREADS		32
 #define MAX_TEST_SIZE		SZ_1M
 #define MAX_SRCS		32
 #define DMA_OUT_RESOURCE_TO	msecs_to_jiffies(50)
 #define DMA_RETRIES		20
 #define SZ_4G			(1ULL << 32)
 #define MAX_SEG_ORDER		20 /* no larger than 1M for kmalloc buffer */

 MODULE_LICENSE(DRIVER_LICENSE);
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESCRIPTION);

 static struct dentry *perf_debugfs_dir;

 static unsigned long max_mw_size;
 module_param(max_mw_size, ulong, 0644);
 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");

 static unsigned int seg_order = 19; /* 512K */
 module_param(seg_order, uint, 0644);
 MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing");

 static unsigned int run_order = 32; /* 4G */
 module_param(run_order, uint, 0644);
 MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer");

 static bool use_dma; /* default to 0 */
 module_param(use_dma, bool, 0644);
 MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");

 struct perf_mw {
 	phys_addr_t	phys_addr;
 	resource_size_t	phys_size;
 	resource_size_t	xlat_align;
 	resource_size_t	xlat_align_size;
 	void __iomem	*vbase;
 	size_t		xlat_size;
 	size_t		buf_size;
 	void		*virt_addr;
 	dma_addr_t	dma_addr;
 };

 struct perf_ctx;

 struct pthr_ctx {
 	struct task_struct	*thread;
 	struct perf_ctx		*perf;
 	atomic_t		dma_sync;
 	struct dma_chan		*dma_chan;
 	int			dma_prep_err;
 	int			src_idx;
 	void			*srcs[MAX_SRCS];
 	wait_queue_head_t       *wq;
 	int			status;
 	u64			copied;
 	u64			diff_us;
 };

 struct perf_ctx {
 	struct ntb_dev		*ntb;
 	spinlock_t		db_lock;
 	struct perf_mw		mw;
 	bool			link_is_up;
 	struct delayed_work	link_work;
 	wait_queue_head_t	link_wq;
 	struct dentry		*debugfs_node_dir;
 	struct dentry		*debugfs_run;
 	struct dentry		*debugfs_threads;
 	u8			perf_threads;
 	/* mutex ensures only one set of threads run at once */
 	struct mutex		run_mutex;
 	struct pthr_ctx		pthr_ctx[MAX_THREADS];
 	atomic_t		tsync;
 	atomic_t                tdone;
 };

 enum {
 	VERSION = 0,
 	MW_SZ_HIGH,
 	MW_SZ_LOW,
 	MAX_SPAD
 };

 static void perf_link_event(void *ctx)
 {
 	struct perf_ctx *perf = ctx;

 	if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) {
 		schedule_delayed_work(&perf->link_work, 2*HZ);
 	} else {
 		dev_dbg(&perf->ntb->pdev->dev, "link down\n");

 		if (!perf->link_is_up)
 			cancel_delayed_work_sync(&perf->link_work);

 		perf->link_is_up = false;
 	}
 }

 static void perf_db_event(void *ctx, int vec)
 {
 	struct perf_ctx *perf = ctx;
 	u64 db_bits, db_mask;

 	db_mask = ntb_db_vector_mask(perf->ntb, vec);
 	db_bits = ntb_db_read(perf->ntb);

 	dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
 		vec, db_mask, db_bits);
 }

 static const struct ntb_ctx_ops perf_ops = {
 	.link_event = perf_link_event,
 	.db_event = perf_db_event,
 };

 static void perf_copy_callback(void *data)
 {
 	struct pthr_ctx *pctx = data;

 	atomic_dec(&pctx->dma_sync);
 }

 static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst,
 			 char *src, size_t size)
 {
 	struct perf_ctx *perf = pctx->perf;
 	struct dma_async_tx_descriptor *txd;
 	struct dma_chan *chan = pctx->dma_chan;
 	struct dma_device *device;
 	struct dmaengine_unmap_data *unmap;
 	dma_cookie_t cookie;
 	size_t src_off, dst_off;
 	struct perf_mw *mw = &perf->mw;
 	void __iomem *vbase;
 	void __iomem *dst_vaddr;
 	dma_addr_t dst_phys;
 	int retries = 0;

 	if (!use_dma) {
 		memcpy_toio(dst, src, size);
 		return size;
 	}

 	if (!chan) {
 		dev_err(&perf->ntb->dev, "DMA engine does not exist\n");
 		return -EINVAL;
 	}

 	device = chan->device;
 	src_off = (uintptr_t)src & ~PAGE_MASK;
 	dst_off = (uintptr_t __force)dst & ~PAGE_MASK;

 	if (!is_dma_copy_aligned(device, src_off, dst_off, size))
 		return -ENODEV;

 	vbase = mw->vbase;
 	dst_vaddr = dst;
 	dst_phys = mw->phys_addr + (dst_vaddr - vbase);

 	unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
 	if (!unmap)
 		return -ENOMEM;

 	unmap->len = size;
 	unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src),
 				      src_off, size, DMA_TO_DEVICE);
 	if (dma_mapping_error(device->dev, unmap->addr[0]))
 		goto err_get_unmap;

 	unmap->to_cnt = 1;

 	do {
 		txd = device->device_prep_dma_memcpy(chan, dst_phys,
 						     unmap->addr[0],
 						     size, DMA_PREP_INTERRUPT);
 		if (!txd) {
 			set_current_state(TASK_INTERRUPTIBLE);
 			schedule_timeout(DMA_OUT_RESOURCE_TO);
 		}
 	} while (!txd && (++retries < DMA_RETRIES));

 	if (!txd) {
 		pctx->dma_prep_err++;
 		goto err_get_unmap;
 	}

 	txd->callback = perf_copy_callback;
 	txd->callback_param = pctx;
 	dma_set_unmap(txd, unmap);

 	cookie = dmaengine_submit(txd);
 	if (dma_submit_error(cookie))
 		goto err_set_unmap;

 	atomic_inc(&pctx->dma_sync);
 	dma_async_issue_pending(chan);

 	return size;

 err_set_unmap:
 	dmaengine_unmap_put(unmap);
 err_get_unmap:
 	dmaengine_unmap_put(unmap);
 	return 0;
 }

 static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src,
 			  u64 buf_size, u64 win_size, u64 total)
 {
 	int chunks, total_chunks, i;
 	int copied_chunks = 0;
 	u64 copied = 0, result;
 	char __iomem *tmp = dst;
 	u64 perf, diff_us;
 	ktime_t kstart, kstop, kdiff;
 	unsigned long last_sleep = jiffies;

 	chunks = div64_u64(win_size, buf_size);
 	total_chunks = div64_u64(total, buf_size);
 	kstart = ktime_get();

 	for (i = 0; i < total_chunks; i++) {
 		result = perf_copy(pctx, tmp, src, buf_size);
 		copied += result;
 		copied_chunks++;
 		if (copied_chunks == chunks) {
 			tmp = dst;
 			copied_chunks = 0;
 		} else
 			tmp += buf_size;

 		/* Probably should schedule every 5s to prevent soft hang. */
 		if (unlikely((jiffies - last_sleep) > 5 * HZ)) {
 			last_sleep = jiffies;
 			set_current_state(TASK_INTERRUPTIBLE);
 			schedule_timeout(1);
 		}

 		if (unlikely(kthread_should_stop()))
 			break;
 	}

 	if (use_dma) {
 		pr_debug("%s: All DMA descriptors submitted\n", current->comm);
 		while (atomic_read(&pctx->dma_sync) != 0) {
 			if (kthread_should_stop())
 				break;
 			msleep(20);
 		}
 	}

 	kstop = ktime_get();
 	kdiff = ktime_sub(kstop, kstart);
 	diff_us = ktime_to_us(kdiff);

 	pr_debug("%s: copied %llu bytes\n", current->comm, copied);

 	pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us);

 	perf = div64_u64(copied, diff_us);

 	pr_debug("%s: MBytes/s: %llu\n", current->comm, perf);

 	pctx->copied = copied;
 	pctx->diff_us = diff_us;

 	return 0;
 }

 static bool perf_dma_filter_fn(struct dma_chan *chan, void *node)
 {
 	return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
 }

 static int ntb_perf_thread(void *data)
 {
 	struct pthr_ctx *pctx = data;
 	struct perf_ctx *perf = pctx->perf;
 	struct pci_dev *pdev = perf->ntb->pdev;
 	struct perf_mw *mw = &perf->mw;
 	char __iomem *dst;
 	u64 win_size, buf_size, total;
 	void *src;
 	int rc, node, i;
 	struct dma_chan *dma_chan = NULL;

 	pr_debug("kthread %s starting...\n", current->comm);

 	node = dev_to_node(&pdev->dev);

 	if (use_dma && !pctx->dma_chan) {
 		dma_cap_mask_t dma_mask;

 		dma_cap_zero(dma_mask);
 		dma_cap_set(DMA_MEMCPY, dma_mask);
 		dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn,
 					       (void *)(unsigned long)node);
 		if (!dma_chan) {
 			pr_warn("%s: cannot acquire DMA channel, quitting\n",
 				current->comm);
 			return -ENODEV;
 		}
 		pctx->dma_chan = dma_chan;
 	}

 	for (i = 0; i < MAX_SRCS; i++) {
 		pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node);
 		if (!pctx->srcs[i]) {
 			rc = -ENOMEM;
 			goto err;
 		}
 	}

 	win_size = mw->phys_size;
 	buf_size = 1ULL << seg_order;
 	total = 1ULL << run_order;

 	if (buf_size > MAX_TEST_SIZE)
 		buf_size = MAX_TEST_SIZE;

 	dst = (char __iomem *)mw->vbase;

 	atomic_inc(&perf->tsync);
 	while (atomic_read(&perf->tsync) != perf->perf_threads)
 		schedule();

 	src = pctx->srcs[pctx->src_idx];
 	pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1);

 	rc = perf_move_data(pctx, dst, src, buf_size, win_size, total);

 	atomic_dec(&perf->tsync);

 	if (rc < 0) {
 		pr_err("%s: failed\n", current->comm);
 		rc = -ENXIO;
 		goto err;
 	}

 	for (i = 0; i < MAX_SRCS; i++) {
 		kfree(pctx->srcs[i]);
 		pctx->srcs[i] = NULL;
 	}

 	atomic_inc(&perf->tdone);
 	wake_up(pctx->wq);
 	rc = 0;
 	goto done;

 err:
 	for (i = 0; i < MAX_SRCS; i++) {
 		kfree(pctx->srcs[i]);
 		pctx->srcs[i] = NULL;
 	}

 	if (dma_chan) {
 		dma_release_channel(dma_chan);
 		pctx->dma_chan = NULL;
 	}

 done:
 	/* Wait until we are told to stop */
 	for (;;) {
 		set_current_state(TASK_INTERRUPTIBLE);
 		if (kthread_should_stop())
 			break;
 		schedule();
 	}
 	__set_current_state(TASK_RUNNING);

 	return rc;
 }

 static void perf_free_mw(struct perf_ctx *perf)
 {
 	struct perf_mw *mw = &perf->mw;
 	struct pci_dev *pdev = perf->ntb->pdev;

 	if (!mw->virt_addr)
 		return;

 	ntb_mw_clear_trans(perf->ntb, 0);
 	dma_free_coherent(&pdev->dev, mw->buf_size,
 			  mw->virt_addr, mw->dma_addr);
 	mw->xlat_size = 0;
 	mw->buf_size = 0;
 	mw->virt_addr = NULL;
 }

 static int perf_set_mw(struct perf_ctx *perf, resource_size_t size)
 {
 	struct perf_mw *mw = &perf->mw;
 	size_t xlat_size, buf_size;
 	int rc;

 	if (!size)
 		return -EINVAL;

 	xlat_size = round_up(size, mw->xlat_align_size);
 	buf_size = round_up(size, mw->xlat_align);

 	if (mw->xlat_size == xlat_size)
 		return 0;

 	if (mw->buf_size)
 		perf_free_mw(perf);

 	mw->xlat_size = xlat_size;
 	mw->buf_size = buf_size;

 	mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size,
 					   &mw->dma_addr, GFP_KERNEL);
 	if (!mw->virt_addr) {
 		mw->xlat_size = 0;
 		mw->buf_size = 0;
 	}

 	rc = ntb_mw_set_trans(perf->ntb, 0, mw->dma_addr, mw->xlat_size);
 	if (rc) {
 		dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n");
 		perf_free_mw(perf);
 		return -EIO;
 	}

 	return 0;
 }

 static void perf_link_work(struct work_struct *work)
 {
 	struct perf_ctx *perf =
 		container_of(work, struct perf_ctx, link_work.work);
 	struct ntb_dev *ndev = perf->ntb;
 	struct pci_dev *pdev = ndev->pdev;
 	u32 val;
 	u64 size;
 	int rc;

 	dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);

 	size = perf->mw.phys_size;

 	if (max_mw_size && size > max_mw_size)
 		size = max_mw_size;

 	ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size));
 	ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size));
 	ntb_peer_spad_write(ndev, VERSION, PERF_VERSION);

 	/* now read what peer wrote */
 	val = ntb_spad_read(ndev, VERSION);
 	if (val != PERF_VERSION) {
 		dev_dbg(&pdev->dev, "Remote version = %#x\n", val);
 		goto out;
 	}

 	val = ntb_spad_read(ndev, MW_SZ_HIGH);
 	size = (u64)val << 32;

 	val = ntb_spad_read(ndev, MW_SZ_LOW);
 	size |= val;

 	dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size);

 	rc = perf_set_mw(perf, size);
 	if (rc)
 		goto out1;

 	perf->link_is_up = true;
 	wake_up(&perf->link_wq);

 	return;

 out1:
 	perf_free_mw(perf);

 out:
 	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
 		schedule_delayed_work(&perf->link_work,
 				      msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT));
 }

 static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf)
 {
 	struct perf_mw *mw;
 	int rc;

 	mw = &perf->mw;

 	rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size,
 			      &mw->xlat_align, &mw->xlat_align_size);
 	if (rc)
 		return rc;

 	perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
 	if (!mw->vbase)
 		return -ENOMEM;

 	return 0;
 }

 static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf,
 				size_t count, loff_t *offp)
 {
 	struct perf_ctx *perf = filp->private_data;
 	char *buf;
 	ssize_t ret, out_off = 0;
 	struct pthr_ctx *pctx;
 	int i;
 	u64 rate;

 	if (!perf)
 		return 0;

 	buf = kmalloc(1024, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;

 	if (mutex_is_locked(&perf->run_mutex)) {
 		out_off = scnprintf(buf, 64, "running\n");
 		goto read_from_buf;
 	}

 	for (i = 0; i < MAX_THREADS; i++) {
 		pctx = &perf->pthr_ctx[i];

 		if (pctx->status == -ENODATA)
 			break;

 		if (pctx->status) {
 			out_off += scnprintf(buf + out_off, 1024 - out_off,
 					    "%d: error %d\n", i,
 					    pctx->status);
 			continue;
 		}

 		rate = div64_u64(pctx->copied, pctx->diff_us);
 		out_off += scnprintf(buf + out_off, 1024 - out_off,
 			"%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
 			i, pctx->copied, pctx->diff_us, rate);
 	}

 read_from_buf:
 	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_off);
 	kfree(buf);

 	return ret;
 }

 static void threads_cleanup(struct perf_ctx *perf)
 {
 	struct pthr_ctx *pctx;
 	int i;

 	for (i = 0; i < MAX_THREADS; i++) {
 		pctx = &perf->pthr_ctx[i];
 		if (pctx->thread) {
 			pctx->status = kthread_stop(pctx->thread);
 			pctx->thread = NULL;
 		}
 	}
 }

 static void perf_clear_thread_status(struct perf_ctx *perf)
 {
 	int i;

 	for (i = 0; i < MAX_THREADS; i++)
 		perf->pthr_ctx[i].status = -ENODATA;
 }

 static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf,
 				 size_t count, loff_t *offp)
 {
 	struct perf_ctx *perf = filp->private_data;
 	int node, i;
 	DECLARE_WAIT_QUEUE_HEAD(wq);

 	if (wait_event_interruptible(perf->link_wq, perf->link_is_up))
 		return -ENOLINK;

 	if (perf->perf_threads == 0)
 		return -EINVAL;

 	if (!mutex_trylock(&perf->run_mutex))
 		return -EBUSY;

 	perf_clear_thread_status(perf);

 	if (perf->perf_threads > MAX_THREADS) {
 		perf->perf_threads = MAX_THREADS;
 		pr_info("Reset total threads to: %u\n", MAX_THREADS);
 	}

 	/* no greater than 1M */
 	if (seg_order > MAX_SEG_ORDER) {
 		seg_order = MAX_SEG_ORDER;
 		pr_info("Fix seg_order to %u\n", seg_order);
 	}

 	if (run_order < seg_order) {
 		run_order = seg_order;
 		pr_info("Fix run_order to %u\n", run_order);
 	}

 	node = dev_to_node(&perf->ntb->pdev->dev);
 	atomic_set(&perf->tdone, 0);

 	/* launch kernel thread */
 	for (i = 0; i < perf->perf_threads; i++) {
 		struct pthr_ctx *pctx;

 		pctx = &perf->pthr_ctx[i];
 		atomic_set(&pctx->dma_sync, 0);
 		pctx->perf = perf;
 		pctx->wq = &wq;
 		pctx->thread =
 			kthread_create_on_node(ntb_perf_thread,
 					       (void *)pctx,
 					       node, "ntb_perf %d", i);
 		if (IS_ERR(pctx->thread)) {
 			pctx->thread = NULL;
 			goto err;
 		} else {
 			wake_up_process(pctx->thread);
 		}
 	}

 	wait_event_interruptible(wq,
 		atomic_read(&perf->tdone) == perf->perf_threads);

 	threads_cleanup(perf);
 	mutex_unlock(&perf->run_mutex);
 	return count;

 err:
 	threads_cleanup(perf);
 	mutex_unlock(&perf->run_mutex);
 	return -ENXIO;
 }

 static const struct file_operations ntb_perf_debugfs_run = {
 	.owner = THIS_MODULE,
 	.open = simple_open,
 	.read = debugfs_run_read,
 	.write = debugfs_run_write,
 };

 static int perf_debugfs_setup(struct perf_ctx *perf)
 {
 	struct pci_dev *pdev = perf->ntb->pdev;

 	if (!debugfs_initialized())
 		return -ENODEV;

 	if (!perf_debugfs_dir) {
 		perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
 		if (!perf_debugfs_dir)
 			return -ENODEV;
 	}

 	perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev),
 						    perf_debugfs_dir);
 	if (!perf->debugfs_node_dir)
 		return -ENODEV;

 	perf->debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR,
 						perf->debugfs_node_dir, perf,
 						&ntb_perf_debugfs_run);
 	if (!perf->debugfs_run)
 		return -ENODEV;

 	perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR,
 						  perf->debugfs_node_dir,
 						  &perf->perf_threads);
 	if (!perf->debugfs_threads)
 		return -ENODEV;

 	return 0;
 }

 static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
 {
 	struct pci_dev *pdev = ntb->pdev;
 	struct perf_ctx *perf;
 	int node;
 	int rc = 0;

 	if (ntb_spad_count(ntb) < MAX_SPAD) {
 		dev_err(&ntb->dev, "Not enough scratch pad registers for %s",
 			DRIVER_NAME);
 		return -EIO;
 	}

 	node = dev_to_node(&pdev->dev);

 	perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
 	if (!perf) {
 		rc = -ENOMEM;
 		goto err_perf;
 	}

 	perf->ntb = ntb;
 	perf->perf_threads = 1;
 	atomic_set(&perf->tsync, 0);
 	mutex_init(&perf->run_mutex);
 	spin_lock_init(&perf->db_lock);
 	perf_setup_mw(ntb, perf);
 	init_waitqueue_head(&perf->link_wq);
 	INIT_DELAYED_WORK(&perf->link_work, perf_link_work);

 	rc = ntb_set_ctx(ntb, perf, &perf_ops);
 	if (rc)
 		goto err_ctx;

 	perf->link_is_up = false;
 	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 	ntb_link_event(ntb);

 	rc = perf_debugfs_setup(perf);
 	if (rc)
 		goto err_ctx;

 	perf_clear_thread_status(perf);

 	return 0;

 err_ctx:
 	cancel_delayed_work_sync(&perf->link_work);
 	kfree(perf);
 err_perf:
 	return rc;
 }

 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
 {
 	struct perf_ctx *perf = ntb->ctx;
 	int i;

 	dev_dbg(&perf->ntb->dev, "%s called\n", __func__);

 	mutex_lock(&perf->run_mutex);

 	cancel_delayed_work_sync(&perf->link_work);

 	ntb_clear_ctx(ntb);
 	ntb_link_disable(ntb);

 	debugfs_remove_recursive(perf_debugfs_dir);
 	perf_debugfs_dir = NULL;

 	if (use_dma) {
 		for (i = 0; i < MAX_THREADS; i++) {
 			struct pthr_ctx *pctx = &perf->pthr_ctx[i];

 			if (pctx->dma_chan)
 				dma_release_channel(pctx->dma_chan);
 		}
 	}

 	kfree(perf);
 }

 static struct ntb_client perf_client = {
 	.ops = {
 		.probe = perf_probe,
 		.remove = perf_remove,
 	},
 };
 module_ntb_client(perf_client);
	/*
	* This file is provided under a dual BSD/GPLv2 license. When using or
	* redistributing this file, you may do so under either license.
	*
	* GPL LICENSE SUMMARY
	*
	* Copyright(c) 2015 Intel Corporation. All rights reserved.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of version 2 of the GNU General Public License as
	* published by the Free Software Foundation.
	*
	* BSD LICENSE
	*
	* Copyright(c) 2015 Intel Corporation. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* * Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* * Redistributions in binary form must reproduce the above copy
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	* * Neither the name of Intel Corporation nor the names of its
	* contributors may be used to endorse or promote products derived
	* from this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	* PCIe NTB Perf Linux driver
	*/

	#include <linux/init.h>
	#include <linux/kernel.h>
	#include <linux/module.h>
	#include <linux/kthread.h>
	#include <linux/time.h>
	#include <linux/timer.h>
	#include <linux/dma-mapping.h>
	#include <linux/pci.h>
	#include <linux/slab.h>
	#include <linux/spinlock.h>
	#include <linux/debugfs.h>
	#include <linux/dmaengine.h>
	#include <linux/delay.h>
	#include <linux/sizes.h>
	#include <linux/ntb.h>
	#include <linux/mutex.h>

	#define DRIVER_NAME "ntb_perf"
	#define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool"

	#define DRIVER_LICENSE "Dual BSD/GPL"
	#define DRIVER_VERSION "1.0"
	#define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>"

	#define PERF_LINK_DOWN_TIMEOUT 10
	#define PERF_VERSION 0xffff0001
	#define MAX_THREADS 32
	#define MAX_TEST_SIZE SZ_1M
	#define MAX_SRCS 32
	#define DMA_OUT_RESOURCE_TO msecs_to_jiffies(50)
	#define DMA_RETRIES 20
	#define SZ_4G (1ULL << 32)
	#define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */

	MODULE_LICENSE(DRIVER_LICENSE);
	MODULE_VERSION(DRIVER_VERSION);
	MODULE_AUTHOR(DRIVER_AUTHOR);
	MODULE_DESCRIPTION(DRIVER_DESCRIPTION);

	static struct dentry *perf_debugfs_dir;

	static unsigned long max_mw_size;
	module_param(max_mw_size, ulong, 0644);
	MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");

	static unsigned int seg_order = 19; /* 512K */
	module_param(seg_order, uint, 0644);
	MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing");

	static unsigned int run_order = 32; /* 4G */
	module_param(run_order, uint, 0644);
	MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer");

	static bool use_dma; /* default to 0 */
	module_param(use_dma, bool, 0644);
	MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");

	struct perf_mw {
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buf_size;
	void *virt_addr;
	dma_addr_t dma_addr;
	};

	struct perf_ctx;

	struct pthr_ctx {
	struct task_struct *thread;
	struct perf_ctx *perf;
	atomic_t dma_sync;
	struct dma_chan *dma_chan;
	int dma_prep_err;
	int src_idx;
	void *srcs[MAX_SRCS];
	wait_queue_head_t *wq;
	int status;
	u64 copied;
	u64 diff_us;
	};

	struct perf_ctx {
	struct ntb_dev *ntb;
	spinlock_t db_lock;
	struct perf_mw mw;
	bool link_is_up;
	struct delayed_work link_work;
	wait_queue_head_t link_wq;
	struct dentry *debugfs_node_dir;
	struct dentry *debugfs_run;
	struct dentry *debugfs_threads;
	u8 perf_threads;
	/* mutex ensures only one set of threads run at once */
	struct mutex run_mutex;
	struct pthr_ctx pthr_ctx[MAX_THREADS];
	atomic_t tsync;
	atomic_t tdone;
	};

	enum {
	VERSION = 0,
	MW_SZ_HIGH,
	MW_SZ_LOW,
	MAX_SPAD
	};

	static void perf_link_event(void *ctx)
	{
	struct perf_ctx *perf = ctx;

	if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) {
	schedule_delayed_work(&perf->link_work, 2*HZ);
	} else {
	dev_dbg(&perf->ntb->pdev->dev, "link down\n");

	if (!perf->link_is_up)
	cancel_delayed_work_sync(&perf->link_work);

	perf->link_is_up = false;
	}
	}

	static void perf_db_event(void *ctx, int vec)
	{
	struct perf_ctx *perf = ctx;
	u64 db_bits, db_mask;

	db_mask = ntb_db_vector_mask(perf->ntb, vec);
	db_bits = ntb_db_read(perf->ntb);

	dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
	vec, db_mask, db_bits);
	}

	static const struct ntb_ctx_ops perf_ops = {
	.link_event = perf_link_event,
	.db_event = perf_db_event,
	};

	static void perf_copy_callback(void *data)
	{
	struct pthr_ctx *pctx = data;

	atomic_dec(&pctx->dma_sync);
	}

	static ssize_t perf_copy(struct pthr_ctx pctx, char __iomem dst,
	char *src, size_t size)
	{
	struct perf_ctx *perf = pctx->perf;
	struct dma_async_tx_descriptor *txd;
	struct dma_chan *chan = pctx->dma_chan;
	struct dma_device *device;
	struct dmaengine_unmap_data *unmap;
	dma_cookie_t cookie;
	size_t src_off, dst_off;
	struct perf_mw *mw = &perf->mw;
	void __iomem *vbase;
	void __iomem *dst_vaddr;
	dma_addr_t dst_phys;
	int retries = 0;

	if (!use_dma) {
	memcpy_toio(dst, src, size);
	return size;
	}

	if (!chan) {
	dev_err(&perf->ntb->dev, "DMA engine does not exist\n");
	return -EINVAL;
	}

	device = chan->device;
	src_off = (uintptr_t)src & ~PAGE_MASK;
	dst_off = (uintptr_t __force)dst & ~PAGE_MASK;

	if (!is_dma_copy_aligned(device, src_off, dst_off, size))
	return -ENODEV;

	vbase = mw->vbase;
	dst_vaddr = dst;
	dst_phys = mw->phys_addr + (dst_vaddr - vbase);

	unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
	if (!unmap)
	return -ENOMEM;

	unmap->len = size;
	unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src),
	src_off, size, DMA_TO_DEVICE);
	if (dma_mapping_error(device->dev, unmap->addr[0]))
	goto err_get_unmap;

	unmap->to_cnt = 1;

	do {
	txd = device->device_prep_dma_memcpy(chan, dst_phys,
	unmap->addr[0],
	size, DMA_PREP_INTERRUPT);
	if (!txd) {
	set_current_state(TASK_INTERRUPTIBLE);
	schedule_timeout(DMA_OUT_RESOURCE_TO);
	}
	} while (!txd && (++retries < DMA_RETRIES));

	if (!txd) {
	pctx->dma_prep_err++;
	goto err_get_unmap;
	}

	txd->callback = perf_copy_callback;
	txd->callback_param = pctx;
	dma_set_unmap(txd, unmap);

	cookie = dmaengine_submit(txd);
	if (dma_submit_error(cookie))
	goto err_set_unmap;

	atomic_inc(&pctx->dma_sync);
	dma_async_issue_pending(chan);

	return size;

	err_set_unmap:
	dmaengine_unmap_put(unmap);
	err_get_unmap:
	dmaengine_unmap_put(unmap);
	return 0;
	}

	static int perf_move_data(struct pthr_ctx pctx, char __iomem dst, char *src,
	u64 buf_size, u64 win_size, u64 total)
	{
	int chunks, total_chunks, i;
	int copied_chunks = 0;
	u64 copied = 0, result;
	char __iomem *tmp = dst;
	u64 perf, diff_us;
	ktime_t kstart, kstop, kdiff;
	unsigned long last_sleep = jiffies;

	chunks = div64_u64(win_size, buf_size);
	total_chunks = div64_u64(total, buf_size);
	kstart = ktime_get();

	for (i = 0; i < total_chunks; i++) {
	result = perf_copy(pctx, tmp, src, buf_size);
	copied += result;
	copied_chunks++;
	if (copied_chunks == chunks) {
	tmp = dst;
	copied_chunks = 0;
	} else
	tmp += buf_size;

	/* Probably should schedule every 5s to prevent soft hang. */
	if (unlikely((jiffies - last_sleep) > 5 * HZ)) {
	last_sleep = jiffies;
	set_current_state(TASK_INTERRUPTIBLE);
	schedule_timeout(1);
	}

	if (unlikely(kthread_should_stop()))
	break;
	}

	if (use_dma) {
	pr_debug("%s: All DMA descriptors submitted\n", current->comm);
	while (atomic_read(&pctx->dma_sync) != 0) {
	if (kthread_should_stop())
	break;
	msleep(20);
	}
	}

	kstop = ktime_get();
	kdiff = ktime_sub(kstop, kstart);
	diff_us = ktime_to_us(kdiff);

	pr_debug("%s: copied %llu bytes\n", current->comm, copied);

	pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us);

	perf = div64_u64(copied, diff_us);

	pr_debug("%s: MBytes/s: %llu\n", current->comm, perf);

	pctx->copied = copied;
	pctx->diff_us = diff_us;

	return 0;
	}

	static bool perf_dma_filter_fn(struct dma_chan chan, void node)
	{
	return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
	}

	static int ntb_perf_thread(void *data)
	{
	struct pthr_ctx *pctx = data;
	struct perf_ctx *perf = pctx->perf;
	struct pci_dev *pdev = perf->ntb->pdev;
	struct perf_mw *mw = &perf->mw;
	char __iomem *dst;
	u64 win_size, buf_size, total;
	void *src;
	int rc, node, i;
	struct dma_chan *dma_chan = NULL;

	pr_debug("kthread %s starting...\n", current->comm);

	node = dev_to_node(&pdev->dev);

	if (use_dma && !pctx->dma_chan) {
	dma_cap_mask_t dma_mask;

	dma_cap_zero(dma_mask);
	dma_cap_set(DMA_MEMCPY, dma_mask);
	dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn,
	(void *)(unsigned long)node);
	if (!dma_chan) {
	pr_warn("%s: cannot acquire DMA channel, quitting\n",
	current->comm);
	return -ENODEV;
	}
	pctx->dma_chan = dma_chan;
	}

	for (i = 0; i < MAX_SRCS; i++) {
	pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node);
	if (!pctx->srcs[i]) {
	rc = -ENOMEM;
	goto err;
	}
	}

	win_size = mw->phys_size;
	buf_size = 1ULL << seg_order;
	total = 1ULL << run_order;

	if (buf_size > MAX_TEST_SIZE)
	buf_size = MAX_TEST_SIZE;

	dst = (char __iomem *)mw->vbase;

	atomic_inc(&perf->tsync);
	while (atomic_read(&perf->tsync) != perf->perf_threads)
	schedule();

	src = pctx->srcs[pctx->src_idx];
	pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1);

	rc = perf_move_data(pctx, dst, src, buf_size, win_size, total);

	atomic_dec(&perf->tsync);

	if (rc < 0) {
	pr_err("%s: failed\n", current->comm);
	rc = -ENXIO;
	goto err;
	}

	for (i = 0; i < MAX_SRCS; i++) {
	kfree(pctx->srcs[i]);
	pctx->srcs[i] = NULL;
	}

	atomic_inc(&perf->tdone);
	wake_up(pctx->wq);
	rc = 0;
	goto done;

	err:
	for (i = 0; i < MAX_SRCS; i++) {
	kfree(pctx->srcs[i]);
	pctx->srcs[i] = NULL;
	}

	if (dma_chan) {
	dma_release_channel(dma_chan);
	pctx->dma_chan = NULL;
	}

	done:
	/* Wait until we are told to stop */
	for (;;) {
	set_current_state(TASK_INTERRUPTIBLE);
	if (kthread_should_stop())
	break;
	schedule();
	}
	__set_current_state(TASK_RUNNING);

	return rc;
	}

	static void perf_free_mw(struct perf_ctx *perf)
	{
	struct perf_mw *mw = &perf->mw;
	struct pci_dev *pdev = perf->ntb->pdev;

	if (!mw->virt_addr)
	return;

	ntb_mw_clear_trans(perf->ntb, 0);
	dma_free_coherent(&pdev->dev, mw->buf_size,
	mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buf_size = 0;
	mw->virt_addr = NULL;
	}

	static int perf_set_mw(struct perf_ctx *perf, resource_size_t size)
	{
	struct perf_mw *mw = &perf->mw;
	size_t xlat_size, buf_size;
	int rc;

	if (!size)
	return -EINVAL;

	xlat_size = round_up(size, mw->xlat_align_size);
	buf_size = round_up(size, mw->xlat_align);

	if (mw->xlat_size == xlat_size)
	return 0;

	if (mw->buf_size)
	perf_free_mw(perf);

	mw->xlat_size = xlat_size;
	mw->buf_size = buf_size;

	mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size,
	&mw->dma_addr, GFP_KERNEL);
	if (!mw->virt_addr) {
	mw->xlat_size = 0;
	mw->buf_size = 0;
	}

	rc = ntb_mw_set_trans(perf->ntb, 0, mw->dma_addr, mw->xlat_size);
	if (rc) {
	dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n");
	perf_free_mw(perf);
	return -EIO;
	}

	return 0;
	}

	static void perf_link_work(struct work_struct *work)
	{
	struct perf_ctx *perf =
	container_of(work, struct perf_ctx, link_work.work);
	struct ntb_dev *ndev = perf->ntb;
	struct pci_dev *pdev = ndev->pdev;
	u32 val;
	u64 size;
	int rc;

	dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);

	size = perf->mw.phys_size;

	if (max_mw_size && size > max_mw_size)
	size = max_mw_size;

	ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size));
	ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size));
	ntb_peer_spad_write(ndev, VERSION, PERF_VERSION);

	/* now read what peer wrote */
	val = ntb_spad_read(ndev, VERSION);
	if (val != PERF_VERSION) {
	dev_dbg(&pdev->dev, "Remote version = %#x\n", val);
	goto out;
	}

	val = ntb_spad_read(ndev, MW_SZ_HIGH);
	size = (u64)val << 32;

	val = ntb_spad_read(ndev, MW_SZ_LOW);
	size \|= val;

	dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size);

	rc = perf_set_mw(perf, size);
	if (rc)
	goto out1;

	perf->link_is_up = true;
	wake_up(&perf->link_wq);

	return;

	out1:
	perf_free_mw(perf);

	out:
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
	schedule_delayed_work(&perf->link_work,
	msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT));
	}

	static int perf_setup_mw(struct ntb_dev ntb, struct perf_ctx perf)
	{
	struct perf_mw *mw;
	int rc;

	mw = &perf->mw;

	rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size,
	&mw->xlat_align, &mw->xlat_align_size);
	if (rc)
	return rc;

	perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
	if (!mw->vbase)
	return -ENOMEM;

	return 0;
	}

	static ssize_t debugfs_run_read(struct file filp, char __user ubuf,
	size_t count, loff_t *offp)
	{
	struct perf_ctx *perf = filp->private_data;
	char *buf;
	ssize_t ret, out_off = 0;
	struct pthr_ctx *pctx;
	int i;
	u64 rate;

	if (!perf)
	return 0;

	buf = kmalloc(1024, GFP_KERNEL);
	if (!buf)
	return -ENOMEM;

	if (mutex_is_locked(&perf->run_mutex)) {
	out_off = scnprintf(buf, 64, "running\n");
	goto read_from_buf;
	}

	for (i = 0; i < MAX_THREADS; i++) {
	pctx = &perf->pthr_ctx[i];

	if (pctx->status == -ENODATA)
	break;

	if (pctx->status) {
	out_off += scnprintf(buf + out_off, 1024 - out_off,
	"%d: error %d\n", i,
	pctx->status);
	continue;
	}

	rate = div64_u64(pctx->copied, pctx->diff_us);
	out_off += scnprintf(buf + out_off, 1024 - out_off,
	"%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
	i, pctx->copied, pctx->diff_us, rate);
	}

	read_from_buf:
	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_off);
	kfree(buf);

	return ret;
	}

	static void threads_cleanup(struct perf_ctx *perf)
	{
	struct pthr_ctx *pctx;
	int i;

	for (i = 0; i < MAX_THREADS; i++) {
	pctx = &perf->pthr_ctx[i];
	if (pctx->thread) {
	pctx->status = kthread_stop(pctx->thread);
	pctx->thread = NULL;
	}
	}
	}

	static void perf_clear_thread_status(struct perf_ctx *perf)
	{
	int i;

	for (i = 0; i < MAX_THREADS; i++)
	perf->pthr_ctx[i].status = -ENODATA;
	}

	static ssize_t debugfs_run_write(struct file filp, const char __user ubuf,
	size_t count, loff_t *offp)
	{
	struct perf_ctx *perf = filp->private_data;
	int node, i;
	DECLARE_WAIT_QUEUE_HEAD(wq);

	if (wait_event_interruptible(perf->link_wq, perf->link_is_up))
	return -ENOLINK;

	if (perf->perf_threads == 0)
	return -EINVAL;

	if (!mutex_trylock(&perf->run_mutex))
	return -EBUSY;

	perf_clear_thread_status(perf);

	if (perf->perf_threads > MAX_THREADS) {
	perf->perf_threads = MAX_THREADS;
	pr_info("Reset total threads to: %u\n", MAX_THREADS);
	}

	/* no greater than 1M */
	if (seg_order > MAX_SEG_ORDER) {
	seg_order = MAX_SEG_ORDER;
	pr_info("Fix seg_order to %u\n", seg_order);
	}

	if (run_order < seg_order) {
	run_order = seg_order;
	pr_info("Fix run_order to %u\n", run_order);
	}

	node = dev_to_node(&perf->ntb->pdev->dev);
	atomic_set(&perf->tdone, 0);

	/* launch kernel thread */
	for (i = 0; i < perf->perf_threads; i++) {
	struct pthr_ctx *pctx;

	pctx = &perf->pthr_ctx[i];
	atomic_set(&pctx->dma_sync, 0);
	pctx->perf = perf;
	pctx->wq = &wq;
	pctx->thread =
	kthread_create_on_node(ntb_perf_thread,
	(void *)pctx,
	node, "ntb_perf %d", i);
	if (IS_ERR(pctx->thread)) {
	pctx->thread = NULL;
	goto err;
	} else {
	wake_up_process(pctx->thread);
	}
	}

	wait_event_interruptible(wq,
	atomic_read(&perf->tdone) == perf->perf_threads);

	threads_cleanup(perf);
	mutex_unlock(&perf->run_mutex);
	return count;

	err:
	threads_cleanup(perf);
	mutex_unlock(&perf->run_mutex);
	return -ENXIO;
	}

	static const struct file_operations ntb_perf_debugfs_run = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.read = debugfs_run_read,
	.write = debugfs_run_write,
	};

	static int perf_debugfs_setup(struct perf_ctx *perf)
	{
	struct pci_dev *pdev = perf->ntb->pdev;

	if (!debugfs_initialized())
	return -ENODEV;

	if (!perf_debugfs_dir) {
	perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
	if (!perf_debugfs_dir)
	return -ENODEV;
	}

	perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev),
	perf_debugfs_dir);
	if (!perf->debugfs_node_dir)
	return -ENODEV;

	perf->debugfs_run = debugfs_create_file("run", S_IRUSR \| S_IWUSR,
	perf->debugfs_node_dir, perf,
	&ntb_perf_debugfs_run);
	if (!perf->debugfs_run)
	return -ENODEV;

	perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR \| S_IWUSR,
	perf->debugfs_node_dir,
	&perf->perf_threads);
	if (!perf->debugfs_threads)
	return -ENODEV;

	return 0;
	}

	static int perf_probe(struct ntb_client client, struct ntb_dev ntb)
	{
	struct pci_dev *pdev = ntb->pdev;
	struct perf_ctx *perf;
	int node;
	int rc = 0;

	if (ntb_spad_count(ntb) < MAX_SPAD) {
	dev_err(&ntb->dev, "Not enough scratch pad registers for %s",
	DRIVER_NAME);
	return -EIO;
	}

	node = dev_to_node(&pdev->dev);

	perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
	if (!perf) {
	rc = -ENOMEM;
	goto err_perf;
	}

	perf->ntb = ntb;
	perf->perf_threads = 1;
	atomic_set(&perf->tsync, 0);
	mutex_init(&perf->run_mutex);
	spin_lock_init(&perf->db_lock);
	perf_setup_mw(ntb, perf);
	init_waitqueue_head(&perf->link_wq);
	INIT_DELAYED_WORK(&perf->link_work, perf_link_work);

	rc = ntb_set_ctx(ntb, perf, &perf_ops);
	if (rc)
	goto err_ctx;

	perf->link_is_up = false;
	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
	ntb_link_event(ntb);

	rc = perf_debugfs_setup(perf);
	if (rc)
	goto err_ctx;

	perf_clear_thread_status(perf);

	return 0;

	err_ctx:
	cancel_delayed_work_sync(&perf->link_work);
	kfree(perf);
	err_perf:
	return rc;
	}

	static void perf_remove(struct ntb_client client, struct ntb_dev ntb)
	{
	struct perf_ctx *perf = ntb->ctx;
	int i;

	dev_dbg(&perf->ntb->dev, "%s called\n", __func__);

	mutex_lock(&perf->run_mutex);

	cancel_delayed_work_sync(&perf->link_work);

	ntb_clear_ctx(ntb);
	ntb_link_disable(ntb);

	debugfs_remove_recursive(perf_debugfs_dir);
	perf_debugfs_dir = NULL;

	if (use_dma) {
	for (i = 0; i < MAX_THREADS; i++) {
	struct pthr_ctx *pctx = &perf->pthr_ctx[i];

	if (pctx->dma_chan)
	dma_release_channel(pctx->dma_chan);
	}
	}

	kfree(perf);
	}

	static struct ntb_client perf_client = {
	.ops = {
	.probe = perf_probe,
	.remove = perf_remove,
	},
	};
	module_ntb_client(perf_client);