drivers/hwspinlock/msm_remote_spinlock.c - kernel/msm-4.9 - Gitiles

 /* Copyright (c) 2008-2009, 2011-2016 The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
  * only version 2 as published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  */

 #include <linux/err.h>
 #include <linux/kernel.h>
 #include <linux/string.h>
 #include <linux/delay.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/msm_remote_spinlock.h>
 #include <linux/slab.h>

 #include <soc/qcom/smem.h>

 /**
  * The local processor (APPS) is PID 0, but because 0 is reserved for an empty
  * lock, the value PID + 1 is used as the APSS token when writing to the lock.
  */
 #define SPINLOCK_TOKEN_APPS 1

 static int is_hw_lock_type;
 static DEFINE_MUTEX(ops_init_lock);

 struct spinlock_ops {
 	void (*lock)(raw_remote_spinlock_t *lock);
 	void (*unlock)(raw_remote_spinlock_t *lock);
 	int (*trylock)(raw_remote_spinlock_t *lock);
 	int (*release)(raw_remote_spinlock_t *lock, uint32_t pid);
 	int (*owner)(raw_remote_spinlock_t *lock);
 	void (*lock_rlock_id)(raw_remote_spinlock_t *lock, uint32_t tid);
 	void (*unlock_rlock)(raw_remote_spinlock_t *lock);
 	int (*get_hw_spinlocks_element)(raw_remote_spinlock_t *lock);
 };

 static struct spinlock_ops current_ops;

 static int remote_spinlock_init_address(int id, _remote_spinlock_t *lock);

 /* ldrex implementation ----------------------------------------------------- */
 static char *ldrex_compatible_string = "qcom,ipc-spinlock-ldrex";

 #ifdef CONFIG_ARM
 static void __raw_remote_ex_spin_lock(raw_remote_spinlock_t *lock)
 {
 	unsigned long tmp;

 	__asm__ __volatile__(
 "1:     ldrex   %0, [%1]\n"
 "       teq     %0, #0\n"
 "       strexeq %0, %2, [%1]\n"
 "       teqeq   %0, #0\n"
 "       bne     1b"
 	: "=&r" (tmp)
 	: "r" (&lock->lock), "r" (SPINLOCK_TOKEN_APPS)
 	: "cc");

 	/*
 	 * Ensure the ordering of read/write operations to ensure the
 	 * proper ownership of the lock during the lock/unlock operations
 	 */
 	smp_mb();
 }

 static int __raw_remote_ex_spin_trylock(raw_remote_spinlock_t *lock)
 {
 	unsigned long tmp;

 	__asm__ __volatile__(
 "       ldrex   %0, [%1]\n"
 "       teq     %0, #0\n"
 "       strexeq %0, %2, [%1]\n"
 	: "=&r" (tmp)
 	: "r" (&lock->lock), "r" (SPINLOCK_TOKEN_APPS)
 	: "cc");

 	if (tmp == 0) {
 		/*
 		 * Ensure the ordering of read/write operations to ensure the
 		 * proper ownership of the lock during the lock/unlock
 		 * operations
 		 */
 		smp_mb();
 		return 1;
 	}
 	return 0;
 }

 static void __raw_remote_ex_spin_unlock(raw_remote_spinlock_t *lock)
 {
 	int lock_owner;

 	/*
 	 * Ensure the ordering of read/write operations to ensure the
 	 * proper ownership of the lock during the lock/unlock operations
 	 */
 	smp_mb();
 	lock_owner = readl_relaxed(&lock->lock);
 	if (lock_owner != SPINLOCK_TOKEN_APPS) {
 		pr_err("%s: spinlock not owned by Apps (actual owner is %d)\n",
 				__func__, lock_owner);
 	}

 	__asm__ __volatile__(
 "       str     %1, [%0]\n"
 	:
 	: "r" (&lock->lock), "r" (0)
 	: "cc");
 }
 #else
 static void __raw_remote_ex_spin_lock(raw_remote_spinlock_t *lock)
 {
 }

 static int __raw_remote_ex_spin_trylock(raw_remote_spinlock_t *lock)
 {
 	return 0;
 }

 static void __raw_remote_ex_spin_unlock(raw_remote_spinlock_t *lock)
 {
 }
 #endif /* CONFIG_ARM */
 /* end ldrex implementation ------------------------------------------------- */

 /* sfpb implementation ------------------------------------------------------ */
 static uint32_t lock_count;
 static phys_addr_t reg_base;
 static uint32_t reg_size;
 static uint32_t lock_offset; /* offset into the hardware block before lock 0 */
 static uint32_t lock_size;

 static void *hw_mutex_reg_base;
 static DEFINE_MUTEX(hw_map_init_lock);
 static int *hw_spinlocks;

 static char *sfpb_compatible_string = "qcom,ipc-spinlock-sfpb";

 static int init_hw_mutex(struct device_node *node)
 {
 	struct resource r;
 	int rc;

 	rc = of_address_to_resource(node, 0, &r);
 	if (rc)
 		BUG();

 	rc = of_property_read_u32(node, "qcom,num-locks", &lock_count);
 	if (rc)
 		BUG();

 	reg_base = r.start;
 	reg_size = (uint32_t)(resource_size(&r));
 	lock_offset = 0;
 	lock_size = reg_size / lock_count;

 	return 0;
 }

 static void find_and_init_hw_mutex(void)
 {
 	struct device_node *node;

 	node = of_find_compatible_node(NULL, NULL, sfpb_compatible_string);
 	BUG_ON(node == NULL);
 	init_hw_mutex(node);
 	hw_mutex_reg_base = ioremap(reg_base, reg_size);
 	BUG_ON(hw_mutex_reg_base == NULL);
 	hw_spinlocks = kcalloc(lock_count, sizeof(int), GFP_KERNEL);
 	BUG_ON(hw_spinlocks == NULL);
 }

 static int remote_spinlock_init_address_hw(int id, _remote_spinlock_t *lock)
 {
 	/*
 	 * Optimistic locking.  Init only needs to be done once by the first
 	 * caller.  After that, serializing inits between different callers
 	 * is unnecessary.  The second check after the lock ensures init
 	 * wasn't previously completed by someone else before the lock could
 	 * be grabbed.
 	 */
 	if (!hw_mutex_reg_base) {
 		mutex_lock(&hw_map_init_lock);
 		if (!hw_mutex_reg_base)
 			find_and_init_hw_mutex();
 		mutex_unlock(&hw_map_init_lock);
 	}

 	if (id >= lock_count)
 		return -EINVAL;

 	*lock = hw_mutex_reg_base + lock_offset + id * lock_size;
 	return 0;
 }

 static unsigned int remote_spinlock_get_lock_id(raw_remote_spinlock_t *lock)
 {
 	unsigned int id;

 	BUG_ON((uintptr_t)lock < (uintptr_t)hw_mutex_reg_base);
 	BUG_ON(((uintptr_t)lock - (uintptr_t)hw_mutex_reg_base) < lock_offset);

 	id = (unsigned int)((uintptr_t)lock - (uintptr_t)hw_mutex_reg_base -
 			lock_offset) / lock_size;
 	BUG_ON(id >= lock_count);
 	return id;
 }

 static void __raw_remote_sfpb_spin_lock(raw_remote_spinlock_t *lock)
 {
 	int owner;
 	unsigned int id = remote_spinlock_get_lock_id(lock);

 	/*
 	 * Wait for other local processor task to release spinlock if it
 	 * already has the remote spinlock locked.  This can only happen in
 	 * test cases since the local spinlock will prevent this when using the
 	 * public APIs.
 	 */
 	while (readl_relaxed(lock) == SPINLOCK_TOKEN_APPS)
 		;

 	/* acquire remote spinlock */
 	do {
 		writel_relaxed(SPINLOCK_TOKEN_APPS, lock);
 		/*
 		 * Ensure the ordering of read/write operations to ensure the
 		 * proper ownership of the lock during the lock/unlock
 		 * operations
 		 */
 		smp_mb();
 		owner = readl_relaxed(lock);
 		hw_spinlocks[id] = owner;
 	} while (owner != SPINLOCK_TOKEN_APPS);
 }

 static int __raw_remote_sfpb_spin_trylock(raw_remote_spinlock_t *lock)
 {
 	int owner;
 	unsigned int id = remote_spinlock_get_lock_id(lock);
 	/*
 	 * If the local processor owns the spinlock, return failure. This can
 	 * only happen in test cases since the local spinlock will prevent this
 	 * when using the public APIs.
 	 */
 	if (readl_relaxed(lock) == SPINLOCK_TOKEN_APPS)
 		return 0;

 	writel_relaxed(SPINLOCK_TOKEN_APPS, lock);
 	/*
 	 * Ensure the ordering of read/write operations to ensure the
 	 * proper ownership of the lock during the lock/unlock operations
 	 */
 	smp_mb();
 	owner = readl_relaxed(lock);
 	hw_spinlocks[id] = owner;
 	return owner == SPINLOCK_TOKEN_APPS;
 }

 static void __raw_remote_sfpb_spin_unlock(raw_remote_spinlock_t *lock)
 {
 	int lock_owner;

 	lock_owner = readl_relaxed(lock);
 	if (lock_owner != SPINLOCK_TOKEN_APPS) {
 		pr_err("%s: spinlock not owned by Apps (actual owner is %d)\n",
 				__func__, lock_owner);
 	}

 	writel_relaxed(0, lock);
 	/*
 	 * Ensure the ordering of read/write operations to ensure the
 	 * proper ownership of the lock during the lock/unlock operations
 	 */
 	smp_mb();
 }

 static void __raw_remote_sfpb_spin_lock_rlock_id(raw_remote_spinlock_t *lock,
 						 uint32_t tid)
 {
 	if (unlikely(!tid)) {
 		pr_err("%s: unsupported rlock tid=0\n", __func__);
 		BUG();
 	}

 	do {
 		writel_relaxed(tid, lock);
 		/*
 		 * Ensure the ordering of read/write operations to ensure the
 		 * proper ownership of the lock during the lock/unlock
 		 * operations
 		 */
 		smp_mb();
 	} while (readl_relaxed(lock) != tid);
 }

 static void __raw_remote_sfpb_spin_unlock_rlock(raw_remote_spinlock_t *lock)
 {
 	writel_relaxed(0, lock);
 	/*
 	 * Ensure the ordering of read/write operations to ensure the
 	 * proper ownership of the lock during the lock/unlock operations
 	 */
 	smp_mb();
 }

 static int __raw_remote_sfpb_get_hw_spinlocks_element(
 		raw_remote_spinlock_t *lock)
 {
 	return hw_spinlocks[remote_spinlock_get_lock_id(lock)];
 }

 /* end sfpb implementation -------------------------------------------------- */

 /* common spinlock API ------------------------------------------------------ */
 /**
  * Release spinlock if it is owned by @pid.
  *
  * This is only to be used for situations where the processor owning
  * the spinlock has crashed and the spinlock must be released.
  *
  * @lock: lock structure
  * @pid: processor ID of processor to release
  */
 static int __raw_remote_gen_spin_release(raw_remote_spinlock_t *lock,
 		uint32_t pid)
 {
 	int ret = 1;

 	/*
 	 * Since 0 is reserved for an empty lock and the PIDs start at 0, the
 	 * value PID + 1 is written to the lock.
 	 */
 	if (readl_relaxed(&lock->lock) == (pid + 1)) {
 		writel_relaxed(0, &lock->lock);
 		/*
 		 * Ensure the ordering of read/write operations to ensure the
 		 * proper ownership of the lock during the lock/unlock
 		 * operations
 		 */
 		wmb();
 		ret = 0;
 	}
 	return ret;
 }

 /**
  * Return owner of the spinlock.
  *
  * @lock: pointer to lock structure
  * @returns: >= 0 owned PID; < 0 for error case
  *
  * Used for testing.  PID's are assumed to be 31 bits or less.
  */
 static int __raw_remote_gen_spin_owner(raw_remote_spinlock_t *lock)
 {
 	int owner;

 	/*
 	 * Ensure the ordering of read/write operations to ensure the
 	 * proper ownership of the lock during the lock/unlock operations
 	 */
 	rmb();

 	owner = readl_relaxed(&lock->lock);
 	if (owner)
 		return owner - 1;
 	else
 		return -ENODEV;
 }


 static int dt_node_is_valid(const struct device_node *node)
 {
 	const char *status;
 	int statlen;

 	status = of_get_property(node, "status", &statlen);
 	if (status == NULL)
 		return 1;

 	if (statlen > 0) {
 		if (!strcmp(status, "okay") || !strcmp(status, "ok"))
 			return 1;
 	}

 	return 0;
 }

 static void initialize_ops(void)
 {
 	struct device_node *node;

 	/*
 	 * of_find_compatible_node() returns a valid pointer even if
 	 * the status property is "disabled", so the validity needs
 	 * to be checked
 	 */
 	node = of_find_compatible_node(NULL, NULL, sfpb_compatible_string);
 	if (node && dt_node_is_valid(node)) {
 		current_ops.lock = __raw_remote_sfpb_spin_lock;
 		current_ops.unlock = __raw_remote_sfpb_spin_unlock;
 		current_ops.trylock = __raw_remote_sfpb_spin_trylock;
 		current_ops.release = __raw_remote_gen_spin_release;
 		current_ops.owner = __raw_remote_gen_spin_owner;
 		current_ops.lock_rlock_id =
 				__raw_remote_sfpb_spin_lock_rlock_id;
 		current_ops.unlock_rlock = __raw_remote_sfpb_spin_unlock_rlock;
 		current_ops.get_hw_spinlocks_element =
 			__raw_remote_sfpb_get_hw_spinlocks_element;
 		is_hw_lock_type = 1;
 		return;
 	}

 	node = of_find_compatible_node(NULL, NULL, ldrex_compatible_string);
 	if (node && dt_node_is_valid(node)) {
 		current_ops.lock = __raw_remote_ex_spin_lock;
 		current_ops.unlock = __raw_remote_ex_spin_unlock;
 		current_ops.trylock = __raw_remote_ex_spin_trylock;
 		current_ops.release = __raw_remote_gen_spin_release;
 		current_ops.owner = __raw_remote_gen_spin_owner;
 		is_hw_lock_type = 0;
 		return;
 	}

 	current_ops.lock = __raw_remote_ex_spin_lock;
 	current_ops.unlock = __raw_remote_ex_spin_unlock;
 	current_ops.trylock = __raw_remote_ex_spin_trylock;
 	current_ops.release = __raw_remote_gen_spin_release;
 	current_ops.owner = __raw_remote_gen_spin_owner;
 	is_hw_lock_type = 0;
 	pr_warn("Falling back to LDREX remote spinlock implementation");
 }

 /**
  * Release all spinlocks owned by @pid.
  *
  * This is only to be used for situations where the processor owning
  * spinlocks has crashed and the spinlocks must be released.
  *
  * @pid - processor ID of processor to release
  */
 static void remote_spin_release_all_locks(uint32_t pid, int count)
 {
 	int n;
 	 _remote_spinlock_t lock;

 	if (pid >= REMOTE_SPINLOCK_NUM_PID) {
 		pr_err("%s: Unsupported PID %d\n", __func__, pid);
 		return;
 	}

 	for (n = 0; n < count; ++n) {
 		if (remote_spinlock_init_address(n, &lock) == 0)
 			_remote_spin_release(&lock, pid);
 	}
 }

 void _remote_spin_release_all(uint32_t pid)
 {
 	remote_spin_release_all_locks(pid, lock_count);
 }

 #define SMEM_SPINLOCK_COUNT 8
 #define SMEM_SPINLOCK_ARRAY_SIZE (SMEM_SPINLOCK_COUNT * sizeof(uint32_t))

 static int remote_spinlock_init_address_smem(int id, _remote_spinlock_t *lock)
 {
 	_remote_spinlock_t spinlock_start;

 	if (id >= SMEM_SPINLOCK_COUNT)
 		return -EINVAL;

 	spinlock_start = smem_find(SMEM_SPINLOCK_ARRAY,
 				    SMEM_SPINLOCK_ARRAY_SIZE,
 				    0,
 				    SMEM_ANY_HOST_FLAG);
 	if (spinlock_start == NULL)
 		return -ENXIO;

 	*lock = spinlock_start + id;

 	lock_count = SMEM_SPINLOCK_COUNT;

 	return 0;
 }

 static int remote_spinlock_init_address(int id, _remote_spinlock_t *lock)
 {
 	if (is_hw_lock_type)
 		return remote_spinlock_init_address_hw(id, lock);
 	else
 		return remote_spinlock_init_address_smem(id, lock);
 }

 int _remote_spin_lock_init(remote_spinlock_id_t id, _remote_spinlock_t *lock)
 {
 	BUG_ON(id == NULL);

 	/*
 	 * Optimistic locking.  Init only needs to be done once by the first
 	 * caller.  After that, serializing inits between different callers
 	 * is unnecessary.  The second check after the lock ensures init
 	 * wasn't previously completed by someone else before the lock could
 	 * be grabbed.
 	 */
 	if (!current_ops.lock) {
 		mutex_lock(&ops_init_lock);
 		if (!current_ops.lock)
 			initialize_ops();
 		mutex_unlock(&ops_init_lock);
 	}

 	if (id[0] == 'S' && id[1] == ':') {
 		/* Single-digit lock ID follows "S:" */
 		BUG_ON(id[3] != '\0');

 		return remote_spinlock_init_address((((uint8_t)id[2])-'0'),
 			lock);
 	} else {
 		return -EINVAL;
 	}
 }

 /*
  * lock comes in as a pointer to a pointer to the lock location, so it must
  * be dereferenced and casted to the right type for the actual lock
  * implementation functions
  */
 void _remote_spin_lock(_remote_spinlock_t *lock)
 {
 	if (unlikely(!current_ops.lock))
 		BUG();
 	current_ops.lock((raw_remote_spinlock_t *)(*lock));
 }
 EXPORT_SYMBOL(_remote_spin_lock);

 void _remote_spin_unlock(_remote_spinlock_t *lock)
 {
 	if (unlikely(!current_ops.unlock))
 		BUG();
 	current_ops.unlock((raw_remote_spinlock_t *)(*lock));
 }
 EXPORT_SYMBOL(_remote_spin_unlock);

 int _remote_spin_trylock(_remote_spinlock_t *lock)
 {
 	if (unlikely(!current_ops.trylock))
 		BUG();
 	return current_ops.trylock((raw_remote_spinlock_t *)(*lock));
 }
 EXPORT_SYMBOL(_remote_spin_trylock);

 int _remote_spin_release(_remote_spinlock_t *lock, uint32_t pid)
 {
 	if (unlikely(!current_ops.release))
 		BUG();
 	return current_ops.release((raw_remote_spinlock_t *)(*lock), pid);
 }
 EXPORT_SYMBOL(_remote_spin_release);

 int _remote_spin_owner(_remote_spinlock_t *lock)
 {
 	if (unlikely(!current_ops.owner))
 		BUG();
 	return current_ops.owner((raw_remote_spinlock_t *)(*lock));
 }
 EXPORT_SYMBOL(_remote_spin_owner);

 void _remote_spin_lock_rlock_id(_remote_spinlock_t *lock, uint32_t tid)
 {
 	if (unlikely(!current_ops.lock_rlock_id))
 		BUG();
 	current_ops.lock_rlock_id((raw_remote_spinlock_t *)(*lock), tid);
 }
 EXPORT_SYMBOL(_remote_spin_lock_rlock_id);

 void _remote_spin_unlock_rlock(_remote_spinlock_t *lock)
 {
 	if (unlikely(!current_ops.unlock_rlock))
 		BUG();
 	current_ops.unlock_rlock((raw_remote_spinlock_t *)(*lock));
 }
 EXPORT_SYMBOL(_remote_spin_unlock_rlock);

 int _remote_spin_get_hw_spinlocks_element(_remote_spinlock_t *lock)
 {
 	return current_ops.get_hw_spinlocks_element(
 			(raw_remote_spinlock_t *)(*lock));
 }
 EXPORT_SYMBOL(_remote_spin_get_hw_spinlocks_element);

 /* end common spinlock API -------------------------------------------------- */
	/* Copyright (c) 2008-2009, 2011-2016 The Linux Foundation. All rights reserved.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License version 2 and
	* only version 2 as published by the Free Software Foundation.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	*/

	#include <linux/err.h>
	#include <linux/kernel.h>
	#include <linux/string.h>
	#include <linux/delay.h>
	#include <linux/of.h>
	#include <linux/of_address.h>
	#include <linux/msm_remote_spinlock.h>
	#include <linux/slab.h>

	#include <soc/qcom/smem.h>

	/**
	* The local processor (APPS) is PID 0, but because 0 is reserved for an empty
	* lock, the value PID + 1 is used as the APSS token when writing to the lock.
	*/
	#define SPINLOCK_TOKEN_APPS 1

	static int is_hw_lock_type;
	static DEFINE_MUTEX(ops_init_lock);

	struct spinlock_ops {
	void (lock)(raw_remote_spinlock_t lock);
	void (unlock)(raw_remote_spinlock_t lock);
	int (trylock)(raw_remote_spinlock_t lock);
	int (release)(raw_remote_spinlock_t lock, uint32_t pid);
	int (owner)(raw_remote_spinlock_t lock);
	void (lock_rlock_id)(raw_remote_spinlock_t lock, uint32_t tid);
	void (unlock_rlock)(raw_remote_spinlock_t lock);
	int (get_hw_spinlocks_element)(raw_remote_spinlock_t lock);
	};

	static struct spinlock_ops current_ops;

	static int remote_spinlock_init_address(int id, _remote_spinlock_t *lock);

	/* ldrex implementation ----------------------------------------------------- */
	static char *ldrex_compatible_string = "qcom,ipc-spinlock-ldrex";

	#ifdef CONFIG_ARM
	static void __raw_remote_ex_spin_lock(raw_remote_spinlock_t *lock)
	{
	unsigned long tmp;

	__asm__ __volatile__(
	"1: ldrex %0, [%1]\n"
	" teq %0, #0\n"
	" strexeq %0, %2, [%1]\n"
	" teqeq %0, #0\n"
	" bne 1b"
	: "=&r" (tmp)
	: "r" (&lock->lock), "r" (SPINLOCK_TOKEN_APPS)
	: "cc");

	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock operations
	*/
	smp_mb();
	}

	static int __raw_remote_ex_spin_trylock(raw_remote_spinlock_t *lock)
	{
	unsigned long tmp;

	__asm__ __volatile__(
	" ldrex %0, [%1]\n"
	" teq %0, #0\n"
	" strexeq %0, %2, [%1]\n"
	: "=&r" (tmp)
	: "r" (&lock->lock), "r" (SPINLOCK_TOKEN_APPS)
	: "cc");

	if (tmp == 0) {
	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock
	* operations
	*/
	smp_mb();
	return 1;
	}
	return 0;
	}

	static void __raw_remote_ex_spin_unlock(raw_remote_spinlock_t *lock)
	{
	int lock_owner;

	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock operations
	*/
	smp_mb();
	lock_owner = readl_relaxed(&lock->lock);
	if (lock_owner != SPINLOCK_TOKEN_APPS) {
	pr_err("%s: spinlock not owned by Apps (actual owner is %d)\n",
	__func__, lock_owner);
	}

	__asm__ __volatile__(
	" str %1, [%0]\n"
	:
	: "r" (&lock->lock), "r" (0)
	: "cc");
	}
	#else
	static void __raw_remote_ex_spin_lock(raw_remote_spinlock_t *lock)
	{
	}

	static int __raw_remote_ex_spin_trylock(raw_remote_spinlock_t *lock)
	{
	return 0;
	}

	static void __raw_remote_ex_spin_unlock(raw_remote_spinlock_t *lock)
	{
	}
	#endif /* CONFIG_ARM */
	/* end ldrex implementation ------------------------------------------------- */

	/* sfpb implementation ------------------------------------------------------ */
	static uint32_t lock_count;
	static phys_addr_t reg_base;
	static uint32_t reg_size;
	static uint32_t lock_offset; /* offset into the hardware block before lock 0 */
	static uint32_t lock_size;

	static void *hw_mutex_reg_base;
	static DEFINE_MUTEX(hw_map_init_lock);
	static int *hw_spinlocks;

	static char *sfpb_compatible_string = "qcom,ipc-spinlock-sfpb";

	static int init_hw_mutex(struct device_node *node)
	{
	struct resource r;
	int rc;

	rc = of_address_to_resource(node, 0, &r);
	if (rc)
	BUG();

	rc = of_property_read_u32(node, "qcom,num-locks", &lock_count);
	if (rc)
	BUG();

	reg_base = r.start;
	reg_size = (uint32_t)(resource_size(&r));
	lock_offset = 0;
	lock_size = reg_size / lock_count;

	return 0;
	}

	static void find_and_init_hw_mutex(void)
	{
	struct device_node *node;

	node = of_find_compatible_node(NULL, NULL, sfpb_compatible_string);
	BUG_ON(node == NULL);
	init_hw_mutex(node);
	hw_mutex_reg_base = ioremap(reg_base, reg_size);
	BUG_ON(hw_mutex_reg_base == NULL);
	hw_spinlocks = kcalloc(lock_count, sizeof(int), GFP_KERNEL);
	BUG_ON(hw_spinlocks == NULL);
	}

	static int remote_spinlock_init_address_hw(int id, _remote_spinlock_t *lock)
	{
	/*
	* Optimistic locking. Init only needs to be done once by the first
	* caller. After that, serializing inits between different callers
	* is unnecessary. The second check after the lock ensures init
	* wasn't previously completed by someone else before the lock could
	* be grabbed.
	*/
	if (!hw_mutex_reg_base) {
	mutex_lock(&hw_map_init_lock);
	if (!hw_mutex_reg_base)
	find_and_init_hw_mutex();
	mutex_unlock(&hw_map_init_lock);
	}

	if (id >= lock_count)
	return -EINVAL;

	lock = hw_mutex_reg_base + lock_offset + id lock_size;
	return 0;
	}

	static unsigned int remote_spinlock_get_lock_id(raw_remote_spinlock_t *lock)
	{
	unsigned int id;

	BUG_ON((uintptr_t)lock < (uintptr_t)hw_mutex_reg_base);
	BUG_ON(((uintptr_t)lock - (uintptr_t)hw_mutex_reg_base) < lock_offset);

	id = (unsigned int)((uintptr_t)lock - (uintptr_t)hw_mutex_reg_base -
	lock_offset) / lock_size;
	BUG_ON(id >= lock_count);
	return id;
	}

	static void __raw_remote_sfpb_spin_lock(raw_remote_spinlock_t *lock)
	{
	int owner;
	unsigned int id = remote_spinlock_get_lock_id(lock);

	/*
	* Wait for other local processor task to release spinlock if it
	* already has the remote spinlock locked. This can only happen in
	* test cases since the local spinlock will prevent this when using the
	* public APIs.
	*/
	while (readl_relaxed(lock) == SPINLOCK_TOKEN_APPS)
	;

	/* acquire remote spinlock */
	do {
	writel_relaxed(SPINLOCK_TOKEN_APPS, lock);
	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock
	* operations
	*/
	smp_mb();
	owner = readl_relaxed(lock);
	hw_spinlocks[id] = owner;
	} while (owner != SPINLOCK_TOKEN_APPS);
	}

	static int __raw_remote_sfpb_spin_trylock(raw_remote_spinlock_t *lock)
	{
	int owner;
	unsigned int id = remote_spinlock_get_lock_id(lock);
	/*
	* If the local processor owns the spinlock, return failure. This can
	* only happen in test cases since the local spinlock will prevent this
	* when using the public APIs.
	*/
	if (readl_relaxed(lock) == SPINLOCK_TOKEN_APPS)
	return 0;

	writel_relaxed(SPINLOCK_TOKEN_APPS, lock);
	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock operations
	*/
	smp_mb();
	owner = readl_relaxed(lock);
	hw_spinlocks[id] = owner;
	return owner == SPINLOCK_TOKEN_APPS;
	}

	static void __raw_remote_sfpb_spin_unlock(raw_remote_spinlock_t *lock)
	{
	int lock_owner;

	lock_owner = readl_relaxed(lock);
	if (lock_owner != SPINLOCK_TOKEN_APPS) {
	pr_err("%s: spinlock not owned by Apps (actual owner is %d)\n",
	__func__, lock_owner);
	}

	writel_relaxed(0, lock);
	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock operations
	*/
	smp_mb();
	}

	static void __raw_remote_sfpb_spin_lock_rlock_id(raw_remote_spinlock_t *lock,
	uint32_t tid)
	{
	if (unlikely(!tid)) {
	pr_err("%s: unsupported rlock tid=0\n", __func__);
	BUG();
	}

	do {
	writel_relaxed(tid, lock);
	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock
	* operations
	*/
	smp_mb();
	} while (readl_relaxed(lock) != tid);
	}

	static void __raw_remote_sfpb_spin_unlock_rlock(raw_remote_spinlock_t *lock)
	{
	writel_relaxed(0, lock);
	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock operations
	*/
	smp_mb();
	}

	static int __raw_remote_sfpb_get_hw_spinlocks_element(
	raw_remote_spinlock_t *lock)
	{
	return hw_spinlocks[remote_spinlock_get_lock_id(lock)];
	}

	/* end sfpb implementation -------------------------------------------------- */

	/* common spinlock API ------------------------------------------------------ */
	/**
	* Release spinlock if it is owned by @pid.
	*
	* This is only to be used for situations where the processor owning
	* the spinlock has crashed and the spinlock must be released.
	*
	* @lock: lock structure
	* @pid: processor ID of processor to release
	*/
	static int __raw_remote_gen_spin_release(raw_remote_spinlock_t *lock,
	uint32_t pid)
	{
	int ret = 1;

	/*
	* Since 0 is reserved for an empty lock and the PIDs start at 0, the
	* value PID + 1 is written to the lock.
	*/
	if (readl_relaxed(&lock->lock) == (pid + 1)) {
	writel_relaxed(0, &lock->lock);
	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock
	* operations
	*/
	wmb();
	ret = 0;
	}
	return ret;
	}

	/**
	* Return owner of the spinlock.
	*
	* @lock: pointer to lock structure
	* @returns: >= 0 owned PID; < 0 for error case
	*
	* Used for testing. PID's are assumed to be 31 bits or less.
	*/
	static int __raw_remote_gen_spin_owner(raw_remote_spinlock_t *lock)
	{
	int owner;

	/*
	* Ensure the ordering of read/write operations to ensure the
	* proper ownership of the lock during the lock/unlock operations
	*/
	rmb();

	owner = readl_relaxed(&lock->lock);
	if (owner)
	return owner - 1;
	else
	return -ENODEV;
	}


	static int dt_node_is_valid(const struct device_node *node)
	{
	const char *status;
	int statlen;

	status = of_get_property(node, "status", &statlen);
	if (status == NULL)
	return 1;

	if (statlen > 0) {
	if (!strcmp(status, "okay") \|\| !strcmp(status, "ok"))
	return 1;
	}

	return 0;
	}

	static void initialize_ops(void)
	{
	struct device_node *node;

	/*
	* of_find_compatible_node() returns a valid pointer even if
	* the status property is "disabled", so the validity needs
	* to be checked
	*/
	node = of_find_compatible_node(NULL, NULL, sfpb_compatible_string);
	if (node && dt_node_is_valid(node)) {
	current_ops.lock = __raw_remote_sfpb_spin_lock;
	current_ops.unlock = __raw_remote_sfpb_spin_unlock;
	current_ops.trylock = __raw_remote_sfpb_spin_trylock;
	current_ops.release = __raw_remote_gen_spin_release;
	current_ops.owner = __raw_remote_gen_spin_owner;
	current_ops.lock_rlock_id =
	__raw_remote_sfpb_spin_lock_rlock_id;
	current_ops.unlock_rlock = __raw_remote_sfpb_spin_unlock_rlock;
	current_ops.get_hw_spinlocks_element =
	__raw_remote_sfpb_get_hw_spinlocks_element;
	is_hw_lock_type = 1;
	return;
	}

	node = of_find_compatible_node(NULL, NULL, ldrex_compatible_string);
	if (node && dt_node_is_valid(node)) {
	current_ops.lock = __raw_remote_ex_spin_lock;
	current_ops.unlock = __raw_remote_ex_spin_unlock;
	current_ops.trylock = __raw_remote_ex_spin_trylock;
	current_ops.release = __raw_remote_gen_spin_release;
	current_ops.owner = __raw_remote_gen_spin_owner;
	is_hw_lock_type = 0;
	return;
	}

	current_ops.lock = __raw_remote_ex_spin_lock;
	current_ops.unlock = __raw_remote_ex_spin_unlock;
	current_ops.trylock = __raw_remote_ex_spin_trylock;
	current_ops.release = __raw_remote_gen_spin_release;
	current_ops.owner = __raw_remote_gen_spin_owner;
	is_hw_lock_type = 0;
	pr_warn("Falling back to LDREX remote spinlock implementation");
	}

	/**
	* Release all spinlocks owned by @pid.
	*
	* This is only to be used for situations where the processor owning
	* spinlocks has crashed and the spinlocks must be released.
	*
	* @pid - processor ID of processor to release
	*/
	static void remote_spin_release_all_locks(uint32_t pid, int count)
	{
	int n;
	_remote_spinlock_t lock;

	if (pid >= REMOTE_SPINLOCK_NUM_PID) {
	pr_err("%s: Unsupported PID %d\n", __func__, pid);
	return;
	}

	for (n = 0; n < count; ++n) {
	if (remote_spinlock_init_address(n, &lock) == 0)
	_remote_spin_release(&lock, pid);
	}
	}

	void _remote_spin_release_all(uint32_t pid)
	{
	remote_spin_release_all_locks(pid, lock_count);
	}

	#define SMEM_SPINLOCK_COUNT 8
	#define SMEM_SPINLOCK_ARRAY_SIZE (SMEM_SPINLOCK_COUNT * sizeof(uint32_t))

	static int remote_spinlock_init_address_smem(int id, _remote_spinlock_t *lock)
	{
	_remote_spinlock_t spinlock_start;

	if (id >= SMEM_SPINLOCK_COUNT)
	return -EINVAL;

	spinlock_start = smem_find(SMEM_SPINLOCK_ARRAY,
	SMEM_SPINLOCK_ARRAY_SIZE,
	0,
	SMEM_ANY_HOST_FLAG);
	if (spinlock_start == NULL)
	return -ENXIO;

	*lock = spinlock_start + id;

	lock_count = SMEM_SPINLOCK_COUNT;

	return 0;
	}

	static int remote_spinlock_init_address(int id, _remote_spinlock_t *lock)
	{
	if (is_hw_lock_type)
	return remote_spinlock_init_address_hw(id, lock);
	else
	return remote_spinlock_init_address_smem(id, lock);
	}

	int _remote_spin_lock_init(remote_spinlock_id_t id, _remote_spinlock_t *lock)
	{
	BUG_ON(id == NULL);

	/*
	* Optimistic locking. Init only needs to be done once by the first
	* caller. After that, serializing inits between different callers
	* is unnecessary. The second check after the lock ensures init
	* wasn't previously completed by someone else before the lock could
	* be grabbed.
	*/
	if (!current_ops.lock) {
	mutex_lock(&ops_init_lock);
	if (!current_ops.lock)
	initialize_ops();
	mutex_unlock(&ops_init_lock);
	}

	if (id[0] == 'S' && id[1] == ':') {
	/* Single-digit lock ID follows "S:" */
	BUG_ON(id[3] != '\0');

	return remote_spinlock_init_address((((uint8_t)id[2])-'0'),
	lock);
	} else {
	return -EINVAL;
	}
	}

	/*
	* lock comes in as a pointer to a pointer to the lock location, so it must
	* be dereferenced and casted to the right type for the actual lock
	* implementation functions
	*/
	void _remote_spin_lock(_remote_spinlock_t *lock)
	{
	if (unlikely(!current_ops.lock))
	BUG();
	current_ops.lock((raw_remote_spinlock_t )(lock));
	}
	EXPORT_SYMBOL(_remote_spin_lock);

	void _remote_spin_unlock(_remote_spinlock_t *lock)
	{
	if (unlikely(!current_ops.unlock))
	BUG();
	current_ops.unlock((raw_remote_spinlock_t )(lock));
	}
	EXPORT_SYMBOL(_remote_spin_unlock);

	int _remote_spin_trylock(_remote_spinlock_t *lock)
	{
	if (unlikely(!current_ops.trylock))
	BUG();
	return current_ops.trylock((raw_remote_spinlock_t )(lock));
	}
	EXPORT_SYMBOL(_remote_spin_trylock);

	int _remote_spin_release(_remote_spinlock_t *lock, uint32_t pid)
	{
	if (unlikely(!current_ops.release))
	BUG();
	return current_ops.release((raw_remote_spinlock_t )(lock), pid);
	}
	EXPORT_SYMBOL(_remote_spin_release);

	int _remote_spin_owner(_remote_spinlock_t *lock)
	{
	if (unlikely(!current_ops.owner))
	BUG();
	return current_ops.owner((raw_remote_spinlock_t )(lock));
	}
	EXPORT_SYMBOL(_remote_spin_owner);

	void _remote_spin_lock_rlock_id(_remote_spinlock_t *lock, uint32_t tid)
	{
	if (unlikely(!current_ops.lock_rlock_id))
	BUG();
	current_ops.lock_rlock_id((raw_remote_spinlock_t )(lock), tid);
	}
	EXPORT_SYMBOL(_remote_spin_lock_rlock_id);

	void _remote_spin_unlock_rlock(_remote_spinlock_t *lock)
	{
	if (unlikely(!current_ops.unlock_rlock))
	BUG();
	current_ops.unlock_rlock((raw_remote_spinlock_t )(lock));
	}
	EXPORT_SYMBOL(_remote_spin_unlock_rlock);

	int _remote_spin_get_hw_spinlocks_element(_remote_spinlock_t *lock)
	{
	return current_ops.get_hw_spinlocks_element(
	(raw_remote_spinlock_t )(lock));
	}
	EXPORT_SYMBOL(_remote_spin_get_hw_spinlocks_element);

	/* end common spinlock API -------------------------------------------------- */