Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Split spinlock implementation out into its own file, so it can be |
| 3 | * compiled in a FTRACE-compatible way. |
| 4 | */ |
| 5 | #include <linux/kernel_stat.h> |
| 6 | #include <linux/spinlock.h> |
Jeremy Fitzhardinge | 994025c | 2008-08-20 17:02:19 -0700 | [diff] [blame] | 7 | #include <linux/debugfs.h> |
| 8 | #include <linux/log2.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 9 | #include <linux/gfp.h> |
Konrad Rzeszutek Wilk | 354e7b7 | 2013-06-05 10:44:47 -0400 | [diff] [blame] | 10 | #include <linux/slab.h> |
Juergen Gross | 782512e | 2018-11-08 08:35:06 +0100 | [diff] [blame] | 11 | #include <linux/atomic.h> |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 12 | |
| 13 | #include <asm/paravirt.h> |
| 14 | |
| 15 | #include <xen/interface/xen.h> |
| 16 | #include <xen/events.h> |
| 17 | |
| 18 | #include "xen-ops.h" |
Jeremy Fitzhardinge | 994025c | 2008-08-20 17:02:19 -0700 | [diff] [blame] | 19 | #include "debugfs.h" |
| 20 | |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 21 | static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; |
| 22 | static DEFINE_PER_CPU(char *, irq_name); |
Juergen Gross | 782512e | 2018-11-08 08:35:06 +0100 | [diff] [blame] | 23 | static DEFINE_PER_CPU(atomic_t, xen_qlock_wait_nest); |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 24 | static bool xen_pvspin = true; |
| 25 | |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 26 | #include <asm/qspinlock.h> |
| 27 | |
| 28 | static void xen_qlock_kick(int cpu) |
| 29 | { |
Ross Lagerwall | 707e59b | 2016-04-22 13:05:31 +0100 | [diff] [blame] | 30 | int irq = per_cpu(lock_kicker_irq, cpu); |
| 31 | |
| 32 | /* Don't kick if the target's kicker interrupt is not initialized. */ |
| 33 | if (irq == -1) |
| 34 | return; |
| 35 | |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 36 | xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); |
| 37 | } |
| 38 | |
| 39 | /* |
| 40 | * Halt the current CPU & release it back to the host |
| 41 | */ |
| 42 | static void xen_qlock_wait(u8 *byte, u8 val) |
| 43 | { |
| 44 | int irq = __this_cpu_read(lock_kicker_irq); |
Juergen Gross | 782512e | 2018-11-08 08:35:06 +0100 | [diff] [blame] | 45 | atomic_t *nest_cnt = this_cpu_ptr(&xen_qlock_wait_nest); |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 46 | |
| 47 | /* If kicker interrupts not initialized yet, just spin */ |
Juergen Gross | 00893f4 | 2018-10-01 07:57:42 +0200 | [diff] [blame] | 48 | if (irq == -1 || in_nmi()) |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 49 | return; |
| 50 | |
Juergen Gross | 782512e | 2018-11-08 08:35:06 +0100 | [diff] [blame] | 51 | /* Detect reentry. */ |
| 52 | atomic_inc(nest_cnt); |
Juergen Gross | 00893f4 | 2018-10-01 07:57:42 +0200 | [diff] [blame] | 53 | |
Juergen Gross | 782512e | 2018-11-08 08:35:06 +0100 | [diff] [blame] | 54 | /* If irq pending already and no nested call clear it. */ |
| 55 | if (atomic_read(nest_cnt) == 1 && xen_test_irq_pending(irq)) { |
Juergen Gross | 95f5d10 | 2018-10-01 07:57:42 +0200 | [diff] [blame] | 56 | xen_clear_irq_pending(irq); |
Juergen Gross | 00893f4 | 2018-10-01 07:57:42 +0200 | [diff] [blame] | 57 | } else if (READ_ONCE(*byte) == val) { |
| 58 | /* Block until irq becomes pending (or a spurious wakeup) */ |
| 59 | xen_poll_irq(irq); |
Juergen Gross | 95f5d10 | 2018-10-01 07:57:42 +0200 | [diff] [blame] | 60 | } |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 61 | |
Juergen Gross | 782512e | 2018-11-08 08:35:06 +0100 | [diff] [blame] | 62 | atomic_dec(nest_cnt); |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 63 | } |
| 64 | |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 65 | static irqreturn_t dummy_handler(int irq, void *dev_id) |
| 66 | { |
| 67 | BUG(); |
| 68 | return IRQ_HANDLED; |
| 69 | } |
| 70 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 71 | void xen_init_lock_cpu(int cpu) |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 72 | { |
| 73 | int irq; |
Konrad Rzeszutek Wilk | 354e7b7 | 2013-06-05 10:44:47 -0400 | [diff] [blame] | 74 | char *name; |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 75 | |
Konrad Rzeszutek Wilk | 3310bbe | 2013-08-26 14:28:06 -0400 | [diff] [blame] | 76 | if (!xen_pvspin) |
| 77 | return; |
| 78 | |
Konrad Rzeszutek Wilk | cb91f8f | 2013-05-06 08:33:15 -0400 | [diff] [blame] | 79 | WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", |
Konrad Rzeszutek Wilk | cb9c6f1 | 2013-04-16 14:33:20 -0400 | [diff] [blame] | 80 | cpu, per_cpu(lock_kicker_irq, cpu)); |
| 81 | |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 82 | name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); |
| 83 | irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, |
| 84 | cpu, |
| 85 | dummy_handler, |
Michael Opdenacker | 9d71cee | 2013-09-07 08:46:49 +0200 | [diff] [blame] | 86 | IRQF_PERCPU|IRQF_NOBALANCING, |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 87 | name, |
| 88 | NULL); |
| 89 | |
| 90 | if (irq >= 0) { |
| 91 | disable_irq(irq); /* make sure it's never delivered */ |
| 92 | per_cpu(lock_kicker_irq, cpu) = irq; |
Konrad Rzeszutek Wilk | 354e7b7 | 2013-06-05 10:44:47 -0400 | [diff] [blame] | 93 | per_cpu(irq_name, cpu) = name; |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 94 | } |
| 95 | |
| 96 | printk("cpu %d spinlock event irq %d\n", cpu, irq); |
| 97 | } |
| 98 | |
Alex Nixon | d68d82a | 2008-08-22 11:52:15 +0100 | [diff] [blame] | 99 | void xen_uninit_lock_cpu(int cpu) |
| 100 | { |
Konrad Rzeszutek Wilk | 3310bbe | 2013-08-26 14:28:06 -0400 | [diff] [blame] | 101 | if (!xen_pvspin) |
| 102 | return; |
| 103 | |
Alex Nixon | d68d82a | 2008-08-22 11:52:15 +0100 | [diff] [blame] | 104 | unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); |
Konrad Rzeszutek Wilk | cb9c6f1 | 2013-04-16 14:33:20 -0400 | [diff] [blame] | 105 | per_cpu(lock_kicker_irq, cpu) = -1; |
Konrad Rzeszutek Wilk | 354e7b7 | 2013-06-05 10:44:47 -0400 | [diff] [blame] | 106 | kfree(per_cpu(irq_name, cpu)); |
| 107 | per_cpu(irq_name, cpu) = NULL; |
Alex Nixon | d68d82a | 2008-08-22 11:52:15 +0100 | [diff] [blame] | 108 | } |
| 109 | |
Jeremy Fitzhardinge | b8fa70b | 2013-08-09 19:51:54 +0530 | [diff] [blame] | 110 | |
Konrad Rzeszutek Wilk | a945928 | 2013-09-12 22:29:44 -0400 | [diff] [blame] | 111 | /* |
| 112 | * Our init of PV spinlocks is split in two init functions due to us |
| 113 | * using paravirt patching and jump labels patching and having to do |
| 114 | * all of this before SMP code is invoked. |
| 115 | * |
| 116 | * The paravirt patching needs to be done _before_ the alternative asm code |
| 117 | * is started, otherwise we would not patch the core kernel code. |
| 118 | */ |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 119 | void __init xen_init_spinlocks(void) |
| 120 | { |
Konrad Rzeszutek Wilk | 70dd499 | 2013-04-16 14:34:45 -0400 | [diff] [blame] | 121 | |
Jeremy Fitzhardinge | b8fa70b | 2013-08-09 19:51:54 +0530 | [diff] [blame] | 122 | if (!xen_pvspin) { |
| 123 | printk(KERN_DEBUG "xen: PV spinlocks disabled\n"); |
| 124 | return; |
| 125 | } |
Konrad Rzeszutek Wilk | e0fc17a | 2014-04-04 14:48:04 -0400 | [diff] [blame] | 126 | printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); |
Peter Zijlstra | cfd8983 | 2016-05-18 20:43:02 +0200 | [diff] [blame] | 127 | |
David Vrabel | e95e6f1 | 2015-04-24 14:56:40 -0400 | [diff] [blame] | 128 | __pv_init_lock_hash(); |
| 129 | pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; |
| 130 | pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); |
| 131 | pv_lock_ops.wait = xen_qlock_wait; |
| 132 | pv_lock_ops.kick = xen_qlock_kick; |
Jeremy Fitzhardinge | d5de884 | 2008-07-23 13:28:58 -0700 | [diff] [blame] | 133 | } |
Jeremy Fitzhardinge | 994025c | 2008-08-20 17:02:19 -0700 | [diff] [blame] | 134 | |
Konrad Rzeszutek Wilk | a945928 | 2013-09-12 22:29:44 -0400 | [diff] [blame] | 135 | /* |
| 136 | * While the jump_label init code needs to happend _after_ the jump labels are |
| 137 | * enabled and before SMP is started. Hence we use pre-SMP initcall level |
| 138 | * init. We cannot do it in xen_init_spinlocks as that is done before |
| 139 | * jump labels are activated. |
| 140 | */ |
| 141 | static __init int xen_init_spinlocks_jump(void) |
| 142 | { |
| 143 | if (!xen_pvspin) |
| 144 | return 0; |
| 145 | |
Konrad Rzeszutek Wilk | e0fc17a | 2014-04-04 14:48:04 -0400 | [diff] [blame] | 146 | if (!xen_domain()) |
| 147 | return 0; |
| 148 | |
Konrad Rzeszutek Wilk | a945928 | 2013-09-12 22:29:44 -0400 | [diff] [blame] | 149 | static_key_slow_inc(¶virt_ticketlocks_enabled); |
| 150 | return 0; |
| 151 | } |
| 152 | early_initcall(xen_init_spinlocks_jump); |
| 153 | |
Jeremy Fitzhardinge | b8fa70b | 2013-08-09 19:51:54 +0530 | [diff] [blame] | 154 | static __init int xen_parse_nopvspin(char *arg) |
| 155 | { |
| 156 | xen_pvspin = false; |
| 157 | return 0; |
| 158 | } |
| 159 | early_param("xen_nopvspin", xen_parse_nopvspin); |
| 160 | |