blob: 8b70e03f939f9c0a78f3708a0393b46d8f61e8c8 [file] [log] [blame]
Dean Nelson94bd2702008-07-29 22:34:05 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
Robin Holta374c572009-04-13 14:40:18 -07006 * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
Dean Nelson94bd2702008-07-29 22:34:05 -07007 */
8
9/*
10 * Cross Partition Communication (XPC) sn2-based functions.
11 *
12 * Architecture specific implementation of common functions.
13 *
14 */
15
Dean Nelsone17d4162008-07-29 22:34:06 -070016#include <linux/delay.h>
Dean Nelson94bd2702008-07-29 22:34:05 -070017#include <asm/uncached.h>
Dean Nelson261f3b42008-07-29 22:34:16 -070018#include <asm/sn/mspec.h>
Dean Nelson94bd2702008-07-29 22:34:05 -070019#include <asm/sn/sn_sal.h>
20#include "xpc.h"
21
Dean Nelsonee6665e2008-07-29 22:34:13 -070022/*
23 * Define the number of u64s required to represent all the C-brick nasids
24 * as a bitmap. The cross-partition kernel modules deal only with
25 * C-brick nasids, thus the need for bitmaps which don't account for
26 * odd-numbered (non C-brick) nasids.
27 */
28#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
29#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
30#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
31
32/*
33 * Memory for XPC's amo variables is allocated by the MSPEC driver. These
34 * pages are located in the lowest granule. The lowest granule uses 4k pages
35 * for cached references and an alternate TLB handler to never provide a
36 * cacheable mapping for the entire region. This will prevent speculative
37 * reading of cached copies of our lines from being issued which will cause
38 * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
39 * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
40 * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
41 * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
42 * partitions (i.e., XPCs) consider themselves currently engaged with the
43 * local XPC and 1 amo variable to request partition deactivation.
44 */
45#define XPC_NOTIFY_IRQ_AMOS_SN2 0
46#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
47 XP_MAX_NPARTITIONS_SN2)
48#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
49 XP_NASID_MASK_WORDS_SN2)
50#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
51
52/*
53 * Buffer used to store a local copy of portions of a remote partition's
54 * reserved page (either its header and part_nasids mask, or its vars).
55 */
Dean Nelsonee6665e2008-07-29 22:34:13 -070056static void *xpc_remote_copy_buffer_base_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -070057static char *xpc_remote_copy_buffer_sn2;
Dean Nelsonee6665e2008-07-29 22:34:13 -070058
Dean Nelson8e85c232008-07-29 22:34:13 -070059static struct xpc_vars_sn2 *xpc_vars_sn2;
60static struct xpc_vars_part_sn2 *xpc_vars_part_sn2;
Dean Nelson94bd2702008-07-29 22:34:05 -070061
Dean Nelson5b8669d2008-07-29 22:34:18 -070062static int
Robin Holta7665b02009-04-13 14:40:19 -070063xpc_setup_partitions_sn2(void)
Dean Nelson5b8669d2008-07-29 22:34:18 -070064{
65 /* nothing needs to be done */
66 return 0;
67}
68
Jack Steiner6f2584f2009-04-02 16:59:10 -070069static void
Robin Holta7665b02009-04-13 14:40:19 -070070xpc_teardown_partitions_sn2(void)
Jack Steiner6f2584f2009-04-02 16:59:10 -070071{
72 /* nothing needs to be done */
73}
74
Dean Nelson6e410172008-07-29 22:34:09 -070075/* SH_IPI_ACCESS shub register value on startup */
Dean Nelson8e85c232008-07-29 22:34:13 -070076static u64 xpc_sh1_IPI_access_sn2;
77static u64 xpc_sh2_IPI_access0_sn2;
78static u64 xpc_sh2_IPI_access1_sn2;
79static u64 xpc_sh2_IPI_access2_sn2;
80static u64 xpc_sh2_IPI_access3_sn2;
Dean Nelson6e410172008-07-29 22:34:09 -070081
82/*
83 * Change protections to allow IPI operations.
84 */
85static void
86xpc_allow_IPI_ops_sn2(void)
87{
88 int node;
89 int nasid;
90
Dean Nelsonea57f802008-07-29 22:34:14 -070091 /* !!! The following should get moved into SAL. */
Dean Nelson6e410172008-07-29 22:34:09 -070092 if (is_shub2()) {
Dean Nelson8e85c232008-07-29 22:34:13 -070093 xpc_sh2_IPI_access0_sn2 =
Dean Nelson6e410172008-07-29 22:34:09 -070094 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0));
Dean Nelson8e85c232008-07-29 22:34:13 -070095 xpc_sh2_IPI_access1_sn2 =
Dean Nelson6e410172008-07-29 22:34:09 -070096 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1));
Dean Nelson8e85c232008-07-29 22:34:13 -070097 xpc_sh2_IPI_access2_sn2 =
Dean Nelson6e410172008-07-29 22:34:09 -070098 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2));
Dean Nelson8e85c232008-07-29 22:34:13 -070099 xpc_sh2_IPI_access3_sn2 =
Dean Nelson6e410172008-07-29 22:34:09 -0700100 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3));
101
102 for_each_online_node(node) {
103 nasid = cnodeid_to_nasid(node);
104 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
105 -1UL);
106 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
107 -1UL);
108 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
109 -1UL);
110 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
111 -1UL);
112 }
113 } else {
Dean Nelson8e85c232008-07-29 22:34:13 -0700114 xpc_sh1_IPI_access_sn2 =
Dean Nelson6e410172008-07-29 22:34:09 -0700115 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS));
116
117 for_each_online_node(node) {
118 nasid = cnodeid_to_nasid(node);
119 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
120 -1UL);
121 }
122 }
123}
124
125/*
126 * Restrict protections to disallow IPI operations.
127 */
128static void
129xpc_disallow_IPI_ops_sn2(void)
130{
131 int node;
132 int nasid;
133
Dean Nelsonea57f802008-07-29 22:34:14 -0700134 /* !!! The following should get moved into SAL. */
Dean Nelson6e410172008-07-29 22:34:09 -0700135 if (is_shub2()) {
136 for_each_online_node(node) {
137 nasid = cnodeid_to_nasid(node);
138 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
Dean Nelson8e85c232008-07-29 22:34:13 -0700139 xpc_sh2_IPI_access0_sn2);
Dean Nelson6e410172008-07-29 22:34:09 -0700140 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
Dean Nelson8e85c232008-07-29 22:34:13 -0700141 xpc_sh2_IPI_access1_sn2);
Dean Nelson6e410172008-07-29 22:34:09 -0700142 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
Dean Nelson8e85c232008-07-29 22:34:13 -0700143 xpc_sh2_IPI_access2_sn2);
Dean Nelson6e410172008-07-29 22:34:09 -0700144 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
Dean Nelson8e85c232008-07-29 22:34:13 -0700145 xpc_sh2_IPI_access3_sn2);
Dean Nelson6e410172008-07-29 22:34:09 -0700146 }
147 } else {
148 for_each_online_node(node) {
149 nasid = cnodeid_to_nasid(node);
150 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
Dean Nelson8e85c232008-07-29 22:34:13 -0700151 xpc_sh1_IPI_access_sn2);
Dean Nelson6e410172008-07-29 22:34:09 -0700152 }
153 }
154}
155
Dean Nelson33ba3c72008-07-29 22:34:07 -0700156/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700157 * The following set of functions are used for the sending and receiving of
158 * IRQs (also known as IPIs). There are two flavors of IRQs, one that is
159 * associated with partition activity (SGI_XPC_ACTIVATE) and the other that
160 * is associated with channel activity (SGI_XPC_NOTIFY).
Dean Nelson33ba3c72008-07-29 22:34:07 -0700161 */
162
163static u64
Dean Nelsonc39838c2008-07-29 22:34:11 -0700164xpc_receive_IRQ_amo_sn2(struct amo *amo)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700165{
166 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR);
167}
168
169static enum xp_retval
Dean Nelsonc39838c2008-07-29 22:34:11 -0700170xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid,
171 int vector)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700172{
173 int ret = 0;
174 unsigned long irq_flags;
175
176 local_irq_save(irq_flags);
177
178 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag);
179 sn_send_IPI_phys(nasid, phys_cpuid, vector, 0);
180
181 /*
182 * We must always use the nofault function regardless of whether we
183 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
184 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700185 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700186 */
187 ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable),
188 xp_nofault_PIOR_target));
189
190 local_irq_restore(irq_flags);
191
Dean Nelson261f3b42008-07-29 22:34:16 -0700192 return (ret == 0) ? xpSuccess : xpPioReadError;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700193}
194
Dean Nelsonc39838c2008-07-29 22:34:11 -0700195static struct amo *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700196xpc_init_IRQ_amo_sn2(int index)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700197{
Dean Nelson8e85c232008-07-29 22:34:13 -0700198 struct amo *amo = xpc_vars_sn2->amos_page + index;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700199
Dean Nelsonc39838c2008-07-29 22:34:11 -0700200 (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700201 return amo;
202}
203
204/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700205 * Functions associated with SGI_XPC_ACTIVATE IRQ.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700206 */
207
208/*
Dean Nelson6e410172008-07-29 22:34:09 -0700209 * Notify the heartbeat check thread that an activate IRQ has been received.
210 */
211static irqreturn_t
212xpc_handle_activate_IRQ_sn2(int irq, void *dev_id)
213{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700214 unsigned long irq_flags;
215
216 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
217 xpc_activate_IRQ_rcvd++;
218 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
219
Dean Nelson6e410172008-07-29 22:34:09 -0700220 wake_up_interruptible(&xpc_activate_IRQ_wq);
221 return IRQ_HANDLED;
222}
223
224/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700225 * Flag the appropriate amo variable and send an IRQ to the specified node.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700226 */
227static void
Dean Nelsona812dcc2008-07-29 22:34:16 -0700228xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid,
229 int to_nasid, int to_phys_cpuid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700230{
Dean Nelsonc39838c2008-07-29 22:34:11 -0700231 struct amo *amos = (struct amo *)__va(amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700232 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700233 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700234
Dean Nelson04de7412008-07-29 22:34:14 -0700235 (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)],
236 BIT_MASK(from_nasid / 2), to_nasid,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700237 to_phys_cpuid, SGI_XPC_ACTIVATE);
238}
239
240static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700241xpc_send_local_activate_IRQ_sn2(int from_nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700242{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700243 unsigned long irq_flags;
Dean Nelson8e85c232008-07-29 22:34:13 -0700244 struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700245 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700246 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700247
248 /* fake the sending and receipt of an activate IRQ from remote nasid */
Dean Nelson04de7412008-07-29 22:34:14 -0700249 FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable),
250 FETCHOP_OR, BIT_MASK(from_nasid / 2));
251
Dean Nelson5b8669d2008-07-29 22:34:18 -0700252 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
253 xpc_activate_IRQ_rcvd++;
254 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
255
Dean Nelson6e410172008-07-29 22:34:09 -0700256 wake_up_interruptible(&xpc_activate_IRQ_wq);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700257}
258
Dean Nelson33ba3c72008-07-29 22:34:07 -0700259/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700260 * Functions associated with SGI_XPC_NOTIFY IRQ.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700261 */
262
263/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700264 * Check to see if any chctl flags were sent from the specified partition.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700265 */
266static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700267xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700268{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700269 union xpc_channel_ctl_flags chctl;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700270 unsigned long irq_flags;
271
Dean Nelson7fb5e592008-07-29 22:34:10 -0700272 chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2.
273 local_chctl_amo_va);
274 if (chctl.all_flags == 0)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700275 return;
276
Dean Nelson7fb5e592008-07-29 22:34:10 -0700277 spin_lock_irqsave(&part->chctl_lock, irq_flags);
278 part->chctl.all_flags |= chctl.all_flags;
279 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700280
Dean Nelson7fb5e592008-07-29 22:34:10 -0700281 dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags="
Randy Dunlapf509e342009-09-04 17:02:59 -0700282 "0x%llx\n", XPC_PARTID(part), chctl.all_flags);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700283
284 xpc_wakeup_channel_mgr(part);
285}
286
287/*
288 * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
289 * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
Dean Nelsonc39838c2008-07-29 22:34:11 -0700290 * than one partition, we use an amo structure per partition to indicate
Dean Nelson7fb5e592008-07-29 22:34:10 -0700291 * whether a partition has sent an IRQ or not. If it has, then wake up the
Dean Nelsona47d5da2008-07-29 22:34:09 -0700292 * associated kthread to handle it.
293 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700294 * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC
Dean Nelsona47d5da2008-07-29 22:34:09 -0700295 * running on other partitions.
296 *
297 * Noteworthy Arguments:
298 *
299 * irq - Interrupt ReQuest number. NOT USED.
300 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700301 * dev_id - partid of IRQ's potential sender.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700302 */
303static irqreturn_t
304xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
305{
306 short partid = (short)(u64)dev_id;
307 struct xpc_partition *part = &xpc_partitions[partid];
308
Dean Nelson261f3b42008-07-29 22:34:16 -0700309 DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700310
311 if (xpc_part_ref(part)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700312 xpc_check_for_sent_chctl_flags_sn2(part);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700313
314 xpc_part_deref(part);
315 }
316 return IRQ_HANDLED;
317}
318
319/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700320 * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor
321 * because the write to their associated amo variable completed after the IRQ
Dean Nelsona47d5da2008-07-29 22:34:09 -0700322 * was received.
323 */
324static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700325xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700326{
327 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
328
329 if (xpc_part_ref(part)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700330 xpc_check_for_sent_chctl_flags_sn2(part);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700331
332 part_sn2->dropped_notify_IRQ_timer.expires = jiffies +
Dean Nelson7fb5e592008-07-29 22:34:10 -0700333 XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700334 add_timer(&part_sn2->dropped_notify_IRQ_timer);
335 xpc_part_deref(part);
336 }
337}
338
339/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700340 * Send a notify IRQ to the remote partition that is associated with the
Dean Nelson33ba3c72008-07-29 22:34:07 -0700341 * specified channel.
342 */
343static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700344xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
345 char *chctl_flag_string, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700346{
347 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -0700348 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700349 union xpc_channel_ctl_flags chctl = { 0 };
Dean Nelson33ba3c72008-07-29 22:34:07 -0700350 enum xp_retval ret;
351
Dean Nelson83469b52008-07-29 22:34:18 -0700352 if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700353 chctl.flags[ch->number] = chctl_flag;
354 ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va,
355 chctl.all_flags,
356 part_sn2->notify_IRQ_nasid,
357 part_sn2->notify_IRQ_phys_cpuid,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700358 SGI_XPC_NOTIFY);
359 dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
Dean Nelson7fb5e592008-07-29 22:34:10 -0700360 chctl_flag_string, ch->partid, ch->number, ret);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700361 if (unlikely(ret != xpSuccess)) {
362 if (irq_flags != NULL)
363 spin_unlock_irqrestore(&ch->lock, *irq_flags);
364 XPC_DEACTIVATE_PARTITION(part, ret);
365 if (irq_flags != NULL)
366 spin_lock_irqsave(&ch->lock, *irq_flags);
367 }
368 }
369}
370
Dean Nelson7fb5e592008-07-29 22:34:10 -0700371#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
372 xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700373
374/*
375 * Make it look like the remote partition, which is associated with the
Dean Nelson7fb5e592008-07-29 22:34:10 -0700376 * specified channel, sent us a notify IRQ. This faked IRQ will be handled
377 * by xpc_check_for_dropped_notify_IRQ_sn2().
Dean Nelson33ba3c72008-07-29 22:34:07 -0700378 */
379static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700380xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
381 char *chctl_flag_string)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700382{
383 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelson7fb5e592008-07-29 22:34:10 -0700384 union xpc_channel_ctl_flags chctl = { 0 };
Dean Nelson33ba3c72008-07-29 22:34:07 -0700385
Dean Nelson7fb5e592008-07-29 22:34:10 -0700386 chctl.flags[ch->number] = chctl_flag;
387 FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va->
388 variable), FETCHOP_OR, chctl.all_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700389 dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
Dean Nelson7fb5e592008-07-29 22:34:10 -0700390 chctl_flag_string, ch->partid, ch->number);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700391}
392
Dean Nelson7fb5e592008-07-29 22:34:10 -0700393#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
394 xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700395
396static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700397xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch,
398 unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700399{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700400 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700401
402 args->reason = ch->reason;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700403 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700404}
405
406static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700407xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700408{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700409 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700410}
411
412static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700413xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700414{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700415 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700416
Dean Nelsonbd3e64c2008-07-29 22:34:19 -0700417 args->entry_size = ch->entry_size;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700418 args->local_nentries = ch->local_nentries;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700419 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700420}
421
422static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700423xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700424{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700425 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700426
427 args->remote_nentries = ch->remote_nentries;
428 args->local_nentries = ch->local_nentries;
Dean Nelson5b8669d2008-07-29 22:34:18 -0700429 args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue);
Dean Nelson7fb5e592008-07-29 22:34:10 -0700430 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700431}
432
433static void
Robin Holtefdd06e2009-04-13 14:40:19 -0700434xpc_send_chctl_opencomplete_sn2(struct xpc_channel *ch,
435 unsigned long *irq_flags)
436{
437 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENCOMPLETE, irq_flags);
438}
439
440static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700441xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700442{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700443 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700444}
445
446static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700447xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700448{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700449 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700450}
451
Jack Steiner6f2584f2009-04-02 16:59:10 -0700452static enum xp_retval
Dean Nelson5b8669d2008-07-29 22:34:18 -0700453xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
454 unsigned long msgqueue_pa)
455{
456 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
Jack Steiner6f2584f2009-04-02 16:59:10 -0700457 return xpSuccess;
Dean Nelson5b8669d2008-07-29 22:34:18 -0700458}
459
Dean Nelson33ba3c72008-07-29 22:34:07 -0700460/*
461 * This next set of functions are used to keep track of when a partition is
462 * potentially engaged in accessing memory belonging to another partition.
463 */
464
465static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700466xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700467{
468 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700469 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700470 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700471 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700472
473 local_irq_save(irq_flags);
474
Dean Nelsonc39838c2008-07-29 22:34:11 -0700475 /* set bit corresponding to our partid in remote partition's amo */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700476 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
Dean Nelson04de7412008-07-29 22:34:14 -0700477 BIT(sn_partition_id));
478
Dean Nelson33ba3c72008-07-29 22:34:07 -0700479 /*
480 * We must always use the nofault function regardless of whether we
481 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
482 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700483 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700484 */
485 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
486 variable),
487 xp_nofault_PIOR_target));
488
489 local_irq_restore(irq_flags);
490}
491
492static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700493xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700494{
Dean Nelsona47d5da2008-07-29 22:34:09 -0700495 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700496 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700497 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700498 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700499 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700500
501 local_irq_save(irq_flags);
502
Dean Nelsonc39838c2008-07-29 22:34:11 -0700503 /* clear bit corresponding to our partid in remote partition's amo */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700504 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
Dean Nelson04de7412008-07-29 22:34:14 -0700505 ~BIT(sn_partition_id));
506
Dean Nelson33ba3c72008-07-29 22:34:07 -0700507 /*
508 * We must always use the nofault function regardless of whether we
509 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
510 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700511 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700512 */
513 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
514 variable),
515 xp_nofault_PIOR_target));
516
517 local_irq_restore(irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700518
Dean Nelson33ba3c72008-07-29 22:34:07 -0700519 /*
Dean Nelsona47d5da2008-07-29 22:34:09 -0700520 * Send activate IRQ to get other side to see that we've cleared our
Dean Nelsonc39838c2008-07-29 22:34:11 -0700521 * bit in their engaged partitions amo.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700522 */
Dean Nelson7fb5e592008-07-29 22:34:10 -0700523 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -0700524 cnodeid_to_nasid(0),
525 part_sn2->activate_IRQ_nasid,
526 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700527}
528
Dean Nelson5b8669d2008-07-29 22:34:18 -0700529static void
530xpc_assume_partition_disengaged_sn2(short partid)
531{
532 struct amo *amo = xpc_vars_sn2->amos_page +
533 XPC_ENGAGED_PARTITIONS_AMO_SN2;
534
535 /* clear bit(s) based on partid mask in our partition's amo */
536 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
537 ~BIT(partid));
538}
539
Dean Nelsona47d5da2008-07-29 22:34:09 -0700540static int
541xpc_partition_engaged_sn2(short partid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700542{
Dean Nelson8e85c232008-07-29 22:34:13 -0700543 struct amo *amo = xpc_vars_sn2->amos_page +
544 XPC_ENGAGED_PARTITIONS_AMO_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700545
Dean Nelsonc39838c2008-07-29 22:34:11 -0700546 /* our partition's amo variable ANDed with partid mask */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700547 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
Dean Nelson04de7412008-07-29 22:34:14 -0700548 BIT(partid)) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700549}
550
Dean Nelsona47d5da2008-07-29 22:34:09 -0700551static int
552xpc_any_partition_engaged_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700553{
Dean Nelson8e85c232008-07-29 22:34:13 -0700554 struct amo *amo = xpc_vars_sn2->amos_page +
555 XPC_ENGAGED_PARTITIONS_AMO_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700556
Dean Nelsonc39838c2008-07-29 22:34:11 -0700557 /* our partition's amo variable */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700558 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700559}
560
Dean Nelson6e410172008-07-29 22:34:09 -0700561/* original protection values for each node */
562static u64 xpc_prot_vec_sn2[MAX_NUMNODES];
563
564/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700565 * Change protections to allow amo operations on non-Shub 1.1 systems.
Dean Nelson6e410172008-07-29 22:34:09 -0700566 */
567static enum xp_retval
Dean Nelsonc39838c2008-07-29 22:34:11 -0700568xpc_allow_amo_ops_sn2(struct amo *amos_page)
Dean Nelson6e410172008-07-29 22:34:09 -0700569{
Dean Nelson6c1c3252008-11-05 17:27:22 -0600570 enum xp_retval ret = xpSuccess;
Dean Nelson6e410172008-07-29 22:34:09 -0700571
572 /*
573 * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST
574 * collides with memory operations. On those systems we call
Dean Nelsonc39838c2008-07-29 22:34:11 -0700575 * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead.
Dean Nelson6e410172008-07-29 22:34:09 -0700576 */
Dean Nelson6c1c3252008-11-05 17:27:22 -0600577 if (!enable_shub_wars_1_1())
578 ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE);
579
580 return ret;
Dean Nelson6e410172008-07-29 22:34:09 -0700581}
582
583/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700584 * Change protections to allow amo operations on Shub 1.1 systems.
Dean Nelson6e410172008-07-29 22:34:09 -0700585 */
586static void
Dean Nelsonc39838c2008-07-29 22:34:11 -0700587xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
Dean Nelson6e410172008-07-29 22:34:09 -0700588{
589 int node;
590 int nasid;
591
592 if (!enable_shub_wars_1_1())
593 return;
594
595 for_each_online_node(node) {
596 nasid = cnodeid_to_nasid(node);
597 /* save current protection values */
598 xpc_prot_vec_sn2[node] =
599 (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid,
600 SH1_MD_DQLP_MMR_DIR_PRIVEC0));
601 /* open up everything */
602 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
603 SH1_MD_DQLP_MMR_DIR_PRIVEC0),
604 -1UL);
605 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
606 SH1_MD_DQRP_MMR_DIR_PRIVEC0),
607 -1UL);
608 }
609}
610
Dean Nelson94bd2702008-07-29 22:34:05 -0700611static enum xp_retval
Dean Nelsona812dcc2008-07-29 22:34:16 -0700612xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa,
Dean Nelson261f3b42008-07-29 22:34:16 -0700613 size_t *len)
614{
615 s64 status;
616 enum xp_retval ret;
617
Randy Dunlapf509e342009-09-04 17:02:59 -0700618 status = sn_partition_reserved_page_pa((u64)buf, cookie,
619 (u64 *)rp_pa, (u64 *)len);
Dean Nelson261f3b42008-07-29 22:34:16 -0700620 if (status == SALRET_OK)
621 ret = xpSuccess;
622 else if (status == SALRET_MORE_PASSES)
623 ret = xpNeedMoreInfo;
624 else
625 ret = xpSalError;
626
627 return ret;
628}
629
630
Dean Nelson5b8669d2008-07-29 22:34:18 -0700631static int
Robin Holta7665b02009-04-13 14:40:19 -0700632xpc_setup_rsvd_page_sn2(struct xpc_rsvd_page *rp)
Dean Nelson94bd2702008-07-29 22:34:05 -0700633{
Dean Nelsonc39838c2008-07-29 22:34:11 -0700634 struct amo *amos_page;
Dean Nelson94bd2702008-07-29 22:34:05 -0700635 int i;
636 int ret;
637
Dean Nelson8e85c232008-07-29 22:34:13 -0700638 xpc_vars_sn2 = XPC_RP_VARS(rp);
Dean Nelson94bd2702008-07-29 22:34:05 -0700639
Robin Holta374c572009-04-13 14:40:18 -0700640 rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700641
Dean Nelsone17d4162008-07-29 22:34:06 -0700642 /* vars_part array follows immediately after vars */
Dean Nelson8e85c232008-07-29 22:34:13 -0700643 xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
644 XPC_RP_VARS_SIZE);
Dean Nelsone17d4162008-07-29 22:34:06 -0700645
Dean Nelson94bd2702008-07-29 22:34:05 -0700646 /*
Dean Nelson8e85c232008-07-29 22:34:13 -0700647 * Before clearing xpc_vars_sn2, see if a page of amos had been
648 * previously allocated. If not we'll need to allocate one and set
649 * permissions so that cross-partition amos are allowed.
Dean Nelson94bd2702008-07-29 22:34:05 -0700650 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700651 * The allocated amo page needs MCA reporting to remain disabled after
Dean Nelson94bd2702008-07-29 22:34:05 -0700652 * XPC has unloaded. To make this work, we keep a copy of the pointer
Dean Nelson8e85c232008-07-29 22:34:13 -0700653 * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure,
Dean Nelson94bd2702008-07-29 22:34:05 -0700654 * which is pointed to by the reserved page, and re-use that saved copy
Dean Nelsonc39838c2008-07-29 22:34:11 -0700655 * on subsequent loads of XPC. This amo page is never freed, and its
Dean Nelson94bd2702008-07-29 22:34:05 -0700656 * memory protections are never restricted.
657 */
Dean Nelson8e85c232008-07-29 22:34:13 -0700658 amos_page = xpc_vars_sn2->amos_page;
Dean Nelson94bd2702008-07-29 22:34:05 -0700659 if (amos_page == NULL) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700660 amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1));
Dean Nelson94bd2702008-07-29 22:34:05 -0700661 if (amos_page == NULL) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700662 dev_err(xpc_part, "can't allocate page of amos\n");
Dean Nelson5b8669d2008-07-29 22:34:18 -0700663 return -ENOMEM;
Dean Nelson94bd2702008-07-29 22:34:05 -0700664 }
665
666 /*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700667 * Open up amo-R/W to cpu. This is done on Shub 1.1 systems
668 * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called.
Dean Nelson94bd2702008-07-29 22:34:05 -0700669 */
Dean Nelsonc39838c2008-07-29 22:34:11 -0700670 ret = xpc_allow_amo_ops_sn2(amos_page);
Dean Nelson6e410172008-07-29 22:34:09 -0700671 if (ret != xpSuccess) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700672 dev_err(xpc_part, "can't allow amo operations\n");
Dean Nelson6e410172008-07-29 22:34:09 -0700673 uncached_free_page(__IA64_UNCACHED_OFFSET |
674 TO_PHYS((u64)amos_page), 1);
Dean Nelson5b8669d2008-07-29 22:34:18 -0700675 return -EPERM;
Dean Nelson94bd2702008-07-29 22:34:05 -0700676 }
677 }
678
Dean Nelson8e85c232008-07-29 22:34:13 -0700679 /* clear xpc_vars_sn2 */
680 memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2));
Dean Nelson94bd2702008-07-29 22:34:05 -0700681
Dean Nelson8e85c232008-07-29 22:34:13 -0700682 xpc_vars_sn2->version = XPC_V_VERSION;
683 xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0);
684 xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0);
Dean Nelsona812dcc2008-07-29 22:34:16 -0700685 xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2);
Dean Nelson8e85c232008-07-29 22:34:13 -0700686 xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page);
687 xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */
Dean Nelson94bd2702008-07-29 22:34:05 -0700688
Dean Nelson8e85c232008-07-29 22:34:13 -0700689 /* clear xpc_vars_part_sn2 */
690 memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) *
Dean Nelson261f3b42008-07-29 22:34:16 -0700691 XP_MAX_NPARTITIONS_SN2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700692
Dean Nelsonc39838c2008-07-29 22:34:11 -0700693 /* initialize the activate IRQ related amo variables */
Dean Nelson04de7412008-07-29 22:34:14 -0700694 for (i = 0; i < xpc_nasid_mask_nlongs; i++)
Dean Nelsonee6665e2008-07-29 22:34:13 -0700695 (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
Dean Nelson94bd2702008-07-29 22:34:05 -0700696
Dean Nelsonc39838c2008-07-29 22:34:11 -0700697 /* initialize the engaged remote partitions related amo variables */
Dean Nelsonee6665e2008-07-29 22:34:13 -0700698 (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
699 (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700700
Dean Nelson5b8669d2008-07-29 22:34:18 -0700701 return 0;
Dean Nelson94bd2702008-07-29 22:34:05 -0700702}
703
Robin Holta374c572009-04-13 14:40:18 -0700704static int
705xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask)
706{
707 return test_bit(partid, heartbeating_to_mask);
708}
709
710static void
711xpc_allow_hb_sn2(short partid)
712{
713 DBUG_ON(xpc_vars_sn2 == NULL);
714 set_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
715}
716
717static void
718xpc_disallow_hb_sn2(short partid)
719{
720 DBUG_ON(xpc_vars_sn2 == NULL);
721 clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
722}
723
724static void
725xpc_disallow_all_hbs_sn2(void)
726{
727 DBUG_ON(xpc_vars_sn2 == NULL);
728 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions);
729}
730
Dean Nelson33ba3c72008-07-29 22:34:07 -0700731static void
732xpc_increment_heartbeat_sn2(void)
733{
Dean Nelson8e85c232008-07-29 22:34:13 -0700734 xpc_vars_sn2->heartbeat++;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700735}
736
737static void
738xpc_offline_heartbeat_sn2(void)
739{
740 xpc_increment_heartbeat_sn2();
Dean Nelson8e85c232008-07-29 22:34:13 -0700741 xpc_vars_sn2->heartbeat_offline = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700742}
743
744static void
745xpc_online_heartbeat_sn2(void)
746{
747 xpc_increment_heartbeat_sn2();
Dean Nelson8e85c232008-07-29 22:34:13 -0700748 xpc_vars_sn2->heartbeat_offline = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700749}
750
751static void
752xpc_heartbeat_init_sn2(void)
753{
Dean Nelson8e85c232008-07-29 22:34:13 -0700754 DBUG_ON(xpc_vars_sn2 == NULL);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700755
Dean Nelson8e85c232008-07-29 22:34:13 -0700756 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700757 xpc_online_heartbeat_sn2();
758}
759
760static void
761xpc_heartbeat_exit_sn2(void)
762{
763 xpc_offline_heartbeat_sn2();
764}
765
Dean Nelson61deb862008-07-29 22:34:17 -0700766static enum xp_retval
767xpc_get_remote_heartbeat_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700768{
769 struct xpc_vars_sn2 *remote_vars;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700770 enum xp_retval ret;
771
Dean Nelsonee6665e2008-07-29 22:34:13 -0700772 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700773
Dean Nelson61deb862008-07-29 22:34:17 -0700774 /* pull the remote vars structure that contains the heartbeat */
775 ret = xp_remote_memcpy(xp_pa(remote_vars),
776 part->sn.sn2.remote_vars_pa,
777 XPC_RP_VARS_SIZE);
778 if (ret != xpSuccess)
779 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700780
Randy Dunlapf509e342009-09-04 17:02:59 -0700781 dev_dbg(xpc_part, "partid=%d, heartbeat=%lld, last_heartbeat=%lld, "
782 "heartbeat_offline=%lld, HB_mask[0]=0x%lx\n", XPC_PARTID(part),
Dean Nelson61deb862008-07-29 22:34:17 -0700783 remote_vars->heartbeat, part->last_heartbeat,
784 remote_vars->heartbeat_offline,
785 remote_vars->heartbeating_to_mask[0]);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700786
Dean Nelson61deb862008-07-29 22:34:17 -0700787 if ((remote_vars->heartbeat == part->last_heartbeat &&
Robin Holta374c572009-04-13 14:40:18 -0700788 !remote_vars->heartbeat_offline) ||
789 !xpc_hb_allowed_sn2(sn_partition_id,
790 remote_vars->heartbeating_to_mask)) {
Dean Nelson61deb862008-07-29 22:34:17 -0700791 ret = xpNoHeartbeat;
792 } else {
Dean Nelson33ba3c72008-07-29 22:34:07 -0700793 part->last_heartbeat = remote_vars->heartbeat;
794 }
Dean Nelson61deb862008-07-29 22:34:17 -0700795
796 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700797}
798
799/*
800 * Get a copy of the remote partition's XPC variables from the reserved page.
801 *
802 * remote_vars points to a buffer that is cacheline aligned for BTE copies and
803 * assumed to be of size XPC_RP_VARS_SIZE.
804 */
805static enum xp_retval
Dean Nelsona812dcc2008-07-29 22:34:16 -0700806xpc_get_remote_vars_sn2(unsigned long remote_vars_pa,
807 struct xpc_vars_sn2 *remote_vars)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700808{
809 enum xp_retval ret;
810
811 if (remote_vars_pa == 0)
812 return xpVarsNotSet;
813
814 /* pull over the cross partition variables */
Dean Nelsona812dcc2008-07-29 22:34:16 -0700815 ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700816 XPC_RP_VARS_SIZE);
817 if (ret != xpSuccess)
818 return ret;
819
820 if (XPC_VERSION_MAJOR(remote_vars->version) !=
821 XPC_VERSION_MAJOR(XPC_V_VERSION)) {
822 return xpBadVersion;
823 }
824
825 return xpSuccess;
826}
827
828static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700829xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
Dean Nelsona812dcc2008-07-29 22:34:16 -0700830 unsigned long remote_rp_pa, int nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700831{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700832 xpc_send_local_activate_IRQ_sn2(nasid);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700833}
834
835static void
836xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
837{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700838 xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700839}
840
841static void
842xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
843{
844 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
845 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700846 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700847 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700848 sizeof(struct amo)));
Dean Nelsona47d5da2008-07-29 22:34:09 -0700849
850 local_irq_save(irq_flags);
851
Dean Nelsonc39838c2008-07-29 22:34:11 -0700852 /* set bit corresponding to our partid in remote partition's amo */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700853 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
Dean Nelson04de7412008-07-29 22:34:14 -0700854 BIT(sn_partition_id));
855
Dean Nelsona47d5da2008-07-29 22:34:09 -0700856 /*
857 * We must always use the nofault function regardless of whether we
858 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
859 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700860 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700861 */
862 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
863 variable),
864 xp_nofault_PIOR_target));
865
866 local_irq_restore(irq_flags);
867
868 /*
869 * Send activate IRQ to get other side to see that we've set our
Dean Nelsonc39838c2008-07-29 22:34:11 -0700870 * bit in their deactivate request amo.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700871 */
Dean Nelson7fb5e592008-07-29 22:34:10 -0700872 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -0700873 cnodeid_to_nasid(0),
874 part_sn2->activate_IRQ_nasid,
875 part_sn2->activate_IRQ_phys_cpuid);
876}
877
878static void
879xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
880{
881 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700882 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700883 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700884 sizeof(struct amo)));
Dean Nelsona47d5da2008-07-29 22:34:09 -0700885
886 local_irq_save(irq_flags);
887
Dean Nelsonc39838c2008-07-29 22:34:11 -0700888 /* clear bit corresponding to our partid in remote partition's amo */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700889 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
Dean Nelson04de7412008-07-29 22:34:14 -0700890 ~BIT(sn_partition_id));
891
Dean Nelsona47d5da2008-07-29 22:34:09 -0700892 /*
893 * We must always use the nofault function regardless of whether we
894 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
895 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700896 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700897 */
898 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
899 variable),
900 xp_nofault_PIOR_target));
901
902 local_irq_restore(irq_flags);
903}
904
905static int
906xpc_partition_deactivation_requested_sn2(short partid)
907{
Dean Nelson8e85c232008-07-29 22:34:13 -0700908 struct amo *amo = xpc_vars_sn2->amos_page +
909 XPC_DEACTIVATE_REQUEST_AMO_SN2;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700910
Dean Nelsonc39838c2008-07-29 22:34:11 -0700911 /* our partition's amo variable ANDed with partid mask */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700912 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
Dean Nelson04de7412008-07-29 22:34:14 -0700913 BIT(partid)) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700914}
915
916/*
917 * Update the remote partition's info.
918 */
919static void
920xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -0700921 unsigned long *remote_rp_ts_jiffies,
Dean Nelsona812dcc2008-07-29 22:34:16 -0700922 unsigned long remote_rp_pa,
923 unsigned long remote_vars_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700924 struct xpc_vars_sn2 *remote_vars)
925{
Dean Nelsona47d5da2008-07-29 22:34:09 -0700926 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
927
Dean Nelson33ba3c72008-07-29 22:34:07 -0700928 part->remote_rp_version = remote_rp_version;
929 dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n",
930 part->remote_rp_version);
931
Dean Nelson81fe7882008-07-29 22:34:15 -0700932 part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies;
933 dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n",
934 part->remote_rp_ts_jiffies);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700935
936 part->remote_rp_pa = remote_rp_pa;
937 dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
938
Dean Nelsona47d5da2008-07-29 22:34:09 -0700939 part_sn2->remote_vars_pa = remote_vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700940 dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700941 part_sn2->remote_vars_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700942
Dean Nelson158bc692009-01-15 13:50:57 -0800943 part->last_heartbeat = remote_vars->heartbeat - 1;
Randy Dunlapf509e342009-09-04 17:02:59 -0700944 dev_dbg(xpc_part, " last_heartbeat = 0x%016llx\n",
Dean Nelson33ba3c72008-07-29 22:34:07 -0700945 part->last_heartbeat);
946
Dean Nelsona47d5da2008-07-29 22:34:09 -0700947 part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700948 dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700949 part_sn2->remote_vars_part_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700950
Dean Nelsona47d5da2008-07-29 22:34:09 -0700951 part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
952 dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n",
953 part_sn2->activate_IRQ_nasid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700954
Dean Nelsona47d5da2008-07-29 22:34:09 -0700955 part_sn2->activate_IRQ_phys_cpuid =
956 remote_vars->activate_IRQ_phys_cpuid;
957 dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n",
958 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700959
Dean Nelsona47d5da2008-07-29 22:34:09 -0700960 part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700961 dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700962 part_sn2->remote_amos_page_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700963
Dean Nelsona47d5da2008-07-29 22:34:09 -0700964 part_sn2->remote_vars_version = remote_vars->version;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700965 dev_dbg(xpc_part, " remote_vars_version = 0x%x\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700966 part_sn2->remote_vars_version);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700967}
968
969/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700970 * Prior code has determined the nasid which generated a activate IRQ.
971 * Inspect that nasid to determine if its partition needs to be activated
972 * or deactivated.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700973 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700974 * A partition is considered "awaiting activation" if our partition
Dean Nelson33ba3c72008-07-29 22:34:07 -0700975 * flags indicate it is not active and it has a heartbeat. A
976 * partition is considered "awaiting deactivation" if our partition
977 * flags indicate it is active but it has no heartbeat or it is not
978 * sending its heartbeat to us.
979 *
980 * To determine the heartbeat, the remote nasid must have a properly
981 * initialized reserved page.
982 */
983static void
Dean Nelson6e410172008-07-29 22:34:09 -0700984xpc_identify_activate_IRQ_req_sn2(int nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700985{
986 struct xpc_rsvd_page *remote_rp;
987 struct xpc_vars_sn2 *remote_vars;
Dean Nelsona812dcc2008-07-29 22:34:16 -0700988 unsigned long remote_rp_pa;
989 unsigned long remote_vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700990 int remote_rp_version;
991 int reactivate = 0;
Dean Nelson81fe7882008-07-29 22:34:15 -0700992 unsigned long remote_rp_ts_jiffies = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700993 short partid;
994 struct xpc_partition *part;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700995 struct xpc_partition_sn2 *part_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700996 enum xp_retval ret;
997
998 /* pull over the reserved page structure */
999
Dean Nelsonee6665e2008-07-29 22:34:13 -07001000 remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001001
1002 ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
1003 if (ret != xpSuccess) {
1004 dev_warn(xpc_part, "unable to get reserved page from nasid %d, "
1005 "which sent interrupt, reason=%d\n", nasid, ret);
1006 return;
1007 }
1008
Robin Holta374c572009-04-13 14:40:18 -07001009 remote_vars_pa = remote_rp->sn.sn2.vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001010 remote_rp_version = remote_rp->version;
Dean Nelson81fe7882008-07-29 22:34:15 -07001011 remote_rp_ts_jiffies = remote_rp->ts_jiffies;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001012
1013 partid = remote_rp->SAL_partid;
1014 part = &xpc_partitions[partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001015 part_sn2 = &part->sn.sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001016
1017 /* pull over the cross partition variables */
1018
Dean Nelsonee6665e2008-07-29 22:34:13 -07001019 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001020
1021 ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
1022 if (ret != xpSuccess) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001023 dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
1024 "which sent interrupt, reason=%d\n", nasid, ret);
1025
1026 XPC_DEACTIVATE_PARTITION(part, ret);
1027 return;
1028 }
1029
Dean Nelson6e410172008-07-29 22:34:09 -07001030 part->activate_IRQ_rcvd++;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001031
1032 dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = "
Randy Dunlapf509e342009-09-04 17:02:59 -07001033 "%lld:0x%lx\n", (int)nasid, (int)partid,
1034 part->activate_IRQ_rcvd,
Dean Nelson33ba3c72008-07-29 22:34:07 -07001035 remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]);
1036
1037 if (xpc_partition_disengaged(part) &&
Dean Nelson83469b52008-07-29 22:34:18 -07001038 part->act_state == XPC_P_AS_INACTIVE) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001039
1040 xpc_update_partition_info_sn2(part, remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -07001041 &remote_rp_ts_jiffies,
1042 remote_rp_pa, remote_vars_pa,
1043 remote_vars);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001044
Dean Nelsona47d5da2008-07-29 22:34:09 -07001045 if (xpc_partition_deactivation_requested_sn2(partid)) {
1046 /*
1047 * Other side is waiting on us to deactivate even though
1048 * we already have.
1049 */
1050 return;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001051 }
1052
1053 xpc_activate_partition(part);
1054 return;
1055 }
1056
1057 DBUG_ON(part->remote_rp_version == 0);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001058 DBUG_ON(part_sn2->remote_vars_version == 0);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001059
Dean Nelson81fe7882008-07-29 22:34:15 -07001060 if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001061
Dean Nelsona47d5da2008-07-29 22:34:09 -07001062 /* the other side rebooted */
Dean Nelson33ba3c72008-07-29 22:34:07 -07001063
Dean Nelsona47d5da2008-07-29 22:34:09 -07001064 DBUG_ON(xpc_partition_engaged_sn2(partid));
1065 DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
Dean Nelson33ba3c72008-07-29 22:34:07 -07001066
1067 xpc_update_partition_info_sn2(part, remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -07001068 &remote_rp_ts_jiffies,
1069 remote_rp_pa, remote_vars_pa,
1070 remote_vars);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001071 reactivate = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001072 }
1073
Dean Nelsona47d5da2008-07-29 22:34:09 -07001074 if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001075 /* still waiting on other side to disengage from us */
1076 return;
1077 }
1078
Dean Nelsona47d5da2008-07-29 22:34:09 -07001079 if (reactivate)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001080 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001081 else if (xpc_partition_deactivation_requested_sn2(partid))
Dean Nelson33ba3c72008-07-29 22:34:07 -07001082 XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001083}
1084
1085/*
Dean Nelsonc39838c2008-07-29 22:34:11 -07001086 * Loop through the activation amo variables and process any bits
Dean Nelson33ba3c72008-07-29 22:34:07 -07001087 * which are set. Each bit indicates a nasid sending a partition
1088 * activation or deactivation request.
1089 *
1090 * Return #of IRQs detected.
1091 */
1092int
Dean Nelson6e410172008-07-29 22:34:09 -07001093xpc_identify_activate_IRQ_sender_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001094{
Dean Nelson04de7412008-07-29 22:34:14 -07001095 int l;
1096 int b;
1097 unsigned long nasid_mask_long;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001098 u64 nasid; /* remote nasid */
1099 int n_IRQs_detected = 0;
Dean Nelsonc39838c2008-07-29 22:34:11 -07001100 struct amo *act_amos;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001101
Dean Nelson8e85c232008-07-29 22:34:13 -07001102 act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001103
Dean Nelson04de7412008-07-29 22:34:14 -07001104 /* scan through activate amo variables looking for non-zero entries */
1105 for (l = 0; l < xpc_nasid_mask_nlongs; l++) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001106
1107 if (xpc_exiting)
1108 break;
1109
Dean Nelson04de7412008-07-29 22:34:14 -07001110 nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]);
1111
1112 b = find_first_bit(&nasid_mask_long, BITS_PER_LONG);
1113 if (b >= BITS_PER_LONG) {
1114 /* no IRQs from nasids in this amo variable */
Dean Nelson33ba3c72008-07-29 22:34:07 -07001115 continue;
1116 }
1117
Dean Nelson04de7412008-07-29 22:34:14 -07001118 dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l,
1119 nasid_mask_long);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001120
1121 /*
1122 * If this nasid has been added to the machine since
1123 * our partition was reset, this will retain the
1124 * remote nasid in our reserved pages machine mask.
1125 * This is used in the event of module reload.
1126 */
Dean Nelson04de7412008-07-29 22:34:14 -07001127 xpc_mach_nasids[l] |= nasid_mask_long;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001128
1129 /* locate the nasid(s) which sent interrupts */
1130
Dean Nelson04de7412008-07-29 22:34:14 -07001131 do {
1132 n_IRQs_detected++;
1133 nasid = (l * BITS_PER_LONG + b) * 2;
Randy Dunlapf509e342009-09-04 17:02:59 -07001134 dev_dbg(xpc_part, "interrupt from nasid %lld\n", nasid);
Dean Nelson04de7412008-07-29 22:34:14 -07001135 xpc_identify_activate_IRQ_req_sn2(nasid);
1136
1137 b = find_next_bit(&nasid_mask_long, BITS_PER_LONG,
1138 b + 1);
1139 } while (b < BITS_PER_LONG);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001140 }
1141 return n_IRQs_detected;
1142}
1143
1144static void
Dean Nelson5b8669d2008-07-29 22:34:18 -07001145xpc_process_activate_IRQ_rcvd_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001146{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001147 unsigned long irq_flags;
1148 int n_IRQs_expected;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001149 int n_IRQs_detected;
1150
Dean Nelson5b8669d2008-07-29 22:34:18 -07001151 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1152 n_IRQs_expected = xpc_activate_IRQ_rcvd;
1153 xpc_activate_IRQ_rcvd = 0;
1154 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1155
Dean Nelson6e410172008-07-29 22:34:09 -07001156 n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2();
Dean Nelson33ba3c72008-07-29 22:34:07 -07001157 if (n_IRQs_detected < n_IRQs_expected) {
Dean Nelsonc39838c2008-07-29 22:34:11 -07001158 /* retry once to help avoid missing amo */
Dean Nelson6e410172008-07-29 22:34:09 -07001159 (void)xpc_identify_activate_IRQ_sender_sn2();
Dean Nelson33ba3c72008-07-29 22:34:07 -07001160 }
1161}
1162
Dean Nelsone17d4162008-07-29 22:34:06 -07001163/*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001164 * Setup the channel structures that are sn2 specific.
Dean Nelsone17d4162008-07-29 22:34:06 -07001165 */
1166static enum xp_retval
Robin Holta7665b02009-04-13 14:40:19 -07001167xpc_setup_ch_structures_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001168{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001169 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001170 struct xpc_channel_sn2 *ch_sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001171 enum xp_retval retval;
1172 int ret;
1173 int cpuid;
1174 int ch_number;
Dean Nelsone17d4162008-07-29 22:34:06 -07001175 struct timer_list *timer;
1176 short partid = XPC_PARTID(part);
1177
Dean Nelsone17d4162008-07-29 22:34:06 -07001178 /* allocate all the required GET/PUT values */
1179
Dean Nelson185c3a12008-07-29 22:34:11 -07001180 part_sn2->local_GPs =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001181 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1182 &part_sn2->local_GPs_base);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001183 if (part_sn2->local_GPs == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001184 dev_err(xpc_chan, "can't get memory for local get/put "
1185 "values\n");
Dean Nelson5b8669d2008-07-29 22:34:18 -07001186 return xpNoMemory;
Dean Nelsone17d4162008-07-29 22:34:06 -07001187 }
1188
Dean Nelson185c3a12008-07-29 22:34:11 -07001189 part_sn2->remote_GPs =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001190 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1191 &part_sn2->remote_GPs_base);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001192 if (part_sn2->remote_GPs == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001193 dev_err(xpc_chan, "can't get memory for remote get/put "
1194 "values\n");
1195 retval = xpNoMemory;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001196 goto out_1;
Dean Nelsone17d4162008-07-29 22:34:06 -07001197 }
1198
Dean Nelsona47d5da2008-07-29 22:34:09 -07001199 part_sn2->remote_GPs_pa = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001200
1201 /* allocate all the required open and close args */
1202
Dean Nelson5b8669d2008-07-29 22:34:18 -07001203 part_sn2->local_openclose_args =
1204 xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE,
1205 GFP_KERNEL, &part_sn2->
1206 local_openclose_args_base);
1207 if (part_sn2->local_openclose_args == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001208 dev_err(xpc_chan, "can't get memory for local connect args\n");
1209 retval = xpNoMemory;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001210 goto out_2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001211 }
1212
Dean Nelsona47d5da2008-07-29 22:34:09 -07001213 part_sn2->remote_openclose_args_pa = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001214
Dean Nelson7fb5e592008-07-29 22:34:10 -07001215 part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001216
Dean Nelson7fb5e592008-07-29 22:34:10 -07001217 part_sn2->notify_IRQ_nasid = 0;
1218 part_sn2->notify_IRQ_phys_cpuid = 0;
1219 part_sn2->remote_chctl_amo_va = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001220
Dean Nelson7fb5e592008-07-29 22:34:10 -07001221 sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001222 ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
Dean Nelson7fb5e592008-07-29 22:34:10 -07001223 IRQF_SHARED, part_sn2->notify_IRQ_owner,
Dean Nelsona47d5da2008-07-29 22:34:09 -07001224 (void *)(u64)partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001225 if (ret != 0) {
1226 dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
1227 "errno=%d\n", -ret);
1228 retval = xpLackOfResources;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001229 goto out_3;
Dean Nelsone17d4162008-07-29 22:34:06 -07001230 }
1231
Dean Nelson7fb5e592008-07-29 22:34:10 -07001232 /* Setup a timer to check for dropped notify IRQs */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001233 timer = &part_sn2->dropped_notify_IRQ_timer;
Dean Nelsone17d4162008-07-29 22:34:06 -07001234 init_timer(timer);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001235 timer->function =
Dean Nelson7fb5e592008-07-29 22:34:10 -07001236 (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001237 timer->data = (unsigned long)part;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001238 timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001239 add_timer(timer);
1240
Dean Nelsone17d4162008-07-29 22:34:06 -07001241 for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
Dean Nelson5b8669d2008-07-29 22:34:18 -07001242 ch_sn2 = &part->channels[ch_number].sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001243
Dean Nelson5b8669d2008-07-29 22:34:18 -07001244 ch_sn2->local_GP = &part_sn2->local_GPs[ch_number];
1245 ch_sn2->local_openclose_args =
1246 &part_sn2->local_openclose_args[ch_number];
Dean Nelsone17d4162008-07-29 22:34:06 -07001247
Dean Nelson5b8669d2008-07-29 22:34:18 -07001248 mutex_init(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001249 }
1250
1251 /*
Dean Nelsone17d4162008-07-29 22:34:06 -07001252 * Setup the per partition specific variables required by the
1253 * remote partition to establish channel connections with us.
1254 *
1255 * The setting of the magic # indicates that these per partition
1256 * specific variables are ready to be used.
1257 */
Dean Nelsona812dcc2008-07-29 22:34:16 -07001258 xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs);
Dean Nelson8e85c232008-07-29 22:34:13 -07001259 xpc_vars_part_sn2[partid].openclose_args_pa =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001260 xp_pa(part_sn2->local_openclose_args);
Dean Nelson8e85c232008-07-29 22:34:13 -07001261 xpc_vars_part_sn2[partid].chctl_amo_pa =
Dean Nelsona812dcc2008-07-29 22:34:16 -07001262 xp_pa(part_sn2->local_chctl_amo_va);
Dean Nelsone17d4162008-07-29 22:34:06 -07001263 cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */
Dean Nelson8e85c232008-07-29 22:34:13 -07001264 xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid);
1265 xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid =
1266 cpu_physical_id(cpuid);
1267 xpc_vars_part_sn2[partid].nchannels = part->nchannels;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001268 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001269
1270 return xpSuccess;
1271
Dean Nelson5b8669d2008-07-29 22:34:18 -07001272 /* setup of ch structures failed */
Dean Nelsone17d4162008-07-29 22:34:06 -07001273out_3:
Dean Nelson5b8669d2008-07-29 22:34:18 -07001274 kfree(part_sn2->local_openclose_args_base);
1275 part_sn2->local_openclose_args = NULL;
1276out_2:
Dean Nelsona47d5da2008-07-29 22:34:09 -07001277 kfree(part_sn2->remote_GPs_base);
1278 part_sn2->remote_GPs = NULL;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001279out_1:
Dean Nelsona47d5da2008-07-29 22:34:09 -07001280 kfree(part_sn2->local_GPs_base);
1281 part_sn2->local_GPs = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001282 return retval;
1283}
1284
1285/*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001286 * Teardown the channel structures that are sn2 specific.
Dean Nelsone17d4162008-07-29 22:34:06 -07001287 */
1288static void
Robin Holta7665b02009-04-13 14:40:19 -07001289xpc_teardown_ch_structures_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001290{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001291 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001292 short partid = XPC_PARTID(part);
1293
1294 /*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001295 * Indicate that the variables specific to the remote partition are no
1296 * longer available for its use.
Dean Nelsone17d4162008-07-29 22:34:06 -07001297 */
Dean Nelson8e85c232008-07-29 22:34:13 -07001298 xpc_vars_part_sn2[partid].magic = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001299
Dean Nelsone17d4162008-07-29 22:34:06 -07001300 /* in case we've still got outstanding timers registered... */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001301 del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
Dean Nelson5b8669d2008-07-29 22:34:18 -07001302 free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001303
Dean Nelson5b8669d2008-07-29 22:34:18 -07001304 kfree(part_sn2->local_openclose_args_base);
1305 part_sn2->local_openclose_args = NULL;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001306 kfree(part_sn2->remote_GPs_base);
1307 part_sn2->remote_GPs = NULL;
1308 kfree(part_sn2->local_GPs_base);
1309 part_sn2->local_GPs = NULL;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001310 part_sn2->local_chctl_amo_va = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001311}
1312
1313/*
1314 * Create a wrapper that hides the underlying mechanism for pulling a cacheline
1315 * (or multiple cachelines) from a remote partition.
1316 *
Dean Nelsona812dcc2008-07-29 22:34:16 -07001317 * src_pa must be a cacheline aligned physical address on the remote partition.
Dean Nelsone17d4162008-07-29 22:34:06 -07001318 * dst must be a cacheline aligned virtual address on this partition.
1319 * cnt must be cacheline sized
1320 */
Dean Nelsonea57f802008-07-29 22:34:14 -07001321/* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */
Dean Nelsone17d4162008-07-29 22:34:06 -07001322static enum xp_retval
1323xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001324 const unsigned long src_pa, size_t cnt)
Dean Nelsone17d4162008-07-29 22:34:06 -07001325{
1326 enum xp_retval ret;
1327
Dean Nelsona812dcc2008-07-29 22:34:16 -07001328 DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa));
1329 DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst));
Dean Nelsone17d4162008-07-29 22:34:06 -07001330 DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
1331
Dean Nelson83469b52008-07-29 22:34:18 -07001332 if (part->act_state == XPC_P_AS_DEACTIVATING)
Dean Nelsone17d4162008-07-29 22:34:06 -07001333 return part->reason;
1334
Dean Nelsona812dcc2008-07-29 22:34:16 -07001335 ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt);
Dean Nelsone17d4162008-07-29 22:34:06 -07001336 if (ret != xpSuccess) {
1337 dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed,"
1338 " ret=%d\n", XPC_PARTID(part), ret);
1339 }
1340 return ret;
1341}
1342
1343/*
1344 * Pull the remote per partition specific variables from the specified
1345 * partition.
1346 */
1347static enum xp_retval
1348xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
1349{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001350 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001351 u8 buffer[L1_CACHE_BYTES * 2];
1352 struct xpc_vars_part_sn2 *pulled_entry_cacheline =
1353 (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
1354 struct xpc_vars_part_sn2 *pulled_entry;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001355 unsigned long remote_entry_cacheline_pa;
1356 unsigned long remote_entry_pa;
Dean Nelsone17d4162008-07-29 22:34:06 -07001357 short partid = XPC_PARTID(part);
1358 enum xp_retval ret;
1359
1360 /* pull the cacheline that contains the variables we're interested in */
1361
Dean Nelsona47d5da2008-07-29 22:34:09 -07001362 DBUG_ON(part_sn2->remote_vars_part_pa !=
1363 L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
Dean Nelsone17d4162008-07-29 22:34:06 -07001364 DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
1365
Dean Nelsona47d5da2008-07-29 22:34:09 -07001366 remote_entry_pa = part_sn2->remote_vars_part_pa +
Dean Nelsone17d4162008-07-29 22:34:06 -07001367 sn_partition_id * sizeof(struct xpc_vars_part_sn2);
1368
1369 remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
1370
1371 pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline
1372 + (remote_entry_pa &
1373 (L1_CACHE_BYTES - 1)));
1374
1375 ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001376 remote_entry_cacheline_pa,
Dean Nelsone17d4162008-07-29 22:34:06 -07001377 L1_CACHE_BYTES);
1378 if (ret != xpSuccess) {
1379 dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
1380 "partition %d, ret=%d\n", partid, ret);
1381 return ret;
1382 }
1383
1384 /* see if they've been set up yet */
1385
Dean Nelson5b8669d2008-07-29 22:34:18 -07001386 if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 &&
1387 pulled_entry->magic != XPC_VP_MAGIC2_SN2) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001388
1389 if (pulled_entry->magic != 0) {
1390 dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
Randy Dunlapf509e342009-09-04 17:02:59 -07001391 "partition %d has bad magic value (=0x%llx)\n",
Dean Nelsone17d4162008-07-29 22:34:06 -07001392 partid, sn_partition_id, pulled_entry->magic);
1393 return xpBadMagic;
1394 }
1395
1396 /* they've not been initialized yet */
1397 return xpRetry;
1398 }
1399
Dean Nelson5b8669d2008-07-29 22:34:18 -07001400 if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001401
1402 /* validate the variables */
1403
1404 if (pulled_entry->GPs_pa == 0 ||
1405 pulled_entry->openclose_args_pa == 0 ||
Dean Nelson7fb5e592008-07-29 22:34:10 -07001406 pulled_entry->chctl_amo_pa == 0) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001407
1408 dev_err(xpc_chan, "partition %d's XPC vars_part for "
1409 "partition %d are not valid\n", partid,
1410 sn_partition_id);
1411 return xpInvalidAddress;
1412 }
1413
1414 /* the variables we imported look to be valid */
1415
Dean Nelsona47d5da2008-07-29 22:34:09 -07001416 part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
1417 part_sn2->remote_openclose_args_pa =
Dean Nelsone17d4162008-07-29 22:34:06 -07001418 pulled_entry->openclose_args_pa;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001419 part_sn2->remote_chctl_amo_va =
Dean Nelsonc39838c2008-07-29 22:34:11 -07001420 (struct amo *)__va(pulled_entry->chctl_amo_pa);
Dean Nelson7fb5e592008-07-29 22:34:10 -07001421 part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid;
1422 part_sn2->notify_IRQ_phys_cpuid =
1423 pulled_entry->notify_IRQ_phys_cpuid;
Dean Nelsone17d4162008-07-29 22:34:06 -07001424
1425 if (part->nchannels > pulled_entry->nchannels)
1426 part->nchannels = pulled_entry->nchannels;
1427
1428 /* let the other side know that we've pulled their variables */
1429
Dean Nelson5b8669d2008-07-29 22:34:18 -07001430 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001431 }
1432
Dean Nelson5b8669d2008-07-29 22:34:18 -07001433 if (pulled_entry->magic == XPC_VP_MAGIC1_SN2)
Dean Nelsone17d4162008-07-29 22:34:06 -07001434 return xpRetry;
1435
1436 return xpSuccess;
1437}
1438
1439/*
1440 * Establish first contact with the remote partititon. This involves pulling
1441 * the XPC per partition variables from the remote partition and waiting for
1442 * the remote partition to pull ours.
1443 */
1444static enum xp_retval
1445xpc_make_first_contact_sn2(struct xpc_partition *part)
1446{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001447 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001448 enum xp_retval ret;
1449
Dean Nelson33ba3c72008-07-29 22:34:07 -07001450 /*
Dean Nelsonc39838c2008-07-29 22:34:11 -07001451 * Register the remote partition's amos with SAL so it can handle
Dean Nelson33ba3c72008-07-29 22:34:07 -07001452 * and cleanup errors within that address range should the remote
1453 * partition go down. We don't unregister this range because it is
1454 * difficult to tell when outstanding writes to the remote partition
1455 * are finished and thus when it is safe to unregister. This should
1456 * not result in wasted space in the SAL xp_addr_region table because
1457 * we should get the same page for remote_amos_page_pa after module
1458 * reloads and system reboots.
1459 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001460 if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -07001461 PAGE_SIZE, 1) < 0) {
1462 dev_warn(xpc_part, "xpc_activating(%d) failed to register "
1463 "xp_addr region\n", XPC_PARTID(part));
1464
1465 ret = xpPhysAddrRegFailed;
1466 XPC_DEACTIVATE_PARTITION(part, ret);
1467 return ret;
1468 }
1469
Dean Nelsona47d5da2008-07-29 22:34:09 -07001470 /*
1471 * Send activate IRQ to get other side to activate if they've not
1472 * already begun to do so.
1473 */
Dean Nelson7fb5e592008-07-29 22:34:10 -07001474 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -07001475 cnodeid_to_nasid(0),
1476 part_sn2->activate_IRQ_nasid,
1477 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001478
Dean Nelsone17d4162008-07-29 22:34:06 -07001479 while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
1480 if (ret != xpRetry) {
1481 XPC_DEACTIVATE_PARTITION(part, ret);
1482 return ret;
1483 }
1484
1485 dev_dbg(xpc_part, "waiting to make first contact with "
1486 "partition %d\n", XPC_PARTID(part));
1487
1488 /* wait a 1/4 of a second or so */
1489 (void)msleep_interruptible(250);
1490
Dean Nelson83469b52008-07-29 22:34:18 -07001491 if (part->act_state == XPC_P_AS_DEACTIVATING)
Dean Nelsone17d4162008-07-29 22:34:06 -07001492 return part->reason;
1493 }
1494
1495 return xpSuccess;
1496}
1497
1498/*
Dean Nelson7fb5e592008-07-29 22:34:10 -07001499 * Get the chctl flags and pull the openclose args and/or remote GPs as needed.
Dean Nelsone17d4162008-07-29 22:34:06 -07001500 */
1501static u64
Dean Nelson7fb5e592008-07-29 22:34:10 -07001502xpc_get_chctl_all_flags_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001503{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001504 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001505 unsigned long irq_flags;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001506 union xpc_channel_ctl_flags chctl;
Dean Nelsone17d4162008-07-29 22:34:06 -07001507 enum xp_retval ret;
1508
1509 /*
Dean Nelson7fb5e592008-07-29 22:34:10 -07001510 * See if there are any chctl flags to be handled.
Dean Nelsone17d4162008-07-29 22:34:06 -07001511 */
1512
Dean Nelson7fb5e592008-07-29 22:34:10 -07001513 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1514 chctl = part->chctl;
1515 if (chctl.all_flags != 0)
1516 part->chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001517
Dean Nelson7fb5e592008-07-29 22:34:10 -07001518 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
Dean Nelsone17d4162008-07-29 22:34:06 -07001519
Dean Nelson7fb5e592008-07-29 22:34:10 -07001520 if (xpc_any_openclose_chctl_flags_set(&chctl)) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001521 ret = xpc_pull_remote_cachelines_sn2(part, part->
1522 remote_openclose_args,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001523 part_sn2->
Dean Nelsone17d4162008-07-29 22:34:06 -07001524 remote_openclose_args_pa,
1525 XPC_OPENCLOSE_ARGS_SIZE);
1526 if (ret != xpSuccess) {
1527 XPC_DEACTIVATE_PARTITION(part, ret);
1528
1529 dev_dbg(xpc_chan, "failed to pull openclose args from "
1530 "partition %d, ret=%d\n", XPC_PARTID(part),
1531 ret);
1532
Dean Nelson7fb5e592008-07-29 22:34:10 -07001533 /* don't bother processing chctl flags anymore */
1534 chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001535 }
1536 }
1537
Dean Nelson7fb5e592008-07-29 22:34:10 -07001538 if (xpc_any_msg_chctl_flags_set(&chctl)) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001539 ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001540 part_sn2->remote_GPs_pa,
Dean Nelsone17d4162008-07-29 22:34:06 -07001541 XPC_GP_SIZE);
1542 if (ret != xpSuccess) {
1543 XPC_DEACTIVATE_PARTITION(part, ret);
1544
1545 dev_dbg(xpc_chan, "failed to pull GPs from partition "
1546 "%d, ret=%d\n", XPC_PARTID(part), ret);
1547
Dean Nelson7fb5e592008-07-29 22:34:10 -07001548 /* don't bother processing chctl flags anymore */
1549 chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001550 }
1551 }
1552
Dean Nelson7fb5e592008-07-29 22:34:10 -07001553 return chctl.all_flags;
Dean Nelsone17d4162008-07-29 22:34:06 -07001554}
1555
Dean Nelsona47d5da2008-07-29 22:34:09 -07001556/*
Dean Nelson185c3a12008-07-29 22:34:11 -07001557 * Allocate the local message queue and the notify queue.
1558 */
1559static enum xp_retval
1560xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch)
1561{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001562 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001563 unsigned long irq_flags;
1564 int nentries;
1565 size_t nbytes;
1566
1567 for (nentries = ch->local_nentries; nentries > 0; nentries--) {
1568
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001569 nbytes = nentries * ch->entry_size;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001570 ch_sn2->local_msgqueue =
1571 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL,
1572 &ch_sn2->local_msgqueue_base);
1573 if (ch_sn2->local_msgqueue == NULL)
Dean Nelson185c3a12008-07-29 22:34:11 -07001574 continue;
1575
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001576 nbytes = nentries * sizeof(struct xpc_notify_sn2);
Dean Nelson5b8669d2008-07-29 22:34:18 -07001577 ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL);
1578 if (ch_sn2->notify_queue == NULL) {
1579 kfree(ch_sn2->local_msgqueue_base);
1580 ch_sn2->local_msgqueue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001581 continue;
1582 }
1583
1584 spin_lock_irqsave(&ch->lock, irq_flags);
1585 if (nentries < ch->local_nentries) {
1586 dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
1587 "partid=%d, channel=%d\n", nentries,
1588 ch->local_nentries, ch->partid, ch->number);
1589
1590 ch->local_nentries = nentries;
1591 }
1592 spin_unlock_irqrestore(&ch->lock, irq_flags);
1593 return xpSuccess;
1594 }
1595
1596 dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
1597 "queue, partid=%d, channel=%d\n", ch->partid, ch->number);
1598 return xpNoMemory;
1599}
1600
1601/*
1602 * Allocate the cached remote message queue.
1603 */
1604static enum xp_retval
1605xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch)
1606{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001607 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001608 unsigned long irq_flags;
1609 int nentries;
1610 size_t nbytes;
1611
1612 DBUG_ON(ch->remote_nentries <= 0);
1613
1614 for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
1615
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001616 nbytes = nentries * ch->entry_size;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001617 ch_sn2->remote_msgqueue =
1618 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2->
1619 remote_msgqueue_base);
1620 if (ch_sn2->remote_msgqueue == NULL)
Dean Nelson185c3a12008-07-29 22:34:11 -07001621 continue;
1622
1623 spin_lock_irqsave(&ch->lock, irq_flags);
1624 if (nentries < ch->remote_nentries) {
1625 dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
1626 "partid=%d, channel=%d\n", nentries,
1627 ch->remote_nentries, ch->partid, ch->number);
1628
1629 ch->remote_nentries = nentries;
1630 }
1631 spin_unlock_irqrestore(&ch->lock, irq_flags);
1632 return xpSuccess;
1633 }
1634
1635 dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
1636 "partid=%d, channel=%d\n", ch->partid, ch->number);
1637 return xpNoMemory;
1638}
1639
1640/*
1641 * Allocate message queues and other stuff associated with a channel.
1642 *
1643 * Note: Assumes all of the channel sizes are filled in.
1644 */
1645static enum xp_retval
Dean Nelson5b8669d2008-07-29 22:34:18 -07001646xpc_setup_msg_structures_sn2(struct xpc_channel *ch)
Dean Nelson185c3a12008-07-29 22:34:11 -07001647{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001648 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001649 enum xp_retval ret;
1650
1651 DBUG_ON(ch->flags & XPC_C_SETUP);
1652
1653 ret = xpc_allocate_local_msgqueue_sn2(ch);
1654 if (ret == xpSuccess) {
1655
1656 ret = xpc_allocate_remote_msgqueue_sn2(ch);
1657 if (ret != xpSuccess) {
Dean Nelson5b8669d2008-07-29 22:34:18 -07001658 kfree(ch_sn2->local_msgqueue_base);
1659 ch_sn2->local_msgqueue = NULL;
1660 kfree(ch_sn2->notify_queue);
1661 ch_sn2->notify_queue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001662 }
1663 }
1664 return ret;
1665}
1666
1667/*
1668 * Free up message queues and other stuff that were allocated for the specified
1669 * channel.
Dean Nelson185c3a12008-07-29 22:34:11 -07001670 */
1671static void
Dean Nelson5b8669d2008-07-29 22:34:18 -07001672xpc_teardown_msg_structures_sn2(struct xpc_channel *ch)
Dean Nelson185c3a12008-07-29 22:34:11 -07001673{
1674 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1675
1676 DBUG_ON(!spin_is_locked(&ch->lock));
Dean Nelson185c3a12008-07-29 22:34:11 -07001677
Dean Nelson5b8669d2008-07-29 22:34:18 -07001678 ch_sn2->remote_msgqueue_pa = 0;
Dean Nelson185c3a12008-07-29 22:34:11 -07001679
1680 ch_sn2->local_GP->get = 0;
1681 ch_sn2->local_GP->put = 0;
1682 ch_sn2->remote_GP.get = 0;
1683 ch_sn2->remote_GP.put = 0;
1684 ch_sn2->w_local_GP.get = 0;
1685 ch_sn2->w_local_GP.put = 0;
1686 ch_sn2->w_remote_GP.get = 0;
1687 ch_sn2->w_remote_GP.put = 0;
1688 ch_sn2->next_msg_to_pull = 0;
1689
1690 if (ch->flags & XPC_C_SETUP) {
1691 dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
1692 ch->flags, ch->partid, ch->number);
1693
Dean Nelson5b8669d2008-07-29 22:34:18 -07001694 kfree(ch_sn2->local_msgqueue_base);
1695 ch_sn2->local_msgqueue = NULL;
1696 kfree(ch_sn2->remote_msgqueue_base);
1697 ch_sn2->remote_msgqueue = NULL;
1698 kfree(ch_sn2->notify_queue);
1699 ch_sn2->notify_queue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001700 }
1701}
1702
1703/*
Dean Nelsona47d5da2008-07-29 22:34:09 -07001704 * Notify those who wanted to be notified upon delivery of their message.
1705 */
1706static void
1707xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
1708{
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001709 struct xpc_notify_sn2 *notify;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001710 u8 notify_type;
1711 s64 get = ch->sn.sn2.w_remote_GP.get - 1;
1712
1713 while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
1714
Dean Nelson5b8669d2008-07-29 22:34:18 -07001715 notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001716
1717 /*
1718 * See if the notify entry indicates it was associated with
1719 * a message who's sender wants to be notified. It is possible
1720 * that it is, but someone else is doing or has done the
1721 * notification.
1722 */
1723 notify_type = notify->type;
1724 if (notify_type == 0 ||
1725 cmpxchg(&notify->type, notify_type, 0) != notify_type) {
1726 continue;
1727 }
1728
1729 DBUG_ON(notify_type != XPC_N_CALL);
1730
1731 atomic_dec(&ch->n_to_notify);
1732
1733 if (notify->func != NULL) {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001734 dev_dbg(xpc_chan, "notify->func() called, notify=0x%p "
Randy Dunlapf509e342009-09-04 17:02:59 -07001735 "msg_number=%lld partid=%d channel=%d\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -07001736 (void *)notify, get, ch->partid, ch->number);
1737
1738 notify->func(reason, ch->partid, ch->number,
1739 notify->key);
1740
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001741 dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p"
Randy Dunlapf509e342009-09-04 17:02:59 -07001742 " msg_number=%lld partid=%d channel=%d\n",
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001743 (void *)notify, get, ch->partid, ch->number);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001744 }
1745 }
1746}
1747
1748static void
1749xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
1750{
1751 xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
1752}
1753
1754/*
1755 * Clear some of the msg flags in the local message queue.
1756 */
1757static inline void
1758xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
1759{
1760 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001761 struct xpc_msg_sn2 *msg;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001762 s64 get;
1763
1764 get = ch_sn2->w_remote_GP.get;
1765 do {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001766 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
1767 (get % ch->local_nentries) *
1768 ch->entry_size);
Robin Holt252523e2009-01-29 14:25:07 -08001769 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsona47d5da2008-07-29 22:34:09 -07001770 msg->flags = 0;
1771 } while (++get < ch_sn2->remote_GP.get);
1772}
1773
1774/*
1775 * Clear some of the msg flags in the remote message queue.
1776 */
1777static inline void
1778xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
1779{
1780 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001781 struct xpc_msg_sn2 *msg;
Robin Holt6e873ec2009-04-02 16:59:14 -07001782 s64 put, remote_nentries = ch->remote_nentries;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001783
Robin Holt252523e2009-01-29 14:25:07 -08001784 /* flags are zeroed when the buffer is allocated */
Robin Holt6e873ec2009-04-02 16:59:14 -07001785 if (ch_sn2->remote_GP.put < remote_nentries)
Robin Holt252523e2009-01-29 14:25:07 -08001786 return;
1787
Robin Holt6e873ec2009-04-02 16:59:14 -07001788 put = max(ch_sn2->w_remote_GP.put, remote_nentries);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001789 do {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001790 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
Robin Holt6e873ec2009-04-02 16:59:14 -07001791 (put % remote_nentries) *
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001792 ch->entry_size);
Robin Holt252523e2009-01-29 14:25:07 -08001793 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1794 DBUG_ON(!(msg->flags & XPC_M_SN2_DONE));
Robin Holt6e873ec2009-04-02 16:59:14 -07001795 DBUG_ON(msg->number != put - remote_nentries);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001796 msg->flags = 0;
1797 } while (++put < ch_sn2->remote_GP.put);
1798}
1799
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001800static int
1801xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch)
1802{
1803 return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
1804}
1805
Dean Nelsona47d5da2008-07-29 22:34:09 -07001806static void
Dean Nelson7fb5e592008-07-29 22:34:10 -07001807xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number)
Dean Nelsona47d5da2008-07-29 22:34:09 -07001808{
1809 struct xpc_channel *ch = &part->channels[ch_number];
1810 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001811 int npayloads_sent;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001812
1813 ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
1814
1815 /* See what, if anything, has changed for each connected channel */
1816
1817 xpc_msgqueue_ref(ch);
1818
1819 if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
1820 ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
1821 /* nothing changed since GPs were last pulled */
1822 xpc_msgqueue_deref(ch);
1823 return;
1824 }
1825
1826 if (!(ch->flags & XPC_C_CONNECTED)) {
1827 xpc_msgqueue_deref(ch);
1828 return;
1829 }
1830
1831 /*
1832 * First check to see if messages recently sent by us have been
1833 * received by the other side. (The remote GET value will have
1834 * changed since we last looked at it.)
1835 */
1836
1837 if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
1838
1839 /*
1840 * We need to notify any senders that want to be notified
1841 * that their sent messages have been received by their
1842 * intended recipients. We need to do this before updating
1843 * w_remote_GP.get so that we don't allocate the same message
1844 * queue entries prematurely (see xpc_allocate_msg()).
1845 */
1846 if (atomic_read(&ch->n_to_notify) > 0) {
1847 /*
1848 * Notify senders that messages sent have been
1849 * received and delivered by the other side.
1850 */
1851 xpc_notify_senders_sn2(ch, xpMsgDelivered,
1852 ch_sn2->remote_GP.get);
1853 }
1854
1855 /*
1856 * Clear msg->flags in previously sent messages, so that
1857 * they're ready for xpc_allocate_msg().
1858 */
1859 xpc_clear_local_msgqueue_flags_sn2(ch);
1860
1861 ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
1862
Randy Dunlapf509e342009-09-04 17:02:59 -07001863 dev_dbg(xpc_chan, "w_remote_GP.get changed to %lld, partid=%d, "
Dean Nelsona47d5da2008-07-29 22:34:09 -07001864 "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
1865 ch->number);
1866
1867 /*
1868 * If anyone was waiting for message queue entries to become
1869 * available, wake them up.
1870 */
1871 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1872 wake_up(&ch->msg_allocate_wq);
1873 }
1874
1875 /*
1876 * Now check for newly sent messages by the other side. (The remote
1877 * PUT value will have changed since we last looked at it.)
1878 */
1879
1880 if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
1881 /*
1882 * Clear msg->flags in previously received messages, so that
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001883 * they're ready for xpc_get_deliverable_payload_sn2().
Dean Nelsona47d5da2008-07-29 22:34:09 -07001884 */
1885 xpc_clear_remote_msgqueue_flags_sn2(ch);
1886
Robin Holt69b3bb62009-01-29 14:25:06 -08001887 smp_wmb(); /* ensure flags have been cleared before bte_copy */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001888 ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
1889
Randy Dunlapf509e342009-09-04 17:02:59 -07001890 dev_dbg(xpc_chan, "w_remote_GP.put changed to %lld, partid=%d, "
Dean Nelsona47d5da2008-07-29 22:34:09 -07001891 "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
1892 ch->number);
1893
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001894 npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch);
1895 if (npayloads_sent > 0) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001896 dev_dbg(xpc_chan, "msgs waiting to be copied and "
1897 "delivered=%d, partid=%d, channel=%d\n",
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001898 npayloads_sent, ch->partid, ch->number);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001899
1900 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001901 xpc_activate_kthreads(ch, npayloads_sent);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001902 }
1903 }
1904
1905 xpc_msgqueue_deref(ch);
1906}
1907
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001908static struct xpc_msg_sn2 *
Dean Nelsone17d4162008-07-29 22:34:06 -07001909xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
1910{
1911 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001912 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001913 unsigned long remote_msg_pa;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001914 struct xpc_msg_sn2 *msg;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001915 u32 msg_index;
1916 u32 nmsgs;
Dean Nelsone17d4162008-07-29 22:34:06 -07001917 u64 msg_offset;
1918 enum xp_retval ret;
1919
Dean Nelsona47d5da2008-07-29 22:34:09 -07001920 if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001921 /* we were interrupted by a signal */
1922 return NULL;
1923 }
1924
Dean Nelsona47d5da2008-07-29 22:34:09 -07001925 while (get >= ch_sn2->next_msg_to_pull) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001926
1927 /* pull as many messages as are ready and able to be pulled */
1928
Dean Nelsona47d5da2008-07-29 22:34:09 -07001929 msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
Dean Nelsone17d4162008-07-29 22:34:06 -07001930
Dean Nelsona47d5da2008-07-29 22:34:09 -07001931 DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
1932 nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
Dean Nelsone17d4162008-07-29 22:34:06 -07001933 if (msg_index + nmsgs > ch->remote_nentries) {
1934 /* ignore the ones that wrap the msg queue for now */
1935 nmsgs = ch->remote_nentries - msg_index;
1936 }
1937
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001938 msg_offset = msg_index * ch->entry_size;
1939 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
Dean Nelson5b8669d2008-07-29 22:34:18 -07001940 msg_offset);
1941 remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset;
Dean Nelsone17d4162008-07-29 22:34:06 -07001942
Dean Nelsona812dcc2008-07-29 22:34:16 -07001943 ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa,
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001944 nmsgs * ch->entry_size);
Dean Nelsone17d4162008-07-29 22:34:06 -07001945 if (ret != xpSuccess) {
1946
1947 dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
Randy Dunlapf509e342009-09-04 17:02:59 -07001948 " msg %lld from partition %d, channel=%d, "
Dean Nelsona47d5da2008-07-29 22:34:09 -07001949 "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
Dean Nelsone17d4162008-07-29 22:34:06 -07001950 ch->partid, ch->number, ret);
1951
1952 XPC_DEACTIVATE_PARTITION(part, ret);
1953
Dean Nelsona47d5da2008-07-29 22:34:09 -07001954 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001955 return NULL;
1956 }
1957
Dean Nelsona47d5da2008-07-29 22:34:09 -07001958 ch_sn2->next_msg_to_pull += nmsgs;
Dean Nelsone17d4162008-07-29 22:34:06 -07001959 }
1960
Dean Nelsona47d5da2008-07-29 22:34:09 -07001961 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001962
1963 /* return the message we were looking for */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001964 msg_offset = (get % ch->remote_nentries) * ch->entry_size;
1965 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset);
Dean Nelsone17d4162008-07-29 22:34:06 -07001966
1967 return msg;
1968}
1969
1970/*
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001971 * Get the next deliverable message's payload.
Dean Nelsone17d4162008-07-29 22:34:06 -07001972 */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001973static void *
1974xpc_get_deliverable_payload_sn2(struct xpc_channel *ch)
Dean Nelsone17d4162008-07-29 22:34:06 -07001975{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001976 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001977 struct xpc_msg_sn2 *msg;
1978 void *payload = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001979 s64 get;
1980
1981 do {
1982 if (ch->flags & XPC_C_DISCONNECTING)
1983 break;
1984
Dean Nelsona47d5da2008-07-29 22:34:09 -07001985 get = ch_sn2->w_local_GP.get;
Robin Holt69b3bb62009-01-29 14:25:06 -08001986 smp_rmb(); /* guarantee that .get loads before .put */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001987 if (get == ch_sn2->w_remote_GP.put)
Dean Nelsone17d4162008-07-29 22:34:06 -07001988 break;
1989
1990 /* There are messages waiting to be pulled and delivered.
1991 * We need to try to secure one for ourselves. We'll do this
1992 * by trying to increment w_local_GP.get and hope that no one
1993 * else beats us to it. If they do, we'll we'll simply have
1994 * to try again for the next one.
1995 */
1996
Dean Nelsona47d5da2008-07-29 22:34:09 -07001997 if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001998 /* we got the entry referenced by get */
1999
Randy Dunlapf509e342009-09-04 17:02:59 -07002000 dev_dbg(xpc_chan, "w_local_GP.get changed to %lld, "
Dean Nelsone17d4162008-07-29 22:34:06 -07002001 "partid=%d, channel=%d\n", get + 1,
2002 ch->partid, ch->number);
2003
2004 /* pull the message from the remote partition */
2005
2006 msg = xpc_pull_remote_msg_sn2(ch, get);
2007
Robin Holt17e21612009-01-29 14:25:07 -08002008 if (msg != NULL) {
2009 DBUG_ON(msg->number != get);
2010 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
2011 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsone17d4162008-07-29 22:34:06 -07002012
Robin Holt17e21612009-01-29 14:25:07 -08002013 payload = &msg->payload;
2014 }
Dean Nelsone17d4162008-07-29 22:34:06 -07002015 break;
2016 }
2017
2018 } while (1);
2019
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002020 return payload;
Dean Nelsone17d4162008-07-29 22:34:06 -07002021}
2022
Dean Nelson33ba3c72008-07-29 22:34:07 -07002023/*
2024 * Now we actually send the messages that are ready to be sent by advancing
Dean Nelson7fb5e592008-07-29 22:34:10 -07002025 * the local message queue's Put value and then send a chctl msgrequest to the
2026 * recipient partition.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002027 */
2028static void
2029xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
2030{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002031 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002032 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002033 s64 put = initial_put + 1;
Dean Nelson7fb5e592008-07-29 22:34:10 -07002034 int send_msgrequest = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002035
2036 while (1) {
2037
2038 while (1) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07002039 if (put == ch_sn2->w_local_GP.put)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002040 break;
2041
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002042 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2043 local_msgqueue + (put %
2044 ch->local_nentries) *
2045 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002046
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002047 if (!(msg->flags & XPC_M_SN2_READY))
Dean Nelson33ba3c72008-07-29 22:34:07 -07002048 break;
2049
2050 put++;
2051 }
2052
2053 if (put == initial_put) {
2054 /* nothing's changed */
2055 break;
2056 }
2057
Dean Nelsona47d5da2008-07-29 22:34:09 -07002058 if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002059 initial_put) {
2060 /* someone else beat us to it */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002061 DBUG_ON(ch_sn2->local_GP->put < initial_put);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002062 break;
2063 }
2064
2065 /* we just set the new value of local_GP->put */
2066
Randy Dunlapf509e342009-09-04 17:02:59 -07002067 dev_dbg(xpc_chan, "local_GP->put changed to %lld, partid=%d, "
Dean Nelson33ba3c72008-07-29 22:34:07 -07002068 "channel=%d\n", put, ch->partid, ch->number);
2069
Dean Nelson7fb5e592008-07-29 22:34:10 -07002070 send_msgrequest = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002071
2072 /*
2073 * We need to ensure that the message referenced by
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002074 * local_GP->put is not XPC_M_SN2_READY or that local_GP->put
Dean Nelson33ba3c72008-07-29 22:34:07 -07002075 * equals w_local_GP.put, so we'll go have a look.
2076 */
2077 initial_put = put;
2078 }
2079
Dean Nelson7fb5e592008-07-29 22:34:10 -07002080 if (send_msgrequest)
2081 xpc_send_chctl_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002082}
2083
2084/*
2085 * Allocate an entry for a message from the message queue associated with the
2086 * specified channel.
2087 */
2088static enum xp_retval
2089xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002090 struct xpc_msg_sn2 **address_of_msg)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002091{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002092 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002093 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002094 enum xp_retval ret;
2095 s64 put;
2096
Dean Nelson33ba3c72008-07-29 22:34:07 -07002097 /*
2098 * Get the next available message entry from the local message queue.
2099 * If none are available, we'll make sure that we grab the latest
2100 * GP values.
2101 */
2102 ret = xpTimeout;
2103
2104 while (1) {
2105
Dean Nelsona47d5da2008-07-29 22:34:09 -07002106 put = ch_sn2->w_local_GP.put;
Robin Holt69b3bb62009-01-29 14:25:06 -08002107 smp_rmb(); /* guarantee that .put loads before .get */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002108 if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07002109
2110 /* There are available message entries. We need to try
2111 * to secure one for ourselves. We'll do this by trying
2112 * to increment w_local_GP.put as long as someone else
2113 * doesn't beat us to it. If they do, we'll have to
2114 * try again.
2115 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002116 if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
2117 put) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07002118 /* we got the entry referenced by put */
2119 break;
2120 }
2121 continue; /* try again */
2122 }
2123
2124 /*
2125 * There aren't any available msg entries at this time.
2126 *
2127 * In waiting for a message entry to become available,
Dean Nelson7fb5e592008-07-29 22:34:10 -07002128 * we set a timeout in case the other side is not sending
2129 * completion interrupts. This lets us fake a notify IRQ
2130 * that will cause the notify IRQ handler to fetch the latest
2131 * GP values as if an interrupt was sent by the other side.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002132 */
2133 if (ret == xpTimeout)
Dean Nelson7fb5e592008-07-29 22:34:10 -07002134 xpc_send_chctl_local_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002135
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002136 if (flags & XPC_NOWAIT)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002137 return xpNoWait;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002138
2139 ret = xpc_allocate_msg_wait(ch);
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002140 if (ret != xpInterrupted && ret != xpTimeout)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002141 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002142 }
2143
2144 /* get the message's address and initialize it */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002145 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
2146 (put % ch->local_nentries) *
2147 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002148
2149 DBUG_ON(msg->flags != 0);
2150 msg->number = put;
2151
Randy Dunlapf509e342009-09-04 17:02:59 -07002152 dev_dbg(xpc_chan, "w_local_GP.put changed to %lld; msg=0x%p, "
2153 "msg_number=%lld, partid=%d, channel=%d\n", put + 1,
Dean Nelson33ba3c72008-07-29 22:34:07 -07002154 (void *)msg, msg->number, ch->partid, ch->number);
2155
2156 *address_of_msg = msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002157 return xpSuccess;
2158}
2159
2160/*
2161 * Common code that does the actual sending of the message by advancing the
Dean Nelson7fb5e592008-07-29 22:34:10 -07002162 * local message queue's Put value and sends a chctl msgrequest to the
2163 * partition the message is being sent to.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002164 */
2165static enum xp_retval
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002166xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload,
2167 u16 payload_size, u8 notify_type, xpc_notify_func func,
2168 void *key)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002169{
2170 enum xp_retval ret = xpSuccess;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002171 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002172 struct xpc_msg_sn2 *msg = msg;
2173 struct xpc_notify_sn2 *notify = notify;
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002174 s64 msg_number;
2175 s64 put;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002176
2177 DBUG_ON(notify_type == XPC_N_CALL && func == NULL);
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002178
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002179 if (XPC_MSG_SIZE(payload_size) > ch->entry_size)
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002180 return xpPayloadTooBig;
2181
2182 xpc_msgqueue_ref(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002183
2184 if (ch->flags & XPC_C_DISCONNECTING) {
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002185 ret = ch->reason;
2186 goto out_1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002187 }
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002188 if (!(ch->flags & XPC_C_CONNECTED)) {
2189 ret = xpNotConnected;
2190 goto out_1;
2191 }
2192
2193 ret = xpc_allocate_msg_sn2(ch, flags, &msg);
2194 if (ret != xpSuccess)
2195 goto out_1;
2196
2197 msg_number = msg->number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002198
2199 if (notify_type != 0) {
2200 /*
2201 * Tell the remote side to send an ACK interrupt when the
2202 * message has been delivered.
2203 */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002204 msg->flags |= XPC_M_SN2_INTERRUPT;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002205
2206 atomic_inc(&ch->n_to_notify);
2207
Dean Nelson5b8669d2008-07-29 22:34:18 -07002208 notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries];
Dean Nelson33ba3c72008-07-29 22:34:07 -07002209 notify->func = func;
2210 notify->key = key;
2211 notify->type = notify_type;
2212
Dean Nelsonea57f802008-07-29 22:34:14 -07002213 /* ??? Is a mb() needed here? */
Dean Nelson33ba3c72008-07-29 22:34:07 -07002214
2215 if (ch->flags & XPC_C_DISCONNECTING) {
2216 /*
2217 * An error occurred between our last error check and
2218 * this one. We will try to clear the type field from
2219 * the notify entry. If we succeed then
2220 * xpc_disconnect_channel() didn't already process
2221 * the notify entry.
2222 */
2223 if (cmpxchg(&notify->type, notify_type, 0) ==
2224 notify_type) {
2225 atomic_dec(&ch->n_to_notify);
2226 ret = ch->reason;
2227 }
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002228 goto out_1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002229 }
2230 }
2231
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002232 memcpy(&msg->payload, payload, payload_size);
2233
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002234 msg->flags |= XPC_M_SN2_READY;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002235
2236 /*
2237 * The preceding store of msg->flags must occur before the following
Dean Nelsona47d5da2008-07-29 22:34:09 -07002238 * load of local_GP->put.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002239 */
Robin Holt69b3bb62009-01-29 14:25:06 -08002240 smp_mb();
Dean Nelson33ba3c72008-07-29 22:34:07 -07002241
2242 /* see if the message is next in line to be sent, if so send it */
2243
Dean Nelson5b8669d2008-07-29 22:34:18 -07002244 put = ch_sn2->local_GP->put;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002245 if (put == msg_number)
2246 xpc_send_msgs_sn2(ch, put);
2247
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002248out_1:
Dean Nelson33ba3c72008-07-29 22:34:07 -07002249 xpc_msgqueue_deref(ch);
2250 return ret;
2251}
2252
2253/*
2254 * Now we actually acknowledge the messages that have been delivered and ack'd
2255 * by advancing the cached remote message queue's Get value and if requested
Dean Nelson7fb5e592008-07-29 22:34:10 -07002256 * send a chctl msgrequest to the message sender's partition.
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002257 *
2258 * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition
2259 * that sent the message.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002260 */
2261static void
2262xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
2263{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002264 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002265 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002266 s64 get = initial_get + 1;
Dean Nelson7fb5e592008-07-29 22:34:10 -07002267 int send_msgrequest = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002268
2269 while (1) {
2270
2271 while (1) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07002272 if (get == ch_sn2->w_local_GP.get)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002273 break;
2274
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002275 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2276 remote_msgqueue + (get %
2277 ch->remote_nentries) *
2278 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002279
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002280 if (!(msg->flags & XPC_M_SN2_DONE))
Dean Nelson33ba3c72008-07-29 22:34:07 -07002281 break;
2282
2283 msg_flags |= msg->flags;
2284 get++;
2285 }
2286
2287 if (get == initial_get) {
2288 /* nothing's changed */
2289 break;
2290 }
2291
Dean Nelsona47d5da2008-07-29 22:34:09 -07002292 if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002293 initial_get) {
2294 /* someone else beat us to it */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002295 DBUG_ON(ch_sn2->local_GP->get <= initial_get);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002296 break;
2297 }
2298
2299 /* we just set the new value of local_GP->get */
2300
Randy Dunlapf509e342009-09-04 17:02:59 -07002301 dev_dbg(xpc_chan, "local_GP->get changed to %lld, partid=%d, "
Dean Nelson33ba3c72008-07-29 22:34:07 -07002302 "channel=%d\n", get, ch->partid, ch->number);
2303
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002304 send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002305
2306 /*
2307 * We need to ensure that the message referenced by
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002308 * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get
Dean Nelson33ba3c72008-07-29 22:34:07 -07002309 * equals w_local_GP.get, so we'll go have a look.
2310 */
2311 initial_get = get;
2312 }
2313
Dean Nelson7fb5e592008-07-29 22:34:10 -07002314 if (send_msgrequest)
2315 xpc_send_chctl_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002316}
2317
2318static void
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002319xpc_received_payload_sn2(struct xpc_channel *ch, void *payload)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002320{
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002321 struct xpc_msg_sn2 *msg;
2322 s64 msg_number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002323 s64 get;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002324
2325 msg = container_of(payload, struct xpc_msg_sn2, payload);
2326 msg_number = msg->number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002327
Randy Dunlapf509e342009-09-04 17:02:59 -07002328 dev_dbg(xpc_chan, "msg=0x%p, msg_number=%lld, partid=%d, channel=%d\n",
Dean Nelson33ba3c72008-07-29 22:34:07 -07002329 (void *)msg, msg_number, ch->partid, ch->number);
2330
Robin Holt252523e2009-01-29 14:25:07 -08002331 DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002332 msg_number % ch->remote_nentries);
Robin Holt252523e2009-01-29 14:25:07 -08002333 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002334 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002335
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002336 msg->flags |= XPC_M_SN2_DONE;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002337
2338 /*
2339 * The preceding store of msg->flags must occur before the following
Dean Nelsona47d5da2008-07-29 22:34:09 -07002340 * load of local_GP->get.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002341 */
Robin Holt69b3bb62009-01-29 14:25:06 -08002342 smp_mb();
Dean Nelson33ba3c72008-07-29 22:34:07 -07002343
2344 /*
2345 * See if this message is next in line to be acknowledged as having
2346 * been delivered.
2347 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002348 get = ch->sn.sn2.local_GP->get;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002349 if (get == msg_number)
2350 xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
2351}
2352
Robin Holta7665b02009-04-13 14:40:19 -07002353static struct xpc_arch_operations xpc_arch_ops_sn2 = {
2354 .setup_partitions = xpc_setup_partitions_sn2,
2355 .teardown_partitions = xpc_teardown_partitions_sn2,
2356 .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2,
2357 .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2,
2358 .setup_rsvd_page = xpc_setup_rsvd_page_sn2,
2359
2360 .allow_hb = xpc_allow_hb_sn2,
2361 .disallow_hb = xpc_disallow_hb_sn2,
2362 .disallow_all_hbs = xpc_disallow_all_hbs_sn2,
2363 .increment_heartbeat = xpc_increment_heartbeat_sn2,
2364 .offline_heartbeat = xpc_offline_heartbeat_sn2,
2365 .online_heartbeat = xpc_online_heartbeat_sn2,
2366 .heartbeat_init = xpc_heartbeat_init_sn2,
2367 .heartbeat_exit = xpc_heartbeat_exit_sn2,
2368 .get_remote_heartbeat = xpc_get_remote_heartbeat_sn2,
2369
2370 .request_partition_activation =
2371 xpc_request_partition_activation_sn2,
2372 .request_partition_reactivation =
2373 xpc_request_partition_reactivation_sn2,
2374 .request_partition_deactivation =
2375 xpc_request_partition_deactivation_sn2,
2376 .cancel_partition_deactivation_request =
2377 xpc_cancel_partition_deactivation_request_sn2,
2378
2379 .setup_ch_structures = xpc_setup_ch_structures_sn2,
2380 .teardown_ch_structures = xpc_teardown_ch_structures_sn2,
2381
2382 .make_first_contact = xpc_make_first_contact_sn2,
2383
2384 .get_chctl_all_flags = xpc_get_chctl_all_flags_sn2,
2385 .send_chctl_closerequest = xpc_send_chctl_closerequest_sn2,
2386 .send_chctl_closereply = xpc_send_chctl_closereply_sn2,
2387 .send_chctl_openrequest = xpc_send_chctl_openrequest_sn2,
2388 .send_chctl_openreply = xpc_send_chctl_openreply_sn2,
2389 .send_chctl_opencomplete = xpc_send_chctl_opencomplete_sn2,
2390 .process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2,
2391
2392 .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2,
2393
2394 .setup_msg_structures = xpc_setup_msg_structures_sn2,
2395 .teardown_msg_structures = xpc_teardown_msg_structures_sn2,
2396
2397 .indicate_partition_engaged = xpc_indicate_partition_engaged_sn2,
2398 .indicate_partition_disengaged = xpc_indicate_partition_disengaged_sn2,
2399 .partition_engaged = xpc_partition_engaged_sn2,
2400 .any_partition_engaged = xpc_any_partition_engaged_sn2,
2401 .assume_partition_disengaged = xpc_assume_partition_disengaged_sn2,
2402
2403 .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2,
2404 .send_payload = xpc_send_payload_sn2,
2405 .get_deliverable_payload = xpc_get_deliverable_payload_sn2,
2406 .received_payload = xpc_received_payload_sn2,
2407 .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2,
2408};
2409
Dean Nelson6e410172008-07-29 22:34:09 -07002410int
Dean Nelson94bd2702008-07-29 22:34:05 -07002411xpc_init_sn2(void)
2412{
Dean Nelson6e410172008-07-29 22:34:09 -07002413 int ret;
Dean Nelsonee6665e2008-07-29 22:34:13 -07002414 size_t buf_size;
Dean Nelson6e410172008-07-29 22:34:09 -07002415
Robin Holta7665b02009-04-13 14:40:19 -07002416 xpc_arch_ops = xpc_arch_ops_sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002417
2418 if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) {
2419 dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is "
2420 "larger than %d\n", XPC_MSG_HDR_MAX_SIZE);
2421 return -E2BIG;
2422 }
Dean Nelson6e410172008-07-29 22:34:09 -07002423
Dean Nelsonee6665e2008-07-29 22:34:13 -07002424 buf_size = max(XPC_RP_VARS_SIZE,
2425 XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
2426 xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
2427 GFP_KERNEL,
2428 &xpc_remote_copy_buffer_base_sn2);
2429 if (xpc_remote_copy_buffer_sn2 == NULL) {
2430 dev_err(xpc_part, "can't get memory for remote copy buffer\n");
2431 return -ENOMEM;
2432 }
2433
Dean Nelsonc39838c2008-07-29 22:34:11 -07002434 /* open up protections for IPI and [potentially] amo operations */
Dean Nelson6e410172008-07-29 22:34:09 -07002435 xpc_allow_IPI_ops_sn2();
Dean Nelsonc39838c2008-07-29 22:34:11 -07002436 xpc_allow_amo_ops_shub_wars_1_1_sn2();
Dean Nelson6e410172008-07-29 22:34:09 -07002437
2438 /*
2439 * This is safe to do before the xpc_hb_checker thread has started
2440 * because the handler releases a wait queue. If an interrupt is
2441 * received before the thread is waiting, it will not go to sleep,
2442 * but rather immediately process the interrupt.
2443 */
2444 ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0,
2445 "xpc hb", NULL);
2446 if (ret != 0) {
2447 dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
2448 "errno=%d\n", -ret);
2449 xpc_disallow_IPI_ops_sn2();
Dean Nelsonee6665e2008-07-29 22:34:13 -07002450 kfree(xpc_remote_copy_buffer_base_sn2);
Dean Nelson6e410172008-07-29 22:34:09 -07002451 }
2452 return ret;
Dean Nelson94bd2702008-07-29 22:34:05 -07002453}
2454
2455void
2456xpc_exit_sn2(void)
2457{
Dean Nelson6e410172008-07-29 22:34:09 -07002458 free_irq(SGI_XPC_ACTIVATE, NULL);
2459 xpc_disallow_IPI_ops_sn2();
Dean Nelsonee6665e2008-07-29 22:34:13 -07002460 kfree(xpc_remote_copy_buffer_base_sn2);
Dean Nelson94bd2702008-07-29 22:34:05 -07002461}