blob: da485ae2f5951e22b02491679e5b0a6a36170ef5 [file] [log] [blame]
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001/*
2 * CCI cache coherent interconnect driver
3 *
4 * Copyright (C) 2013 ARM Ltd.
5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
12 * kind, whether express or implied; without even the implied warranty
13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 */
16
17#include <linux/arm-cci.h>
18#include <linux/io.h>
19#include <linux/module.h>
20#include <linux/of_address.h>
Punit Agrawalb91c8f22013-08-22 14:41:51 +010021#include <linux/of_irq.h>
22#include <linux/of_platform.h>
23#include <linux/platform_device.h>
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010024#include <linux/slab.h>
Punit Agrawalb91c8f22013-08-22 14:41:51 +010025#include <linux/spinlock.h>
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010026
27#include <asm/cacheflush.h>
Punit Agrawalb91c8f22013-08-22 14:41:51 +010028#include <asm/irq_regs.h>
29#include <asm/pmu.h>
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010030#include <asm/smp_plat.h>
31
Punit Agrawalb91c8f22013-08-22 14:41:51 +010032#define DRIVER_NAME "CCI-400"
33#define DRIVER_NAME_PMU DRIVER_NAME " PMU"
34#define PMU_NAME "CCI_400"
35
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010036#define CCI_PORT_CTRL 0x0
37#define CCI_CTRL_STATUS 0xc
38
39#define CCI_ENABLE_SNOOP_REQ 0x1
40#define CCI_ENABLE_DVM_REQ 0x2
41#define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ)
42
43struct cci_nb_ports {
44 unsigned int nb_ace;
45 unsigned int nb_ace_lite;
46};
47
48enum cci_ace_port_type {
49 ACE_INVALID_PORT = 0x0,
50 ACE_PORT,
51 ACE_LITE_PORT,
52};
53
54struct cci_ace_port {
55 void __iomem *base;
Nicolas Pitre62158f82013-05-21 23:34:41 -040056 unsigned long phys;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010057 enum cci_ace_port_type type;
58 struct device_node *dn;
59};
60
61static struct cci_ace_port *ports;
62static unsigned int nb_cci_ports;
63
64static void __iomem *cci_ctrl_base;
Nicolas Pitre62158f82013-05-21 23:34:41 -040065static unsigned long cci_ctrl_phys;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010066
Punit Agrawalb91c8f22013-08-22 14:41:51 +010067#ifdef CONFIG_HW_PERF_EVENTS
68
69#define CCI_PMCR 0x0100
70#define CCI_PID2 0x0fe8
71
72#define CCI_PMCR_CEN 0x00000001
73#define CCI_PMCR_NCNT_MASK 0x0000f800
74#define CCI_PMCR_NCNT_SHIFT 11
75
76#define CCI_PID2_REV_MASK 0xf0
77#define CCI_PID2_REV_SHIFT 4
78
79/* Port ids */
80#define CCI_PORT_S0 0
81#define CCI_PORT_S1 1
82#define CCI_PORT_S2 2
83#define CCI_PORT_S3 3
84#define CCI_PORT_S4 4
85#define CCI_PORT_M0 5
86#define CCI_PORT_M1 6
87#define CCI_PORT_M2 7
88
89#define CCI_REV_R0 0
90#define CCI_REV_R1 1
Punit Agrawal6fb0c4a2014-02-19 12:17:02 +000091#define CCI_REV_R1_PX 5
Punit Agrawalb91c8f22013-08-22 14:41:51 +010092
93#define CCI_PMU_EVT_SEL 0x000
94#define CCI_PMU_CNTR 0x004
95#define CCI_PMU_CNTR_CTRL 0x008
96#define CCI_PMU_OVRFLW 0x00c
97
98#define CCI_PMU_OVRFLW_FLAG 1
99
100#define CCI_PMU_CNTR_BASE(idx) ((idx) * SZ_4K)
101
102/*
103 * Instead of an event id to monitor CCI cycles, a dedicated counter is
104 * provided. Use 0xff to represent CCI cycles and hope that no future revisions
105 * make use of this event in hardware.
106 */
107enum cci400_perf_events {
108 CCI_PMU_CYCLES = 0xff
109};
110
111#define CCI_PMU_EVENT_MASK 0xff
112#define CCI_PMU_EVENT_SOURCE(event) ((event >> 5) & 0x7)
113#define CCI_PMU_EVENT_CODE(event) (event & 0x1f)
114
115#define CCI_PMU_MAX_HW_EVENTS 5 /* CCI PMU has 4 counters + 1 cycle counter */
116
117#define CCI_PMU_CYCLE_CNTR_IDX 0
118#define CCI_PMU_CNTR0_IDX 1
119#define CCI_PMU_CNTR_LAST(cci_pmu) (CCI_PMU_CYCLE_CNTR_IDX + cci_pmu->num_events - 1)
120
121/*
122 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8
123 * ports and bits 4:0 are event codes. There are different event codes
124 * associated with each port type.
125 *
126 * Additionally, the range of events associated with the port types changed
127 * between Rev0 and Rev1.
128 *
129 * The constants below define the range of valid codes for each port type for
130 * the different revisions and are used to validate the event to be monitored.
131 */
132
133#define CCI_REV_R0_SLAVE_PORT_MIN_EV 0x00
134#define CCI_REV_R0_SLAVE_PORT_MAX_EV 0x13
135#define CCI_REV_R0_MASTER_PORT_MIN_EV 0x14
136#define CCI_REV_R0_MASTER_PORT_MAX_EV 0x1a
137
138#define CCI_REV_R1_SLAVE_PORT_MIN_EV 0x00
139#define CCI_REV_R1_SLAVE_PORT_MAX_EV 0x14
140#define CCI_REV_R1_MASTER_PORT_MIN_EV 0x00
141#define CCI_REV_R1_MASTER_PORT_MAX_EV 0x11
142
143struct pmu_port_event_ranges {
144 u8 slave_min;
145 u8 slave_max;
146 u8 master_min;
147 u8 master_max;
148};
149
150static struct pmu_port_event_ranges port_event_range[] = {
151 [CCI_REV_R0] = {
152 .slave_min = CCI_REV_R0_SLAVE_PORT_MIN_EV,
153 .slave_max = CCI_REV_R0_SLAVE_PORT_MAX_EV,
154 .master_min = CCI_REV_R0_MASTER_PORT_MIN_EV,
155 .master_max = CCI_REV_R0_MASTER_PORT_MAX_EV,
156 },
157 [CCI_REV_R1] = {
158 .slave_min = CCI_REV_R1_SLAVE_PORT_MIN_EV,
159 .slave_max = CCI_REV_R1_SLAVE_PORT_MAX_EV,
160 .master_min = CCI_REV_R1_MASTER_PORT_MIN_EV,
161 .master_max = CCI_REV_R1_MASTER_PORT_MAX_EV,
162 },
163};
164
165struct cci_pmu_drv_data {
166 void __iomem *base;
167 struct arm_pmu *cci_pmu;
168 int nr_irqs;
169 int irqs[CCI_PMU_MAX_HW_EVENTS];
170 unsigned long active_irqs;
171 struct perf_event *events[CCI_PMU_MAX_HW_EVENTS];
172 unsigned long used_mask[BITS_TO_LONGS(CCI_PMU_MAX_HW_EVENTS)];
173 struct pmu_port_event_ranges *port_ranges;
174 struct pmu_hw_events hw_events;
175};
176static struct cci_pmu_drv_data *pmu;
177
178static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)
179{
180 int i;
181
182 for (i = 0; i < nr_irqs; i++)
183 if (irq == irqs[i])
184 return true;
185
186 return false;
187}
188
189static int probe_cci_revision(void)
190{
191 int rev;
192 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK;
193 rev >>= CCI_PID2_REV_SHIFT;
194
Punit Agrawal6fb0c4a2014-02-19 12:17:02 +0000195 if (rev < CCI_REV_R1_PX)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100196 return CCI_REV_R0;
Punit Agrawal6fb0c4a2014-02-19 12:17:02 +0000197 else
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100198 return CCI_REV_R1;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100199}
200
201static struct pmu_port_event_ranges *port_range_by_rev(void)
202{
203 int rev = probe_cci_revision();
204
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100205 return &port_event_range[rev];
206}
207
208static int pmu_is_valid_slave_event(u8 ev_code)
209{
210 return pmu->port_ranges->slave_min <= ev_code &&
211 ev_code <= pmu->port_ranges->slave_max;
212}
213
214static int pmu_is_valid_master_event(u8 ev_code)
215{
216 return pmu->port_ranges->master_min <= ev_code &&
217 ev_code <= pmu->port_ranges->master_max;
218}
219
220static int pmu_validate_hw_event(u8 hw_event)
221{
222 u8 ev_source = CCI_PMU_EVENT_SOURCE(hw_event);
223 u8 ev_code = CCI_PMU_EVENT_CODE(hw_event);
224
225 switch (ev_source) {
226 case CCI_PORT_S0:
227 case CCI_PORT_S1:
228 case CCI_PORT_S2:
229 case CCI_PORT_S3:
230 case CCI_PORT_S4:
231 /* Slave Interface */
232 if (pmu_is_valid_slave_event(ev_code))
233 return hw_event;
234 break;
235 case CCI_PORT_M0:
236 case CCI_PORT_M1:
237 case CCI_PORT_M2:
238 /* Master Interface */
239 if (pmu_is_valid_master_event(ev_code))
240 return hw_event;
241 break;
242 }
243
244 return -ENOENT;
245}
246
247static int pmu_is_valid_counter(struct arm_pmu *cci_pmu, int idx)
248{
249 return CCI_PMU_CYCLE_CNTR_IDX <= idx &&
250 idx <= CCI_PMU_CNTR_LAST(cci_pmu);
251}
252
253static u32 pmu_read_register(int idx, unsigned int offset)
254{
255 return readl_relaxed(pmu->base + CCI_PMU_CNTR_BASE(idx) + offset);
256}
257
258static void pmu_write_register(u32 value, int idx, unsigned int offset)
259{
260 return writel_relaxed(value, pmu->base + CCI_PMU_CNTR_BASE(idx) + offset);
261}
262
263static void pmu_disable_counter(int idx)
264{
265 pmu_write_register(0, idx, CCI_PMU_CNTR_CTRL);
266}
267
268static void pmu_enable_counter(int idx)
269{
270 pmu_write_register(1, idx, CCI_PMU_CNTR_CTRL);
271}
272
273static void pmu_set_event(int idx, unsigned long event)
274{
275 event &= CCI_PMU_EVENT_MASK;
276 pmu_write_register(event, idx, CCI_PMU_EVT_SEL);
277}
278
279static u32 pmu_get_max_counters(void)
280{
281 u32 n_cnts = (readl_relaxed(cci_ctrl_base + CCI_PMCR) &
282 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT;
283
284 /* add 1 for cycle counter */
285 return n_cnts + 1;
286}
287
288static struct pmu_hw_events *pmu_get_hw_events(void)
289{
290 return &pmu->hw_events;
291}
292
293static int pmu_get_event_idx(struct pmu_hw_events *hw, struct perf_event *event)
294{
295 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
296 struct hw_perf_event *hw_event = &event->hw;
297 unsigned long cci_event = hw_event->config_base & CCI_PMU_EVENT_MASK;
298 int idx;
299
300 if (cci_event == CCI_PMU_CYCLES) {
301 if (test_and_set_bit(CCI_PMU_CYCLE_CNTR_IDX, hw->used_mask))
302 return -EAGAIN;
303
304 return CCI_PMU_CYCLE_CNTR_IDX;
305 }
306
307 for (idx = CCI_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx)
308 if (!test_and_set_bit(idx, hw->used_mask))
309 return idx;
310
311 /* No counters available */
312 return -EAGAIN;
313}
314
315static int pmu_map_event(struct perf_event *event)
316{
317 int mapping;
318 u8 config = event->attr.config & CCI_PMU_EVENT_MASK;
319
320 if (event->attr.type < PERF_TYPE_MAX)
321 return -ENOENT;
322
323 if (config == CCI_PMU_CYCLES)
324 mapping = config;
325 else
326 mapping = pmu_validate_hw_event(config);
327
328 return mapping;
329}
330
331static int pmu_request_irq(struct arm_pmu *cci_pmu, irq_handler_t handler)
332{
333 int i;
334 struct platform_device *pmu_device = cci_pmu->plat_device;
335
336 if (unlikely(!pmu_device))
337 return -ENODEV;
338
339 if (pmu->nr_irqs < 1) {
340 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n");
341 return -ENODEV;
342 }
343
344 /*
345 * Register all available CCI PMU interrupts. In the interrupt handler
346 * we iterate over the counters checking for interrupt source (the
347 * overflowing counter) and clear it.
348 *
349 * This should allow handling of non-unique interrupt for the counters.
350 */
351 for (i = 0; i < pmu->nr_irqs; i++) {
352 int err = request_irq(pmu->irqs[i], handler, IRQF_SHARED,
353 "arm-cci-pmu", cci_pmu);
354 if (err) {
355 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",
356 pmu->irqs[i]);
357 return err;
358 }
359
360 set_bit(i, &pmu->active_irqs);
361 }
362
363 return 0;
364}
365
366static irqreturn_t pmu_handle_irq(int irq_num, void *dev)
367{
368 unsigned long flags;
369 struct arm_pmu *cci_pmu = (struct arm_pmu *)dev;
370 struct pmu_hw_events *events = cci_pmu->get_hw_events();
371 struct perf_sample_data data;
372 struct pt_regs *regs;
373 int idx, handled = IRQ_NONE;
374
375 raw_spin_lock_irqsave(&events->pmu_lock, flags);
376 regs = get_irq_regs();
377 /*
378 * Iterate over counters and update the corresponding perf events.
379 * This should work regardless of whether we have per-counter overflow
380 * interrupt or a combined overflow interrupt.
381 */
382 for (idx = CCI_PMU_CYCLE_CNTR_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) {
383 struct perf_event *event = events->events[idx];
384 struct hw_perf_event *hw_counter;
385
386 if (!event)
387 continue;
388
389 hw_counter = &event->hw;
390
391 /* Did this counter overflow? */
392 if (!pmu_read_register(idx, CCI_PMU_OVRFLW) & CCI_PMU_OVRFLW_FLAG)
393 continue;
394
395 pmu_write_register(CCI_PMU_OVRFLW_FLAG, idx, CCI_PMU_OVRFLW);
396
397 handled = IRQ_HANDLED;
398
399 armpmu_event_update(event);
400 perf_sample_data_init(&data, 0, hw_counter->last_period);
401 if (!armpmu_event_set_period(event))
402 continue;
403
404 if (perf_event_overflow(event, &data, regs))
405 cci_pmu->disable(event);
406 }
407 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
408
409 return IRQ_RETVAL(handled);
410}
411
412static void pmu_free_irq(struct arm_pmu *cci_pmu)
413{
414 int i;
415
416 for (i = 0; i < pmu->nr_irqs; i++) {
417 if (!test_and_clear_bit(i, &pmu->active_irqs))
418 continue;
419
420 free_irq(pmu->irqs[i], cci_pmu);
421 }
422}
423
424static void pmu_enable_event(struct perf_event *event)
425{
426 unsigned long flags;
427 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
428 struct pmu_hw_events *events = cci_pmu->get_hw_events();
429 struct hw_perf_event *hw_counter = &event->hw;
430 int idx = hw_counter->idx;
431
432 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
433 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
434 return;
435 }
436
437 raw_spin_lock_irqsave(&events->pmu_lock, flags);
438
439 /* Configure the event to count, unless you are counting cycles */
440 if (idx != CCI_PMU_CYCLE_CNTR_IDX)
441 pmu_set_event(idx, hw_counter->config_base);
442
443 pmu_enable_counter(idx);
444
445 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
446}
447
448static void pmu_disable_event(struct perf_event *event)
449{
450 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
451 struct hw_perf_event *hw_counter = &event->hw;
452 int idx = hw_counter->idx;
453
454 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
455 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
456 return;
457 }
458
459 pmu_disable_counter(idx);
460}
461
462static void pmu_start(struct arm_pmu *cci_pmu)
463{
464 u32 val;
465 unsigned long flags;
466 struct pmu_hw_events *events = cci_pmu->get_hw_events();
467
468 raw_spin_lock_irqsave(&events->pmu_lock, flags);
469
470 /* Enable all the PMU counters. */
471 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN;
472 writel(val, cci_ctrl_base + CCI_PMCR);
473
474 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
475}
476
477static void pmu_stop(struct arm_pmu *cci_pmu)
478{
479 u32 val;
480 unsigned long flags;
481 struct pmu_hw_events *events = cci_pmu->get_hw_events();
482
483 raw_spin_lock_irqsave(&events->pmu_lock, flags);
484
485 /* Disable all the PMU counters. */
486 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN;
487 writel(val, cci_ctrl_base + CCI_PMCR);
488
489 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
490}
491
492static u32 pmu_read_counter(struct perf_event *event)
493{
494 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
495 struct hw_perf_event *hw_counter = &event->hw;
496 int idx = hw_counter->idx;
497 u32 value;
498
499 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
500 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
501 return 0;
502 }
503 value = pmu_read_register(idx, CCI_PMU_CNTR);
504
505 return value;
506}
507
508static void pmu_write_counter(struct perf_event *event, u32 value)
509{
510 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
511 struct hw_perf_event *hw_counter = &event->hw;
512 int idx = hw_counter->idx;
513
514 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx)))
515 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
516 else
517 pmu_write_register(value, idx, CCI_PMU_CNTR);
518}
519
520static int cci_pmu_init(struct arm_pmu *cci_pmu, struct platform_device *pdev)
521{
522 *cci_pmu = (struct arm_pmu){
523 .name = PMU_NAME,
524 .max_period = (1LLU << 32) - 1,
525 .get_hw_events = pmu_get_hw_events,
526 .get_event_idx = pmu_get_event_idx,
527 .map_event = pmu_map_event,
528 .request_irq = pmu_request_irq,
529 .handle_irq = pmu_handle_irq,
530 .free_irq = pmu_free_irq,
531 .enable = pmu_enable_event,
532 .disable = pmu_disable_event,
533 .start = pmu_start,
534 .stop = pmu_stop,
535 .read_counter = pmu_read_counter,
536 .write_counter = pmu_write_counter,
537 };
538
539 cci_pmu->plat_device = pdev;
540 cci_pmu->num_events = pmu_get_max_counters();
541
542 return armpmu_register(cci_pmu, -1);
543}
544
545static const struct of_device_id arm_cci_pmu_matches[] = {
546 {
547 .compatible = "arm,cci-400-pmu",
548 },
549 {},
550};
551
552static int cci_pmu_probe(struct platform_device *pdev)
553{
554 struct resource *res;
555 int i, ret, irq;
556
557 pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);
558 if (!pmu)
559 return -ENOMEM;
560
561 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100562 pmu->base = devm_ioremap_resource(&pdev->dev, res);
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100563 if (IS_ERR(pmu->base))
564 return -ENOMEM;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100565
566 /*
567 * CCI PMU has 5 overflow signals - one per counter; but some may be tied
568 * together to a common interrupt.
569 */
570 pmu->nr_irqs = 0;
571 for (i = 0; i < CCI_PMU_MAX_HW_EVENTS; i++) {
572 irq = platform_get_irq(pdev, i);
573 if (irq < 0)
574 break;
575
576 if (is_duplicate_irq(irq, pmu->irqs, pmu->nr_irqs))
577 continue;
578
579 pmu->irqs[pmu->nr_irqs++] = irq;
580 }
581
582 /*
583 * Ensure that the device tree has as many interrupts as the number
584 * of counters.
585 */
586 if (i < CCI_PMU_MAX_HW_EVENTS) {
587 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n",
588 i, CCI_PMU_MAX_HW_EVENTS);
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100589 return -EINVAL;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100590 }
591
592 pmu->port_ranges = port_range_by_rev();
593 if (!pmu->port_ranges) {
594 dev_warn(&pdev->dev, "CCI PMU version not supported\n");
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100595 return -EINVAL;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100596 }
597
598 pmu->cci_pmu = devm_kzalloc(&pdev->dev, sizeof(*(pmu->cci_pmu)), GFP_KERNEL);
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100599 if (!pmu->cci_pmu)
600 return -ENOMEM;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100601
602 pmu->hw_events.events = pmu->events;
603 pmu->hw_events.used_mask = pmu->used_mask;
604 raw_spin_lock_init(&pmu->hw_events.pmu_lock);
605
606 ret = cci_pmu_init(pmu->cci_pmu, pdev);
607 if (ret)
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100608 return ret;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100609
610 return 0;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100611}
612
613static int cci_platform_probe(struct platform_device *pdev)
614{
615 if (!cci_probed())
616 return -ENODEV;
617
618 return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
619}
620
621#endif /* CONFIG_HW_PERF_EVENTS */
622
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100623struct cpu_port {
624 u64 mpidr;
625 u32 port;
626};
Nicolas Pitre62158f82013-05-21 23:34:41 -0400627
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100628/*
629 * Use the port MSB as valid flag, shift can be made dynamic
630 * by computing number of bits required for port indexes.
631 * Code disabling CCI cpu ports runs with D-cache invalidated
632 * and SCTLR bit clear so data accesses must be kept to a minimum
633 * to improve performance; for now shift is left static to
634 * avoid one more data access while disabling the CCI port.
635 */
636#define PORT_VALID_SHIFT 31
637#define PORT_VALID (0x1 << PORT_VALID_SHIFT)
638
639static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr)
640{
641 port->port = PORT_VALID | index;
642 port->mpidr = mpidr;
643}
644
645static inline bool cpu_port_is_valid(struct cpu_port *port)
646{
647 return !!(port->port & PORT_VALID);
648}
649
650static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr)
651{
652 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK);
653}
654
655static struct cpu_port cpu_port[NR_CPUS];
656
657/**
658 * __cci_ace_get_port - Function to retrieve the port index connected to
659 * a cpu or device.
660 *
661 * @dn: device node of the device to look-up
662 * @type: port type
663 *
664 * Return value:
665 * - CCI port index if success
666 * - -ENODEV if failure
667 */
668static int __cci_ace_get_port(struct device_node *dn, int type)
669{
670 int i;
671 bool ace_match;
672 struct device_node *cci_portn;
673
674 cci_portn = of_parse_phandle(dn, "cci-control-port", 0);
675 for (i = 0; i < nb_cci_ports; i++) {
676 ace_match = ports[i].type == type;
677 if (ace_match && cci_portn == ports[i].dn)
678 return i;
679 }
680 return -ENODEV;
681}
682
683int cci_ace_get_port(struct device_node *dn)
684{
685 return __cci_ace_get_port(dn, ACE_LITE_PORT);
686}
687EXPORT_SYMBOL_GPL(cci_ace_get_port);
688
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100689static void cci_ace_init_ports(void)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100690{
Sudeep KarkadaNagesha78b4d6e2013-06-17 14:51:48 +0100691 int port, cpu;
692 struct device_node *cpun;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100693
694 /*
695 * Port index look-up speeds up the function disabling ports by CPU,
696 * since the logical to port index mapping is done once and does
697 * not change after system boot.
698 * The stashed index array is initialized for all possible CPUs
699 * at probe time.
700 */
Sudeep KarkadaNagesha78b4d6e2013-06-17 14:51:48 +0100701 for_each_possible_cpu(cpu) {
702 /* too early to use cpu->of_node */
703 cpun = of_get_cpu_node(cpu, NULL);
704
705 if (WARN(!cpun, "Missing cpu device node\n"))
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100706 continue;
707
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100708 port = __cci_ace_get_port(cpun, ACE_PORT);
709 if (port < 0)
710 continue;
711
712 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu));
713 }
714
715 for_each_possible_cpu(cpu) {
716 WARN(!cpu_port_is_valid(&cpu_port[cpu]),
717 "CPU %u does not have an associated CCI port\n",
718 cpu);
719 }
720}
721/*
722 * Functions to enable/disable a CCI interconnect slave port
723 *
724 * They are called by low-level power management code to disable slave
725 * interfaces snoops and DVM broadcast.
726 * Since they may execute with cache data allocation disabled and
727 * after the caches have been cleaned and invalidated the functions provide
728 * no explicit locking since they may run with D-cache disabled, so normal
729 * cacheable kernel locks based on ldrex/strex may not work.
730 * Locking has to be provided by BSP implementations to ensure proper
731 * operations.
732 */
733
734/**
735 * cci_port_control() - function to control a CCI port
736 *
737 * @port: index of the port to setup
738 * @enable: if true enables the port, if false disables it
739 */
740static void notrace cci_port_control(unsigned int port, bool enable)
741{
742 void __iomem *base = ports[port].base;
743
744 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL);
745 /*
746 * This function is called from power down procedures
747 * and must not execute any instruction that might
748 * cause the processor to be put in a quiescent state
749 * (eg wfi). Hence, cpu_relax() can not be added to this
750 * read loop to optimize power, since it might hide possibly
751 * disruptive operations.
752 */
753 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1)
754 ;
755}
756
757/**
758 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU
759 * reference
760 *
761 * @mpidr: mpidr of the CPU whose CCI port should be disabled
762 *
763 * Disabling a CCI port for a CPU implies disabling the CCI port
764 * controlling that CPU cluster. Code disabling CPU CCI ports
765 * must make sure that the CPU running the code is the last active CPU
766 * in the cluster ie all other CPUs are quiescent in a low power state.
767 *
768 * Return:
769 * 0 on success
770 * -ENODEV on port look-up failure
771 */
772int notrace cci_disable_port_by_cpu(u64 mpidr)
773{
774 int cpu;
775 bool is_valid;
776 for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
777 is_valid = cpu_port_is_valid(&cpu_port[cpu]);
778 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) {
779 cci_port_control(cpu_port[cpu].port, false);
780 return 0;
781 }
782 }
783 return -ENODEV;
784}
785EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu);
786
787/**
Nicolas Pitre62158f82013-05-21 23:34:41 -0400788 * cci_enable_port_for_self() - enable a CCI port for calling CPU
789 *
790 * Enabling a CCI port for the calling CPU implies enabling the CCI
791 * port controlling that CPU's cluster. Caller must make sure that the
792 * CPU running the code is the first active CPU in the cluster and all
793 * other CPUs are quiescent in a low power state or waiting for this CPU
794 * to complete the CCI initialization.
795 *
796 * Because this is called when the MMU is still off and with no stack,
797 * the code must be position independent and ideally rely on callee
798 * clobbered registers only. To achieve this we must code this function
799 * entirely in assembler.
800 *
801 * On success this returns with the proper CCI port enabled. In case of
802 * any failure this never returns as the inability to enable the CCI is
803 * fatal and there is no possible recovery at this stage.
804 */
805asmlinkage void __naked cci_enable_port_for_self(void)
806{
807 asm volatile ("\n"
Arnd Bergmannf4902492013-06-03 15:15:36 +0200808" .arch armv7-a\n"
Nicolas Pitre62158f82013-05-21 23:34:41 -0400809" mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n"
810" and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n"
811" adr r1, 5f \n"
812" ldr r2, [r1] \n"
813" add r1, r1, r2 @ &cpu_port \n"
814" add ip, r1, %[sizeof_cpu_port] \n"
815
816 /* Loop over the cpu_port array looking for a matching MPIDR */
817"1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n"
818" cmp r2, r0 @ compare MPIDR \n"
819" bne 2f \n"
820
821 /* Found a match, now test port validity */
822" ldr r3, [r1, %[offsetof_cpu_port_port]] \n"
823" tst r3, #"__stringify(PORT_VALID)" \n"
824" bne 3f \n"
825
826 /* no match, loop with the next cpu_port entry */
827"2: add r1, r1, %[sizeof_struct_cpu_port] \n"
828" cmp r1, ip @ done? \n"
829" blo 1b \n"
830
831 /* CCI port not found -- cheaply try to stall this CPU */
832"cci_port_not_found: \n"
833" wfi \n"
834" wfe \n"
835" b cci_port_not_found \n"
836
837 /* Use matched port index to look up the corresponding ports entry */
838"3: bic r3, r3, #"__stringify(PORT_VALID)" \n"
839" adr r0, 6f \n"
840" ldmia r0, {r1, r2} \n"
841" sub r1, r1, r0 @ virt - phys \n"
842" ldr r0, [r0, r2] @ *(&ports) \n"
843" mov r2, %[sizeof_struct_ace_port] \n"
844" mla r0, r2, r3, r0 @ &ports[index] \n"
845" sub r0, r0, r1 @ virt_to_phys() \n"
846
847 /* Enable the CCI port */
848" ldr r0, [r0, %[offsetof_port_phys]] \n"
Victor Kamenskyfdb07ae2013-10-15 21:50:34 -0700849" mov r3, %[cci_enable_req]\n"
Nicolas Pitre62158f82013-05-21 23:34:41 -0400850" str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n"
851
852 /* poll the status reg for completion */
853" adr r1, 7f \n"
854" ldr r0, [r1] \n"
855" ldr r0, [r0, r1] @ cci_ctrl_base \n"
856"4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n"
Victor Kamenskyfdb07ae2013-10-15 21:50:34 -0700857" tst r1, %[cci_control_status_bits] \n"
Nicolas Pitre62158f82013-05-21 23:34:41 -0400858" bne 4b \n"
859
860" mov r0, #0 \n"
861" bx lr \n"
862
863" .align 2 \n"
864"5: .word cpu_port - . \n"
865"6: .word . \n"
866" .word ports - 6b \n"
867"7: .word cci_ctrl_phys - . \n"
868 : :
869 [sizeof_cpu_port] "i" (sizeof(cpu_port)),
Victor Kamenskyfdb07ae2013-10-15 21:50:34 -0700870 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ),
871 [cci_control_status_bits] "i" cpu_to_le32(1),
Nicolas Pitre62158f82013-05-21 23:34:41 -0400872#ifndef __ARMEB__
873 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)),
874#else
875 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4),
876#endif
877 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)),
878 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)),
879 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)),
880 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) );
881
882 unreachable();
883}
884
885/**
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100886 * __cci_control_port_by_device() - function to control a CCI port by device
887 * reference
888 *
889 * @dn: device node pointer of the device whose CCI port should be
890 * controlled
891 * @enable: if true enables the port, if false disables it
892 *
893 * Return:
894 * 0 on success
895 * -ENODEV on port look-up failure
896 */
897int notrace __cci_control_port_by_device(struct device_node *dn, bool enable)
898{
899 int port;
900
901 if (!dn)
902 return -ENODEV;
903
904 port = __cci_ace_get_port(dn, ACE_LITE_PORT);
905 if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n",
906 dn->full_name))
907 return -ENODEV;
908 cci_port_control(port, enable);
909 return 0;
910}
911EXPORT_SYMBOL_GPL(__cci_control_port_by_device);
912
913/**
914 * __cci_control_port_by_index() - function to control a CCI port by port index
915 *
916 * @port: port index previously retrieved with cci_ace_get_port()
917 * @enable: if true enables the port, if false disables it
918 *
919 * Return:
920 * 0 on success
921 * -ENODEV on port index out of range
922 * -EPERM if operation carried out on an ACE PORT
923 */
924int notrace __cci_control_port_by_index(u32 port, bool enable)
925{
926 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT)
927 return -ENODEV;
928 /*
929 * CCI control for ports connected to CPUS is extremely fragile
930 * and must be made to go through a specific and controlled
931 * interface (ie cci_disable_port_by_cpu(); control by general purpose
932 * indexing is therefore disabled for ACE ports.
933 */
934 if (ports[port].type == ACE_PORT)
935 return -EPERM;
936
937 cci_port_control(port, enable);
938 return 0;
939}
940EXPORT_SYMBOL_GPL(__cci_control_port_by_index);
941
942static const struct cci_nb_ports cci400_ports = {
943 .nb_ace = 2,
944 .nb_ace_lite = 3
945};
946
947static const struct of_device_id arm_cci_matches[] = {
948 {.compatible = "arm,cci-400", .data = &cci400_ports },
949 {},
950};
951
952static const struct of_device_id arm_cci_ctrl_if_matches[] = {
953 {.compatible = "arm,cci-400-ctrl-if", },
954 {},
955};
956
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100957static int cci_probe(void)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100958{
959 struct cci_nb_ports const *cci_config;
960 int ret, i, nb_ace = 0, nb_ace_lite = 0;
961 struct device_node *np, *cp;
Nicolas Pitre62158f82013-05-21 23:34:41 -0400962 struct resource res;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100963 const char *match_str;
964 bool is_ace;
965
966 np = of_find_matching_node(NULL, arm_cci_matches);
967 if (!np)
968 return -ENODEV;
969
970 cci_config = of_match_node(arm_cci_matches, np)->data;
971 if (!cci_config)
972 return -ENODEV;
973
974 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite;
975
Lorenzo Pieralisi7c762032014-01-27 10:50:37 +0000976 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL);
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100977 if (!ports)
978 return -ENOMEM;
979
Nicolas Pitre62158f82013-05-21 23:34:41 -0400980 ret = of_address_to_resource(np, 0, &res);
981 if (!ret) {
982 cci_ctrl_base = ioremap(res.start, resource_size(&res));
983 cci_ctrl_phys = res.start;
984 }
985 if (ret || !cci_ctrl_base) {
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +0100986 WARN(1, "unable to ioremap CCI ctrl\n");
987 ret = -ENXIO;
988 goto memalloc_err;
989 }
990
991 for_each_child_of_node(np, cp) {
992 if (!of_match_node(arm_cci_ctrl_if_matches, cp))
993 continue;
994
995 i = nb_ace + nb_ace_lite;
996
997 if (i >= nb_cci_ports)
998 break;
999
1000 if (of_property_read_string(cp, "interface-type",
1001 &match_str)) {
1002 WARN(1, "node %s missing interface-type property\n",
1003 cp->full_name);
1004 continue;
1005 }
1006 is_ace = strcmp(match_str, "ace") == 0;
1007 if (!is_ace && strcmp(match_str, "ace-lite")) {
1008 WARN(1, "node %s containing invalid interface-type property, skipping it\n",
1009 cp->full_name);
1010 continue;
1011 }
1012
Nicolas Pitre62158f82013-05-21 23:34:41 -04001013 ret = of_address_to_resource(cp, 0, &res);
1014 if (!ret) {
1015 ports[i].base = ioremap(res.start, resource_size(&res));
1016 ports[i].phys = res.start;
1017 }
1018 if (ret || !ports[i].base) {
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001019 WARN(1, "unable to ioremap CCI port %d\n", i);
1020 continue;
1021 }
1022
1023 if (is_ace) {
1024 if (WARN_ON(nb_ace >= cci_config->nb_ace))
1025 continue;
1026 ports[i].type = ACE_PORT;
1027 ++nb_ace;
1028 } else {
1029 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite))
1030 continue;
1031 ports[i].type = ACE_LITE_PORT;
1032 ++nb_ace_lite;
1033 }
1034 ports[i].dn = cp;
1035 }
1036
1037 /* initialize a stashed array of ACE ports to speed-up look-up */
1038 cci_ace_init_ports();
1039
1040 /*
1041 * Multi-cluster systems may need this data when non-coherent, during
1042 * cluster power-up/power-down. Make sure it reaches main memory.
1043 */
1044 sync_cache_w(&cci_ctrl_base);
Nicolas Pitre62158f82013-05-21 23:34:41 -04001045 sync_cache_w(&cci_ctrl_phys);
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001046 sync_cache_w(&ports);
1047 sync_cache_w(&cpu_port);
1048 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports);
1049 pr_info("ARM CCI driver probed\n");
1050 return 0;
1051
1052memalloc_err:
1053
1054 kfree(ports);
1055 return ret;
1056}
1057
1058static int cci_init_status = -EAGAIN;
1059static DEFINE_MUTEX(cci_probing);
1060
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001061static int cci_init(void)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001062{
1063 if (cci_init_status != -EAGAIN)
1064 return cci_init_status;
1065
1066 mutex_lock(&cci_probing);
1067 if (cci_init_status == -EAGAIN)
1068 cci_init_status = cci_probe();
1069 mutex_unlock(&cci_probing);
1070 return cci_init_status;
1071}
1072
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001073#ifdef CONFIG_HW_PERF_EVENTS
1074static struct platform_driver cci_pmu_driver = {
1075 .driver = {
1076 .name = DRIVER_NAME_PMU,
1077 .of_match_table = arm_cci_pmu_matches,
1078 },
1079 .probe = cci_pmu_probe,
1080};
1081
1082static struct platform_driver cci_platform_driver = {
1083 .driver = {
1084 .name = DRIVER_NAME,
1085 .of_match_table = arm_cci_matches,
1086 },
1087 .probe = cci_platform_probe,
1088};
1089
1090static int __init cci_platform_init(void)
1091{
1092 int ret;
1093
1094 ret = platform_driver_register(&cci_pmu_driver);
1095 if (ret)
1096 return ret;
1097
1098 return platform_driver_register(&cci_platform_driver);
1099}
1100
1101#else
1102
1103static int __init cci_platform_init(void)
1104{
1105 return 0;
1106}
1107
1108#endif
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001109/*
1110 * To sort out early init calls ordering a helper function is provided to
1111 * check if the CCI driver has beed initialized. Function check if the driver
1112 * has been initialized, if not it calls the init function that probes
1113 * the driver and updates the return value.
1114 */
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001115bool cci_probed(void)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001116{
1117 return cci_init() == 0;
1118}
1119EXPORT_SYMBOL_GPL(cci_probed);
1120
1121early_initcall(cci_init);
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001122core_initcall(cci_platform_init);
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001123MODULE_LICENSE("GPL");
1124MODULE_DESCRIPTION("ARM CCI support");