| /* Copyright (c) 2016-2017, The Linux Foundation. All rights reserved. |
| * |
| * This program is free software; you can redistribute it and/or modify |
| * it under the terms of the GNU General Public License version 2 and |
| * only version 2 as published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| */ |
| |
| #define pr_fmt(fmt) "%s:%s " fmt, KBUILD_MODNAME, __func__ |
| |
| #include <linux/atomic.h> |
| #include <linux/bitmap.h> |
| #include <linux/delay.h> |
| #include <linux/interrupt.h> |
| #include <linux/jiffies.h> |
| #include <linux/kernel.h> |
| #include <linux/list.h> |
| #include <linux/mailbox_client.h> /* For dev_err */ |
| #include <linux/mailbox_controller.h> |
| #include <linux/module.h> |
| #include <linux/of.h> |
| #include <linux/of_address.h> |
| #include <linux/of_irq.h> |
| #include <linux/platform_device.h> |
| #include <linux/spinlock.h> |
| |
| #include <asm-generic/io.h> |
| |
| #include <soc/qcom/tcs.h> |
| |
| #include <dt-bindings/soc/qcom,tcs-mbox.h> |
| |
| #include "mailbox.h" |
| |
| #define CREATE_TRACE_POINTS |
| #include <trace/events/rpmh.h> |
| |
| #define MAX_CMDS_PER_TCS 16 |
| #define MAX_TCS_PER_TYPE 3 |
| #define MAX_TCS_SLOTS (MAX_CMDS_PER_TCS * MAX_TCS_PER_TYPE) |
| |
| #define TCS_DRV_TCS_OFFSET 672 |
| #define TCS_DRV_CMD_OFFSET 20 |
| |
| /* DRV Configuration Information Register */ |
| #define DRV_PRNT_CHLD_CONFIG 0x0C |
| #define DRV_NUM_TCS_MASK 0x3F |
| #define DRV_NUM_TCS_SHIFT 6 |
| #define DRV_NCPT_MASK 0x1F |
| #define DRV_NCPT_SHIFT 27 |
| |
| /* Register offsets */ |
| #define TCS_DRV_IRQ_ENABLE 0x00 |
| #define TCS_DRV_IRQ_STATUS 0x04 |
| #define TCS_DRV_IRQ_CLEAR 0x08 |
| #define TCS_DRV_CMD_WAIT_FOR_CMPL 0x10 |
| #define TCS_DRV_CONTROL 0x14 |
| #define TCS_DRV_STATUS 0x18 |
| #define TCS_DRV_CMD_ENABLE 0x1C |
| #define TCS_DRV_CMD_MSGID 0x30 |
| #define TCS_DRV_CMD_ADDR 0x34 |
| #define TCS_DRV_CMD_DATA 0x38 |
| #define TCS_DRV_CMD_STATUS 0x3C |
| #define TCS_DRV_CMD_RESP_DATA 0x40 |
| |
| #define TCS_AMC_MODE_ENABLE BIT(16) |
| #define TCS_AMC_MODE_TRIGGER BIT(24) |
| |
| /* TCS CMD register bit mask */ |
| #define CMD_MSGID_LEN 8 |
| #define CMD_MSGID_RESP_REQ BIT(8) |
| #define CMD_MSGID_WRITE BIT(16) |
| #define CMD_STATUS_ISSUED BIT(8) |
| #define CMD_STATUS_COMPL BIT(16) |
| |
| /* Control/Hidden TCS */ |
| #define TCS_HIDDEN_MAX_SLOTS 2 |
| #define TCS_HIDDEN_CMD0_DRV_DATA 0x38 |
| #define TCS_HIDDEN_CMD_SHIFT 0x08 |
| |
| #define TCS_TYPE_NR 4 |
| #define MAX_POOL_SIZE (MAX_TCS_PER_TYPE * TCS_TYPE_NR) |
| #define TCS_M_INIT 0xFFFF |
| |
| struct tcs_drv; |
| |
| struct tcs_response { |
| struct tcs_drv *drv; |
| struct mbox_chan *chan; |
| struct tcs_mbox_msg *msg; |
| u32 m; /* m-th TCS */ |
| int err; |
| int idx; |
| bool in_use; |
| struct list_head list; |
| }; |
| |
| struct tcs_response_pool { |
| struct tcs_response resp[MAX_POOL_SIZE]; |
| spinlock_t lock; |
| DECLARE_BITMAP(avail, MAX_POOL_SIZE); |
| }; |
| |
| /* One per TCS type of a controller */ |
| struct tcs_mbox { |
| struct tcs_drv *drv; |
| u32 *cmd_addr; |
| int type; |
| u32 tcs_mask; |
| u32 tcs_offset; |
| int num_tcs; |
| int ncpt; /* num cmds per tcs */ |
| DECLARE_BITMAP(slots, MAX_TCS_SLOTS); |
| spinlock_t tcs_lock; /* TCS type lock */ |
| }; |
| |
| /* One per MBOX controller */ |
| struct tcs_drv { |
| struct mbox_controller mbox; |
| const char *name; |
| void __iomem *base; /* start address of the RSC's registers */ |
| void __iomem *reg_base; /* start address for DRV specific register */ |
| int drv_id; |
| struct platform_device *pdev; |
| struct tcs_mbox tcs[TCS_TYPE_NR]; |
| int num_assigned; |
| int num_tcs; |
| struct tasklet_struct tasklet; |
| struct list_head response_pending; |
| spinlock_t drv_lock; |
| struct tcs_response_pool *resp_pool; |
| atomic_t tcs_in_use[MAX_POOL_SIZE]; |
| /* Debug info */ |
| u64 tcs_last_sent_ts[MAX_POOL_SIZE]; |
| u64 tcs_last_recv_ts[MAX_POOL_SIZE]; |
| atomic_t tcs_send_count[MAX_POOL_SIZE]; |
| atomic_t tcs_irq_count[MAX_POOL_SIZE]; |
| }; |
| |
| static int tcs_response_pool_init(struct tcs_drv *drv) |
| { |
| struct tcs_response_pool *pool; |
| int i; |
| |
| pool = devm_kzalloc(&drv->pdev->dev, sizeof(*pool), GFP_KERNEL); |
| if (!pool) |
| return -ENOMEM; |
| |
| for (i = 0; i < MAX_POOL_SIZE; i++) { |
| pool->resp[i].drv = drv; |
| pool->resp[i].idx = i; |
| pool->resp[i].m = TCS_M_INIT; |
| INIT_LIST_HEAD(&pool->resp[i].list); |
| } |
| |
| spin_lock_init(&pool->lock); |
| drv->resp_pool = pool; |
| |
| return 0; |
| } |
| |
| static struct tcs_response *setup_response(struct tcs_drv *drv, |
| struct tcs_mbox_msg *msg, struct mbox_chan *chan, |
| u32 m, int err) |
| { |
| struct tcs_response_pool *pool = drv->resp_pool; |
| struct tcs_response *resp = ERR_PTR(-ENOMEM); |
| int pos; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&pool->lock, flags); |
| pos = find_first_zero_bit(pool->avail, MAX_POOL_SIZE); |
| if (pos != MAX_POOL_SIZE) { |
| bitmap_set(pool->avail, pos, 1); |
| resp = &pool->resp[pos]; |
| resp->chan = chan; |
| resp->msg = msg; |
| resp->m = m; |
| resp->err = err; |
| resp->in_use = false; |
| } |
| spin_unlock_irqrestore(&pool->lock, flags); |
| |
| if (pos == MAX_POOL_SIZE) |
| pr_err("response pool is full\n"); |
| |
| return resp; |
| } |
| |
| static void free_response(struct tcs_response *resp) |
| { |
| struct tcs_response_pool *pool = resp->drv->resp_pool; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&pool->lock, flags); |
| resp->err = -EINVAL; |
| bitmap_clear(pool->avail, resp->idx, 1); |
| spin_unlock_irqrestore(&pool->lock, flags); |
| } |
| |
| static inline struct tcs_response *get_response(struct tcs_drv *drv, u32 m, |
| bool for_use) |
| { |
| struct tcs_response_pool *pool = drv->resp_pool; |
| struct tcs_response *resp = NULL; |
| int pos = 0; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&pool->lock, flags); |
| do { |
| pos = find_next_bit(pool->avail, MAX_POOL_SIZE, pos); |
| if (pos == MAX_POOL_SIZE) |
| break; |
| |
| resp = &pool->resp[pos]; |
| if (resp->m == m && !resp->in_use) { |
| resp->in_use = for_use; |
| break; |
| } |
| pos++; |
| udelay(1); |
| } while (1); |
| spin_unlock_irqrestore(&pool->lock, flags); |
| |
| return resp; |
| } |
| |
| static void print_response(struct tcs_drv *drv, int m) |
| { |
| struct tcs_response *resp; |
| struct tcs_mbox_msg *msg; |
| int i; |
| |
| resp = get_response(drv, m, false); |
| if (!resp) |
| return; |
| |
| msg = resp->msg; |
| pr_warn("Response object idx=%d:\n\tfor-tcs=%d\tin-use=%d\n", |
| resp->idx, resp->m, resp->in_use); |
| pr_warn("Msg: state=%d\n", msg->state); |
| for (i = 0; i < msg->num_payload; i++) |
| pr_warn("addr=0x%x data=0x%x complete=0x%x\n", |
| msg->payload[i].addr, |
| msg->payload[i].data, |
| msg->payload[i].complete); |
| } |
| |
| static inline u32 read_drv_config(void __iomem *base) |
| { |
| return le32_to_cpu(readl_relaxed(base + DRV_PRNT_CHLD_CONFIG)); |
| } |
| |
| static inline u32 read_tcs_reg(void __iomem *base, int reg, int m, int n) |
| { |
| return le32_to_cpu(readl_relaxed(base + reg + |
| TCS_DRV_TCS_OFFSET * m + TCS_DRV_CMD_OFFSET * n)); |
| } |
| |
| static inline void write_tcs_reg(void __iomem *base, int reg, int m, int n, |
| u32 data) |
| { |
| writel_relaxed(cpu_to_le32(data), base + reg + |
| TCS_DRV_TCS_OFFSET * m + TCS_DRV_CMD_OFFSET * n); |
| } |
| |
| static inline void write_tcs_reg_sync(void __iomem *base, int reg, int m, int n, |
| u32 data) |
| { |
| do { |
| write_tcs_reg(base, reg, m, n, data); |
| if (data == read_tcs_reg(base, reg, m, n)) |
| break; |
| udelay(1); |
| } while (1); |
| } |
| |
| static inline bool tcs_is_free(struct tcs_drv *drv, int m) |
| { |
| void __iomem *base = drv->reg_base; |
| |
| return read_tcs_reg(base, TCS_DRV_STATUS, m, 0) && |
| !atomic_read(&drv->tcs_in_use[m]); |
| } |
| |
| static inline struct tcs_mbox *get_tcs_from_index(struct tcs_drv *drv, int m) |
| { |
| struct tcs_mbox *tcs = NULL; |
| int i; |
| |
| for (i = 0; i < drv->num_tcs; i++) { |
| tcs = &drv->tcs[i]; |
| if (tcs->tcs_mask & (u32)BIT(m)) |
| break; |
| } |
| |
| if (i == drv->num_tcs) { |
| WARN(1, "Incorrect TCS index %d", m); |
| tcs = NULL; |
| } |
| |
| return tcs; |
| } |
| |
| static inline struct tcs_mbox *get_tcs_of_type(struct tcs_drv *drv, int type) |
| { |
| int i; |
| struct tcs_mbox *tcs; |
| |
| for (i = 0; i < TCS_TYPE_NR; i++) |
| if (type == drv->tcs[i].type) |
| break; |
| |
| if (i == TCS_TYPE_NR) |
| return ERR_PTR(-EINVAL); |
| |
| tcs = &drv->tcs[i]; |
| if (!tcs->num_tcs) |
| return ERR_PTR(-EINVAL); |
| |
| return tcs; |
| } |
| |
| static inline struct tcs_mbox *get_tcs_for_msg(struct tcs_drv *drv, |
| struct tcs_mbox_msg *msg) |
| { |
| int type = -1; |
| |
| /* Which box are we dropping this in and do we trigger the TCS */ |
| switch (msg->state) { |
| case RPMH_SLEEP_STATE: |
| type = SLEEP_TCS; |
| break; |
| case RPMH_WAKE_ONLY_STATE: |
| type = WAKE_TCS; |
| break; |
| case RPMH_ACTIVE_ONLY_STATE: |
| type = ACTIVE_TCS; |
| break; |
| case RPMH_AWAKE_STATE: |
| /* |
| * Awake state is only used when the DRV has no separate |
| * TCS for ACTIVE requests. Switch to WAKE TCS to send |
| * active votes. Otherwise, the caller should be explicit |
| * about the state. |
| */ |
| if (IS_ERR(get_tcs_of_type(drv, ACTIVE_TCS))) |
| type = WAKE_TCS; |
| break; |
| } |
| |
| if (msg->is_read) |
| type = ACTIVE_TCS; |
| |
| if (type < 0) |
| return ERR_PTR(-EINVAL); |
| |
| return get_tcs_of_type(drv, type); |
| } |
| |
| static inline void send_tcs_response(struct tcs_response *resp) |
| { |
| struct tcs_drv *drv = resp->drv; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&drv->drv_lock, flags); |
| INIT_LIST_HEAD(&resp->list); |
| list_add_tail(&resp->list, &drv->response_pending); |
| spin_unlock_irqrestore(&drv->drv_lock, flags); |
| |
| tasklet_schedule(&drv->tasklet); |
| } |
| |
| static inline void enable_tcs_irq(struct tcs_drv *drv, int m, bool enable) |
| { |
| void __iomem *base = drv->reg_base; |
| u32 data; |
| |
| /* Enable interrupts for non-ACTIVE TCS */ |
| data = read_tcs_reg(base, TCS_DRV_IRQ_ENABLE, 0, 0); |
| if (enable) |
| data |= BIT(m); |
| else |
| data &= ~BIT(m); |
| write_tcs_reg(base, TCS_DRV_IRQ_ENABLE, 0, 0, data); |
| } |
| |
| /** |
| * tcs_irq_handler: TX Done / Recv data handler |
| */ |
| static irqreturn_t tcs_irq_handler(int irq, void *p) |
| { |
| struct tcs_drv *drv = p; |
| void __iomem *base = drv->reg_base; |
| int m, i; |
| u32 irq_status, sts; |
| struct tcs_mbox *tcs; |
| struct tcs_response *resp; |
| struct tcs_cmd *cmd; |
| u32 data; |
| |
| /* Know which TCSes were triggered */ |
| irq_status = read_tcs_reg(base, TCS_DRV_IRQ_STATUS, 0, 0); |
| |
| for (m = 0; m < drv->num_tcs; m++) { |
| if (!(irq_status & (u32)BIT(m))) |
| continue; |
| atomic_inc(&drv->tcs_irq_count[m]); |
| |
| resp = get_response(drv, m, true); |
| if (!resp) { |
| pr_err("No resp request for TCS-%d\n", m); |
| goto no_resp; |
| } |
| |
| /* Check if all commands were completed */ |
| resp->err = 0; |
| for (i = 0; i < resp->msg->num_payload; i++) { |
| cmd = &resp->msg->payload[i]; |
| sts = read_tcs_reg(base, TCS_DRV_CMD_STATUS, m, i); |
| if ((!(sts & CMD_STATUS_ISSUED)) || |
| ((resp->msg->is_complete || cmd->complete) && |
| (!(sts & CMD_STATUS_COMPL)))) |
| resp->err = -EIO; |
| } |
| |
| /* Check for response if this was a read request */ |
| if (resp->msg->is_read) { |
| /* Respond the data back in the same req data */ |
| data = read_tcs_reg(base, TCS_DRV_CMD_RESP_DATA, m, 0); |
| resp->msg->payload[0].data = data; |
| mbox_chan_received_data(resp->chan, resp->msg); |
| } |
| |
| trace_rpmh_notify_irq(drv->name, m, resp->msg->payload[0].addr, |
| resp->err); |
| |
| /* Clear the AMC mode for non-ACTIVE TCSes */ |
| tcs = get_tcs_from_index(drv, m); |
| if (tcs && tcs->type != ACTIVE_TCS) { |
| data = read_tcs_reg(base, TCS_DRV_CONTROL, m, 0); |
| data &= ~TCS_AMC_MODE_ENABLE; |
| write_tcs_reg(base, TCS_DRV_CONTROL, m, 0, data); |
| /* |
| * Disable interrupt for this TCS to avoid being |
| * spammed with interrupts coming when the solver |
| * sends its wake votes. |
| */ |
| enable_tcs_irq(drv, m, false); |
| } else { |
| /* Clear the enable bit for the commands */ |
| write_tcs_reg(base, TCS_DRV_CMD_ENABLE, m, 0, 0); |
| } |
| |
| no_resp: |
| /* Record the recvd time stamp */ |
| drv->tcs_last_recv_ts[m] = arch_counter_get_cntvct(); |
| |
| /* Clear the TCS IRQ status */ |
| write_tcs_reg(base, TCS_DRV_IRQ_CLEAR, 0, 0, BIT(m)); |
| |
| /* Notify the client that this request is completed. */ |
| atomic_set(&drv->tcs_in_use[m], 0); |
| |
| /* Clean up response object and notify mbox in tasklet */ |
| if (resp) |
| send_tcs_response(resp); |
| } |
| |
| return IRQ_HANDLED; |
| } |
| |
| static inline void mbox_notify_tx_done(struct mbox_chan *chan, |
| struct tcs_mbox_msg *msg, int m, int err) |
| { |
| struct tcs_drv *drv = container_of(chan->mbox, struct tcs_drv, mbox); |
| |
| trace_rpmh_notify(drv->name, m, msg->payload[0].addr, err); |
| mbox_chan_txdone(chan, err); |
| } |
| |
| static void respond_tx_done(struct tcs_response *resp) |
| { |
| struct mbox_chan *chan = resp->chan; |
| struct tcs_mbox_msg *msg = resp->msg; |
| int err = resp->err; |
| int m = resp->m; |
| |
| free_response(resp); |
| mbox_notify_tx_done(chan, msg, m, err); |
| } |
| |
| /** |
| * tcs_notify_tx_done: TX Done for requests that do not trigger TCS |
| */ |
| static void tcs_notify_tx_done(unsigned long data) |
| { |
| struct tcs_drv *drv = (struct tcs_drv *)data; |
| struct tcs_response *resp; |
| unsigned long flags; |
| |
| do { |
| spin_lock_irqsave(&drv->drv_lock, flags); |
| if (list_empty(&drv->response_pending)) { |
| spin_unlock_irqrestore(&drv->drv_lock, flags); |
| break; |
| } |
| resp = list_first_entry(&drv->response_pending, |
| struct tcs_response, list); |
| list_del(&resp->list); |
| spin_unlock_irqrestore(&drv->drv_lock, flags); |
| respond_tx_done(resp); |
| } while (1); |
| } |
| |
| static void __tcs_buffer_write(struct tcs_drv *drv, int d, int m, int n, |
| struct tcs_mbox_msg *msg, bool trigger) |
| { |
| u32 msgid, cmd_msgid = 0; |
| u32 cmd_enable = 0; |
| u32 cmd_complete; |
| u32 enable = TCS_AMC_MODE_ENABLE; |
| struct tcs_cmd *cmd; |
| int i; |
| void __iomem *base = drv->reg_base; |
| |
| /* We have homologous command set i.e pure read or write, not a mix */ |
| cmd_msgid = CMD_MSGID_LEN; |
| cmd_msgid |= (msg->is_complete) ? CMD_MSGID_RESP_REQ : 0; |
| cmd_msgid |= (!msg->is_read) ? CMD_MSGID_WRITE : 0; |
| |
| /* Read the send-after-prev complete flag for those already in TCS */ |
| cmd_complete = read_tcs_reg(base, TCS_DRV_CMD_WAIT_FOR_CMPL, m, 0); |
| |
| for (i = 0; i < msg->num_payload; i++) { |
| cmd = &msg->payload[i]; |
| cmd_enable |= BIT(n + i); |
| cmd_complete |= cmd->complete << (n + i); |
| msgid = cmd_msgid; |
| msgid |= (cmd->complete) ? CMD_MSGID_RESP_REQ : 0; |
| write_tcs_reg(base, TCS_DRV_CMD_MSGID, m, n + i, msgid); |
| write_tcs_reg(base, TCS_DRV_CMD_ADDR, m, n + i, cmd->addr); |
| write_tcs_reg(base, TCS_DRV_CMD_DATA, m, n + i, cmd->data); |
| trace_rpmh_send_msg(drv->name, m, n + i, msgid, cmd->addr, |
| cmd->data, cmd->complete, trigger); |
| } |
| |
| /* Write the send-after-prev completion bits for the batch */ |
| write_tcs_reg(base, TCS_DRV_CMD_WAIT_FOR_CMPL, m, 0, cmd_complete); |
| |
| /* Enable the new commands in TCS */ |
| cmd_enable |= read_tcs_reg(base, TCS_DRV_CMD_ENABLE, m, 0); |
| write_tcs_reg(base, TCS_DRV_CMD_ENABLE, m, 0, cmd_enable); |
| |
| if (trigger) { |
| /* HW req: Clear the DRV_CONTROL and enable TCS again */ |
| write_tcs_reg_sync(base, TCS_DRV_CONTROL, m, 0, 0); |
| write_tcs_reg_sync(base, TCS_DRV_CONTROL, m, 0, enable); |
| /* Enable the AMC mode on the TCS */ |
| enable |= TCS_AMC_MODE_TRIGGER; |
| write_tcs_reg_sync(base, TCS_DRV_CONTROL, m, 0, enable); |
| } |
| } |
| |
| /** |
| * tcs_drv_is_idle: Check if any of the AMCs are busy. |
| * |
| * @mbox: The mailbox controller. |
| * |
| * Returns true if the AMCs are not engaged or absent. |
| */ |
| static bool tcs_drv_is_idle(struct mbox_controller *mbox) |
| { |
| int m; |
| struct tcs_drv *drv = container_of(mbox, struct tcs_drv, mbox); |
| struct tcs_mbox *tcs = get_tcs_of_type(drv, ACTIVE_TCS); |
| |
| /* Check for WAKE TCS if there are no ACTIVE TCS */ |
| if (IS_ERR(tcs)) |
| tcs = get_tcs_of_type(drv, WAKE_TCS); |
| |
| for (m = tcs->tcs_offset; m < tcs->tcs_offset + tcs->num_tcs; m++) |
| if (!tcs_is_free(drv, m)) |
| return false; |
| |
| return true; |
| } |
| |
| static int check_for_req_inflight(struct tcs_drv *drv, struct tcs_mbox *tcs, |
| struct tcs_mbox_msg *msg) |
| { |
| u32 curr_enabled, addr; |
| int i, j, k; |
| void __iomem *base = drv->reg_base; |
| int m = tcs->tcs_offset; |
| |
| for (i = 0; i < tcs->num_tcs; i++, m++) { |
| if (tcs_is_free(drv, m)) |
| continue; |
| |
| curr_enabled = read_tcs_reg(base, TCS_DRV_CMD_ENABLE, m, 0); |
| |
| for (j = 0; j < MAX_CMDS_PER_TCS; j++) { |
| if (!(curr_enabled & (u32)BIT(j))) |
| continue; |
| |
| addr = read_tcs_reg(base, TCS_DRV_CMD_ADDR, m, j); |
| for (k = 0; k < msg->num_payload; k++) { |
| if (addr == msg->payload[k].addr) |
| return -EBUSY; |
| } |
| } |
| } |
| |
| return 0; |
| } |
| |
| static int find_free_tcs(struct tcs_mbox *tcs) |
| { |
| int slot = -EBUSY; |
| int m = 0; |
| |
| /* Loop until we find a free AMC */ |
| for (m = 0; m < tcs->num_tcs; m++) { |
| if (tcs_is_free(tcs->drv, tcs->tcs_offset + m)) { |
| slot = m * tcs->ncpt; |
| break; |
| } |
| } |
| |
| return slot; |
| } |
| |
| static int find_match(struct tcs_mbox *tcs, struct tcs_cmd *cmd, int len) |
| { |
| bool found = false; |
| int i = 0, j; |
| |
| /* Check for already cached commands */ |
| while ((i = find_next_bit(tcs->slots, MAX_TCS_SLOTS, i)) < |
| MAX_TCS_SLOTS) { |
| if (tcs->cmd_addr[i] != cmd[0].addr) { |
| i++; |
| continue; |
| } |
| /* sanity check to ensure the seq is same */ |
| for (j = 1; j < len; j++) { |
| WARN((tcs->cmd_addr[i + j] != cmd[j].addr), |
| "Message does not match previous sequence.\n"); |
| return -EINVAL; |
| } |
| found = true; |
| break; |
| } |
| |
| return found ? i : -1; |
| } |
| |
| static int find_slots(struct tcs_mbox *tcs, struct tcs_mbox_msg *msg) |
| { |
| int slot; |
| int n = 0; |
| |
| /* For active requests find the first free AMC. */ |
| if (tcs->type == ACTIVE_TCS) |
| return find_free_tcs(tcs); |
| |
| /* Find if we already have the msg in our TCS */ |
| slot = find_match(tcs, msg->payload, msg->num_payload); |
| if (slot >= 0) |
| return slot; |
| |
| /* Do over, until we can fit the full payload in a TCS */ |
| do { |
| slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, |
| n, msg->num_payload, 0); |
| if (slot == MAX_TCS_SLOTS) |
| break; |
| n += tcs->ncpt; |
| } while (slot + msg->num_payload - 1 >= n); |
| |
| return (slot != MAX_TCS_SLOTS) ? slot : -ENOMEM; |
| } |
| |
| static int tcs_mbox_write(struct mbox_chan *chan, struct tcs_mbox_msg *msg, |
| bool trigger) |
| { |
| struct tcs_drv *drv = container_of(chan->mbox, struct tcs_drv, mbox); |
| int d = drv->drv_id; |
| struct tcs_mbox *tcs; |
| int i, slot, offset, m, n, ret; |
| struct tcs_response *resp = NULL; |
| unsigned long flags; |
| |
| tcs = get_tcs_for_msg(drv, msg); |
| if (IS_ERR(tcs)) |
| return PTR_ERR(tcs); |
| |
| if (trigger) { |
| resp = setup_response(drv, msg, chan, TCS_M_INIT, 0); |
| if (IS_ERR_OR_NULL(resp)) |
| return -EBUSY; |
| } |
| |
| /* Identify the sequential slots that we can write to */ |
| spin_lock_irqsave(&tcs->tcs_lock, flags); |
| slot = find_slots(tcs, msg); |
| if (slot < 0) { |
| spin_unlock_irqrestore(&tcs->tcs_lock, flags); |
| if (resp) |
| free_response(resp); |
| return slot; |
| } |
| |
| /* Figure out the TCS-m and CMD-n to write to */ |
| offset = slot / tcs->ncpt; |
| m = offset + tcs->tcs_offset; |
| n = slot % tcs->ncpt; |
| |
| if (trigger) { |
| /* Block, if we have an address from the msg in flight */ |
| ret = check_for_req_inflight(drv, tcs, msg); |
| if (ret) { |
| spin_unlock_irqrestore(&tcs->tcs_lock, flags); |
| if (resp) |
| free_response(resp); |
| return ret; |
| } |
| |
| resp->m = m; |
| /* Mark the TCS as busy */ |
| atomic_set(&drv->tcs_in_use[m], 1); |
| atomic_inc(&drv->tcs_send_count[m]); |
| /* Enable interrupt for active votes through wake TCS */ |
| if (tcs->type != ACTIVE_TCS) |
| enable_tcs_irq(drv, m, true); |
| drv->tcs_last_sent_ts[m] = arch_counter_get_cntvct(); |
| } else { |
| /* Mark the slots as in-use, before we unlock */ |
| if (tcs->type == SLEEP_TCS || tcs->type == WAKE_TCS) |
| bitmap_set(tcs->slots, slot, msg->num_payload); |
| |
| /* Copy the addresses of the resources over to the slots */ |
| for (i = 0; tcs->cmd_addr && i < msg->num_payload; i++) |
| tcs->cmd_addr[slot + i] = msg->payload[i].addr; |
| } |
| |
| /* Write to the TCS or AMC */ |
| __tcs_buffer_write(drv, d, m, n, msg, trigger); |
| |
| spin_unlock_irqrestore(&tcs->tcs_lock, flags); |
| |
| return 0; |
| } |
| |
| static void __tcs_buffer_invalidate(void __iomem *base, int m) |
| { |
| write_tcs_reg(base, TCS_DRV_CMD_ENABLE, m, 0, 0); |
| } |
| |
| static int tcs_mbox_invalidate(struct mbox_chan *chan) |
| { |
| struct tcs_drv *drv = container_of(chan->mbox, struct tcs_drv, mbox); |
| struct tcs_mbox *tcs; |
| int m, i; |
| int inv_types[] = { WAKE_TCS, SLEEP_TCS }; |
| int type = 0; |
| unsigned long flags; |
| |
| do { |
| tcs = get_tcs_of_type(drv, inv_types[type]); |
| if (IS_ERR(tcs)) |
| return PTR_ERR(tcs); |
| |
| spin_lock_irqsave(&tcs->tcs_lock, flags); |
| for (i = 0; i < tcs->num_tcs; i++) { |
| m = i + tcs->tcs_offset; |
| __tcs_buffer_invalidate(drv->reg_base, m); |
| } |
| /* Mark the TCS as free */ |
| bitmap_zero(tcs->slots, MAX_TCS_SLOTS); |
| spin_unlock_irqrestore(&tcs->tcs_lock, flags); |
| } while (++type < ARRAY_SIZE(inv_types)); |
| |
| return 0; |
| } |
| |
| static void print_tcs_regs(struct tcs_drv *drv, int m) |
| { |
| int n; |
| struct tcs_mbox *tcs = get_tcs_from_index(drv, m); |
| void __iomem *base = drv->reg_base; |
| u32 enable, addr, data, msgid; |
| |
| if (!tcs || tcs_is_free(drv, m)) |
| return; |
| |
| enable = read_tcs_reg(base, TCS_DRV_CMD_ENABLE, m, 0); |
| if (!enable) |
| return; |
| |
| pr_warn("TCS-%d contents:\n", m); |
| for (n = 0; n < tcs->ncpt; n++) { |
| if (!(enable & BIT(n))) |
| continue; |
| addr = read_tcs_reg(base, TCS_DRV_CMD_ADDR, m, n); |
| data = read_tcs_reg(base, TCS_DRV_CMD_DATA, m, n); |
| msgid = read_tcs_reg(base, TCS_DRV_CMD_MSGID, m, n); |
| pr_warn("\tn=%d addr=0x%x data=0x%x hdr=0x%x\n", |
| n, addr, data, msgid); |
| } |
| } |
| |
| static void dump_tcs_stats(struct tcs_drv *drv) |
| { |
| int i; |
| unsigned long long curr = arch_counter_get_cntvct(); |
| |
| for (i = 0; i < drv->num_tcs; i++) { |
| if (!atomic_read(&drv->tcs_in_use[i])) |
| continue; |
| pr_warn("Time: %llu: TCS-%d:\n\tReq Sent:%d Last Sent:%llu\n\tResp Recv:%d Last Recvd:%llu\n", |
| curr, i, |
| atomic_read(&drv->tcs_send_count[i]), |
| drv->tcs_last_sent_ts[i], |
| atomic_read(&drv->tcs_irq_count[i]), |
| drv->tcs_last_recv_ts[i]); |
| print_tcs_regs(drv, i); |
| print_response(drv, i); |
| } |
| } |
| |
| static void chan_debug(struct mbox_chan *chan) |
| { |
| struct tcs_drv *drv = container_of(chan->mbox, struct tcs_drv, mbox); |
| |
| dump_tcs_stats(drv); |
| } |
| |
| /** |
| * chan_tcs_write: Validate the incoming message and write to the |
| * appropriate TCS block. |
| * |
| * @chan: the MBOX channel |
| * @data: the tcs_mbox_msg* |
| * |
| * Returns a negative error for invalid message structure and invalid |
| * message combination, -EBUSY if there is an other active request for |
| * the channel in process, otherwise bubbles up internal error. |
| */ |
| static int chan_tcs_write(struct mbox_chan *chan, void *data) |
| { |
| struct tcs_drv *drv = container_of(chan->mbox, struct tcs_drv, mbox); |
| struct tcs_mbox_msg *msg = data; |
| const struct device *dev = chan->cl->dev; |
| int ret = 0; |
| |
| if (!msg) { |
| dev_err(dev, "Payload error\n"); |
| ret = -EINVAL; |
| goto tx_fail; |
| } |
| |
| if (!msg->payload || msg->num_payload > MAX_RPMH_PAYLOAD) { |
| dev_err(dev, "Payload error\n"); |
| ret = -EINVAL; |
| goto tx_fail; |
| } |
| |
| if (msg->invalidate || msg->is_control) { |
| dev_err(dev, "Incorrect API\n"); |
| ret = -EINVAL; |
| goto tx_fail; |
| } |
| |
| if (msg->state != RPMH_ACTIVE_ONLY_STATE && |
| msg->state != RPMH_AWAKE_STATE) { |
| dev_err(dev, "Incorrect API\n"); |
| ret = -EINVAL; |
| goto tx_fail; |
| } |
| |
| /* Read requests should always be single */ |
| if (msg->is_read && msg->num_payload > 1) { |
| dev_err(dev, "Incorrect read request\n"); |
| ret = -EINVAL; |
| goto tx_fail; |
| } |
| |
| /* |
| * Since we are re-purposing the wake TCS, invalidate previous |
| * contents to avoid confusion. |
| */ |
| if (msg->state == RPMH_AWAKE_STATE) |
| tcs_mbox_invalidate(chan); |
| |
| /* Post the message to the TCS and trigger */ |
| ret = tcs_mbox_write(chan, msg, true); |
| |
| tx_fail: |
| /* If there was an error in the request, schedule a response */ |
| if (ret < 0 && ret != -EBUSY) { |
| struct tcs_response *resp = setup_response( |
| drv, msg, chan, TCS_M_INIT, ret); |
| |
| dev_err(dev, "Error sending RPMH message %d\n", ret); |
| if (resp) |
| send_tcs_response(resp); |
| ret = 0; |
| } |
| |
| /* If we were just busy waiting for TCS, dump the state and return */ |
| if (ret == -EBUSY) { |
| pr_info_ratelimited("TCS Busy, retrying RPMH message send\n"); |
| ret = -EAGAIN; |
| } |
| |
| return ret; |
| } |
| |
| static void __tcs_write_hidden(struct tcs_drv *drv, int d, |
| struct tcs_mbox_msg *msg) |
| { |
| int i; |
| void __iomem *addr = drv->base + TCS_HIDDEN_CMD0_DRV_DATA; |
| |
| for (i = 0; i < msg->num_payload; i++) { |
| /* Only data is write capable */ |
| writel_relaxed(cpu_to_le32(msg->payload[i].data), addr); |
| trace_rpmh_control_msg(drv->name, msg->payload[i].data); |
| addr += TCS_HIDDEN_CMD_SHIFT; |
| } |
| } |
| |
| static int tcs_control_write(struct mbox_chan *chan, struct tcs_mbox_msg *msg) |
| { |
| const struct device *dev = chan->cl->dev; |
| struct tcs_drv *drv = container_of(chan->mbox, struct tcs_drv, mbox); |
| struct tcs_mbox *tcs; |
| unsigned long flags; |
| |
| tcs = get_tcs_of_type(drv, CONTROL_TCS); |
| if (IS_ERR(tcs)) |
| return PTR_ERR(tcs); |
| |
| if (msg->num_payload != tcs->ncpt) { |
| dev_err(dev, "Request must fit the control TCS size\n"); |
| return -EINVAL; |
| } |
| |
| spin_lock_irqsave(&tcs->tcs_lock, flags); |
| __tcs_write_hidden(tcs->drv, drv->drv_id, msg); |
| spin_unlock_irqrestore(&tcs->tcs_lock, flags); |
| |
| return 0; |
| } |
| |
| /** |
| * chan_tcs_ctrl_write: Write message to the controller, no ACK sent. |
| * |
| * @chan: the MBOX channel |
| * @data: the tcs_mbox_msg* |
| */ |
| static int chan_tcs_ctrl_write(struct mbox_chan *chan, void *data) |
| { |
| struct tcs_mbox_msg *msg = data; |
| const struct device *dev = chan->cl->dev; |
| int ret = -EINVAL; |
| |
| if (!msg) { |
| dev_err(dev, "Payload error\n"); |
| goto tx_done; |
| } |
| |
| if (msg->num_payload > MAX_RPMH_PAYLOAD) { |
| dev_err(dev, "Payload error\n"); |
| goto tx_done; |
| } |
| |
| /* Invalidate sleep/wake TCS */ |
| if (msg->invalidate) { |
| ret = tcs_mbox_invalidate(chan); |
| goto tx_done; |
| } |
| |
| /* Control slots are unique. They carry specific data. */ |
| if (msg->is_control) { |
| ret = tcs_control_write(chan, msg); |
| goto tx_done; |
| } |
| |
| /* Post the message to the TCS without trigger */ |
| ret = tcs_mbox_write(chan, msg, false); |
| |
| tx_done: |
| return ret; |
| } |
| |
| static int chan_init(struct mbox_chan *chan) |
| { |
| return 0; |
| } |
| |
| static void chan_shutdown(struct mbox_chan *chan) |
| { } |
| |
| static const struct mbox_chan_ops mbox_ops = { |
| .send_data = chan_tcs_write, |
| .send_controller_data = chan_tcs_ctrl_write, |
| .startup = chan_init, |
| .shutdown = chan_shutdown, |
| }; |
| |
| static struct mbox_chan *of_tcs_mbox_xlate(struct mbox_controller *mbox, |
| const struct of_phandle_args *sp) |
| { |
| struct tcs_drv *drv = container_of(mbox, struct tcs_drv, mbox); |
| struct mbox_chan *chan; |
| |
| if (drv->num_assigned >= mbox->num_chans) { |
| pr_err("TCS-Mbox out of channel memory\n"); |
| return ERR_PTR(-ENOMEM); |
| } |
| |
| chan = &mbox->chans[drv->num_assigned++]; |
| chan->con_priv = drv; |
| |
| return chan; |
| } |
| |
| static int tcs_drv_probe(struct platform_device *pdev) |
| { |
| struct device_node *dn = pdev->dev.of_node; |
| struct device_node *np; |
| struct tcs_drv *drv; |
| struct mbox_chan *chans; |
| struct tcs_mbox *tcs; |
| struct of_phandle_args p; |
| int irq; |
| u32 val[8] = { 0 }; |
| int num_chans = 0; |
| int st = 0; |
| int i, j, ret, nelem; |
| u32 config, max_tcs, ncpt; |
| int tcs_type_count[TCS_TYPE_NR] = { 0 }; |
| struct resource *res; |
| |
| drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); |
| if (!drv) |
| return -ENOMEM; |
| |
| ret = of_property_read_u32(dn, "qcom,drv-id", &drv->drv_id); |
| if (ret) |
| return ret; |
| |
| res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| if (!res) |
| return -EINVAL; |
| drv->base = devm_ioremap_resource(&pdev->dev, res); |
| if (IS_ERR(drv->base)) |
| return PTR_ERR(drv->base); |
| |
| res = platform_get_resource(pdev, IORESOURCE_MEM, 1); |
| if (!res) |
| return -EINVAL; |
| drv->reg_base = devm_ioremap_resource(&pdev->dev, res); |
| if (IS_ERR(drv->reg_base)) |
| return PTR_ERR(drv->reg_base); |
| |
| config = read_drv_config(drv->base); |
| max_tcs = config & (DRV_NUM_TCS_MASK << |
| (DRV_NUM_TCS_SHIFT * drv->drv_id)); |
| max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->drv_id); |
| ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); |
| ncpt = ncpt >> DRV_NCPT_SHIFT; |
| |
| nelem = of_property_count_elems_of_size(dn, "qcom,tcs-config", |
| sizeof(u32)); |
| if (!nelem || (nelem % 2) || (nelem > 2 * TCS_TYPE_NR)) |
| return -EINVAL; |
| |
| ret = of_property_read_u32_array(dn, "qcom,tcs-config", val, nelem); |
| if (ret) |
| return ret; |
| |
| /* Ensure we have exactly not more than one of each type in DT */ |
| for (i = 0; i < (nelem / 2); i++) { |
| if (val[2 * i] >= TCS_TYPE_NR) |
| return -EINVAL; |
| tcs_type_count[val[2 * i]]++; |
| if (tcs_type_count[val[2 * i]] > 1) |
| return -EINVAL; |
| } |
| |
| /* Ensure we have each type specified in DT */ |
| for (i = 0; i < ARRAY_SIZE(tcs_type_count); i++) |
| if (!tcs_type_count[i]) |
| return -EINVAL; |
| |
| for (i = 0; i < (nelem / 2); i++) { |
| tcs = &drv->tcs[val[2 * i]]; |
| tcs->drv = drv; |
| tcs->type = val[2 * i]; |
| tcs->num_tcs = val[2 * i + 1]; |
| tcs->ncpt = (tcs->type == CONTROL_TCS) ? TCS_HIDDEN_MAX_SLOTS |
| : ncpt; |
| spin_lock_init(&tcs->tcs_lock); |
| |
| if (tcs->num_tcs <= 0 || tcs->type == CONTROL_TCS) |
| continue; |
| |
| if (tcs->num_tcs > MAX_TCS_PER_TYPE) |
| return -EINVAL; |
| |
| if (st > max_tcs) |
| return -EINVAL; |
| |
| tcs->tcs_mask = ((1 << tcs->num_tcs) - 1) << st; |
| tcs->tcs_offset = st; |
| st += tcs->num_tcs; |
| |
| tcs->cmd_addr = devm_kzalloc(&pdev->dev, sizeof(u32) * |
| tcs->num_tcs * tcs->ncpt, GFP_KERNEL); |
| if (!tcs->cmd_addr) |
| return -ENOMEM; |
| |
| } |
| |
| /* Allocate only that many channels specified in DT for our MBOX */ |
| for_each_node_with_property(np, "mboxes") { |
| if (!of_device_is_available(np)) |
| continue; |
| i = of_count_phandle_with_args(np, "mboxes", "#mbox-cells"); |
| for (j = 0; j < i; j++) { |
| ret = of_parse_phandle_with_args(np, "mboxes", |
| "#mbox-cells", j, &p); |
| if (!ret && p.np == pdev->dev.of_node) |
| break; |
| } |
| num_chans++; |
| } |
| |
| if (!num_chans) { |
| pr_err("%s: No clients for controller (%s)\n", __func__, |
| dn->full_name); |
| return -ENODEV; |
| } |
| |
| chans = devm_kzalloc(&pdev->dev, num_chans * sizeof(*chans), |
| GFP_KERNEL); |
| if (!chans) |
| return -ENOMEM; |
| |
| for (i = 0; i < num_chans; i++) { |
| chans[i].mbox = &drv->mbox; |
| chans[i].txdone_method = TXDONE_BY_IRQ; |
| } |
| |
| drv->mbox.dev = &pdev->dev; |
| drv->mbox.ops = &mbox_ops; |
| drv->mbox.chans = chans; |
| drv->mbox.num_chans = num_chans; |
| drv->mbox.txdone_irq = true; |
| drv->mbox.of_xlate = of_tcs_mbox_xlate; |
| drv->mbox.is_idle = tcs_drv_is_idle; |
| drv->mbox.debug = chan_debug; |
| drv->num_tcs = st; |
| drv->pdev = pdev; |
| INIT_LIST_HEAD(&drv->response_pending); |
| spin_lock_init(&drv->drv_lock); |
| tasklet_init(&drv->tasklet, tcs_notify_tx_done, (unsigned long)drv); |
| |
| drv->name = of_get_property(pdev->dev.of_node, "label", NULL); |
| if (!drv->name) |
| drv->name = dev_name(&pdev->dev); |
| |
| ret = tcs_response_pool_init(drv); |
| if (ret) |
| return ret; |
| |
| irq = of_irq_get(dn, 0); |
| if (irq < 0) |
| return irq; |
| |
| ret = devm_request_irq(&pdev->dev, irq, tcs_irq_handler, |
| IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, |
| drv->name, drv); |
| if (ret) |
| return ret; |
| |
| /* Enable interrupts for AMC TCS */ |
| write_tcs_reg(drv->reg_base, TCS_DRV_IRQ_ENABLE, 0, 0, |
| drv->tcs[ACTIVE_TCS].tcs_mask); |
| |
| for (i = 0; i < ARRAY_SIZE(drv->tcs_in_use); i++) |
| atomic_set(&drv->tcs_in_use[i], 0); |
| |
| ret = mbox_controller_register(&drv->mbox); |
| if (ret) |
| return ret; |
| |
| pr_debug("Mailbox controller (%s, drv=%d) registered\n", |
| dn->full_name, drv->drv_id); |
| |
| return 0; |
| } |
| |
| static const struct of_device_id tcs_drv_match[] = { |
| { .compatible = "qcom,tcs-drv", }, |
| { } |
| }; |
| |
| static struct platform_driver tcs_mbox_driver = { |
| .probe = tcs_drv_probe, |
| .driver = { |
| .name = KBUILD_MODNAME, |
| .of_match_table = tcs_drv_match, |
| }, |
| }; |
| |
| static int __init tcs_mbox_driver_init(void) |
| { |
| return platform_driver_register(&tcs_mbox_driver); |
| } |
| arch_initcall(tcs_mbox_driver_init); |