Merge "hwmon: epm_adc: Add Marker support" into msm-3.4
diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c
index e316742..4579e1f 100644
--- a/drivers/gpu/msm/adreno.c
+++ b/drivers/gpu/msm/adreno.c
@@ -211,6 +211,8 @@
512, 0, 2, SZ_128K, 0x3FF037, 0x3FF016 },
};
+static unsigned int adreno_isidle(struct kgsl_device *device);
+
/**
* adreno_perfcounter_init: Reserve kernel performance counters
* @device: device to configure
@@ -818,6 +820,27 @@
return cmds - cmds_orig;
}
+/**
+ * adreno_use_default_setstate() - Use CPU instead of the GPU to manage the mmu?
+ * @adreno_dev: the device
+ *
+ * In many cases it is preferable to poke the iommu or gpummu directly rather
+ * than using the GPU command stream. If we are idle or trying to go to a low
+ * power state, using the command stream will be slower and asynchronous, which
+ * needlessly complicates the power state transitions. Additionally,
+ * the hardware simulators do not support command stream MMU operations so
+ * the command stream can never be used if we are capturing CFF data.
+ *
+ */
+static bool adreno_use_default_setstate(struct adreno_device *adreno_dev)
+{
+ return (adreno_isidle(&adreno_dev->dev) ||
+ adreno_dev->drawctxt_active == NULL ||
+ KGSL_STATE_ACTIVE != adreno_dev->dev.state ||
+ atomic_read(&adreno_dev->dev.active_cnt) == 0 ||
+ adreno_dev->dev.cff_dump_enable);
+}
+
static void adreno_iommu_setstate(struct kgsl_device *device,
unsigned int context_id,
uint32_t flags)
@@ -832,10 +855,7 @@
struct adreno_context *adreno_ctx = NULL;
struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
- if (!adreno_dev->drawctxt_active ||
- KGSL_STATE_ACTIVE != device->state ||
- !atomic_read(&device->active_cnt) ||
- device->cff_dump_enable) {
+ if (adreno_use_default_setstate(adreno_dev)) {
kgsl_mmu_device_setstate(&device->mmu, flags);
return;
}
@@ -845,7 +865,7 @@
if (context == NULL)
return;
- adreno_ctx = context->devctxt;
+ adreno_ctx = ADRENO_CONTEXT(context);
if (kgsl_mmu_enable_clk(&device->mmu,
KGSL_IOMMU_CONTEXT_USER))
@@ -886,8 +906,7 @@
adreno_ringbuffer_issuecmds(device, adreno_ctx, KGSL_CMD_FLAGS_PMODE,
&link[0], sizedwords);
- kgsl_mmu_disable_clk_on_ts(&device->mmu,
- rb->timestamp[KGSL_MEMSTORE_GLOBAL], true);
+ kgsl_mmu_disable_clk_on_ts(&device->mmu, rb->global_ts, true);
kgsl_context_put(context);
}
@@ -915,11 +934,11 @@
* writes For CFF dump we must idle and use the registers so that it is
* easier to filter out the mmu accesses from the dump
*/
- if (!device->cff_dump_enable && adreno_dev->drawctxt_active) {
+ if (!adreno_use_default_setstate(adreno_dev)) {
context = kgsl_context_get(device, context_id);
if (context == NULL)
return;
- adreno_ctx = context->devctxt;
+ adreno_ctx = ADRENO_CONTEXT(context);
if (flags & KGSL_MMUFLAGS_PTUPDATE) {
/* wait for graphics pipe to be idle */
@@ -1703,7 +1722,6 @@
static int adreno_init(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
- struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
int i;
if (KGSL_STATE_DUMP_AND_FT != device->state)
@@ -1744,8 +1762,6 @@
adreno_gpulist[adreno_dev->gpulist_index].sync_lock_pfp_ver))
device->mmu.flags |= KGSL_MMU_FLAGS_IOMMU_SYNC;
- rb->timestamp[KGSL_MEMSTORE_GLOBAL] = 0;
-
/* Initialize ft detection register offsets */
ft_detect_regs[0] = adreno_getreg(adreno_dev,
ADRENO_REG_RBBM_STATUS);
@@ -1860,6 +1876,9 @@
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+ if (adreno_dev->drawctxt_active)
+ kgsl_context_put(&adreno_dev->drawctxt_active->base);
+
adreno_dev->drawctxt_active = NULL;
adreno_ringbuffer_stop(&adreno_dev->ringbuffer);
@@ -1892,7 +1911,7 @@
{
unsigned int ft_status = *((unsigned int *) data);
struct kgsl_context *context = ptr;
- struct adreno_context *adreno_context = context->devctxt;
+ struct adreno_context *adreno_context = ADRENO_CONTEXT(context);
if (ft_status) {
context->reset_status =
@@ -1930,20 +1949,16 @@
static int _set_max_ts(int id, void *ptr, void *data)
{
struct kgsl_device *device = data;
- struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
- struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
struct kgsl_context *context = ptr;
- struct adreno_context *drawctxt = context->devctxt;
+ struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
- if (drawctxt->flags & CTXT_FLAGS_GPU_HANG) {
+ if (drawctxt && drawctxt->flags & CTXT_FLAGS_GPU_HANG) {
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id,
- soptimestamp),
- rb->timestamp[context->id]);
+ soptimestamp), drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id,
- eoptimestamp),
- rb->timestamp[context->id]);
+ eoptimestamp), drawctxt->timestamp);
}
return 0;
@@ -2199,7 +2214,7 @@
}
if (context) {
- adreno_context = context->devctxt;
+ adreno_context = ADRENO_CONTEXT(context);
if (adreno_context->flags & CTXT_FLAGS_PREAMBLE) {
if (ft_data->ib1) {
ret = _find_hanging_ib_sequence(rb,
@@ -2266,6 +2281,9 @@
return -EINVAL;
}
+ if (adreno_dev->drawctxt_active)
+ kgsl_context_put(&adreno_dev->drawctxt_active->base);
+
adreno_dev->drawctxt_active = NULL;
/* Stop the ringbuffer */
@@ -2328,11 +2346,9 @@
}
reset_done:
- if (context) {
- struct adreno_context *adreno_context = context->devctxt;
- kgsl_mmu_setstate(&device->mmu, adreno_context->pagetable,
- KGSL_MEMSTORE_GLOBAL);
- }
+ if (context)
+ kgsl_mmu_setstate(&device->mmu, context->pagetable,
+ KGSL_MEMSTORE_GLOBAL);
/* If iommu is used then we need to make sure that the iommu clocks
* are on since there could be commands in pipeline that touch iommu */
@@ -2451,7 +2467,7 @@
}
} else {
no_context_ft = 0;
- adreno_context = context->devctxt;
+ adreno_context = ADRENO_CONTEXT(context);
adreno_context->flags |= CTXT_FLAGS_GPU_HANG;
/*
* set the invalid ts flag to 0 for this context since we have
@@ -2507,16 +2523,16 @@
goto play_good_cmds;
}
- /* Do not try the reply if hang is due to a pagefault */
- if (adreno_context && adreno_context->pagefault) {
+ /* Do not try to replay if hang is due to a pagefault */
+ if (context && test_bit(KGSL_CONTEXT_PAGEFAULT, &context->priv)) {
/* Resume MMU */
mmu->mmu_ops->mmu_pagefault_resume(mmu);
- if ((ft_data->context_id == adreno_context->id) &&
- (ft_data->global_eop == adreno_context->pagefault_ts)) {
+ if ((ft_data->context_id == context->id) &&
+ (ft_data->global_eop == context->pagefault_ts)) {
ft_data->ft_policy &= ~KGSL_FT_REPLAY;
KGSL_FT_ERR(device, "MMU fault skipping replay\n");
}
- adreno_context->pagefault = 0;
+ clear_bit(KGSL_CONTEXT_PAGEFAULT, &context->priv);
}
if (ft_data->ft_policy & KGSL_FT_REPLAY) {
@@ -2604,6 +2620,10 @@
adreno_context->flags = (adreno_context->flags &
~CTXT_FLAGS_GPU_HANG) | CTXT_FLAGS_GPU_HANG_FT;
}
+
+ if (last_active_ctx)
+ _kgsl_context_get(&last_active_ctx->base);
+
adreno_dev->drawctxt_active = last_active_ctx;
}
@@ -2631,10 +2651,7 @@
struct kgsl_context *last_ctx = kgsl_context_get(device,
ft_data->last_valid_ctx_id);
- if (last_ctx)
- adreno_dev->drawctxt_active = last_ctx->devctxt;
-
- kgsl_context_put(last_ctx);
+ adreno_dev->drawctxt_active = ADRENO_CONTEXT(last_ctx);
}
done:
@@ -2653,7 +2670,6 @@
int ret = 0;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
- unsigned int timestamp;
/*
* If GPU FT is turned off do not run FT.
@@ -2670,8 +2686,8 @@
"Bad context_id: %u, global_eop: 0x%x\n",
ft_data->ib1, ft_data->context_id, ft_data->global_eop);
- timestamp = rb->timestamp[KGSL_MEMSTORE_GLOBAL];
- KGSL_FT_INFO(device, "Last issued global timestamp: %x\n", timestamp);
+ KGSL_FT_INFO(device, "Last issued global timestamp: %x\n",
+ rb->global_ts);
/* We may need to replay commands multiple times based on whether
* multiple contexts hang the GPU */
@@ -2700,14 +2716,12 @@
/* Restore correct states after fault tolerance */
if (adreno_dev->drawctxt_active)
device->mmu.hwpagetable =
- adreno_dev->drawctxt_active->pagetable;
+ adreno_dev->drawctxt_active->base.pagetable;
else
device->mmu.hwpagetable = device->mmu.defaultpagetable;
- rb->timestamp[KGSL_MEMSTORE_GLOBAL] = timestamp;
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
- eoptimestamp),
- rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
+ eoptimestamp), rb->global_ts);
/* switch to NULL ctxt */
if (adreno_dev->drawctxt_active != NULL)
@@ -2977,11 +2991,10 @@
/* Caller must hold the device mutex. */
int adreno_idle(struct kgsl_device *device)
{
- struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
- unsigned int rbbm_status;
unsigned long wait_time;
unsigned long wait_time_part;
unsigned int prev_reg_val[FT_DETECT_REGS_COUNT];
+ struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
memset(prev_reg_val, 0, sizeof(prev_reg_val));
@@ -2999,23 +3012,15 @@
wait_time_part = jiffies + msecs_to_jiffies(KGSL_TIMEOUT_PART);
while (time_before(jiffies, wait_time)) {
- adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS,
- &rbbm_status);
- if (adreno_is_a2xx(adreno_dev)) {
- if (rbbm_status == 0x110)
- return 0;
- } else {
- if (!(rbbm_status & 0x80000000))
- return 0;
- }
+ if (adreno_isidle(device))
+ return 0;
- /* Dont wait for timeout, detect hang faster.
- */
+ /* Dont wait for timeout, detect hang faster. */
if (time_after(jiffies, wait_time_part)) {
- wait_time_part = jiffies +
- msecs_to_jiffies(KGSL_TIMEOUT_PART);
- if ((adreno_ft_detect(device, prev_reg_val)))
- goto err;
+ wait_time_part = jiffies +
+ msecs_to_jiffies(KGSL_TIMEOUT_PART);
+ if ((adreno_ft_detect(device, prev_reg_val)))
+ goto err;
}
}
@@ -3064,9 +3069,8 @@
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
- WARN_ON(device->state == KGSL_STATE_INIT);
/* If the device isn't active, don't force it on. */
- if (device->state == KGSL_STATE_ACTIVE) {
+ if (kgsl_pwrctrl_isenabled(device)) {
/* Is the ring buffer is empty? */
unsigned int rptr = adreno_get_rptr(rb);
if (rptr == rb->wptr) {
@@ -3109,21 +3113,20 @@
phys_addr_t pt_base, unsigned int gpuaddr, unsigned int size)
{
struct kgsl_context *context;
- struct adreno_context *adreno_context = NULL;
int next = 0;
struct kgsl_memdesc *desc = NULL;
-
read_lock(&device->context_lock);
while (1) {
context = idr_get_next(&device->context_idr, &next);
if (context == NULL)
break;
- adreno_context = (struct adreno_context *)context->devctxt;
-
- if (kgsl_mmu_pt_equal(&device->mmu, adreno_context->pagetable,
+ if (kgsl_mmu_pt_equal(&device->mmu, context->pagetable,
pt_base)) {
+ struct adreno_context *adreno_context;
+
+ adreno_context = ADRENO_CONTEXT(context);
desc = &adreno_context->gpustate;
if (kgsl_gpuaddr_in_memdesc(desc, gpuaddr, size))
break;
@@ -3257,9 +3260,10 @@
static unsigned int _get_context_id(struct kgsl_context *k_ctxt)
{
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
+
if (k_ctxt != NULL) {
- struct adreno_context *a_ctxt = k_ctxt->devctxt;
- if (k_ctxt->id == KGSL_CONTEXT_INVALID || a_ctxt == NULL)
+ struct adreno_context *a_ctxt = ADRENO_CONTEXT(k_ctxt);
+ if (kgsl_context_detached(k_ctxt))
context_id = KGSL_CONTEXT_INVALID;
else if (a_ctxt->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = k_ctxt->id;
@@ -3330,7 +3334,8 @@
*/
if (context && device->state != KGSL_STATE_SLUMBER) {
- adreno_ringbuffer_issuecmds(device, context->devctxt,
+ adreno_ringbuffer_issuecmds(device,
+ ADRENO_CONTEXT(context),
KGSL_CMD_FLAGS_GET_INT, NULL, 0);
}
}
@@ -3390,6 +3395,7 @@
unsigned int curr_context_id = 0;
static struct adreno_context *curr_context;
static struct kgsl_context *context;
+ static char pid_name[TASK_COMM_LEN] = "unknown";
if (!adreno_dev->fast_hang_detect)
fast_hang_detected = 0;
@@ -3402,7 +3408,7 @@
if (is_adreno_rbbm_status_idle(device) &&
(kgsl_readtimestamp(device, NULL, KGSL_TIMESTAMP_RETIRED)
- == rb->timestamp[KGSL_MEMSTORE_GLOBAL])) {
+ == rb->global_ts)) {
/*
* On A2XX if the RPTR != WPTR and the device is idle, then
@@ -3453,8 +3459,13 @@
if (curr_global_ts == prev_global_ts) {
- /* Get the current context here */
- if (context == NULL) {
+ /* If we don't already have a good context, get it. */
+ if (kgsl_context_detached(context)) {
+ kgsl_context_put(context);
+ context = NULL;
+ curr_context = NULL;
+ strlcpy(pid_name, "unknown", sizeof(pid_name));
+
kgsl_sharedmem_readl(&device->memstore,
&curr_context_id,
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
@@ -3464,8 +3475,12 @@
context = kgsl_context_get(device, curr_context_id);
if (context != NULL) {
- curr_context = context->devctxt;
+ struct task_struct *task;
+ curr_context = ADRENO_CONTEXT(context);
curr_context->ib_gpu_time_used = 0;
+ task = find_task_by_vpid(context->pid);
+ if (task)
+ get_task_comm(pid_name, task);
} else {
KGSL_DRV_ERR(device,
"Fault tolerance no context found\n");
@@ -3489,8 +3504,7 @@
KGSL_FT_ERR(device,
"Proc %s, ctxt_id %d ts %d triggered fault tolerance"
" on global ts %d\n",
- curr_context ? curr_context->pid_name : "",
- curr_context ? curr_context->id : 0,
+ pid_name, context ? context->id : 0,
(kgsl_readtimestamp(device, context,
KGSL_TIMESTAMP_RETIRED) + 1),
curr_global_ts + 1);
@@ -3502,7 +3516,7 @@
curr_context->ib_gpu_time_used += KGSL_TIMEOUT_PART;
KGSL_FT_INFO(device,
"Proc %s used GPU Time %d ms on timestamp 0x%X\n",
- curr_context->pid_name, curr_context->ib_gpu_time_used,
+ pid_name, curr_context->ib_gpu_time_used,
curr_global_ts+1);
if ((long_ib_detected) &&
@@ -3518,8 +3532,7 @@
"Proc %s, ctxt_id %d ts %d"
"used GPU for %d ms long ib "
"detected on global ts %d\n",
- curr_context->pid_name,
- curr_context->id,
+ pid_name, context->id,
(kgsl_readtimestamp(device,
context,
KGSL_TIMESTAMP_RETIRED)+1),
@@ -3541,6 +3554,7 @@
kgsl_context_put(context);
context = NULL;
curr_context = NULL;
+ strlcpy(pid_name, "unknown", sizeof(pid_name));
adreno_dev->long_ib = 0;
adreno_dev->long_ib_ts = 0;
}
@@ -3563,7 +3577,7 @@
if (context_id == KGSL_CONTEXT_INVALID)
return -EINVAL;
- ts_issued = adreno_dev->ringbuffer.timestamp[context_id];
+ ts_issued = adreno_context_timestamp(context, &adreno_dev->ringbuffer);
if (timestamp_cmp(timestamp, ts_issued) <= 0)
return 0;
@@ -3598,7 +3612,8 @@
unsigned int msecs)
{
static unsigned int io_cnt;
- struct adreno_context *adreno_ctx = context ? context->devctxt : NULL;
+ struct adreno_context *adreno_ctx = context ? ADRENO_CONTEXT(context) :
+ NULL;
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
unsigned int context_id = _get_context_id(context);
unsigned int time_elapsed = 0;
@@ -3606,8 +3621,6 @@
int ts_compare = 1;
int io, ret = -ETIMEDOUT;
- /* Get out early if the context has already been destroyed */
-
if (context_id == KGSL_CONTEXT_INVALID) {
KGSL_DRV_WARN(device, "context was detached");
return -EINVAL;
@@ -3745,9 +3758,9 @@
switch (type) {
case KGSL_TIMESTAMP_QUEUED: {
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
- struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
- timestamp = rb->timestamp[context_id];
+ timestamp = adreno_context_timestamp(context,
+ &adreno_dev->ringbuffer);
break;
}
case KGSL_TIMESTAMP_CONSUMED:
@@ -3919,6 +3932,7 @@
/* Optional functions */
.setstate = adreno_setstate,
.drawctxt_create = adreno_drawctxt_create,
+ .drawctxt_detach = adreno_drawctxt_detach,
.drawctxt_destroy = adreno_drawctxt_destroy,
.setproperty = adreno_setproperty,
.postmortem_dump = adreno_dump,
diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h
index b53d16f..f83d268 100644
--- a/drivers/gpu/msm/adreno.h
+++ b/drivers/gpu/msm/adreno.h
@@ -25,6 +25,9 @@
#define ADRENO_DEVICE(device) \
KGSL_CONTAINER_OF(device, struct adreno_device, dev)
+#define ADRENO_CONTEXT(device) \
+ KGSL_CONTAINER_OF(device, struct adreno_context, base)
+
#define ADRENO_CHIPID_CORE(_id) (((_id) >> 24) & 0xFF)
#define ADRENO_CHIPID_MAJOR(_id) (((_id) >> 16) & 0xFF)
#define ADRENO_CHIPID_MINOR(_id) (((_id) >> 8) & 0xFF)
@@ -501,6 +504,26 @@
}
/**
+ * adreno_context_timestamp() - Return the last queued timestamp for the context
+ * @k_ctxt: Pointer to the KGSL context to query
+ * @rb: Pointer to the ringbuffer structure for the GPU
+ *
+ * Return the last queued context for the given context. This is used to verify
+ * that incoming requests are not using an invalid (unsubmitted) timestamp
+ */
+static inline int adreno_context_timestamp(struct kgsl_context *k_ctxt,
+ struct adreno_ringbuffer *rb)
+{
+ if (k_ctxt) {
+ struct adreno_context *a_ctxt = ADRENO_CONTEXT(k_ctxt);
+
+ if (a_ctxt->flags & CTXT_FLAGS_PER_CONTEXT_TS)
+ return a_ctxt->timestamp;
+ }
+ return rb->global_ts;
+}
+
+/**
* adreno_encode_istore_size - encode istore size in CP format
* @adreno_dev - The 3D device.
*
diff --git a/drivers/gpu/msm/adreno_a2xx.c b/drivers/gpu/msm/adreno_a2xx.c
index 42137fe..3d72c5c 100644
--- a/drivers/gpu/msm/adreno_a2xx.c
+++ b/drivers/gpu/msm/adreno_a2xx.c
@@ -1355,7 +1355,7 @@
tmp_ctx.gmem_base = adreno_dev->gmem_base;
result = kgsl_allocate(&drawctxt->context_gmem_shadow.gmemshadow,
- drawctxt->pagetable, drawctxt->context_gmem_shadow.size);
+ drawctxt->base.pagetable, drawctxt->context_gmem_shadow.size);
if (result)
return result;
@@ -1364,7 +1364,7 @@
drawctxt->flags |= CTXT_FLAGS_GMEM_SHADOW;
/* blank out gmem shadow. */
- kgsl_sharedmem_set(drawctxt->dev_priv->device,
+ kgsl_sharedmem_set(drawctxt->base.device,
&drawctxt->context_gmem_shadow.gmemshadow, 0, 0,
drawctxt->context_gmem_shadow.size);
@@ -1389,7 +1389,7 @@
kgsl_cache_range_op(&drawctxt->context_gmem_shadow.gmemshadow,
KGSL_CACHE_OP_FLUSH);
- kgsl_cffdump_syncmem(drawctxt->dev_priv,
+ kgsl_cffdump_syncmem(drawctxt->base.device,
&drawctxt->context_gmem_shadow.gmemshadow,
drawctxt->context_gmem_shadow.gmemshadow.gpuaddr,
drawctxt->context_gmem_shadow.gmemshadow.size, false);
@@ -1410,13 +1410,13 @@
*/
ret = kgsl_allocate(&drawctxt->gpustate,
- drawctxt->pagetable, _context_size(adreno_dev));
+ drawctxt->base.pagetable, _context_size(adreno_dev));
if (ret)
return ret;
- kgsl_sharedmem_set(drawctxt->dev_priv->device, &drawctxt->gpustate, 0,
- 0, _context_size(adreno_dev));
+ kgsl_sharedmem_set(drawctxt->base.device, &drawctxt->gpustate,
+ 0, 0, _context_size(adreno_dev));
tmp_ctx.cmd = tmp_ctx.start
= (unsigned int *)((char *)drawctxt->gpustate.hostptr + CMD_OFFSET);
@@ -1440,8 +1440,8 @@
kgsl_cache_range_op(&drawctxt->gpustate,
KGSL_CACHE_OP_FLUSH);
- kgsl_cffdump_syncmem(drawctxt->dev_priv, &drawctxt->gpustate,
- drawctxt->gpustate.gpuaddr,
+ kgsl_cffdump_syncmem(drawctxt->base.device,
+ &drawctxt->gpustate, drawctxt->gpustate.gpuaddr,
drawctxt->gpustate.size, false);
done:
@@ -1516,7 +1516,7 @@
"Current active context has caused gpu hang\n");
if (!(context->flags & CTXT_FLAGS_PREAMBLE)) {
- kgsl_cffdump_syncmem(context->dev_priv, &context->gpustate,
+ kgsl_cffdump_syncmem(context->base.device, &context->gpustate,
context->reg_save[1],
context->reg_save[2] << 2, true);
/* save registers and constants. */
@@ -1525,7 +1525,7 @@
context->reg_save, 3);
if (context->flags & CTXT_FLAGS_SHADER_SAVE) {
- kgsl_cffdump_syncmem(context->dev_priv,
+ kgsl_cffdump_syncmem(context->base.device,
&context->gpustate,
context->shader_save[1],
context->shader_save[2] << 2, true);
@@ -1534,7 +1534,7 @@
KGSL_CMD_FLAGS_PMODE,
context->shader_save, 3);
- kgsl_cffdump_syncmem(context->dev_priv,
+ kgsl_cffdump_syncmem(context->base.device,
&context->gpustate,
context->shader_fixup[1],
context->shader_fixup[2] << 2, true);
@@ -1552,7 +1552,7 @@
if ((context->flags & CTXT_FLAGS_GMEM_SAVE) &&
(context->flags & CTXT_FLAGS_GMEM_SHADOW)) {
- kgsl_cffdump_syncmem(context->dev_priv, &context->gpustate,
+ kgsl_cffdump_syncmem(context->base.device, &context->gpustate,
context->context_gmem_shadow.gmem_save[1],
context->context_gmem_shadow.gmem_save[2] << 2, true);
/* save gmem.
@@ -1562,7 +1562,7 @@
KGSL_CMD_FLAGS_PMODE,
context->context_gmem_shadow.gmem_save, 3);
- kgsl_cffdump_syncmem(context->dev_priv, &context->gpustate,
+ kgsl_cffdump_syncmem(context->base.device, &context->gpustate,
context->chicken_restore[1],
context->chicken_restore[2] << 2, true);
@@ -1586,9 +1586,18 @@
unsigned int cmds[5];
if (context == NULL) {
- /* No context - set the default apgetable and thats it */
+ /* No context - set the default pagetable and thats it */
+ unsigned int id;
+ /*
+ * If there isn't a current context, the kgsl_mmu_setstate
+ * will use the CPU path so we don't need to give
+ * it a valid context id.
+ */
+ id = (adreno_dev->drawctxt_active != NULL)
+ ? adreno_dev->drawctxt_active->base.id
+ : KGSL_CONTEXT_INVALID;
kgsl_mmu_setstate(&device->mmu, device->mmu.defaultpagetable,
- adreno_dev->drawctxt_active->id);
+ id);
return;
}
@@ -1597,16 +1606,17 @@
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
- cmds[4] = context->id;
+ cmds[4] = context->base.id;
adreno_ringbuffer_issuecmds(device, context, KGSL_CMD_FLAGS_NONE,
cmds, 5);
- kgsl_mmu_setstate(&device->mmu, context->pagetable, context->id);
+ kgsl_mmu_setstate(&device->mmu, context->base.pagetable,
+ context->base.id);
/* restore gmem.
* (note: changes shader. shader must not already be restored.)
*/
if (context->flags & CTXT_FLAGS_GMEM_RESTORE) {
- kgsl_cffdump_syncmem(context->dev_priv, &context->gpustate,
+ kgsl_cffdump_syncmem(context->base.device, &context->gpustate,
context->context_gmem_shadow.gmem_restore[1],
context->context_gmem_shadow.gmem_restore[2] << 2,
true);
@@ -1616,7 +1626,7 @@
context->context_gmem_shadow.gmem_restore, 3);
if (!(context->flags & CTXT_FLAGS_PREAMBLE)) {
- kgsl_cffdump_syncmem(context->dev_priv,
+ kgsl_cffdump_syncmem(context->base.device,
&context->gpustate,
context->chicken_restore[1],
context->chicken_restore[2] << 2, true);
@@ -1631,7 +1641,7 @@
}
if (!(context->flags & CTXT_FLAGS_PREAMBLE)) {
- kgsl_cffdump_syncmem(context->dev_priv, &context->gpustate,
+ kgsl_cffdump_syncmem(context->base.device, &context->gpustate,
context->reg_restore[1],
context->reg_restore[2] << 2, true);
@@ -1641,7 +1651,7 @@
/* restore shader instructions & partitioning. */
if (context->flags & CTXT_FLAGS_SHADER_RESTORE) {
- kgsl_cffdump_syncmem(context->dev_priv,
+ kgsl_cffdump_syncmem(context->base.device,
&context->gpustate,
context->shader_restore[1],
context->shader_restore[2] << 2, true);
diff --git a/drivers/gpu/msm/adreno_a3xx.c b/drivers/gpu/msm/adreno_a3xx.c
index 7e5638c..670f0cf 100644
--- a/drivers/gpu/msm/adreno_a3xx.c
+++ b/drivers/gpu/msm/adreno_a3xx.c
@@ -2321,7 +2321,7 @@
tmp_ctx.gmem_base = adreno_dev->gmem_base;
result = kgsl_allocate(&drawctxt->context_gmem_shadow.gmemshadow,
- drawctxt->pagetable, drawctxt->context_gmem_shadow.size);
+ drawctxt->base.pagetable, drawctxt->context_gmem_shadow.size);
if (result)
return result;
@@ -2355,7 +2355,7 @@
*/
ret = kgsl_allocate(&drawctxt->gpustate,
- drawctxt->pagetable, CONTEXT_SIZE);
+ drawctxt->base.pagetable, CONTEXT_SIZE);
if (ret)
return ret;
@@ -2420,7 +2420,7 @@
* already be saved.)
*/
- kgsl_cffdump_syncmem(context->dev_priv,
+ kgsl_cffdump_syncmem(context->base.device,
&context->gpustate,
context->context_gmem_shadow.gmem_save[1],
context->context_gmem_shadow.gmem_save[2] << 2, true);
@@ -2441,8 +2441,17 @@
if (context == NULL) {
/* No context - set the default pagetable and thats it */
+ unsigned int id;
+ /*
+ * If there isn't a current context, the kgsl_mmu_setstate
+ * will use the CPU path so we don't need to give
+ * it a valid context id.
+ */
+ id = (adreno_dev->drawctxt_active != NULL)
+ ? adreno_dev->drawctxt_active->base.id
+ : KGSL_CONTEXT_INVALID;
kgsl_mmu_setstate(&device->mmu, device->mmu.defaultpagetable,
- adreno_dev->drawctxt_active->id);
+ id);
return;
}
@@ -2451,10 +2460,11 @@
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
- cmds[4] = context->id;
+ cmds[4] = context->base.id;
adreno_ringbuffer_issuecmds(device, context, KGSL_CMD_FLAGS_NONE,
cmds, 5);
- kgsl_mmu_setstate(&device->mmu, context->pagetable, context->id);
+ kgsl_mmu_setstate(&device->mmu, context->base.pagetable,
+ context->base.id);
/*
* Restore GMEM. (note: changes shader.
@@ -2462,7 +2472,7 @@
*/
if (context->flags & CTXT_FLAGS_GMEM_RESTORE) {
- kgsl_cffdump_syncmem(context->dev_priv,
+ kgsl_cffdump_syncmem(context->base.device,
&context->gpustate,
context->context_gmem_shadow.gmem_restore[1],
context->context_gmem_shadow.gmem_restore[2] << 2,
diff --git a/drivers/gpu/msm/adreno_drawctxt.c b/drivers/gpu/msm/adreno_drawctxt.c
index b32cdae..bf173a7 100644
--- a/drivers/gpu/msm/adreno_drawctxt.c
+++ b/drivers/gpu/msm/adreno_drawctxt.c
@@ -134,34 +134,32 @@
/**
* adreno_drawctxt_create - create a new adreno draw context
- * @device - KGSL device to create the context on
- * @pagetable - Pagetable for the context
- * @context- Generic KGSL context structure
- * @flags - flags for the context (passed from user space)
+ * @dev_priv: the owner of the context
+ * @flags: flags for the context (passed from user space)
*
- * Create a new draw context for the 3D core. Return 0 on success,
- * or error code on failure.
+ * Create and return a new draw context for the 3D core.
*/
-int adreno_drawctxt_create(struct kgsl_device *device,
- struct kgsl_pagetable *pagetable,
- struct kgsl_context *context, uint32_t *flags)
+struct kgsl_context *
+adreno_drawctxt_create(struct kgsl_device_private *dev_priv,
+ uint32_t *flags)
{
struct adreno_context *drawctxt;
+ struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
- struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
int ret;
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
-
if (drawctxt == NULL)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
- drawctxt->pid = task_pid_nr(current);
- strlcpy(drawctxt->pid_name, current->comm, TASK_COMM_LEN);
- drawctxt->pagetable = pagetable;
+ ret = kgsl_context_init(dev_priv, &drawctxt->base);
+ if (ret != 0) {
+ kfree(drawctxt);
+ return ERR_PTR(ret);
+ }
+
drawctxt->bin_base_offset = 0;
- drawctxt->id = context->id;
- rb->timestamp[context->id] = 0;
+ drawctxt->timestamp = 0;
*flags &= (KGSL_CONTEXT_PREAMBLE |
KGSL_CONTEXT_NO_GMEM_ALLOC |
@@ -192,50 +190,47 @@
drawctxt->type =
(*flags & KGSL_CONTEXT_TYPE_MASK) >> KGSL_CONTEXT_TYPE_SHIFT;
- drawctxt->dev_priv = context->dev_priv;
ret = adreno_dev->gpudev->ctxt_create(adreno_dev, drawctxt);
if (ret)
goto err;
kgsl_sharedmem_writel(device, &device->memstore,
- KGSL_MEMSTORE_OFFSET(drawctxt->id, ref_wait_ts),
+ KGSL_MEMSTORE_OFFSET(drawctxt->base.id, ref_wait_ts),
KGSL_INIT_REFTIMESTAMP);
kgsl_sharedmem_writel(device, &device->memstore,
- KGSL_MEMSTORE_OFFSET(drawctxt->id, ts_cmp_enable), 0);
+ KGSL_MEMSTORE_OFFSET(drawctxt->base.id, ts_cmp_enable),
+ 0);
kgsl_sharedmem_writel(device, &device->memstore,
- KGSL_MEMSTORE_OFFSET(drawctxt->id, soptimestamp), 0);
+ KGSL_MEMSTORE_OFFSET(drawctxt->base.id, soptimestamp),
+ 0);
kgsl_sharedmem_writel(device, &device->memstore,
- KGSL_MEMSTORE_OFFSET(drawctxt->id, eoptimestamp), 0);
+ KGSL_MEMSTORE_OFFSET(drawctxt->base.id, eoptimestamp),
+ 0);
- context->devctxt = drawctxt;
- return 0;
+ return &drawctxt->base;
err:
- kfree(drawctxt);
- return ret;
+ kgsl_context_put(&drawctxt->base);
+ return ERR_PTR(ret);
}
/**
- * adreno_drawctxt_destroy - destroy a draw context
- * @device - KGSL device that owns the context
- * @context- Generic KGSL context container for the context
+ * adreno_drawctxt_detach(): detach a context from the GPU
+ * @context: Generic KGSL context container for the context
*
- * Destroy an existing context. Return 0 on success or error
- * code on failure.
*/
-
-/* destroy a drawing context */
-
-void adreno_drawctxt_destroy(struct kgsl_device *device,
- struct kgsl_context *context)
+void adreno_drawctxt_detach(struct kgsl_context *context)
{
- struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+ struct kgsl_device *device;
+ struct adreno_device *adreno_dev;
struct adreno_context *drawctxt;
- if (context == NULL || context->devctxt == NULL)
+ if (context == NULL)
return;
- drawctxt = context->devctxt;
+ device = context->device;
+ adreno_dev = ADRENO_DEVICE(device);
+ drawctxt = ADRENO_CONTEXT(context);
/* deactivate context */
if (adreno_dev->drawctxt_active == drawctxt) {
/* no need to save GMEM or shader, the context is
@@ -256,9 +251,17 @@
kgsl_sharedmem_free(&drawctxt->gpustate);
kgsl_sharedmem_free(&drawctxt->context_gmem_shadow.gmemshadow);
+}
+
+void adreno_drawctxt_destroy(struct kgsl_context *context)
+{
+ struct adreno_context *drawctxt;
+ if (context == NULL)
+ return;
+
+ drawctxt = ADRENO_CONTEXT(context);
kfree(drawctxt);
- context->devctxt = NULL;
}
/**
@@ -274,10 +277,12 @@
struct kgsl_context *context,
unsigned int offset)
{
- struct adreno_context *drawctxt = context->devctxt;
+ struct adreno_context *drawctxt;
- if (drawctxt)
- drawctxt->bin_base_offset = offset;
+ if (context == NULL)
+ return;
+ drawctxt = ADRENO_CONTEXT(context);
+ drawctxt->bin_base_offset = offset;
}
/**
@@ -316,12 +321,22 @@
KGSL_CTXT_INFO(device, "from %d to %d flags %d\n",
adreno_dev->drawctxt_active ?
- adreno_dev->drawctxt_active->id : 0,
- drawctxt ? drawctxt->id : 0, flags);
+ adreno_dev->drawctxt_active->base.id : 0,
+ drawctxt ? drawctxt->base.id : 0, flags);
/* Save the old context */
adreno_dev->gpudev->ctxt_save(adreno_dev, adreno_dev->drawctxt_active);
+ /* Put the old instance of the active drawctxt */
+ if (adreno_dev->drawctxt_active) {
+ kgsl_context_put(&adreno_dev->drawctxt_active->base);
+ adreno_dev->drawctxt_active = NULL;
+ }
+
+ /* Get a refcount to the new instance */
+ if (drawctxt)
+ _kgsl_context_get(&drawctxt->base);
+
/* Set the new context */
adreno_dev->gpudev->ctxt_restore(adreno_dev, drawctxt);
adreno_dev->drawctxt_active = drawctxt;
diff --git a/drivers/gpu/msm/adreno_drawctxt.h b/drivers/gpu/msm/adreno_drawctxt.h
index 2b8e600..88d1b8c 100644
--- a/drivers/gpu/msm/adreno_drawctxt.h
+++ b/drivers/gpu/msm/adreno_drawctxt.h
@@ -13,8 +13,6 @@
#ifndef __ADRENO_DRAWCTXT_H
#define __ADRENO_DRAWCTXT_H
-#include <linux/sched.h>
-
#include "adreno_pm4types.h"
#include "a2xx_reg.h"
@@ -96,15 +94,11 @@
};
struct adreno_context {
- pid_t pid;
- char pid_name[TASK_COMM_LEN];
- unsigned int id;
+ struct kgsl_context base;
unsigned int ib_gpu_time_used;
+ unsigned int timestamp;
uint32_t flags;
- uint32_t pagefault;
- unsigned long pagefault_ts;
unsigned int type;
- struct kgsl_pagetable *pagetable;
struct kgsl_memdesc gpustate;
unsigned int reg_restore[3];
unsigned int shader_save[3];
@@ -131,16 +125,15 @@
struct kgsl_memdesc constant_load_commands[3];
struct kgsl_memdesc cond_execs[4];
struct kgsl_memdesc hlsqcontrol_restore_commands[1];
- struct kgsl_device_private *dev_priv;
};
-int adreno_drawctxt_create(struct kgsl_device *device,
- struct kgsl_pagetable *pagetable,
- struct kgsl_context *context,
+
+struct kgsl_context *adreno_drawctxt_create(struct kgsl_device_private *,
uint32_t *flags);
-void adreno_drawctxt_destroy(struct kgsl_device *device,
- struct kgsl_context *context);
+void adreno_drawctxt_detach(struct kgsl_context *context);
+
+void adreno_drawctxt_destroy(struct kgsl_context *context);
void adreno_drawctxt_switch(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt,
diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c
index f6c05b5..999b782 100644
--- a/drivers/gpu/msm/adreno_ringbuffer.c
+++ b/drivers/gpu/msm/adreno_ringbuffer.c
@@ -138,7 +138,7 @@
if (context && context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
- context, context->id);
+ context, context->base.id);
return -EDEADLK;
}
wait_time = jiffies + wait_timeout;
@@ -575,6 +575,8 @@
/* overlay structure on memptrs memory */
rb->memptrs = (struct kgsl_rbmemptrs *) rb->memptrs_desc.hostptr;
+ rb->global_ts = 0;
+
return 0;
}
@@ -594,11 +596,11 @@
memset(rb, 0, sizeof(struct adreno_ringbuffer));
}
-static uint32_t
+static int
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
- int sizedwords, uint32_t timestamp)
+ int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
@@ -607,6 +609,7 @@
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
+ unsigned int timestamp;
/*
* if the context was not created with per context timestamp
@@ -616,20 +619,7 @@
*/
if ((context && (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)) &&
!(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
- context_id = context->id;
-
- if ((context && (context->flags & CTXT_FLAGS_USER_GENERATED_TS)) &&
- !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
- if (timestamp_cmp(rb->timestamp[context_id],
- timestamp) >= 0) {
- KGSL_DRV_ERR(rb->device,
- "Invalid user generated ts <%d:0x%x>, "
- "less than last issued ts <%d:0x%x>\n",
- context_id, timestamp, context_id,
- rb->timestamp[context_id]);
- return -ERANGE;
- }
- }
+ context_id = context->base.id;
/* reserve space to temporarily turn off protected mode
* error checking if needed
@@ -669,13 +659,8 @@
total_sizedwords += 2;
ringcmds = adreno_ringbuffer_allocspace(rb, context, total_sizedwords);
- if (!ringcmds) {
- /*
- * We could not allocate space in ringbuffer, just return the
- * last timestamp
- */
- return rb->timestamp[context_id];
- }
+ if (!ringcmds)
+ return -ENOSPC;
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
@@ -690,26 +675,19 @@
}
/* always increment the global timestamp. once. */
- rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
+ rb->global_ts++;
- /*
- * If global timestamp then we are not using per context ts for
- * this submission
- */
- if (context_id != KGSL_MEMSTORE_GLOBAL) {
- if (context->flags & CTXT_FLAGS_USER_GENERATED_TS)
- rb->timestamp[context_id] = timestamp;
- else
- rb->timestamp[context_id]++;
- }
- timestamp = rb->timestamp[context_id];
+ if (KGSL_MEMSTORE_GLOBAL != context_id)
+ timestamp = context->timestamp;
+ else
+ timestamp = rb->global_ts;
/* scratchpad ts for fault tolerance */
GSL_RB_WRITE(rb->device, ringcmds, rcmd_gpu,
cp_type0_packet(adreno_getreg(adreno_dev,
ADRENO_REG_CP_TIMESTAMP), 1));
GSL_RB_WRITE(rb->device, ringcmds, rcmd_gpu,
- rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
+ rb->global_ts);
/* start-of-pipeline timestamp */
GSL_RB_WRITE(rb->device, ringcmds, rcmd_gpu,
@@ -781,7 +759,7 @@
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(rb->device, ringcmds, rcmd_gpu,
- rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
+ rb->global_ts);
}
if (adreno_is_a20x(adreno_dev)) {
@@ -857,7 +835,7 @@
adreno_ringbuffer_submit(rb);
- return timestamp;
+ return 0;
}
unsigned int
@@ -877,7 +855,7 @@
flags |= KGSL_CMD_FLAGS_INTERNAL_ISSUE;
return adreno_ringbuffer_addcmds(rb, drawctxt, flags, cmds,
- sizedwords, 0);
+ sizedwords);
}
static bool _parse_ibs(struct kgsl_device_private *dev_priv, uint gpuaddr,
@@ -1097,12 +1075,9 @@
ret = -EINVAL;
goto done;
}
- drawctxt = context->devctxt;
+ drawctxt = ADRENO_CONTEXT(context);
if (drawctxt->flags & CTXT_FLAGS_GPU_HANG) {
- KGSL_CTXT_ERR(device, "proc %s failed fault tolerance"
- " will not accept commands for context %d\n",
- drawctxt->pid_name, drawctxt->id);
ret = -EDEADLK;
goto done;
}
@@ -1116,10 +1091,6 @@
start_index = 1;
if (drawctxt->flags & CTXT_FLAGS_SKIP_EOF) {
- KGSL_CTXT_ERR(device,
- "proc %s triggered fault tolerance"
- " skipping commands for context till EOF %d\n",
- drawctxt->pid_name, drawctxt->id);
if (flags & KGSL_CMD_FLAGS_EOF)
drawctxt->flags &= ~CTXT_FLAGS_SKIP_EOF;
if (start_index)
@@ -1172,10 +1143,30 @@
adreno_drawctxt_switch(adreno_dev, drawctxt, flags);
- *timestamp = adreno_ringbuffer_addcmds(&adreno_dev->ringbuffer,
+ if (drawctxt->flags & CTXT_FLAGS_USER_GENERATED_TS) {
+ if (timestamp_cmp(drawctxt->timestamp, *timestamp) >= 0) {
+ KGSL_DRV_ERR(device,
+ "Invalid user generated ts <%d:0x%x>, "
+ "less than last issued ts <%d:0x%x>\n",
+ context->id, *timestamp, context->id,
+ drawctxt->timestamp);
+ return -ERANGE;
+ }
+ drawctxt->timestamp = *timestamp;
+ } else
+ drawctxt->timestamp++;
+
+ ret = adreno_ringbuffer_addcmds(&adreno_dev->ringbuffer,
drawctxt,
(flags & KGSL_CMD_FLAGS_EOF),
- &link[0], (cmds - link), *timestamp);
+ &link[0], (cmds - link));
+ if (ret)
+ goto done;
+
+ if (drawctxt->flags & CTXT_FLAGS_PER_CONTEXT_TS)
+ *timestamp = drawctxt->timestamp;
+ else
+ *timestamp = adreno_dev->ringbuffer.global_ts;
#ifdef CONFIG_MSM_KGSL_CFF_DUMP
/*
@@ -1282,7 +1273,7 @@
k_ctxt = kgsl_context_get(device, ft_data->context_id);
if (k_ctxt) {
- a_ctxt = k_ctxt->devctxt;
+ a_ctxt = ADRENO_CONTEXT(k_ctxt);
if (a_ctxt->flags & CTXT_FLAGS_PREAMBLE)
_turn_preamble_on_for_ib_seq(rb, rb_rptr);
kgsl_context_put(k_ctxt);
@@ -1319,7 +1310,7 @@
k_ctxt = kgsl_context_get(rb->device, val2);
if (k_ctxt) {
- a_ctxt = k_ctxt->devctxt;
+ a_ctxt = ADRENO_CONTEXT(k_ctxt);
/* If we are changing to a good context and were not
* copying commands then copy over commands to the good
diff --git a/drivers/gpu/msm/adreno_ringbuffer.h b/drivers/gpu/msm/adreno_ringbuffer.h
index e9fb050..9634e32 100644
--- a/drivers/gpu/msm/adreno_ringbuffer.h
+++ b/drivers/gpu/msm/adreno_ringbuffer.h
@@ -55,7 +55,7 @@
unsigned int wptr; /* write pointer offset in dwords from baseaddr */
- unsigned int timestamp[KGSL_MEMSTORE_MAX];
+ unsigned int global_ts;
};
diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c
index 4c1ef54..966e035 100644
--- a/drivers/gpu/msm/kgsl.c
+++ b/drivers/gpu/msm/kgsl.c
@@ -448,20 +448,24 @@
entry->priv = NULL;
}
-/* Allocate a new context id */
-
-static struct kgsl_context *
-kgsl_create_context(struct kgsl_device_private *dev_priv)
+/**
+ * kgsl_context_init() - helper to initialize kgsl_context members
+ * @dev_priv: the owner of the context
+ * @context: the newly created context struct, should be allocated by
+ * the device specific drawctxt_create function.
+ *
+ * This is a helper function for the device specific drawctxt_create
+ * function to initialize the common members of its context struct.
+ * If this function succeeds, reference counting is active in the context
+ * struct and the caller should kgsl_context_put() it on error.
+ * If it fails, the caller should just free the context structure
+ * it passed in.
+ */
+int kgsl_context_init(struct kgsl_device_private *dev_priv,
+ struct kgsl_context *context)
{
+ int ret = 0, id;
struct kgsl_device *device = dev_priv->device;
- struct kgsl_context *context;
- int ret, id = 0;
-
- context = kzalloc(sizeof(*context), GFP_KERNEL);
-
- if (context == NULL)
- return ERR_PTR(-ENOMEM);
-
while (1) {
if (idr_pre_get(&device->context_idr, GFP_KERNEL) == 0) {
@@ -480,30 +484,26 @@
}
if (ret)
- goto func_end;
+ goto fail;
/* MAX - 1, there is one memdesc in memstore for device info */
if (id >= KGSL_MEMSTORE_MAX) {
KGSL_DRV_INFO(device, "cannot have more than %d "
"ctxts due to memstore limitation\n",
KGSL_MEMSTORE_MAX);
- write_lock(&device->context_lock);
- idr_remove(&device->context_idr, id);
- write_unlock(&device->context_lock);
ret = -ENOSPC;
- goto func_end;
+ goto fail_free_id;
}
kref_init(&context->refcount);
- context->dev_priv = dev_priv;
+ context->device = dev_priv->device;
+ context->pagetable = dev_priv->process_priv->pagetable;
+
+ context->pid = dev_priv->process_priv->pid;
ret = kgsl_sync_timeline_create(context);
- if (ret) {
- write_lock(&device->context_lock);
- idr_remove(&dev_priv->device->context_idr, id);
- write_unlock(&device->context_lock);
- goto func_end;
- }
+ if (ret)
+ goto fail_free_id;
/* Initialize the pending event list */
INIT_LIST_HEAD(&context->events);
@@ -518,15 +518,15 @@
*/
INIT_LIST_HEAD(&context->events_list);
-
-func_end:
- if (ret) {
- kfree(context);
- return ERR_PTR(ret);
- }
-
- return context;
+ return 0;
+fail_free_id:
+ write_lock(&device->context_lock);
+ idr_remove(&dev_priv->device->context_idr, id);
+ write_unlock(&device->context_lock);
+fail:
+ return ret;
}
+EXPORT_SYMBOL(kgsl_context_init);
/**
* kgsl_context_detach - Release the "master" context reference
@@ -536,37 +536,36 @@
* has requested for it to be destroyed. The context itself may
* exist a bit longer until its reference count goes to zero.
* Other code referencing the context can detect that it has been
- * detached because the context id will be set to KGSL_CONTEXT_INVALID.
+ * detached by checking the KGSL_CONTEXT_DETACHED bit in
+ * context->priv.
*/
void
kgsl_context_detach(struct kgsl_context *context)
{
- int id;
struct kgsl_device *device;
if (context == NULL)
return;
- device = context->dev_priv->device;
- trace_kgsl_context_detach(device, context);
- id = context->id;
- if (device->ftbl->drawctxt_destroy)
- device->ftbl->drawctxt_destroy(device, context);
- /*device specific drawctxt_destroy MUST clean up devctxt */
- BUG_ON(context->devctxt);
+ device = context->device;
+
+ /*
+ * Mark the context as detached to keep others from using
+ * the context before it gets fully removed, and to make sure
+ * we don't try to detach twice.
+ */
+ if (test_and_set_bit(KGSL_CONTEXT_DETACHED, &context->priv))
+ return;
+
+ trace_kgsl_context_detach(device, context);
+
+ device->ftbl->drawctxt_detach(context);
/*
* Cancel events after the device-specific context is
- * destroyed, to avoid possibly freeing memory while
+ * detached, to avoid possibly freeing memory while
* it is still in use by the GPU.
*/
kgsl_cancel_events_ctxt(device, context);
- write_lock(&device->context_lock);
- context->id = KGSL_CONTEXT_INVALID;
- idr_remove(&device->context_idr, id);
- write_unlock(&device->context_lock);
-
- context->dev_priv = NULL;
-
kgsl_context_put(context);
}
@@ -575,8 +574,19 @@
{
struct kgsl_context *context = container_of(kref, struct kgsl_context,
refcount);
+ struct kgsl_device *device = context->device;
+
+ trace_kgsl_context_destroy(device, context);
+
+ write_lock(&device->context_lock);
+ if (context->id != KGSL_CONTEXT_INVALID) {
+ idr_remove(&device->context_idr, context->id);
+ context->id = KGSL_CONTEXT_INVALID;
+ }
+ write_unlock(&device->context_lock);
kgsl_sync_timeline_destroy(context);
- kfree(context);
+
+ device->ftbl->drawctxt_destroy(context);
}
struct kgsl_device *kgsl_get_device(int dev_idx)
@@ -975,7 +985,7 @@
if (context == NULL)
break;
- if (context->dev_priv == dev_priv)
+ if (context->pid == private->pid)
kgsl_context_detach(context);
next = next + 1;
@@ -1623,27 +1633,16 @@
int result = 0;
struct kgsl_drawctxt_create *param = data;
struct kgsl_context *context = NULL;
+ struct kgsl_device *device = dev_priv->device;
- context = kgsl_create_context(dev_priv);
-
+ context = device->ftbl->drawctxt_create(dev_priv, ¶m->flags);
if (IS_ERR(context)) {
result = PTR_ERR(context);
goto done;
}
-
- if (dev_priv->device->ftbl->drawctxt_create) {
- result = dev_priv->device->ftbl->drawctxt_create(
- dev_priv->device, dev_priv->process_priv->pagetable,
- context, ¶m->flags);
- if (result)
- goto done;
- }
trace_kgsl_context_create(dev_priv->device, context, param->flags);
param->drawctxt_id = context->id;
done:
- if (result && !IS_ERR(context))
- kgsl_context_detach(context);
-
return result;
}
@@ -2600,7 +2599,7 @@
if (!entry)
return -EINVAL;
- kgsl_cffdump_syncmem(dev_priv, &entry->memdesc, param->gpuaddr,
+ kgsl_cffdump_syncmem(dev_priv->device, &entry->memdesc, param->gpuaddr,
param->len, true);
kgsl_mem_entry_put(entry);
diff --git a/drivers/gpu/msm/kgsl_cffdump.c b/drivers/gpu/msm/kgsl_cffdump.c
index b07a1cad..43bcc30 100644
--- a/drivers/gpu/msm/kgsl_cffdump.c
+++ b/drivers/gpu/msm/kgsl_cffdump.c
@@ -410,29 +410,19 @@
cffdump_printline(-1, cff_opcode, op1, op2, op3, op4, op5);
}
-void kgsl_cffdump_syncmem(struct kgsl_device_private *dev_priv,
- struct kgsl_memdesc *memdesc, uint gpuaddr, uint sizebytes,
- bool clean_cache)
+void kgsl_cffdump_syncmem(struct kgsl_device *device,
+ struct kgsl_memdesc *memdesc, uint gpuaddr,
+ uint sizebytes, bool clean_cache)
{
- struct kgsl_device *device = dev_priv->device;
const void *src;
if (!device->cff_dump_enable)
return;
+ BUG_ON(memdesc == NULL);
+
total_syncmem += sizebytes;
- if (memdesc == NULL) {
- struct kgsl_mem_entry *entry;
- entry = kgsl_sharedmem_find_region(dev_priv->process_priv,
- gpuaddr, sizebytes);
- if (entry == NULL) {
- KGSL_CORE_ERR("did not find mapping "
- "for gpuaddr: 0x%08x\n", gpuaddr);
- return;
- }
- memdesc = &entry->memdesc;
- }
src = (uint *)kgsl_gpuaddr_to_vaddr(memdesc, gpuaddr);
if (memdesc->hostptr == NULL) {
KGSL_CORE_ERR(
diff --git a/drivers/gpu/msm/kgsl_cffdump.h b/drivers/gpu/msm/kgsl_cffdump.h
index ab5a345..2852e0f 100644
--- a/drivers/gpu/msm/kgsl_cffdump.h
+++ b/drivers/gpu/msm/kgsl_cffdump.h
@@ -28,7 +28,7 @@
void kgsl_cffdump_destroy(void);
void kgsl_cffdump_open(struct kgsl_device *device);
void kgsl_cffdump_close(struct kgsl_device *device);
-void kgsl_cffdump_syncmem(struct kgsl_device_private *dev_priv,
+void kgsl_cffdump_syncmem(struct kgsl_device *,
struct kgsl_memdesc *memdesc, uint physaddr, uint sizebytes,
bool clean_cache);
void kgsl_cffdump_setmem(struct kgsl_device *device, uint addr,
@@ -74,7 +74,7 @@
return;
}
-static inline void kgsl_cffdump_syncmem(struct kgsl_device_private *dev_priv,
+static inline void kgsl_cffdump_syncmem(struct kgsl_device *device,
struct kgsl_memdesc *memdesc, uint physaddr, uint sizebytes,
bool clean_cache)
{
diff --git a/drivers/gpu/msm/kgsl_device.h b/drivers/gpu/msm/kgsl_device.h
index d0e40db..1f8bcbe 100644
--- a/drivers/gpu/msm/kgsl_device.h
+++ b/drivers/gpu/msm/kgsl_device.h
@@ -15,6 +15,7 @@
#include <linux/idr.h>
#include <linux/pm_qos.h>
+#include <linux/sched.h>
#include "kgsl.h"
#include "kgsl_mmu.h"
@@ -103,11 +104,10 @@
calling the hook */
void (*setstate) (struct kgsl_device *device, unsigned int context_id,
uint32_t flags);
- int (*drawctxt_create) (struct kgsl_device *device,
- struct kgsl_pagetable *pagetable, struct kgsl_context *context,
- uint32_t *flags);
- void (*drawctxt_destroy) (struct kgsl_device *device,
- struct kgsl_context *context);
+ struct kgsl_context *(*drawctxt_create) (struct kgsl_device_private *,
+ uint32_t *flags);
+ void (*drawctxt_detach) (struct kgsl_context *context);
+ void (*drawctxt_destroy) (struct kgsl_context *context);
long (*ioctl) (struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
int (*setproperty) (struct kgsl_device *device,
@@ -254,31 +254,44 @@
.ver_minor = DRIVER_VERSION_MINOR
+/* bits for struct kgsl_context.priv */
+/* the context has been destroyed by userspace and is no longer using the gpu */
+#define KGSL_CONTEXT_DETACHED 0
+/* the context has caused a pagefault */
+#define KGSL_CONTEXT_PAGEFAULT 1
+
/**
* struct kgsl_context - Master structure for a KGSL context object
- * @refcount - kref object for reference counting the context
- * @id - integer identifier for the context
- * @dev_priv - pointer to the owning device instance
- * @devctxt - pointer to the device specific context information
- * @reset_status - status indication whether a gpu reset occured and whether
+ * @refcount: kref object for reference counting the context
+ * @id: integer identifier for the context
+ * @priv: in-kernel context flags, use KGSL_CONTEXT_* values
+ * @dev_priv: pointer to the owning device instance
+ * @reset_status: status indication whether a gpu reset occured and whether
* this context was responsible for causing it
- * @wait_on_invalid_ts - flag indicating if this context has tried to wait on a
+ * @wait_on_invalid_ts: flag indicating if this context has tried to wait on a
* bad timestamp
- * @timeline - sync timeline used to create fences that can be signaled when a
+ * @timeline: sync timeline used to create fences that can be signaled when a
* sync_pt timestamp expires
- * @events - list head of pending events for this context
- * @events_list - list node for the list of all contexts that have pending events
+ * @events: list head of pending events for this context
+ * @events_list: list node for the list of all contexts that have pending events
+ * @pid: process that owns this context.
+ * @pagefault: flag set if this context caused a pagefault.
+ * @pagefault_ts: global timestamp of the pagefault, if KGSL_CONTEXT_PAGEFAULT
+ * is set.
*/
struct kgsl_context {
struct kref refcount;
uint32_t id;
- struct kgsl_device_private *dev_priv;
- void *devctxt;
+ pid_t pid;
+ unsigned long priv;
+ struct kgsl_device *device;
+ struct kgsl_pagetable *pagetable;
unsigned int reset_status;
bool wait_on_invalid_ts;
struct sync_timeline *timeline;
struct list_head events;
struct list_head events_list;
+ unsigned int pagefault_ts;
};
struct kgsl_process_private {
@@ -429,6 +442,9 @@
void kgsl_context_destroy(struct kref *kref);
+int kgsl_context_init(struct kgsl_device_private *, struct kgsl_context
+ *context);
+
/**
* kgsl_context_put() - Release context reference count
* @context: Pointer to the KGSL context to be released
@@ -442,6 +458,22 @@
if (context)
kref_put(&context->refcount, kgsl_context_destroy);
}
+
+/**
+ * kgsl_context_detached() - check if a context is detached
+ * @context: the context
+ *
+ * Check if a context has been destroyed by userspace and is only waiting
+ * for reference counts to go away. This check is used to weed out
+ * contexts that shouldn't use the gpu so NULL is considered detached.
+ */
+static inline bool kgsl_context_detached(struct kgsl_context *context)
+{
+ return (context == NULL || test_bit(KGSL_CONTEXT_DETACHED,
+ &context->priv));
+}
+
+
/**
* kgsl_context_get() - get a pointer to a KGSL context
* @device: Pointer to the KGSL device that owns the context
@@ -462,7 +494,10 @@
context = idr_find(&device->context_idr, id);
- if (context)
+ /* Don't return a context that has been detached */
+ if (kgsl_context_detached(context))
+ context = NULL;
+ else
kref_get(&context->refcount);
read_unlock(&device->context_lock);
@@ -471,6 +506,20 @@
}
/**
+* _kgsl_context_get() - lightweight function to just increment the ref count
+* @context: Pointer to the KGSL context
+*
+* Get a reference to the specified KGSL context structure. This is a
+* lightweight way to just increase the refcount on a known context rather than
+* walking through kgsl_context_get and searching the iterator
+*/
+static inline void _kgsl_context_get(struct kgsl_context *context)
+{
+ if (context)
+ kref_get(&context->refcount);
+}
+
+/**
* kgsl_context_get_owner() - get a pointer to a KGSL context in a specific
* process
* @dev_priv: Pointer to the process struct
@@ -489,8 +538,8 @@
context = kgsl_context_get(dev_priv->device, id);
- /* Verify that the context belongs to the dev_priv instance */
- if (context && context->dev_priv != dev_priv) {
+ /* Verify that the context belongs to current calling process. */
+ if (context != NULL && context->pid != dev_priv->process_priv->pid) {
kgsl_context_put(context);
return NULL;
}
diff --git a/drivers/gpu/msm/kgsl_events.c b/drivers/gpu/msm/kgsl_events.c
index 0b75175..40974b0 100644
--- a/drivers/gpu/msm/kgsl_events.c
+++ b/drivers/gpu/msm/kgsl_events.c
@@ -147,7 +147,7 @@
* Increment the refcount to avoid freeing the context while
* cancelling its events
*/
- kref_get(&context->refcount);
+ _kgsl_context_get(context);
/* Remove ourselves from the master pending list */
list_del_init(&context->events_list);
@@ -324,7 +324,7 @@
* Increment the refcount to make sure that the list_del_init
* is called with a valid context's list
*/
- kref_get(&context->refcount);
+ _kgsl_context_get(context);
/*
* If kgsl_timestamp_expired_context returns 0 then it no longer
* has any pending events and can be removed from the list
diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c
index 7af4575..14aae9f 100644
--- a/drivers/gpu/msm/kgsl_iommu.c
+++ b/drivers/gpu/msm/kgsl_iommu.c
@@ -327,7 +327,7 @@
unsigned int no_page_fault_log = 0;
unsigned int curr_context_id = 0;
unsigned int curr_global_ts = 0;
- static struct kgsl_context *context;
+ struct kgsl_context *context;
ret = get_iommu_unit(dev, &mmu, &iommu_unit);
if (ret)
@@ -401,23 +401,16 @@
context = kgsl_context_get(device, curr_context_id);
if (context != NULL) {
- struct adreno_context *drawctxt = context->devctxt;
-
kgsl_sharedmem_readl(&device->memstore, &curr_global_ts,
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp));
- /*
- * Store pagefault's timestamp in adreno context,
- * this information will be used in GFT
- */
-
- if (drawctxt != NULL) {
- drawctxt->pagefault = 1;
- drawctxt->pagefault_ts = curr_global_ts;
- }
+ /* save pagefault timestamp for GFT */
+ set_bit(KGSL_CONTEXT_PAGEFAULT, &context->priv);
+ context->pagefault_ts = curr_global_ts;
kgsl_context_put(context);
+ context = NULL;
}
trace_kgsl_mmu_pagefault(iommu_dev->kgsldev, addr,
diff --git a/drivers/gpu/msm/kgsl_pwrctrl.c b/drivers/gpu/msm/kgsl_pwrctrl.c
index 5e6d24b..20391be 100644
--- a/drivers/gpu/msm/kgsl_pwrctrl.c
+++ b/drivers/gpu/msm/kgsl_pwrctrl.c
@@ -1214,6 +1214,11 @@
}
}
+bool kgsl_pwrctrl_isenabled(struct kgsl_device *device)
+{
+ struct kgsl_pwrctrl *pwr = &device->pwrctrl;
+ return (test_bit(KGSL_PWRFLAGS_CLK_ON, &pwr->power_flags) != 0);
+}
/**
* kgsl_pre_hwaccess - Enforce preconditions for touching registers
@@ -1230,7 +1235,7 @@
/* In order to touch a register you must hold the device mutex...*/
BUG_ON(!mutex_is_locked(&device->mutex));
/* and have the clock on! */
- BUG_ON(!test_bit(KGSL_PWRFLAGS_CLK_ON, &device->pwrctrl.power_flags));
+ BUG_ON(!kgsl_pwrctrl_isenabled(device));
}
EXPORT_SYMBOL(kgsl_pre_hwaccess);
diff --git a/drivers/gpu/msm/kgsl_pwrctrl.h b/drivers/gpu/msm/kgsl_pwrctrl.h
index 3bf65ee..b7d9226 100644
--- a/drivers/gpu/msm/kgsl_pwrctrl.h
+++ b/drivers/gpu/msm/kgsl_pwrctrl.h
@@ -110,6 +110,8 @@
void kgsl_pwrctrl_uninit_sysfs(struct kgsl_device *device);
void kgsl_pwrctrl_enable(struct kgsl_device *device);
void kgsl_pwrctrl_disable(struct kgsl_device *device);
+bool kgsl_pwrctrl_isenabled(struct kgsl_device *device);
+
static inline unsigned long kgsl_get_clkrate(struct clk *clk)
{
return (clk != NULL) ? clk_get_rate(clk) : 0;
diff --git a/drivers/gpu/msm/kgsl_snapshot.c b/drivers/gpu/msm/kgsl_snapshot.c
index 35f24d9..6094e04 100644
--- a/drivers/gpu/msm/kgsl_snapshot.c
+++ b/drivers/gpu/msm/kgsl_snapshot.c
@@ -109,7 +109,7 @@
struct kgsl_device *device;
if (context)
- device = context->dev_priv->device;
+ device = context->device;
else
device = (struct kgsl_device *)data;
diff --git a/drivers/gpu/msm/kgsl_sync.c b/drivers/gpu/msm/kgsl_sync.c
index 1853996c..62ecdeb 100644
--- a/drivers/gpu/msm/kgsl_sync.c
+++ b/drivers/gpu/msm/kgsl_sync.c
@@ -239,7 +239,7 @@
char ktimeline_name[sizeof(context->timeline->name)] = {};
snprintf(ktimeline_name, sizeof(ktimeline_name),
"%s_%.15s(%d)-%.15s(%d)-%d",
- context->dev_priv->device->name,
+ context->device->name,
current->group_leader->comm, current->group_leader->pid,
current->comm, current->pid, context->id);
@@ -250,7 +250,7 @@
ktimeline = (struct kgsl_sync_timeline *) context->timeline;
ktimeline->last_timestamp = 0;
- ktimeline->device = context->dev_priv->device;
+ ktimeline->device = context->device;
ktimeline->context_id = context->id;
return 0;
diff --git a/drivers/gpu/msm/kgsl_trace.h b/drivers/gpu/msm/kgsl_trace.h
index abb7c35..5981163 100644
--- a/drivers/gpu/msm/kgsl_trace.h
+++ b/drivers/gpu/msm/kgsl_trace.h
@@ -678,6 +678,28 @@
)
);
+TRACE_EVENT(kgsl_context_destroy,
+
+ TP_PROTO(struct kgsl_device *device, struct kgsl_context *context),
+
+ TP_ARGS(device, context),
+
+ TP_STRUCT__entry(
+ __string(device_name, device->name)
+ __field(unsigned int, id)
+ ),
+
+ TP_fast_assign(
+ __assign_str(device_name, device->name);
+ __entry->id = context->id;
+ ),
+
+ TP_printk(
+ "d_name=%s ctx=%u",
+ __get_str(device_name), __entry->id
+ )
+);
+
TRACE_EVENT(kgsl_mmu_pagefault,
TP_PROTO(struct kgsl_device *device, unsigned int page,
diff --git a/drivers/gpu/msm/z180.c b/drivers/gpu/msm/z180.c
index cc1819d..883417f 100644
--- a/drivers/gpu/msm/z180.c
+++ b/drivers/gpu/msm/z180.c
@@ -859,11 +859,30 @@
return status;
}
-static void
-z180_drawctxt_destroy(struct kgsl_device *device,
- struct kgsl_context *context)
+struct kgsl_context *
+z180_drawctxt_create(struct kgsl_device_private *dev_priv,
+ uint32_t *flags)
{
- struct z180_device *z180_dev = Z180_DEVICE(device);
+ int ret;
+ struct kgsl_context *context = kzalloc(sizeof(*context), GFP_KERNEL);
+ if (context == NULL)
+ return ERR_PTR(-ENOMEM);
+ ret = kgsl_context_init(dev_priv, context);
+ if (ret != 0) {
+ kfree(context);
+ return ERR_PTR(ret);
+ }
+ return context;
+}
+
+static void
+z180_drawctxt_detach(struct kgsl_context *context)
+{
+ struct kgsl_device *device;
+ struct z180_device *z180_dev;
+
+ device = context->device;
+ z180_dev = Z180_DEVICE(device);
z180_idle(device);
@@ -875,6 +894,12 @@
}
}
+static void
+z180_drawctxt_destroy(struct kgsl_context *context)
+{
+ kfree(context);
+}
+
static void z180_power_stats(struct kgsl_device *device,
struct kgsl_power_stats *stats)
{
@@ -941,7 +966,8 @@
.gpuid = z180_gpuid,
.irq_handler = z180_irq_handler,
/* Optional functions */
- .drawctxt_create = NULL,
+ .drawctxt_create = z180_drawctxt_create,
+ .drawctxt_detach = z180_drawctxt_detach,
.drawctxt_destroy = z180_drawctxt_destroy,
.ioctl = NULL,
.postmortem_dump = z180_dump,