prima: Changes to detect thread stuck issues in wlan driver
This fix contains the necessary changes to detect thread stuck issues
in prima driver. The changes are to post messages to MC thread, TX
thread and RX thread for every few seconds, which elicits a response
from these threads. A loss of response from any of these threads for
consecutive three times is regarded as thread stuck.
Also added Support for deferrable timer.
Change-Id: I3184db4dd7472f0c65dd607af7c197c52139cdb0
CRs-Fixed: 862223
diff --git a/CORE/SVC/inc/wlan_logging_sock_svc.h b/CORE/SVC/inc/wlan_logging_sock_svc.h
index 2cd6745..80ac7c5 100644
--- a/CORE/SVC/inc/wlan_logging_sock_svc.h
+++ b/CORE/SVC/inc/wlan_logging_sock_svc.h
@@ -63,5 +63,6 @@
void wlan_logging_set_log_level(void);
-
+bool wlan_is_logger_thread(int threadId);
+void wlan_logging_reset_thread_stuck_count(int threadId);
#endif /* WLAN_LOGGING_SOCK_SVC_H */
diff --git a/CORE/SVC/src/logging/wlan_logging_sock_svc.c b/CORE/SVC/src/logging/wlan_logging_sock_svc.c
index da13972..e333b42 100644
--- a/CORE/SVC/src/logging/wlan_logging_sock_svc.c
+++ b/CORE/SVC/src/logging/wlan_logging_sock_svc.c
@@ -42,6 +42,7 @@
#include "vos_memory.h"
#include <linux/ratelimit.h>
#include <asm/arch_timer.h>
+#include <vos_sched.h>
#define LOGGING_TRACE(level, args...) \
VOS_TRACE(VOS_MODULE_ID_SVC, level, ## args)
@@ -72,6 +73,9 @@
static DEFINE_RATELIMIT_STATE(errCnt, \
NL_BDCAST_RATELIMIT_INTERVAL, \
NL_BDCAST_RATELIMIT_BURST);
+/* Timer value for detecting thread stuck issues */
+#define THREAD_STUCK_TIMER_VAL 5000 // 5 seconds
+#define THREAD_STUCK_COUNT 3
struct log_msg {
struct list_head node;
@@ -135,6 +139,14 @@
struct log_msg *pcur_node;
/* Event flag used for wakeup and post indication*/
unsigned long event_flag;
+ /* Timer to detect thread stuck issue */
+ vos_timer_t threadStuckTimer;
+ /* Count for each thread to determine thread stuck */
+ unsigned int mcThreadStuckCount;
+ unsigned int txThreadStuckCount;
+ unsigned int rxThreadStuckCount;
+ /* lock to synchronize access to the thread stuck counts */
+ spinlock_t thread_stuck_lock;
/* Indicates logger thread is activated */
bool is_active;
/* data structure for log complete event*/
@@ -714,6 +726,51 @@
}
/**
+ * wlan_logging_detect_thread_stuck_cb()- Call back of the
+ * thread stuck timer to detect thread stuck
+ * by probing the MC, TX, RX threads and take action if
+ * Thread doesnt respond.
+ * @priv: timer data.
+ * This function is called when the thread stuck timer expire
+ * to detect thread stuck
+ * and probe threads.
+ *
+ * Return: void
+ */
+static void wlan_logging_detect_thread_stuck_cb(void *priv)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&gwlan_logging.thread_stuck_lock, flags);
+
+ if ((gwlan_logging.mcThreadStuckCount == THREAD_STUCK_COUNT) ||
+ (gwlan_logging.txThreadStuckCount == THREAD_STUCK_COUNT) ||
+ (gwlan_logging.rxThreadStuckCount == THREAD_STUCK_COUNT)) {
+ spin_unlock_irqrestore(&gwlan_logging.thread_stuck_lock, flags);
+ pr_err("%s: %s Thread Stuck !!!\n", __func__,
+ ((gwlan_logging.mcThreadStuckCount == THREAD_STUCK_COUNT)?
+ "MC" : (gwlan_logging.mcThreadStuckCount ==
+ THREAD_STUCK_COUNT)? "TX" : "RX"));
+ VOS_BUG(0);
+ return;
+ }
+
+ /* Increment the thread stuck count for all threads */
+ gwlan_logging.mcThreadStuckCount++;
+ gwlan_logging.txThreadStuckCount++;
+ gwlan_logging.rxThreadStuckCount++;
+
+ spin_unlock_irqrestore(&gwlan_logging.thread_stuck_lock, flags);
+ vos_probe_threads();
+
+ /* Restart the timer */
+ if (VOS_STATUS_SUCCESS !=
+ vos_timer_start(&gwlan_logging.threadStuckTimer,
+ THREAD_STUCK_TIMER_VAL))
+ pr_err("%s: Unable to start thread stuck timer\n", __func__);
+}
+
+/**
* wlan_logging_thread() - The WLAN Logger thread
* @Arg - pointer to the HDD context
*
@@ -730,14 +787,33 @@
daemonize("wlan_logging_thread");
#endif
+ /* Initialize the timer to detect thread stuck issues */
+ if (vos_timer_init(&gwlan_logging.threadStuckTimer, VOS_TIMER_TYPE_SW,
+ wlan_logging_detect_thread_stuck_cb, NULL)) {
+ pr_err("%s: Unable to initialize thread stuck timer\n", __func__);
+ } else {
+ if (VOS_STATUS_SUCCESS !=
+ vos_timer_start(&gwlan_logging.threadStuckTimer,
+ THREAD_STUCK_TIMER_VAL))
+ pr_err("%s: Unable to start thread stuck timer\n",
+ __func__);
+ else
+ pr_info("%s: Successfully started thread stuck timer\n",
+ __func__);
+ }
+
+
while (!gwlan_logging.exit) {
ret_wait_status = wait_event_interruptible(
gwlan_logging.wait_queue,
(test_bit(HOST_LOG_POST_MASK, &gwlan_logging.event_flag) ||
gwlan_logging.exit ||
- test_bit(LOGGER_MGMT_DATA_PKT_POST_MASK,&gwlan_logging.event_flag) ||
- test_bit(LOGGER_FW_LOG_PKT_POST_MASK, &gwlan_logging.event_flag) ||
- test_bit(LOGGER_FATAL_EVENT_POST_MASK, &gwlan_logging.event_flag)));
+ test_bit(LOGGER_MGMT_DATA_PKT_POST_MASK,
+ &gwlan_logging.event_flag) ||
+ test_bit(LOGGER_FW_LOG_PKT_POST_MASK,
+ &gwlan_logging.event_flag) ||
+ test_bit(LOGGER_FATAL_EVENT_POST_MASK,
+ &gwlan_logging.event_flag)));
if (ret_wait_status == -ERESTARTSYS) {
pr_err("%s: wait_event return -ERESTARTSYS", __func__);
@@ -782,6 +858,8 @@
}
}
+ vos_timer_destroy(&gwlan_logging.threadStuckTimer);
+
complete_and_exit(&gwlan_logging.shutdown_comp, 0);
return 0;
@@ -1206,7 +1284,6 @@
return status;
}
-
void wlan_process_done_indication(uint8 type, uint32 reason_code)
{
if ((type == WLAN_QXDM_LOGGING) && (wlan_is_log_report_in_progress() == TRUE))
@@ -1217,4 +1294,47 @@
}
}
+/**
+ * wlan_is_logger_thread()- Check if threadid is
+ * of logger thread
+ *
+ * @threadId: passed threadid
+ *
+ * This function is called to check if threadid is
+ * of logger thread.
+ *
+ * Return: true if threadid is of logger thread.
+ */
+bool wlan_is_logger_thread(int threadId)
+{
+ return ((gwlan_logging.thread) &&
+ (threadId == gwlan_logging.thread->pid));
+}
+
+/**
+ * wlan_logging_reset_thread_stuck_count()- Callback to
+ * probe msg sent to Threads.
+ *
+ * @threadId: passed threadid
+ *
+ * This function is called to by the thread after
+ * processing the probe msg, with their own thread id.
+ *
+ * Return: void.
+ */
+void wlan_logging_reset_thread_stuck_count(int threadId)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&gwlan_logging.thread_stuck_lock, flags);
+ if (vos_sched_is_mc_thread(threadId))
+ gwlan_logging.mcThreadStuckCount = 0;
+ else if (vos_sched_is_tx_thread(threadId))
+ gwlan_logging.txThreadStuckCount = 0;
+ else if (vos_sched_is_rx_thread(threadId))
+ gwlan_logging.rxThreadStuckCount = 0;
+
+ spin_unlock_irqrestore(&gwlan_logging.thread_stuck_lock, flags);
+}
+
#endif /* WLAN_LOGGING_SOCK_SVC_ENABLE */
diff --git a/CORE/SYS/common/inc/wlan_qct_sys.h b/CORE/SYS/common/inc/wlan_qct_sys.h
index 038aa6d..f85d29d 100644
--- a/CORE/SYS/common/inc/wlan_qct_sys.h
+++ b/CORE/SYS/common/inc/wlan_qct_sys.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012-2013 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2012-2015 The Linux Foundation. All rights reserved.
*
* Previously licensed under the ISC license by Qualcomm Atheros, Inc.
*
@@ -70,6 +70,8 @@
--------------------------------------------------------------------------*/
typedef v_VOID_t ( * sysResponseCback ) ( v_VOID_t *pUserData );
+typedef v_VOID_t ( * sysThreadProbeCback ) ( int threadId );
+
@@ -87,6 +89,7 @@
SYS_MSG_ID_MC_STOP,
SYS_MSG_ID_FTM_RSP,
+ SYS_MSG_ID_RX_THR_PROBE,
} SYS_MSG_ID;
/*---------------------------------------------------------------------------
diff --git a/CORE/SYS/common/src/wlan_qct_sys.c b/CORE/SYS/common/src/wlan_qct_sys.c
index 8bdef80..f55bd8c 100644
--- a/CORE/SYS/common/src/wlan_qct_sys.c
+++ b/CORE/SYS/common/src/wlan_qct_sys.c
@@ -382,9 +382,10 @@
// function that is in the message.
case SYS_MSG_ID_MC_THR_PROBE:
{
- VOS_TRACE(VOS_MODULE_ID_SYS, VOS_TRACE_LEVEL_ERROR,
- " Received SYS_MSG_ID_MC_THR_PROBE message msgType = %d [0x%08x]",
- pMsg->type, pMsg->type);
+#ifdef WLAN_LOGGING_SOCK_SVC_ENABLE
+ if(pMsg->callback)
+ ((sysThreadProbeCback)pMsg->callback)(current->pid);
+#endif
break;
}
@@ -405,7 +406,6 @@
WLANFTM_McProcessMsg((v_VOID_t *)pMsg->bodyptr);
break;
}
-
default:
{
VOS_TRACE( VOS_MODULE_ID_SYS, VOS_TRACE_LEVEL_ERROR,
@@ -471,13 +471,10 @@
// function that is in the message.
case SYS_MSG_ID_TX_THR_PROBE:
{
- /* Handling for this message is not needed now so adding
- * debug print and VOS_ASSERT*/
- VOS_TRACE( VOS_MODULE_ID_SYS, VOS_TRACE_LEVEL_ERROR,
- " Received SYS_MSG_ID_TX_THR_PROBE message msgType= %d [0x%08x]",
- pMsg->type, pMsg->type );
- VOS_ASSERT(0);
-
+#ifdef WLAN_LOGGING_SOCK_SVC_ENABLE
+ if(pMsg->callback)
+ ((sysThreadProbeCback)pMsg->callback)(current->pid);
+#endif
break;
}
@@ -550,6 +547,15 @@
break;
}
+ case SYS_MSG_ID_RX_THR_PROBE:
+ {
+#ifdef WLAN_LOGGING_SOCK_SVC_ENABLE
+ if(pMsg->callback)
+ ((sysThreadProbeCback)pMsg->callback)(current->pid);
+#endif
+ break;
+ }
+
default:
{
VOS_TRACE( VOS_MODULE_ID_SYS, VOS_TRACE_LEVEL_ERROR,
diff --git a/CORE/VOSS/inc/vos_api.h b/CORE/VOSS/inc/vos_api.h
index 100ac75..7e5d0da 100644
--- a/CORE/VOSS/inc/vos_api.h
+++ b/CORE/VOSS/inc/vos_api.h
@@ -436,4 +436,5 @@
v_BOOL_t vos_isUnloadInProgress(void);
v_BOOL_t vos_isLoadUnloadInProgress(void);
+void vos_probe_threads(void);
#endif // if !defined __VOS_NVITEM_H
diff --git a/CORE/VOSS/src/vos_api.c b/CORE/VOSS/src/vos_api.c
index 21484db..5239b43 100644
--- a/CORE/VOSS/src/vos_api.c
+++ b/CORE/VOSS/src/vos_api.c
@@ -2903,3 +2903,37 @@
return (WLAN_HDD_UNLOAD_IN_PROGRESS == pHddCtx->isLoadUnloadInProgress);
}
+/**
+ * vos_probe_threads() - VOS API to post messages
+ * to all the threads to detect if they are active or not
+ *
+ * Return none.
+ *
+ */
+void vos_probe_threads(void)
+{
+ vos_msg_t msg;
+
+ msg.callback = wlan_logging_reset_thread_stuck_count;
+ /* Post Message to MC Thread */
+ sysBuildMessageHeader(SYS_MSG_ID_MC_THR_PROBE, &msg);
+ if (VOS_STATUS_SUCCESS != vos_mq_post_message(VOS_MQ_ID_SYS, &msg)) {
+ pr_err("%s: Unable to post SYS_MSG_ID_MC_THR_PROBE message to MC thread\n",
+ __func__);
+ }
+
+ /* Post Message to Tx Thread */
+ sysBuildMessageHeader(SYS_MSG_ID_TX_THR_PROBE, &msg);
+ if (VOS_STATUS_SUCCESS != vos_tx_mq_serialize(VOS_MQ_ID_SYS, &msg)) {
+ pr_err("%s: Unable to post SYS_MSG_ID_TX_THR_PROBE message to TX thread\n",
+ __func__);
+ }
+
+ /* Post Message to Rx Thread */
+ sysBuildMessageHeader(SYS_MSG_ID_RX_THR_PROBE, &msg);
+ if (VOS_STATUS_SUCCESS != vos_rx_mq_serialize(VOS_MQ_ID_SYS, &msg)) {
+ pr_err("%s: Unable to post SYS_MSG_ID_RX_THR_PROBE message to RX thread\n",
+ __func__);
+ }
+}
+
diff --git a/CORE/VOSS/src/vos_sched.c b/CORE/VOSS/src/vos_sched.c
index 27a4dbe..ca858ba 100644
--- a/CORE/VOSS/src/vos_sched.c
+++ b/CORE/VOSS/src/vos_sched.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012-2014 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2012-2015 The Linux Foundation. All rights reserved.
*
* Previously licensed under the ISC license by Qualcomm Atheros, Inc.
*
@@ -1782,6 +1782,24 @@
}
return ((gpVosSchedContext->RxThread) && (threadID == gpVosSchedContext->RxThread->pid));
}
+
+/*-------------------------------------------------------------------------
+ This helper function helps determine if thread id is of MC thread
+ ------------------------------------------------------------------------*/
+int vos_sched_is_mc_thread(int threadID)
+{
+ // Make sure that Vos Scheduler context has been initialized
+ VOS_ASSERT( NULL != gpVosSchedContext);
+ if (gpVosSchedContext == NULL)
+ {
+ VOS_TRACE(VOS_MODULE_ID_VOSS, VOS_TRACE_LEVEL_ERROR,
+ "%s: gpVosSchedContext == NULL",__func__);
+ return 0;
+ }
+ return ((gpVosSchedContext->McThread) &&
+ (threadID == gpVosSchedContext->McThread->pid));
+}
+
/*-------------------------------------------------------------------------
Helper function to get the scheduler context
------------------------------------------------------------------------*/
diff --git a/CORE/VOSS/src/vos_sched.h b/CORE/VOSS/src/vos_sched.h
index f9c3b65..7dcd7a5 100644
--- a/CORE/VOSS/src/vos_sched.h
+++ b/CORE/VOSS/src/vos_sched.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012-2014 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2012-2015 The Linux Foundation. All rights reserved.
*
* Previously licensed under the ISC license by Qualcomm Atheros, Inc.
*
@@ -349,6 +349,7 @@
int vos_sched_is_tx_thread(int threadID);
int vos_sched_is_rx_thread(int threadID);
+int vos_sched_is_mc_thread(int threadID);
/*---------------------------------------------------------------------------
\brief vos_sched_open() - initialize the vOSS Scheduler
diff --git a/CORE/VOSS/src/vos_timer.c b/CORE/VOSS/src/vos_timer.c
index 73f80bd..f67ba33 100644
--- a/CORE/VOSS/src/vos_timer.c
+++ b/CORE/VOSS/src/vos_timer.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012-2013 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2012-2013, 2015 The Linux Foundation. All rights reserved.
*
* Previously licensed under the ISC license by Qualcomm Atheros, Inc.
*
@@ -44,6 +44,7 @@
#include <vos_api.h>
#include "wlan_qct_sys.h"
#include "vos_sched.h"
+#include <wlan_logging_sock_svc.h>
/*--------------------------------------------------------------------------
Preprocessor definitions and constants
@@ -215,7 +216,16 @@
if(vos_rx_mq_serialize( VOS_MQ_ID_SYS, &msg ) == VOS_STATUS_SUCCESS)
return;
}
- else
+#ifdef WLAN_LOGGING_SOCK_SVC_ENABLE
+ else if (wlan_is_logger_thread(threadId))
+ {
+ VOS_TRACE(VOS_MODULE_ID_VOSS, VOS_TRACE_LEVEL_INFO,
+ "TIMER callback: running on Logger thread");
+ callback(NULL);
+ return;
+ }
+#endif
+ else
{
VOS_TRACE( VOS_MODULE_ID_VOSS, VOS_TRACE_LEVEL_INFO,
"TIMER callback: running on MC thread");
@@ -228,7 +238,7 @@
if(vos_mq_post_message( VOS_MQ_ID_SYS, &msg ) == VOS_STATUS_SUCCESS)
return;
- }
+ }
VOS_TRACE(VOS_MODULE_ID_VOSS, VOS_TRACE_LEVEL_ERROR,
"%s: Could not enqueue timer to any queue", __func__);
@@ -443,7 +453,10 @@
// set the various members of the timer structure
// with arguments passed or with default values
spin_lock_init(&timer->platformInfo.spinlock);
- init_timer(&(timer->platformInfo.Timer));
+ if (VOS_TIMER_TYPE_SW == timerType)
+ init_timer_deferrable(&(timer->platformInfo.Timer));
+ else
+ init_timer(&(timer->platformInfo.Timer));
timer->platformInfo.Timer.function = vos_linux_timer_callback;
timer->platformInfo.Timer.data = (unsigned long)timer;
timer->callback = callback;
@@ -471,7 +484,10 @@
// set the various members of the timer structure
// with arguments passed or with default values
spin_lock_init(&timer->platformInfo.spinlock);
- init_timer(&(timer->platformInfo.Timer));
+ if (VOS_TIMER_TYPE_SW == timerType)
+ init_timer_deferrable(&(timer->platformInfo.Timer));
+ else
+ init_timer(&(timer->platformInfo.Timer));
timer->platformInfo.Timer.function = vos_linux_timer_callback;
timer->platformInfo.Timer.data = (unsigned long)timer;
timer->callback = callback;