Tighten the safe points for code cache resets to happen.

Add a new flag in the Thread struct to track the whereabout of the top frame
in each Java thread. It is not safe to blow away the code cache if any thread
is in the JIT'ed land.
diff --git a/vm/Globals.h b/vm/Globals.h
index db1cb94..bac28f6 100644
--- a/vm/Globals.h
+++ b/vm/Globals.h
@@ -738,6 +738,12 @@
     /* Flag to indicate that the code cache is full */
     bool codeCacheFull;
 
+    /* Delay count for the next code cache reset request */
+    int delayCodeCacheReset;
+
+    /* Number of times that the code cache has been reset */
+    int numCodeCacheReset;
+
     /* true/false: compile/reject opcodes specified in the -Xjitop list */
     bool includeSelectedOp;
 
diff --git a/vm/SignalCatcher.c b/vm/SignalCatcher.c
index e187aeb..90211fd 100644
--- a/vm/SignalCatcher.c
+++ b/vm/SignalCatcher.c
@@ -262,10 +262,16 @@
 #endif
 #if defined(WITH_JIT) && defined(WITH_JIT_TUNING)
         } else if (rcvd == SIGUSR2) {
-            gDvmJit.printMe ^= true;
-            dvmCompilerDumpStats();
-            /* Stress-test unchain all */
-            dvmJitUnchainAll();
+            static int codeCacheResetCount = 0;
+            if ((--codeCacheResetCount & 7) == 0) {
+                gDvmJit.codeCacheFull = true;
+            } else {
+                dvmCompilerDumpStats();
+                /* Stress-test unchain all */
+                dvmJitUnchainAll();
+                LOGD("Send %d more signals to rest the code cache",
+                     codeCacheResetCount & 7);
+            }
 #endif
         } else {
             LOGE("unexpected signal %d\n", rcvd);
diff --git a/vm/Thread.c b/vm/Thread.c
index 75965d1..e4ee8e3 100644
--- a/vm/Thread.c
+++ b/vm/Thread.c
@@ -3268,6 +3268,25 @@
 }
 
 /*
+ * Convert ThreadStatus to a string.
+ */
+const char* dvmGetThreadStatusStr(ThreadStatus status)
+{
+    switch (status) {
+    case THREAD_ZOMBIE:         return "ZOMBIE";
+    case THREAD_RUNNING:        return "RUNNABLE";
+    case THREAD_TIMED_WAIT:     return "TIMED_WAIT";
+    case THREAD_MONITOR:        return "MONITOR";
+    case THREAD_WAIT:           return "WAIT";
+    case THREAD_INITIALIZING:   return "INITIALIZING";
+    case THREAD_STARTING:       return "STARTING";
+    case THREAD_NATIVE:         return "NATIVE";
+    case THREAD_VMWAIT:         return "VMWAIT";
+    default:                    return "UNKNOWN";
+    }
+}
+
+/*
  * Print information about the specified thread.
  *
  * Works best when the thread in question is "self" or has been suspended.
@@ -3277,11 +3296,6 @@
 void dvmDumpThreadEx(const DebugOutputTarget* target, Thread* thread,
     bool isRunning)
 {
-    /* tied to ThreadStatus enum */
-    static const char* kStatusNames[] = {
-        "ZOMBIE", "RUNNABLE", "TIMED_WAIT", "MONITOR", "WAIT",
-        "INITIALIZING", "STARTING", "NATIVE", "VMWAIT"
-    };
     Object* threadObj;
     Object* groupObj;
     StringObject* nameStr;
@@ -3333,11 +3347,10 @@
     if (groupName == NULL)
         groupName = strdup("(BOGUS GROUP)");
 
-    assert(thread->status < NELEM(kStatusNames));
     dvmPrintDebugMessage(target,
         "\"%s\"%s prio=%d tid=%d %s\n",
         threadName, isDaemon ? " daemon" : "",
-        priority, thread->threadId, kStatusNames[thread->status]);
+        priority, thread->threadId, dvmGetThreadStatusStr(thread->status));
     dvmPrintDebugMessage(target,
         "  | group=\"%s\" sCount=%d dsCount=%d s=%c obj=%p self=%p\n",
         groupName, thread->suspendCount, thread->dbgSuspendCount,
diff --git a/vm/Thread.h b/vm/Thread.h
index a6ad832..5e0ad4d 100644
--- a/vm/Thread.h
+++ b/vm/Thread.h
@@ -157,6 +157,16 @@
     /* internal reference tracking */
     ReferenceTable  internalLocalRefTable;
 
+#if defined(WITH_JIT)
+    /*
+     * Whether the current top VM frame is in the interpreter or JIT cache:
+     *   NULL    : in the interpreter
+     *   non-NULL: entry address of the JIT'ed code (the actual value doesn't
+     *             matter)
+     */
+    void*       inJitCodeCache;
+#endif
+
     /* JNI local reference tracking */
 #ifdef USE_INDIRECT_REF
     IndirectRefTable jniLocalRefTable;
@@ -426,6 +436,11 @@
 char* dvmGetThreadName(Thread* thread);
 
 /*
+ * Convert ThreadStatus to a string.
+ */
+const char* dvmGetThreadStatusStr(ThreadStatus status);
+
+/*
  * Return true if a thread is on the internal list.  If it is, the
  * thread is part of the GC's root set.
  */
diff --git a/vm/compiler/Compiler.c b/vm/compiler/Compiler.c
index bf44467..3886cce 100644
--- a/vm/compiler/Compiler.c
+++ b/vm/compiler/Compiler.c
@@ -59,9 +59,12 @@
 
     dvmLockMutex(&gDvmJit.compilerLock);
 
-    /* Queue full */
-    if (gDvmJit.compilerQueueLength == COMPILER_WORK_QUEUE_SIZE ||
-        gDvmJit.codeCacheFull == true) {
+    /*
+     * Return if queue is full.
+     * If the code cache is full, we will allow the work order to be added and
+     * we use that to trigger code cache reset.
+     */
+    if (gDvmJit.compilerQueueLength == COMPILER_WORK_QUEUE_SIZE) {
         result = false;
         goto done;
     }
@@ -128,6 +131,9 @@
         return false;
     }
 
+    // For debugging only
+    // LOGD("Code cache starts at %p", gDvmJit.codeCache);
+
     /* Copy the template code into the beginning of the code cache */
     int templateSize = (intptr_t) dmvCompilerTemplateEnd -
                        (intptr_t) dvmCompilerTemplateStart;
@@ -144,45 +150,101 @@
     return true;
 }
 
+static void crawlDalvikStack(Thread *thread, bool print)
+{
+    void *fp = thread->curFrame;
+    StackSaveArea* saveArea = NULL;
+    int stackLevel = 0;
+
+    if (print) {
+        LOGD("Crawling tid %d (%s / %p %s)", thread->systemTid,
+             dvmGetThreadStatusStr(thread->status),
+             thread->inJitCodeCache,
+             thread->inJitCodeCache ? "jit" : "interp");
+    }
+    /* Crawl the Dalvik stack frames to clear the returnAddr field */
+    while (fp != NULL) {
+        saveArea = SAVEAREA_FROM_FP(fp);
+
+        if (print) {
+            if (dvmIsBreakFrame(fp)) {
+                LOGD("  #%d: break frame (%p)",
+                     stackLevel, saveArea->returnAddr);
+            }
+            else {
+                LOGD("  #%d: %s.%s%s (%p)",
+                     stackLevel,
+                     saveArea->method->clazz->descriptor,
+                     saveArea->method->name,
+                     dvmIsNativeMethod(saveArea->method) ?
+                         " (native)" : "",
+                     saveArea->returnAddr);
+            }
+        }
+        stackLevel++;
+        saveArea->returnAddr = NULL;
+        assert(fp != saveArea->prevFrame);
+        fp = saveArea->prevFrame;
+    }
+    /* Make sure the stack is fully unwound to the bottom */
+    assert(saveArea == NULL ||
+           (u1 *) (saveArea+1) == thread->interpStackStart);
+}
+
 static void resetCodeCache(void)
 {
-    Thread* self = dvmThreadSelf();
     Thread* thread;
+    u8 startTime = dvmGetRelativeTimeUsec();
+    int inJit = 0;
 
-    LOGD("Reset the JIT code cache (%d bytes used)", gDvmJit.codeCacheByteUsed);
+    LOGD("Reset the JIT code cache (%d bytes used / %d time(s))",
+         gDvmJit.codeCacheByteUsed, ++gDvmJit.numCodeCacheReset);
 
     /* Stop the world */
     dvmSuspendAllThreads(SUSPEND_FOR_CC_RESET);
 
+    /* If any thread is found stuck in the JIT state, don't reset the cache */
+    for (thread = gDvm.threadList; thread != NULL; thread = thread->next) {
+        if (thread->inJitCodeCache) {
+            inJit++;
+            /*
+             * STOPSHIP
+             * Change the verbose mode to false after the new code receives
+             * more QA love.
+             */
+            crawlDalvikStack(thread, true);
+        }
+    }
+
+    if (inJit) {
+        /* Wait a while for the busy threads to rest and try again */
+        gDvmJit.delayCodeCacheReset = 256;
+        goto done;
+    }
+
+    /* Drain the work queue to free the work order */
+    while (workQueueLength()) {
+        CompilerWorkOrder work = workDequeue();
+        free(work.info);
+    }
+
     /* Wipe out the returnAddr field that soon will point to stale code */
     for (thread = gDvm.threadList; thread != NULL; thread = thread->next) {
-        if (thread == self)
-            continue;
-
-        /* Crawl the Dalvik stack frames */
-        StackSaveArea *ssaPtr = ((StackSaveArea *) thread->curFrame) - 1;
-        while (ssaPtr != ((StackSaveArea *) NULL) - 1) {
-            ssaPtr->returnAddr = NULL;
-            ssaPtr = ((StackSaveArea *) ssaPtr->prevFrame) - 1;
-        };
+        crawlDalvikStack(thread, false);
     }
 
     /* Reset the JitEntry table contents to the initial unpopulated state */
     dvmJitResetTable();
 
-#if 0
     /*
-     * Uncomment the following code when testing/debugging.
-     *
      * Wipe out the code cache content to force immediate crashes if
      * stale JIT'ed code is invoked.
      */
-    memset(gDvmJit.codeCache,
-           (intptr_t) gDvmJit.codeCache + gDvmJit.codeCacheByteUsed,
-           0);
+    memset((char *) gDvmJit.codeCache + gDvmJit.templateSize,
+           0,
+           gDvmJit.codeCacheByteUsed - gDvmJit.templateSize);
     cacheflush((intptr_t) gDvmJit.codeCache,
                (intptr_t) gDvmJit.codeCache + gDvmJit.codeCacheByteUsed, 0);
-#endif
 
     /* Reset the current mark of used bytes to the end of template code */
     gDvmJit.codeCacheByteUsed = gDvmJit.templateSize;
@@ -197,6 +259,10 @@
     /* All clear now */
     gDvmJit.codeCacheFull = false;
 
+    LOGD("Code cache reset takes %lld usec",
+         dvmGetRelativeTimeUsec() - startTime);
+
+done:
     /* Resume all threads */
     dvmResumeAllThreads(SUSPEND_FOR_CC_RESET);
 }
@@ -263,7 +329,15 @@
                  */
 #if 0
                 if (gDvmJit.codeCacheFull == true) {
-                    resetCodeCache();
+                    if (gDvmJit.delayCodeCacheReset == 0) {
+                        resetCodeCache();
+                        assert(workQueueLength() == 0 ||
+                               gDvmJit.delayCodeCacheReset != 0);
+                    } else {
+                        LOGD("Delay the next %d tries to reset code cache",
+                             gDvmJit.delayCodeCacheReset);
+                        gDvmJit.delayCodeCacheReset--;
+                    }
                 }
 #endif
             } while (workQueueLength() != 0);
diff --git a/vm/compiler/codegen/arm/Assemble.c b/vm/compiler/codegen/arm/Assemble.c
index 7c16ff9..6a59c7e 100644
--- a/vm/compiler/codegen/arm/Assemble.c
+++ b/vm/compiler/codegen/arm/Assemble.c
@@ -1464,6 +1464,7 @@
     /* Stop the world */
     dvmSuspendAllThreads(SUSPEND_FOR_IC_PATCH);
 
+
     COMPILER_TRACE_CHAINING(
         LOGD("Jit Runtime: predicted chain %p from %s to %s (%s) patched",
              cellDest, cellDest->clazz ? cellDest->clazz->descriptor : "NULL",
@@ -1542,11 +1543,14 @@
                 case kChainingCellInvokePredicted:
                     targetOffset = 0;
                     predChainCell = (PredictedChainingCell *) pChainCells;
-                    /* Reset the cell to the init state */
-                    predChainCell->branch = PREDICTED_CHAIN_BX_PAIR_INIT;
+                    /*
+                     * There could be a race on another mutator thread to use
+                     * this particular predicted cell and the check has passed
+                     * the clazz comparison. So we cannot safely wipe the
+                     * method and branch but it is safe to clear the clazz,
+                     * which serves as the key.
+                     */
                     predChainCell->clazz = PREDICTED_CHAIN_CLAZZ_INIT;
-                    predChainCell->method = PREDICTED_CHAIN_METHOD_INIT;
-                    predChainCell->counter = PREDICTED_CHAIN_COUNTER_INIT;
                     break;
 #if defined(WITH_SELF_VERIFICATION)
                 case kChainingCellBackwardBranch:
@@ -1611,6 +1615,7 @@
         }
         cacheflush((long)lowAddress, (long)highAddress, 0);
         dvmUnlockMutex(&gDvmJit.tableLock);
+        gDvmJit.translationChains = 0;
     }
 }
 
diff --git a/vm/compiler/codegen/arm/CodegenDriver.c b/vm/compiler/codegen/arm/CodegenDriver.c
index f92e347..1993c9d 100644
--- a/vm/compiler/codegen/arm/CodegenDriver.c
+++ b/vm/compiler/codegen/arm/CodegenDriver.c
@@ -4103,7 +4103,8 @@
 {
     bool res;
 
-    if (gDvmJit.codeCacheFull) {
+    if (gDvmJit.codeCacheFull &&
+        (work->kind != kWorkOrderICPatch)) {
         return false;
     }
 
diff --git a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_CHAIN.S b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_CHAIN.S
index 20bb3ab..d6e6763 100644
--- a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_CHAIN.S
+++ b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_CHAIN.S
@@ -16,7 +16,7 @@
     SAVEAREA_FROM_FP(r10, r1)           @ r10<- stack save area
     add     r12, lr, #2                 @ setup the punt-to-interp address
     sub     r10, r10, r2, lsl #2        @ r10<- bottom (newsave - outsSize)
-    ldr     r8, [r8]                    @ r3<- suspendCount (int)
+    ldr     r8, [r8]                    @ r8<- suspendCount (int)
     cmp     r10, r9                     @ bottom < interpStackEnd?
     bxlt    r12                         @ return to raise stack overflow excep.
     @ r1 = newFP, r0 = methodToCall, r3 = returnCell, rPC = dalvikCallsite
diff --git a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S
index 0ecccfa..945203e 100644
--- a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S
+++ b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S
@@ -30,7 +30,9 @@
 
     @ go ahead and transfer control to the native code
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    mov     r2, #0
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in the jit code cache
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
     mov     r9, r3                      @ r9<- glue->self (preserve)
@@ -55,12 +57,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the mode properly
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 
diff --git a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NO_OPT.S b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NO_OPT.S
index 88b38e0..c4f766a 100644
--- a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NO_OPT.S
+++ b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NO_OPT.S
@@ -13,7 +13,7 @@
     sub     r1, r1, r7, lsl #2          @ r1<- newFp (old savearea - regsSize)
     SAVEAREA_FROM_FP(r10, r1)           @ r10<- stack save area
     sub     r10, r10, r2, lsl #2        @ r10<- bottom (newsave - outsSize)
-    ldr     r8, [r8]                    @ r3<- suspendCount (int)
+    ldr     r8, [r8]                    @ r8<- suspendCount (int)
     cmp     r10, r9                     @ bottom < interpStackEnd?
     bxlt    lr                          @ return to raise stack overflow excep.
     @ r1 = newFP, r0 = methodToCall, r3 = returnCell, rPC = dalvikCallsite
diff --git a/vm/compiler/template/armv5te/TEMPLATE_RETURN.S b/vm/compiler/template/armv5te/TEMPLATE_RETURN.S
index 1c85b19..aa9884a 100644
--- a/vm/compiler/template/armv5te/TEMPLATE_RETURN.S
+++ b/vm/compiler/template/armv5te/TEMPLATE_RETURN.S
@@ -35,6 +35,7 @@
     str     r0, [rGLUE, #offGlue_methodClassDex]
     cmp     r8, #0                      @ check the suspendCount
     movne   r9, #0                      @ clear the chaining cell address
+    str     r9, [r3, #offThread_inJitCodeCache] @ in code cache or not
     cmp     r9, #0                      @ chaining cell exists?
     blxne   r9                          @ jump to the chaining cell
 #if defined(EXIT_STATS)
diff --git a/vm/compiler/template/armv5te/footer.S b/vm/compiler/template/armv5te/footer.S
index dfbf9a8..c1487dd 100644
--- a/vm/compiler/template/armv5te/footer.S
+++ b/vm/compiler/template/armv5te/footer.S
@@ -10,7 +10,9 @@
     @ Prep for the native call
     @ r1 = newFP, r0 = methodToCall
     ldr     r3, [rGLUE, #offGlue_self]      @ r3<- glue->self
+    mov     r2, #0
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in jit code cache
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
@@ -36,12 +38,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the new mode
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 /*
@@ -49,6 +55,9 @@
  * r0  Faulting Dalvik PC
  */
 .LhandleException:
+    ldr     r3, [rGLUE, #offGlue_self]  @ r3<- glue->self
+    mov     r2, #0
+    str     r2, [r3, #offThread_inJitCodeCache] @ in interpreter land
     ldr     r1, .LdvmMterpCommonExceptionThrown @ PIC way of getting &func
     ldr     rIBASE, .LdvmAsmInstructionStart    @ same as above
     mov     rPC, r0                 @ reload the faulting Dalvik address
diff --git a/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S b/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S
index af487fd..36d3ea1 100644
--- a/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S
+++ b/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S
@@ -209,6 +209,7 @@
     str     r0, [rGLUE, #offGlue_methodClassDex]
     cmp     r8, #0                      @ check the suspendCount
     movne   r9, #0                      @ clear the chaining cell address
+    str     r9, [r3, #offThread_inJitCodeCache] @ in code cache or not
     cmp     r9, #0                      @ chaining cell exists?
     blxne   r9                          @ jump to the chaining cell
 #if defined(EXIT_STATS)
@@ -420,7 +421,9 @@
 
     @ go ahead and transfer control to the native code
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    mov     r2, #0
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in the jit code cache
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
     mov     r9, r3                      @ r9<- glue->self (preserve)
@@ -445,12 +448,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the mode properly
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 
@@ -1395,7 +1402,9 @@
     @ Prep for the native call
     @ r1 = newFP, r0 = methodToCall
     ldr     r3, [rGLUE, #offGlue_self]      @ r3<- glue->self
+    mov     r2, #0
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in jit code cache
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
@@ -1421,12 +1430,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the new mode
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 /*
@@ -1434,6 +1447,9 @@
  * r0  Faulting Dalvik PC
  */
 .LhandleException:
+    ldr     r3, [rGLUE, #offGlue_self]  @ r3<- glue->self
+    mov     r2, #0
+    str     r2, [r3, #offThread_inJitCodeCache] @ in interpreter land
     ldr     r1, .LdvmMterpCommonExceptionThrown @ PIC way of getting &func
     ldr     rIBASE, .LdvmAsmInstructionStart    @ same as above
     mov     rPC, r0                 @ reload the faulting Dalvik address
diff --git a/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S b/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S
index 5715b9b..48dd707 100644
--- a/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S
+++ b/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S
@@ -209,6 +209,7 @@
     str     r0, [rGLUE, #offGlue_methodClassDex]
     cmp     r8, #0                      @ check the suspendCount
     movne   r9, #0                      @ clear the chaining cell address
+    str     r9, [r3, #offThread_inJitCodeCache] @ in code cache or not
     cmp     r9, #0                      @ chaining cell exists?
     blxne   r9                          @ jump to the chaining cell
 #if defined(EXIT_STATS)
@@ -420,7 +421,9 @@
 
     @ go ahead and transfer control to the native code
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    mov     r2, #0
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in the jit code cache
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
     mov     r9, r3                      @ r9<- glue->self (preserve)
@@ -445,12 +448,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the mode properly
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 
@@ -1120,7 +1127,9 @@
     @ Prep for the native call
     @ r1 = newFP, r0 = methodToCall
     ldr     r3, [rGLUE, #offGlue_self]      @ r3<- glue->self
+    mov     r2, #0
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in jit code cache
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
@@ -1146,12 +1155,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the new mode
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 /*
@@ -1159,6 +1172,9 @@
  * r0  Faulting Dalvik PC
  */
 .LhandleException:
+    ldr     r3, [rGLUE, #offGlue_self]  @ r3<- glue->self
+    mov     r2, #0
+    str     r2, [r3, #offThread_inJitCodeCache] @ in interpreter land
     ldr     r1, .LdvmMterpCommonExceptionThrown @ PIC way of getting &func
     ldr     rIBASE, .LdvmAsmInstructionStart    @ same as above
     mov     rPC, r0                 @ reload the faulting Dalvik address
diff --git a/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S b/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S
index e3edf7d..469919a 100644
--- a/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S
+++ b/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S
@@ -209,6 +209,7 @@
     str     r0, [rGLUE, #offGlue_methodClassDex]
     cmp     r8, #0                      @ check the suspendCount
     movne   r9, #0                      @ clear the chaining cell address
+    str     r9, [r3, #offThread_inJitCodeCache] @ in code cache or not
     cmp     r9, #0                      @ chaining cell exists?
     blxne   r9                          @ jump to the chaining cell
 #if defined(EXIT_STATS)
@@ -420,7 +421,9 @@
 
     @ go ahead and transfer control to the native code
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    mov     r2, #0
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in the jit code cache
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
     mov     r9, r3                      @ r9<- glue->self (preserve)
@@ -445,12 +448,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the mode properly
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 
@@ -1395,7 +1402,9 @@
     @ Prep for the native call
     @ r1 = newFP, r0 = methodToCall
     ldr     r3, [rGLUE, #offGlue_self]      @ r3<- glue->self
+    mov     r2, #0
     ldr     r9, [r3, #offThread_jniLocal_topCookie] @ r9<- thread->localRef->...
+    str     r2, [r3, #offThread_inJitCodeCache] @ not in jit code cache
     str     r1, [r3, #offThread_curFrame]   @ self->curFrame = newFp
     str     r9, [r1, #(offStackSaveArea_localRefCookie - sizeofStackSaveArea)]
                                         @ newFp->localRefCookie=top
@@ -1421,12 +1430,16 @@
     @ r0 = dalvikCallsitePC
     bne     .LhandleException           @ no, handle exception
 
+    str     r2, [r9, #offThread_inJitCodeCache] @ set the new mode
     cmp     r2, #0                      @ return chaining cell still exists?
     bxne    r2                          @ yes - go ahead
 
     @ continue executing the next instruction through the interpreter
     ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
     add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+#if defined(EXIT_STATS)
+    mov     r0, #kCallsiteInterpreted
+#endif
     mov     pc, r1
 
 /*
@@ -1434,6 +1447,9 @@
  * r0  Faulting Dalvik PC
  */
 .LhandleException:
+    ldr     r3, [rGLUE, #offGlue_self]  @ r3<- glue->self
+    mov     r2, #0
+    str     r2, [r3, #offThread_inJitCodeCache] @ in interpreter land
     ldr     r1, .LdvmMterpCommonExceptionThrown @ PIC way of getting &func
     ldr     rIBASE, .LdvmAsmInstructionStart    @ same as above
     mov     rPC, r0                 @ reload the faulting Dalvik address
diff --git a/vm/interp/Interp.c b/vm/interp/Interp.c
index 2a7476c..6c6abce 100644
--- a/vm/interp/Interp.c
+++ b/vm/interp/Interp.c
@@ -1248,6 +1248,8 @@
         dvmJitToBackwardBranch,
 #endif
     };
+
+    assert(self->inJitCodeCache == NULL);
 #endif
 
 
diff --git a/vm/mterp/Mterp.c b/vm/mterp/Mterp.c
index dbae516..d14f6b2 100644
--- a/vm/mterp/Mterp.c
+++ b/vm/mterp/Mterp.c
@@ -106,6 +106,13 @@
     //LOGI("first instruction is 0x%04x\n", glue->pc[0]);
 
     changeInterp = dvmMterpStdRun(glue);
+
+#if defined(WITH_JIT)
+    if (glue->jitState != kJitSingleStep) {
+        glue->self->inJitCodeCache = NULL;
+    }
+#endif
+
     if (!changeInterp) {
         /* this is a "normal" exit; we're not coming back */
 #ifdef LOG_INSTR
diff --git a/vm/mterp/armv5te/entry.S b/vm/mterp/armv5te/entry.S
index f9e01a3..174a37a 100644
--- a/vm/mterp/armv5te/entry.S
+++ b/vm/mterp/armv5te/entry.S
@@ -68,9 +68,12 @@
 
 #if defined(WITH_JIT)
 .Lno_singleStep:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     /* Entry is always a possible trace start */
     GET_JIT_PROF_TABLE(r0)
     FETCH_INST()
+    mov    r1, #0                       @ prepare the value for the new state
+    str    r1, [r10, #offThread_inJitCodeCache] @ back to the interp land
     cmp    r0,#0
     bne    common_updateProfile
     GET_INST_OPCODE(ip)
diff --git a/vm/mterp/armv5te/footer.S b/vm/mterp/armv5te/footer.S
index a58527d..7476e29 100644
--- a/vm/mterp/armv5te/footer.S
+++ b/vm/mterp/armv5te/footer.S
@@ -55,12 +55,15 @@
  */
     .global dvmJitToInterpPunt
 dvmJitToInterpPunt:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    rPC, r0
 #ifdef EXIT_STATS
     mov    r0,lr
     bl     dvmBumpPunt;
 #endif
     EXPORT_PC()
+    mov    r0, #0
+    str    r0, [r10, #offThread_inJitCodeCache] @ Back to the interp land
     adrl   rIBASE, dvmAsmInstructionStart
     FETCH_INST()
     GET_INST_OPCODE(ip)
@@ -82,6 +85,7 @@
     str    r1, [rGLUE, #offGlue_entryPoint]
     mov    rPC,r0
     EXPORT_PC()
+
     adrl   rIBASE, dvmAsmInstructionStart
     mov    r2,#kJitSingleStep     @ Ask for single step and then revert
     str    r2,[rGLUE,#offGlue_jitState]
@@ -97,9 +101,11 @@
     .global dvmJitToTraceSelect
 dvmJitToTraceSelect:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
     mov    r0,rPC
-    bl     dvmJitGetCodeAddr        @ Is there a translation?
+    bl     dvmJitGetCodeAddr       @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    2f
     mov    r1,rINST
@@ -137,12 +143,14 @@
     .global dvmJitToInterpNormal
 dvmJitToInterpNormal:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
 #ifdef EXIT_STATS
     bl     dvmBumpNormal
 #endif
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    toInterpreter            @ go if not, otherwise do chain
     mov    r1,rINST
@@ -162,8 +170,10 @@
 #ifdef EXIT_STATS
     bl     dvmBumpNoChain
 #endif
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     mov    r1, rPC                  @ arg1 of translation may need this
     mov    lr, #0                   @  in case target is HANDLER_INTERPRET
     cmp    r0,#0
@@ -210,12 +220,14 @@
  * jump to it now).
  */
     GET_JIT_THRESHOLD(r1)
+    ldr     r10, [rGLUE, #offGlue_self] @ callee saved r10 <- glue->self
     strb    r1,[r0,r3,lsr #21] @ reset counter
     EXPORT_PC()
     mov     r0,rPC
     bl      dvmJitGetCodeAddr           @ r0<- dvmJitGetCodeAddr(rPC)
-    mov    r1, rPC                      @ arg1 of translation may need this
-    mov    lr, #0                       @  in case target is HANDLER_INTERPRET
+    str     r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
+    mov     r1, rPC                     @ arg1 of translation may need this
+    mov     lr, #0                      @  in case target is HANDLER_INTERPRET
     cmp     r0,#0
 #if !defined(WITH_SELF_VERIFICATION)
     bxne    r0                          @ jump to the translation
@@ -640,12 +652,13 @@
     ldr     r1, [r10, #offClassObject_pDvmDex]   @ r1<- method->clazz->pDvmDex
     str     rFP, [r3, #offThread_curFrame]  @ self->curFrame = fp
 #if defined(WITH_JIT)
-    ldr     r3, [r0, #offStackSaveArea_returnAddr] @ r3 = saveArea->returnAddr
+    ldr     r10, [r0, #offStackSaveArea_returnAddr] @ r10 = saveArea->returnAddr
     GET_JIT_PROF_TABLE(r0)
     mov     rPC, r9                     @ publish new rPC
     str     r1, [rGLUE, #offGlue_methodClassDex]
-    cmp     r3, #0                      @ caller is compiled code
-    blxne   r3
+    str     r10, [r3, #offThread_inJitCodeCache]  @ may return to JIT'ed land
+    cmp     r10, #0                      @ caller is compiled code
+    blxne   r10
     GET_INST_OPCODE(ip)                 @ extract opcode from rINST
     cmp     r0,#0
     bne     common_updateProfile
diff --git a/vm/mterp/common/asm-constants.h b/vm/mterp/common/asm-constants.h
index 6046bb3..d6ff482 100644
--- a/vm/mterp/common/asm-constants.h
+++ b/vm/mterp/common/asm-constants.h
@@ -214,6 +214,17 @@
 MTERP_OFFSET(offThread_stackOverflowed, Thread, stackOverflowed, 40)
 MTERP_OFFSET(offThread_curFrame,        Thread, curFrame, 44)
 MTERP_OFFSET(offThread_exception,       Thread, exception, 48)
+
+#if defined(WITH_JIT)
+MTERP_OFFSET(offThread_inJitCodeCache,  Thread, inJitCodeCache, 76)
+#ifdef USE_INDIRECT_REF
+MTERP_OFFSET(offThread_jniLocal_topCookie, \
+                                Thread, jniLocalRefTable.segmentState.all, 80)
+#else
+MTERP_OFFSET(offThread_jniLocal_topCookie, \
+                                Thread, jniLocalRefTable.nextEntry, 80)
+#endif
+#else
 #ifdef USE_INDIRECT_REF
 MTERP_OFFSET(offThread_jniLocal_topCookie, \
                                 Thread, jniLocalRefTable.segmentState.all, 76)
@@ -221,6 +232,7 @@
 MTERP_OFFSET(offThread_jniLocal_topCookie, \
                                 Thread, jniLocalRefTable.nextEntry, 76)
 #endif
+#endif
 
 /* Object fields */
 MTERP_OFFSET(offObject_clazz,           Object, clazz, 0)
diff --git a/vm/mterp/out/InterpAsm-armv4t.S b/vm/mterp/out/InterpAsm-armv4t.S
index 759956f..ba248de 100644
--- a/vm/mterp/out/InterpAsm-armv4t.S
+++ b/vm/mterp/out/InterpAsm-armv4t.S
@@ -311,9 +311,12 @@
 
 #if defined(WITH_JIT)
 .Lno_singleStep:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     /* Entry is always a possible trace start */
     GET_JIT_PROF_TABLE(r0)
     FETCH_INST()
+    mov    r1, #0                       @ prepare the value for the new state
+    str    r1, [r10, #offThread_inJitCodeCache] @ back to the interp land
     cmp    r0,#0
     bne    common_updateProfile
     GET_INST_OPCODE(ip)
@@ -9579,12 +9582,15 @@
  */
     .global dvmJitToInterpPunt
 dvmJitToInterpPunt:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    rPC, r0
 #ifdef EXIT_STATS
     mov    r0,lr
     bl     dvmBumpPunt;
 #endif
     EXPORT_PC()
+    mov    r0, #0
+    str    r0, [r10, #offThread_inJitCodeCache] @ Back to the interp land
     adrl   rIBASE, dvmAsmInstructionStart
     FETCH_INST()
     GET_INST_OPCODE(ip)
@@ -9606,6 +9612,7 @@
     str    r1, [rGLUE, #offGlue_entryPoint]
     mov    rPC,r0
     EXPORT_PC()
+
     adrl   rIBASE, dvmAsmInstructionStart
     mov    r2,#kJitSingleStep     @ Ask for single step and then revert
     str    r2,[rGLUE,#offGlue_jitState]
@@ -9621,9 +9628,11 @@
     .global dvmJitToTraceSelect
 dvmJitToTraceSelect:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
     mov    r0,rPC
-    bl     dvmJitGetCodeAddr        @ Is there a translation?
+    bl     dvmJitGetCodeAddr       @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    2f
     mov    r1,rINST
@@ -9661,12 +9670,14 @@
     .global dvmJitToInterpNormal
 dvmJitToInterpNormal:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
 #ifdef EXIT_STATS
     bl     dvmBumpNormal
 #endif
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    toInterpreter            @ go if not, otherwise do chain
     mov    r1,rINST
@@ -9686,8 +9697,10 @@
 #ifdef EXIT_STATS
     bl     dvmBumpNoChain
 #endif
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     mov    r1, rPC                  @ arg1 of translation may need this
     mov    lr, #0                   @  in case target is HANDLER_INTERPRET
     cmp    r0,#0
@@ -9734,12 +9747,14 @@
  * jump to it now).
  */
     GET_JIT_THRESHOLD(r1)
+    ldr     r10, [rGLUE, #offGlue_self] @ callee saved r10 <- glue->self
     strb    r1,[r0,r3,lsr #21] @ reset counter
     EXPORT_PC()
     mov     r0,rPC
     bl      dvmJitGetCodeAddr           @ r0<- dvmJitGetCodeAddr(rPC)
-    mov    r1, rPC                      @ arg1 of translation may need this
-    mov    lr, #0                       @  in case target is HANDLER_INTERPRET
+    str     r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
+    mov     r1, rPC                     @ arg1 of translation may need this
+    mov     lr, #0                      @  in case target is HANDLER_INTERPRET
     cmp     r0,#0
 #if !defined(WITH_SELF_VERIFICATION)
     bxne    r0                          @ jump to the translation
@@ -10164,12 +10179,13 @@
     ldr     r1, [r10, #offClassObject_pDvmDex]   @ r1<- method->clazz->pDvmDex
     str     rFP, [r3, #offThread_curFrame]  @ self->curFrame = fp
 #if defined(WITH_JIT)
-    ldr     r3, [r0, #offStackSaveArea_returnAddr] @ r3 = saveArea->returnAddr
+    ldr     r10, [r0, #offStackSaveArea_returnAddr] @ r10 = saveArea->returnAddr
     GET_JIT_PROF_TABLE(r0)
     mov     rPC, r9                     @ publish new rPC
     str     r1, [rGLUE, #offGlue_methodClassDex]
-    cmp     r3, #0                      @ caller is compiled code
-    blxne   r3
+    str     r10, [r3, #offThread_inJitCodeCache]  @ may return to JIT'ed land
+    cmp     r10, #0                      @ caller is compiled code
+    blxne   r10
     GET_INST_OPCODE(ip)                 @ extract opcode from rINST
     cmp     r0,#0
     bne     common_updateProfile
diff --git a/vm/mterp/out/InterpAsm-armv5te-vfp.S b/vm/mterp/out/InterpAsm-armv5te-vfp.S
index d9e7cdb..966e4f0 100644
--- a/vm/mterp/out/InterpAsm-armv5te-vfp.S
+++ b/vm/mterp/out/InterpAsm-armv5te-vfp.S
@@ -311,9 +311,12 @@
 
 #if defined(WITH_JIT)
 .Lno_singleStep:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     /* Entry is always a possible trace start */
     GET_JIT_PROF_TABLE(r0)
     FETCH_INST()
+    mov    r1, #0                       @ prepare the value for the new state
+    str    r1, [r10, #offThread_inJitCodeCache] @ back to the interp land
     cmp    r0,#0
     bne    common_updateProfile
     GET_INST_OPCODE(ip)
@@ -9097,12 +9100,15 @@
  */
     .global dvmJitToInterpPunt
 dvmJitToInterpPunt:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    rPC, r0
 #ifdef EXIT_STATS
     mov    r0,lr
     bl     dvmBumpPunt;
 #endif
     EXPORT_PC()
+    mov    r0, #0
+    str    r0, [r10, #offThread_inJitCodeCache] @ Back to the interp land
     adrl   rIBASE, dvmAsmInstructionStart
     FETCH_INST()
     GET_INST_OPCODE(ip)
@@ -9124,6 +9130,7 @@
     str    r1, [rGLUE, #offGlue_entryPoint]
     mov    rPC,r0
     EXPORT_PC()
+
     adrl   rIBASE, dvmAsmInstructionStart
     mov    r2,#kJitSingleStep     @ Ask for single step and then revert
     str    r2,[rGLUE,#offGlue_jitState]
@@ -9139,9 +9146,11 @@
     .global dvmJitToTraceSelect
 dvmJitToTraceSelect:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
     mov    r0,rPC
-    bl     dvmJitGetCodeAddr        @ Is there a translation?
+    bl     dvmJitGetCodeAddr       @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    2f
     mov    r1,rINST
@@ -9179,12 +9188,14 @@
     .global dvmJitToInterpNormal
 dvmJitToInterpNormal:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
 #ifdef EXIT_STATS
     bl     dvmBumpNormal
 #endif
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    toInterpreter            @ go if not, otherwise do chain
     mov    r1,rINST
@@ -9204,8 +9215,10 @@
 #ifdef EXIT_STATS
     bl     dvmBumpNoChain
 #endif
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     mov    r1, rPC                  @ arg1 of translation may need this
     mov    lr, #0                   @  in case target is HANDLER_INTERPRET
     cmp    r0,#0
@@ -9252,12 +9265,14 @@
  * jump to it now).
  */
     GET_JIT_THRESHOLD(r1)
+    ldr     r10, [rGLUE, #offGlue_self] @ callee saved r10 <- glue->self
     strb    r1,[r0,r3,lsr #21] @ reset counter
     EXPORT_PC()
     mov     r0,rPC
     bl      dvmJitGetCodeAddr           @ r0<- dvmJitGetCodeAddr(rPC)
-    mov    r1, rPC                      @ arg1 of translation may need this
-    mov    lr, #0                       @  in case target is HANDLER_INTERPRET
+    str     r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
+    mov     r1, rPC                     @ arg1 of translation may need this
+    mov     lr, #0                      @  in case target is HANDLER_INTERPRET
     cmp     r0,#0
 #if !defined(WITH_SELF_VERIFICATION)
     bxne    r0                          @ jump to the translation
@@ -9682,12 +9697,13 @@
     ldr     r1, [r10, #offClassObject_pDvmDex]   @ r1<- method->clazz->pDvmDex
     str     rFP, [r3, #offThread_curFrame]  @ self->curFrame = fp
 #if defined(WITH_JIT)
-    ldr     r3, [r0, #offStackSaveArea_returnAddr] @ r3 = saveArea->returnAddr
+    ldr     r10, [r0, #offStackSaveArea_returnAddr] @ r10 = saveArea->returnAddr
     GET_JIT_PROF_TABLE(r0)
     mov     rPC, r9                     @ publish new rPC
     str     r1, [rGLUE, #offGlue_methodClassDex]
-    cmp     r3, #0                      @ caller is compiled code
-    blxne   r3
+    str     r10, [r3, #offThread_inJitCodeCache]  @ may return to JIT'ed land
+    cmp     r10, #0                      @ caller is compiled code
+    blxne   r10
     GET_INST_OPCODE(ip)                 @ extract opcode from rINST
     cmp     r0,#0
     bne     common_updateProfile
diff --git a/vm/mterp/out/InterpAsm-armv5te.S b/vm/mterp/out/InterpAsm-armv5te.S
index eb95134..da56e6c 100644
--- a/vm/mterp/out/InterpAsm-armv5te.S
+++ b/vm/mterp/out/InterpAsm-armv5te.S
@@ -311,9 +311,12 @@
 
 #if defined(WITH_JIT)
 .Lno_singleStep:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     /* Entry is always a possible trace start */
     GET_JIT_PROF_TABLE(r0)
     FETCH_INST()
+    mov    r1, #0                       @ prepare the value for the new state
+    str    r1, [r10, #offThread_inJitCodeCache] @ back to the interp land
     cmp    r0,#0
     bne    common_updateProfile
     GET_INST_OPCODE(ip)
@@ -9573,12 +9576,15 @@
  */
     .global dvmJitToInterpPunt
 dvmJitToInterpPunt:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    rPC, r0
 #ifdef EXIT_STATS
     mov    r0,lr
     bl     dvmBumpPunt;
 #endif
     EXPORT_PC()
+    mov    r0, #0
+    str    r0, [r10, #offThread_inJitCodeCache] @ Back to the interp land
     adrl   rIBASE, dvmAsmInstructionStart
     FETCH_INST()
     GET_INST_OPCODE(ip)
@@ -9600,6 +9606,7 @@
     str    r1, [rGLUE, #offGlue_entryPoint]
     mov    rPC,r0
     EXPORT_PC()
+
     adrl   rIBASE, dvmAsmInstructionStart
     mov    r2,#kJitSingleStep     @ Ask for single step and then revert
     str    r2,[rGLUE,#offGlue_jitState]
@@ -9615,9 +9622,11 @@
     .global dvmJitToTraceSelect
 dvmJitToTraceSelect:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
     mov    r0,rPC
-    bl     dvmJitGetCodeAddr        @ Is there a translation?
+    bl     dvmJitGetCodeAddr       @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    2f
     mov    r1,rINST
@@ -9655,12 +9664,14 @@
     .global dvmJitToInterpNormal
 dvmJitToInterpNormal:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
 #ifdef EXIT_STATS
     bl     dvmBumpNormal
 #endif
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    toInterpreter            @ go if not, otherwise do chain
     mov    r1,rINST
@@ -9680,8 +9691,10 @@
 #ifdef EXIT_STATS
     bl     dvmBumpNoChain
 #endif
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     mov    r1, rPC                  @ arg1 of translation may need this
     mov    lr, #0                   @  in case target is HANDLER_INTERPRET
     cmp    r0,#0
@@ -9728,12 +9741,14 @@
  * jump to it now).
  */
     GET_JIT_THRESHOLD(r1)
+    ldr     r10, [rGLUE, #offGlue_self] @ callee saved r10 <- glue->self
     strb    r1,[r0,r3,lsr #21] @ reset counter
     EXPORT_PC()
     mov     r0,rPC
     bl      dvmJitGetCodeAddr           @ r0<- dvmJitGetCodeAddr(rPC)
-    mov    r1, rPC                      @ arg1 of translation may need this
-    mov    lr, #0                       @  in case target is HANDLER_INTERPRET
+    str     r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
+    mov     r1, rPC                     @ arg1 of translation may need this
+    mov     lr, #0                      @  in case target is HANDLER_INTERPRET
     cmp     r0,#0
 #if !defined(WITH_SELF_VERIFICATION)
     bxne    r0                          @ jump to the translation
@@ -10158,12 +10173,13 @@
     ldr     r1, [r10, #offClassObject_pDvmDex]   @ r1<- method->clazz->pDvmDex
     str     rFP, [r3, #offThread_curFrame]  @ self->curFrame = fp
 #if defined(WITH_JIT)
-    ldr     r3, [r0, #offStackSaveArea_returnAddr] @ r3 = saveArea->returnAddr
+    ldr     r10, [r0, #offStackSaveArea_returnAddr] @ r10 = saveArea->returnAddr
     GET_JIT_PROF_TABLE(r0)
     mov     rPC, r9                     @ publish new rPC
     str     r1, [rGLUE, #offGlue_methodClassDex]
-    cmp     r3, #0                      @ caller is compiled code
-    blxne   r3
+    str     r10, [r3, #offThread_inJitCodeCache]  @ may return to JIT'ed land
+    cmp     r10, #0                      @ caller is compiled code
+    blxne   r10
     GET_INST_OPCODE(ip)                 @ extract opcode from rINST
     cmp     r0,#0
     bne     common_updateProfile
diff --git a/vm/mterp/out/InterpAsm-armv7-a.S b/vm/mterp/out/InterpAsm-armv7-a.S
index 28b3537..606b064 100644
--- a/vm/mterp/out/InterpAsm-armv7-a.S
+++ b/vm/mterp/out/InterpAsm-armv7-a.S
@@ -311,9 +311,12 @@
 
 #if defined(WITH_JIT)
 .Lno_singleStep:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     /* Entry is always a possible trace start */
     GET_JIT_PROF_TABLE(r0)
     FETCH_INST()
+    mov    r1, #0                       @ prepare the value for the new state
+    str    r1, [r10, #offThread_inJitCodeCache] @ back to the interp land
     cmp    r0,#0
     bne    common_updateProfile
     GET_INST_OPCODE(ip)
@@ -9033,12 +9036,15 @@
  */
     .global dvmJitToInterpPunt
 dvmJitToInterpPunt:
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    rPC, r0
 #ifdef EXIT_STATS
     mov    r0,lr
     bl     dvmBumpPunt;
 #endif
     EXPORT_PC()
+    mov    r0, #0
+    str    r0, [r10, #offThread_inJitCodeCache] @ Back to the interp land
     adrl   rIBASE, dvmAsmInstructionStart
     FETCH_INST()
     GET_INST_OPCODE(ip)
@@ -9060,6 +9066,7 @@
     str    r1, [rGLUE, #offGlue_entryPoint]
     mov    rPC,r0
     EXPORT_PC()
+
     adrl   rIBASE, dvmAsmInstructionStart
     mov    r2,#kJitSingleStep     @ Ask for single step and then revert
     str    r2,[rGLUE,#offGlue_jitState]
@@ -9075,9 +9082,11 @@
     .global dvmJitToTraceSelect
 dvmJitToTraceSelect:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
     mov    r0,rPC
-    bl     dvmJitGetCodeAddr        @ Is there a translation?
+    bl     dvmJitGetCodeAddr       @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    2f
     mov    r1,rINST
@@ -9115,12 +9124,14 @@
     .global dvmJitToInterpNormal
 dvmJitToInterpNormal:
     ldr    rPC,[lr, #-1]           @ get our target PC
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     add    rINST,lr,#-5            @ save start of chain branch
 #ifdef EXIT_STATS
     bl     dvmBumpNormal
 #endif
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     cmp    r0,#0
     beq    toInterpreter            @ go if not, otherwise do chain
     mov    r1,rINST
@@ -9140,8 +9151,10 @@
 #ifdef EXIT_STATS
     bl     dvmBumpNoChain
 #endif
+    ldr    r10, [rGLUE, #offGlue_self]  @ callee saved r10 <- glue->self
     mov    r0,rPC
     bl     dvmJitGetCodeAddr        @ Is there a translation?
+    str    r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
     mov    r1, rPC                  @ arg1 of translation may need this
     mov    lr, #0                   @  in case target is HANDLER_INTERPRET
     cmp    r0,#0
@@ -9188,12 +9201,14 @@
  * jump to it now).
  */
     GET_JIT_THRESHOLD(r1)
+    ldr     r10, [rGLUE, #offGlue_self] @ callee saved r10 <- glue->self
     strb    r1,[r0,r3,lsr #21] @ reset counter
     EXPORT_PC()
     mov     r0,rPC
     bl      dvmJitGetCodeAddr           @ r0<- dvmJitGetCodeAddr(rPC)
-    mov    r1, rPC                      @ arg1 of translation may need this
-    mov    lr, #0                       @  in case target is HANDLER_INTERPRET
+    str     r0, [r10, #offThread_inJitCodeCache] @ set the inJitCodeCache flag
+    mov     r1, rPC                     @ arg1 of translation may need this
+    mov     lr, #0                      @  in case target is HANDLER_INTERPRET
     cmp     r0,#0
 #if !defined(WITH_SELF_VERIFICATION)
     bxne    r0                          @ jump to the translation
@@ -9618,12 +9633,13 @@
     ldr     r1, [r10, #offClassObject_pDvmDex]   @ r1<- method->clazz->pDvmDex
     str     rFP, [r3, #offThread_curFrame]  @ self->curFrame = fp
 #if defined(WITH_JIT)
-    ldr     r3, [r0, #offStackSaveArea_returnAddr] @ r3 = saveArea->returnAddr
+    ldr     r10, [r0, #offStackSaveArea_returnAddr] @ r10 = saveArea->returnAddr
     GET_JIT_PROF_TABLE(r0)
     mov     rPC, r9                     @ publish new rPC
     str     r1, [rGLUE, #offGlue_methodClassDex]
-    cmp     r3, #0                      @ caller is compiled code
-    blxne   r3
+    str     r10, [r3, #offThread_inJitCodeCache]  @ may return to JIT'ed land
+    cmp     r10, #0                      @ caller is compiled code
+    blxne   r10
     GET_INST_OPCODE(ip)                 @ extract opcode from rINST
     cmp     r0,#0
     bne     common_updateProfile