diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index c837bcd..73aa5ef 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -77,7 +77,7 @@
 extern void _PyGC_Fini(PyThreadState *tstate);
 extern void _PyType_Fini(void);
 extern void _Py_HashRandomization_Fini(void);
-extern void _PyUnicode_Fini(void);
+extern void _PyUnicode_Fini(PyThreadState *tstate);
 extern void _PyLong_Fini(void);
 extern void _PyFaulthandler_Fini(void);
 extern void _PyHash_Fini(void);
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 5ae0af8..89e45d0 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -15929,34 +15929,37 @@
 
 
 void
-_PyUnicode_Fini(void)
+_PyUnicode_Fini(PyThreadState *tstate)
 {
+    if (_Py_IsMainInterpreter(tstate)) {
 #if defined(WITH_VALGRIND) || defined(__INSURE__)
-    /* Insure++ is a memory analysis tool that aids in discovering
-     * memory leaks and other memory problems.  On Python exit, the
-     * interned string dictionaries are flagged as being in use at exit
-     * (which it is).  Under normal circumstances, this is fine because
-     * the memory will be automatically reclaimed by the system.  Under
-     * memory debugging, it's a huge source of useless noise, so we
-     * trade off slower shutdown for less distraction in the memory
-     * reports.  -baw
-     */
-    unicode_release_interned();
+        /* Insure++ is a memory analysis tool that aids in discovering
+         * memory leaks and other memory problems.  On Python exit, the
+         * interned string dictionaries are flagged as being in use at exit
+         * (which it is).  Under normal circumstances, this is fine because
+         * the memory will be automatically reclaimed by the system.  Under
+         * memory debugging, it's a huge source of useless noise, so we
+         * trade off slower shutdown for less distraction in the memory
+         * reports.  -baw
+         */
+        unicode_release_interned();
 #endif /* __INSURE__ */
 
-    Py_CLEAR(unicode_empty);
+        Py_CLEAR(unicode_empty);
 
-    for (Py_ssize_t i = 0; i < 256; i++) {
-        Py_CLEAR(unicode_latin1[i]);
+        for (Py_ssize_t i = 0; i < 256; i++) {
+            Py_CLEAR(unicode_latin1[i]);
+        }
+        _PyUnicode_ClearStaticStrings();
+        (void)PyUnicode_ClearFreeList();
     }
-    _PyUnicode_ClearStaticStrings();
-    (void)PyUnicode_ClearFreeList();
 
     PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE();
     PyMem_RawFree(interp->fs_codec.encoding);
     interp->fs_codec.encoding = NULL;
     PyMem_RawFree(interp->fs_codec.errors);
     interp->fs_codec.errors = NULL;
+    interp->config.filesystem_errors = _Py_ERROR_UNKNOWN;
 }
 
 
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 8c508e3..5f3c49a 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1182,9 +1182,6 @@
         _PySet_Fini();
         _PyBytes_Fini();
         _PyLong_Fini();
-    }
-
-    if (is_main_interp) {
         _PyFloat_Fini();
         _PyDict_Fini();
         _PySlice_Fini();
@@ -1197,9 +1194,12 @@
         _PyArg_Fini();
         _PyAsyncGen_Fini();
         _PyContext_Fini();
+    }
 
-        /* Cleanup Unicode implementation */
-        _PyUnicode_Fini();
+    /* Cleanup Unicode implementation */
+    _PyUnicode_Fini(tstate);
+
+    if (is_main_interp) {
         _Py_ClearFileSystemEncoding();
     }
 }