Need for speed: Patch #921466 : sys.path_importer_cache is now used to cache valid and
invalid file paths for the built-in import machinery which leads to
fewer open calls on startup.
Also fix issue with PEP 302 style import hooks which lead to more open()
calls than necessary.
diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
index ddf2a72..26c797f 100644
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -340,11 +340,13 @@
importer = None
sys.path_importer_cache.setdefault(path_item, importer)
- if importer is None:
+ # The boolean values are used for caching valid and invalid
+ # file paths for the built-in import machinery
+ if importer in (None, True, False):
try:
importer = ImpImporter(path_item)
except ImportError:
- pass
+ importer = None
return importer
diff --git a/Misc/NEWS b/Misc/NEWS
index be5e4ea..a18e9e7 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@
Core and builtins
-----------------
+- Patch #921466: sys.path_importer_cache is now used to cache valid and
+ invalid file paths for the built-in import machinery which leads to
+ fewer open calls on startup.
+
- Patch #1442927: ``long(str, base)`` is now up to 6x faster for non-power-
of-2 bases. The largest speedup is for inputs with about 1000 decimal
digits. Conversion from non-power-of-2 bases remains quadratic-time in
diff --git a/Python/import.c b/Python/import.c
index 862f33c..e09365b 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1240,7 +1240,33 @@
if (importer == NULL)
return NULL;
/* Note: importer is a borrowed reference */
- if (importer != Py_None) {
+ if (importer == Py_False) {
+ /* Cached as not being a valid dir. */
+ Py_XDECREF(copy);
+ continue;
+ }
+ else if (importer == Py_True) {
+ /* Cached as being a valid dir, so just
+ * continue below. */
+ }
+ else if (importer == Py_None) {
+ /* No importer was found, so it has to be a file.
+ * Check if the directory is valid. */
+#ifdef HAVE_STAT
+ if (stat(buf, &statbuf) != 0) {
+ /* Directory does not exist. */
+ PyDict_SetItem(path_importer_cache,
+ v, Py_False);
+ Py_XDECREF(copy);
+ continue;
+ } else {
+ PyDict_SetItem(path_importer_cache,
+ v, Py_True);
+ }
+#endif
+ }
+ else {
+ /* A real import hook importer was found. */
PyObject *loader;
loader = PyObject_CallMethod(importer,
"find_module",
@@ -1253,9 +1279,11 @@
return &importhookdescr;
}
Py_DECREF(loader);
+ Py_XDECREF(copy);
+ continue;
}
- /* no hook was successful, use builtin import */
}
+ /* no hook was found, use builtin import */
if (len > 0 && buf[len-1] != SEP
#ifdef ALTSEP