Issue #18076: Introduce imoportlib.util.decode_source().
The helper function makes it easier to implement
imoprtlib.abc.InspectLoader.get_source() by making that function
require just the raw bytes for source code and handling all other
details.
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 0668dbd..37fe083 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -723,6 +723,18 @@
return data
+def decode_source(source_bytes):
+ """Decode bytes representing source code and return the string.
+
+ Universal newline support is used in the decoding.
+ """
+ import tokenize # To avoid bootstrap issues.
+ source_bytes_readline = _io.BytesIO(source_bytes).readline
+ encoding = tokenize.detect_encoding(source_bytes_readline)
+ newline_decoder = _io.IncrementalNewlineDecoder(None, True)
+ return newline_decoder.decode(source_bytes.decode(encoding[0]))
+
+
# Loaders #####################################################################
class BuiltinImporter:
@@ -965,11 +977,7 @@
except OSError as exc:
raise ImportError("source not available through get_data()",
name=fullname) from exc
- import tokenize
- readsource = _io.BytesIO(source_bytes).readline
- encoding = tokenize.detect_encoding(readsource)
- newline_decoder = _io.IncrementalNewlineDecoder(None, True)
- return newline_decoder.decode(source_bytes.decode(encoding[0]))
+ return decode_source(source_bytes)
def source_to_code(self, data, path, *, _optimize=-1):
"""Return the code object compiled from source.
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index 06f4d2f..7727f9d 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -2,6 +2,7 @@
from ._bootstrap import MAGIC_NUMBER
from ._bootstrap import cache_from_source
+from ._bootstrap import decode_source
from ._bootstrap import module_to_load
from ._bootstrap import set_loader
from ._bootstrap import set_package