bpo-13153: Use OS native encoding for converting between Python and Tcl. (GH-16545)
On Windows use UTF-16 (or UTF-32 for 32-bit Tcl_UniChar) with the
"surrogatepass" error handler for converting to/from Tcl Unicode objects.
On Linux use UTF-8 with the "surrogateescape" error handler for converting
to/from Tcl String objects.
Converting strings from Tcl to Python and back now never fails
(except MemoryError).
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index b969f8c..adeed74 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -358,21 +358,6 @@
Font(text, font=text.cget('font')).measure('0')
self.width = pixel_width // zero_char_width
- def _filename_to_unicode(self, filename):
- """Return filename as BMP unicode so displayable in Tk."""
- # Decode bytes to unicode.
- if isinstance(filename, bytes):
- try:
- filename = filename.decode(self.filesystemencoding)
- except UnicodeDecodeError:
- try:
- filename = filename.decode(self.encoding)
- except UnicodeDecodeError:
- # byte-to-byte conversion
- filename = filename.decode('iso8859-1')
- # Replace non-BMP char with diamond questionmark.
- return re.sub('[\U00010000-\U0010FFFF]', '\ufffd', filename)
-
def new_callback(self, event):
dirname, basename = self.io.defaultfilename()
self.flist.new(dirname)
@@ -963,10 +948,8 @@
menu.delete(0, END) # clear, and rebuild:
for i, file_name in enumerate(rf_list):
file_name = file_name.rstrip() # zap \n
- # make unicode string to display non-ASCII chars correctly
- ufile_name = self._filename_to_unicode(file_name)
callback = instance.__recent_file_callback(file_name)
- menu.add_command(label=ulchars[i] + " " + ufile_name,
+ menu.add_command(label=ulchars[i] + " " + file_name,
command=callback,
underline=0)
@@ -1004,16 +987,10 @@
def short_title(self):
filename = self.io.filename
- if filename:
- filename = os.path.basename(filename)
- else:
- filename = "untitled"
- # return unicode string to display non-ASCII chars correctly
- return self._filename_to_unicode(filename)
+ return os.path.basename(filename) if filename else "untitled"
def long_title(self):
- # return unicode string to display non-ASCII chars correctly
- return self._filename_to_unicode(self.io.filename or "")
+ return self.io.filename or ""
def center_insert_event(self, event):
self.center()
diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py
index 4af4ff0..240db71 100644
--- a/Lib/idlelib/idle_test/test_editor.py
+++ b/Lib/idlelib/idle_test/test_editor.py
@@ -30,18 +30,6 @@
e._close()
-class EditorFunctionTest(unittest.TestCase):
-
- def test_filename_to_unicode(self):
- func = Editor._filename_to_unicode
- class dummy():
- filesystemencoding = 'utf-8'
- pairs = (('abc', 'abc'), ('a\U00011111c', 'a\ufffdc'),
- (b'abc', 'abc'), (b'a\xf0\x91\x84\x91c', 'a\ufffdc'))
- for inp, out in pairs:
- self.assertEqual(func(dummy, inp), out)
-
-
class TestGetLineIndent(unittest.TestCase):
def test_empty_lines(self):
for tabwidth in [1, 2, 4, 6, 8]:
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 2e4dfad..bc87d24 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -679,14 +679,6 @@
self.more = 0
# at the moment, InteractiveInterpreter expects str
assert isinstance(source, str)
- #if isinstance(source, str):
- # from idlelib import iomenu
- # try:
- # source = source.encode(iomenu.encoding)
- # except UnicodeError:
- # self.tkconsole.resetoutput()
- # self.write("Unsupported characters in input\n")
- # return
# InteractiveInterpreter.runsource() calls its runcode() method,
# which is overridden (see below)
return InteractiveInterpreter.runsource(self, source, filename)
@@ -1298,16 +1290,6 @@
self.set_line_and_column()
def write(self, s, tags=()):
- if isinstance(s, str) and len(s) and max(s) > '\uffff':
- # Tk doesn't support outputting non-BMP characters
- # Let's assume what printed string is not very long,
- # find first non-BMP character and construct informative
- # UnicodeEncodeError exception.
- for start, char in enumerate(s):
- if char > '\uffff':
- break
- raise UnicodeEncodeError("UCS-2", char, start, start+1,
- 'Non-BMP character not supported in Tk')
try:
self.text.mark_gravity("iomark", "right")
count = OutputWindow.write(self, s, tags, "iomark")
diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py
index de73bf8..e99d0d2 100644
--- a/Lib/idlelib/runscript.py
+++ b/Lib/idlelib/runscript.py
@@ -147,8 +147,7 @@
interp = self.shell.interp
if pyshell.use_subprocess and restart:
interp.restart_subprocess(
- with_cwd=False, filename=
- self.editwin._filename_to_unicode(filename))
+ with_cwd=False, filename=filename)
dirname = os.path.dirname(filename)
argv = [filename]
if self.cli_args: