| from .. import util |
| |
| machinery = util.import_importlib('importlib.machinery') |
| |
| import codecs |
| import importlib.util |
| import re |
| import sys |
| import types |
| # Because sys.path gets essentially blanked, need to have unicodedata already |
| # imported for the parser to use. |
| import unicodedata |
| import unittest |
| import warnings |
| |
| |
| CODING_RE = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) |
| |
| |
| class EncodingTest: |
| |
| """PEP 3120 makes UTF-8 the default encoding for source code |
| [default encoding]. |
| |
| PEP 263 specifies how that can change on a per-file basis. Either the first |
| or second line can contain the encoding line [encoding first line] |
| encoding second line]. If the file has the BOM marker it is considered UTF-8 |
| implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is |
| an error [BOM and utf-8][BOM conflict]. |
| |
| """ |
| |
| variable = '\u00fc' |
| character = '\u00c9' |
| source_line = "{0} = '{1}'\n".format(variable, character) |
| module_name = '_temp' |
| |
| def run_test(self, source): |
| with util.create_modules(self.module_name) as mapping: |
| with open(mapping[self.module_name], 'wb') as file: |
| file.write(source) |
| loader = self.machinery.SourceFileLoader(self.module_name, |
| mapping[self.module_name]) |
| return self.load(loader) |
| |
| def create_source(self, encoding): |
| encoding_line = "# coding={0}".format(encoding) |
| assert CODING_RE.match(encoding_line) |
| source_lines = [encoding_line.encode('utf-8')] |
| source_lines.append(self.source_line.encode(encoding)) |
| return b'\n'.join(source_lines) |
| |
| def test_non_obvious_encoding(self): |
| # Make sure that an encoding that has never been a standard one for |
| # Python works. |
| encoding_line = "# coding=koi8-r" |
| assert CODING_RE.match(encoding_line) |
| source = "{0}\na=42\n".format(encoding_line).encode("koi8-r") |
| self.run_test(source) |
| |
| # [default encoding] |
| def test_default_encoding(self): |
| self.run_test(self.source_line.encode('utf-8')) |
| |
| # [encoding first line] |
| def test_encoding_on_first_line(self): |
| encoding = 'Latin-1' |
| source = self.create_source(encoding) |
| self.run_test(source) |
| |
| # [encoding second line] |
| def test_encoding_on_second_line(self): |
| source = b"#/usr/bin/python\n" + self.create_source('Latin-1') |
| self.run_test(source) |
| |
| # [BOM] |
| def test_bom(self): |
| self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8')) |
| |
| # [BOM and utf-8] |
| def test_bom_and_utf_8(self): |
| source = codecs.BOM_UTF8 + self.create_source('utf-8') |
| self.run_test(source) |
| |
| # [BOM conflict] |
| def test_bom_conflict(self): |
| source = codecs.BOM_UTF8 + self.create_source('latin-1') |
| with self.assertRaises(SyntaxError): |
| self.run_test(source) |
| |
| |
| class EncodingTestPEP451(EncodingTest): |
| |
| def load(self, loader): |
| module = types.ModuleType(self.module_name) |
| module.__spec__ = importlib.util.spec_from_loader(self.module_name, loader) |
| loader.exec_module(module) |
| return module |
| |
| |
| (Frozen_EncodingTestPEP451, |
| Source_EncodingTestPEP451 |
| ) = util.test_both(EncodingTestPEP451, machinery=machinery) |
| |
| |
| class EncodingTestPEP302(EncodingTest): |
| |
| def load(self, loader): |
| with warnings.catch_warnings(): |
| warnings.simplefilter('ignore', DeprecationWarning) |
| return loader.load_module(self.module_name) |
| |
| |
| (Frozen_EncodingTestPEP302, |
| Source_EncodingTestPEP302 |
| ) = util.test_both(EncodingTestPEP302, machinery=machinery) |
| |
| |
| class LineEndingTest: |
| |
| r"""Source written with the three types of line endings (\n, \r\n, \r) |
| need to be readable [cr][crlf][lf].""" |
| |
| def run_test(self, line_ending): |
| module_name = '_temp' |
| source_lines = [b"a = 42", b"b = -13", b''] |
| source = line_ending.join(source_lines) |
| with util.create_modules(module_name) as mapping: |
| with open(mapping[module_name], 'wb') as file: |
| file.write(source) |
| loader = self.machinery.SourceFileLoader(module_name, |
| mapping[module_name]) |
| return self.load(loader, module_name) |
| |
| # [cr] |
| def test_cr(self): |
| self.run_test(b'\r') |
| |
| # [crlf] |
| def test_crlf(self): |
| self.run_test(b'\r\n') |
| |
| # [lf] |
| def test_lf(self): |
| self.run_test(b'\n') |
| |
| |
| class LineEndingTestPEP451(LineEndingTest): |
| |
| def load(self, loader, module_name): |
| module = types.ModuleType(module_name) |
| module.__spec__ = importlib.util.spec_from_loader(module_name, loader) |
| loader.exec_module(module) |
| return module |
| |
| |
| (Frozen_LineEndingTestPEP451, |
| Source_LineEndingTestPEP451 |
| ) = util.test_both(LineEndingTestPEP451, machinery=machinery) |
| |
| |
| class LineEndingTestPEP302(LineEndingTest): |
| |
| def load(self, loader, module_name): |
| with warnings.catch_warnings(): |
| warnings.simplefilter('ignore', DeprecationWarning) |
| return loader.load_module(module_name) |
| |
| |
| (Frozen_LineEndingTestPEP302, |
| Source_LineEndingTestPEP302 |
| ) = util.test_both(LineEndingTestPEP302, machinery=machinery) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |