Branch merge
diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst
index 04e9b98..2416807 100644
--- a/Doc/howto/sockets.rst
+++ b/Doc/howto/sockets.rst
@@ -23,8 +23,8 @@
working. It doesn't cover the fine points (and there are a lot of them), but I
hope it will give you enough background to begin using them decently.
-I'm only going to talk about INET sockets, but they account for at least 99% of
-the sockets in use. And I'll only talk about STREAM sockets - unless you really
+I'm only going to talk about INET (i.e. IPv4) sockets, but they account for at least 99% of
+the sockets in use. And I'll only talk about STREAM (i.e. TCP) sockets - unless you really
know what you're doing (in which case this HOWTO isn't for you!), you'll get
better behavior and performance from a STREAM socket than anything else. I will
try to clear up the mystery of what a socket is, as well as some hints on how to
@@ -208,10 +208,10 @@
totalsent = totalsent + sent
def myreceive(self):
- msg = ''
+ msg = b''
while len(msg) < MSGLEN:
chunk = self.sock.recv(MSGLEN-len(msg))
- if chunk == '':
+ if chunk == b'':
raise RuntimeError("socket connection broken")
msg = msg + chunk
return msg
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 0ad788b..74c5264 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -36,6 +36,8 @@
typedef struct _alias *alias_ty;
+typedef struct _withitem *withitem_ty;
+
enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3,
Suite_kind=4};
@@ -128,8 +130,7 @@
} If;
struct {
- expr_ty context_expr;
- expr_ty optional_vars;
+ asdl_seq *items;
asdl_seq *body;
} With;
@@ -383,6 +384,11 @@
identifier asname;
};
+struct _withitem {
+ expr_ty context_expr;
+ expr_ty optional_vars;
+};
+
#define Module(a0, a1) _Py_Module(a0, a1)
mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
@@ -421,9 +427,9 @@
#define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
int col_offset, PyArena *arena);
-#define With(a0, a1, a2, a3, a4, a5) _Py_With(a0, a1, a2, a3, a4, a5)
-stmt_ty _Py_With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body,
- int lineno, int col_offset, PyArena *arena);
+#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
+stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
+ PyArena *arena);
#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
PyArena *arena);
@@ -547,6 +553,9 @@
keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
#define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena);
+#define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2)
+withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena
+ *arena);
PyObject* PyAST_mod2obj(mod_ty t);
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
diff --git a/Lib/plistlib.py b/Lib/plistlib.py
index 82d456a..2e7e512 100644
--- a/Lib/plistlib.py
+++ b/Lib/plistlib.py
@@ -68,13 +68,15 @@
usually is a dictionary).
"""
didOpen = False
- if isinstance(pathOrFile, str):
- pathOrFile = open(pathOrFile, 'rb')
- didOpen = True
- p = PlistParser()
- rootObject = p.parse(pathOrFile)
- if didOpen:
- pathOrFile.close()
+ try:
+ if isinstance(pathOrFile, str):
+ pathOrFile = open(pathOrFile, 'rb')
+ didOpen = True
+ p = PlistParser()
+ rootObject = p.parse(pathOrFile)
+ finally:
+ if didOpen:
+ pathOrFile.close()
return rootObject
@@ -83,15 +85,17 @@
file name or a (writable) file object.
"""
didOpen = False
- if isinstance(pathOrFile, str):
- pathOrFile = open(pathOrFile, 'wb')
- didOpen = True
- writer = PlistWriter(pathOrFile)
- writer.writeln("<plist version=\"1.0\">")
- writer.writeValue(rootObject)
- writer.writeln("</plist>")
- if didOpen:
- pathOrFile.close()
+ try:
+ if isinstance(pathOrFile, str):
+ pathOrFile = open(pathOrFile, 'wb')
+ didOpen = True
+ writer = PlistWriter(pathOrFile)
+ writer.writeln("<plist version=\"1.0\">")
+ writer.writeValue(rootObject)
+ writer.writeln("</plist>")
+ finally:
+ if didOpen:
+ pathOrFile.close()
def readPlistFromBytes(data):
@@ -352,7 +356,6 @@
def __repr__(self):
return "%s(%s)" % (self.__class__.__name__, repr(self.data))
-
class PlistParser:
def __init__(self):
@@ -362,11 +365,11 @@
def parse(self, fileobj):
from xml.parsers.expat import ParserCreate
- parser = ParserCreate()
- parser.StartElementHandler = self.handleBeginElement
- parser.EndElementHandler = self.handleEndElement
- parser.CharacterDataHandler = self.handleData
- parser.ParseFile(fileobj)
+ self.parser = ParserCreate()
+ self.parser.StartElementHandler = self.handleBeginElement
+ self.parser.EndElementHandler = self.handleEndElement
+ self.parser.CharacterDataHandler = self.handleData
+ self.parser.ParseFile(fileobj)
return self.root
def handleBeginElement(self, element, attrs):
@@ -385,12 +388,18 @@
def addObject(self, value):
if self.currentKey is not None:
+ if not isinstance(self.stack[-1], type({})):
+ raise ValueError("unexpected element at line %d" %
+ self.parser.CurrentLineNumber)
self.stack[-1][self.currentKey] = value
self.currentKey = None
elif not self.stack:
# this is the root object
self.root = value
else:
+ if not isinstance(self.stack[-1], type([])):
+ raise ValueError("unexpected element at line %d" %
+ self.parser.CurrentLineNumber)
self.stack[-1].append(value)
def getData(self):
@@ -405,9 +414,15 @@
self.addObject(d)
self.stack.append(d)
def end_dict(self):
+ if self.currentKey:
+ raise ValueError("missing value for key '%s' at line %d" %
+ (self.currentKey,self.parser.CurrentLineNumber))
self.stack.pop()
def end_key(self):
+ if self.currentKey or not isinstance(self.stack[-1], type({})):
+ raise ValueError("unexpected key at line %d" %
+ self.parser.CurrentLineNumber)
self.currentKey = self.getData()
def begin_array(self, attrs):
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index c5128d8..6e0cf06 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -397,39 +397,14 @@
else:
import select
_has_poll = hasattr(select, 'poll')
- import fcntl
- import pickle
-
- try:
- import _posixsubprocess
- except ImportError:
- _posixsubprocess = None
- warnings.warn("The _posixsubprocess module is not being used. "
- "Child process reliability may suffer if your "
- "program uses threads.", RuntimeWarning)
+ import _posixsubprocess
+ _create_pipe = _posixsubprocess.cloexec_pipe
# When select or poll has indicated that the file is writable,
# we can write up to _PIPE_BUF bytes without risk of blocking.
# POSIX defines PIPE_BUF as >= 512.
_PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
- _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1)
-
- def _set_cloexec(fd, cloexec):
- old = fcntl.fcntl(fd, fcntl.F_GETFD)
- if cloexec:
- fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC)
- else:
- fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC)
-
- if _posixsubprocess:
- _create_pipe = _posixsubprocess.cloexec_pipe
- else:
- def _create_pipe():
- fds = os.pipe()
- _set_cloexec(fds[0], True)
- _set_cloexec(fds[1], True)
- return fds
__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
"getoutput", "check_output", "CalledProcessError", "DEVNULL"]
@@ -1267,140 +1242,33 @@
errpipe_read, errpipe_write = _create_pipe()
try:
try:
+ # We must avoid complex work that could involve
+ # malloc or free in the child process to avoid
+ # potential deadlocks, thus we do all this here.
+ # and pass it to fork_exec()
- if _posixsubprocess:
- # We must avoid complex work that could involve
- # malloc or free in the child process to avoid
- # potential deadlocks, thus we do all this here.
- # and pass it to fork_exec()
-
- if env:
- env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
- for k, v in env.items()]
- else:
- env_list = None # Use execv instead of execve.
- executable = os.fsencode(executable)
- if os.path.dirname(executable):
- executable_list = (executable,)
- else:
- # This matches the behavior of os._execvpe().
- executable_list = tuple(
- os.path.join(os.fsencode(dir), executable)
- for dir in os.get_exec_path(env))
- fds_to_keep = set(pass_fds)
- fds_to_keep.add(errpipe_write)
- self.pid = _posixsubprocess.fork_exec(
- args, executable_list,
- close_fds, sorted(fds_to_keep), cwd, env_list,
- p2cread, p2cwrite, c2pread, c2pwrite,
- errread, errwrite,
- errpipe_read, errpipe_write,
- restore_signals, start_new_session, preexec_fn)
+ if env:
+ env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
+ for k, v in env.items()]
else:
- # Pure Python implementation: It is not thread safe.
- # This implementation may deadlock in the child if your
- # parent process has any other threads running.
-
- gc_was_enabled = gc.isenabled()
- # Disable gc to avoid bug where gc -> file_dealloc ->
- # write to stderr -> hang. See issue1336
- gc.disable()
- try:
- self.pid = os.fork()
- except:
- if gc_was_enabled:
- gc.enable()
- raise
- self._child_created = True
- if self.pid == 0:
- # Child
- try:
- # Close parent's pipe ends
- if p2cwrite != -1:
- os.close(p2cwrite)
- if c2pread != -1:
- os.close(c2pread)
- if errread != -1:
- os.close(errread)
- os.close(errpipe_read)
-
- # Dup fds for child
- def _dup2(a, b):
- # dup2() removes the CLOEXEC flag but
- # we must do it ourselves if dup2()
- # would be a no-op (issue #10806).
- if a == b:
- _set_cloexec(a, False)
- elif a != -1:
- os.dup2(a, b)
- _dup2(p2cread, 0)
- _dup2(c2pwrite, 1)
- _dup2(errwrite, 2)
-
- # Close pipe fds. Make sure we don't close the
- # same fd more than once, or standard fds.
- closed = set()
- for fd in [p2cread, c2pwrite, errwrite]:
- if fd > 2 and fd not in closed:
- os.close(fd)
- closed.add(fd)
-
- # Close all other fds, if asked for
- if close_fds:
- fds_to_keep = set(pass_fds)
- fds_to_keep.add(errpipe_write)
- self._close_fds(fds_to_keep)
-
-
- if cwd is not None:
- os.chdir(cwd)
-
- # This is a copy of Python/pythonrun.c
- # _Py_RestoreSignals(). If that were exposed
- # as a sys._py_restoresignals func it would be
- # better.. but this pure python implementation
- # isn't likely to be used much anymore.
- if restore_signals:
- signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ')
- for sig in signals:
- if hasattr(signal, sig):
- signal.signal(getattr(signal, sig),
- signal.SIG_DFL)
-
- if start_new_session and hasattr(os, 'setsid'):
- os.setsid()
-
- if preexec_fn:
- preexec_fn()
-
- if env is None:
- os.execvp(executable, args)
- else:
- os.execvpe(executable, args, env)
-
- except:
- try:
- exc_type, exc_value = sys.exc_info()[:2]
- if isinstance(exc_value, OSError):
- errno_num = exc_value.errno
- else:
- errno_num = 0
- message = '%s:%x:%s' % (exc_type.__name__,
- errno_num, exc_value)
- message = message.encode(errors="surrogatepass")
- os.write(errpipe_write, message)
- except Exception:
- # We MUST not allow anything odd happening
- # above to prevent us from exiting below.
- pass
-
- # This exitcode won't be reported to applications
- # so it really doesn't matter what we return.
- os._exit(255)
-
- # Parent
- if gc_was_enabled:
- gc.enable()
+ env_list = None # Use execv instead of execve.
+ executable = os.fsencode(executable)
+ if os.path.dirname(executable):
+ executable_list = (executable,)
+ else:
+ # This matches the behavior of os._execvpe().
+ executable_list = tuple(
+ os.path.join(os.fsencode(dir), executable)
+ for dir in os.get_exec_path(env))
+ fds_to_keep = set(pass_fds)
+ fds_to_keep.add(errpipe_write)
+ self.pid = _posixsubprocess.fork_exec(
+ args, executable_list,
+ close_fds, sorted(fds_to_keep), cwd, env_list,
+ p2cread, p2cwrite, c2pread, c2pwrite,
+ errread, errwrite,
+ errpipe_read, errpipe_write,
+ restore_signals, start_new_session, preexec_fn)
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
diff --git a/Lib/test/support.py b/Lib/test/support.py
index b03069c..6724e9b 100644
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -48,7 +48,7 @@
"threading_cleanup", "reap_children", "cpython_only", "check_impl_detail",
"get_attribute", "swap_item", "swap_attr", "requires_IEEE_754",
"TestHandler", "Matcher", "can_symlink", "skip_unless_symlink",
- "import_fresh_module", "requires_zlib"
+ "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE"
]
class Error(Exception):
@@ -409,6 +409,13 @@
IPV6_ENABLED = _is_ipv6_enabled()
+
+# A constant likely larger than the underlying OS pipe buffer size.
+# Windows limit seems to be around 512B, and most Unix kernels have a 64K pipe
+# buffer size: take 1M to be sure.
+PIPE_MAX_SIZE = 1024 * 1024
+
+
# decorator for skipping tests on non-IEEE 754 platforms
requires_IEEE_754 = unittest.skipUnless(
float.__getformat__("double").startswith("IEEE"),
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index da07060..fde2cfe 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -38,6 +38,9 @@
"while v:pass",
# If
"if v:pass",
+ # With
+ "with x as y: pass",
+ "with x as y, z as q: pass",
# Raise
"raise Exception('string')",
# TryExcept
@@ -341,6 +344,8 @@
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]),
('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index c0dc6ec..0ffb4a1 100644
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -2683,7 +2683,7 @@
# The buffered IO layer must check for pending signal
# handlers, which in this case will invoke alarm_interrupt().
self.assertRaises(ZeroDivisionError,
- wio.write, item * (1024 * 1024))
+ wio.write, item * (support.PIPE_MAX_SIZE // len(item)))
t.join()
# We got one byte, get another one and check that it isn't a
# repeat of the first one.
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
index dfa4725..5f980d0 100644
--- a/Lib/test/test_plistlib.py
+++ b/Lib/test/test_plistlib.py
@@ -175,6 +175,32 @@
self.assertEqual(test1, result1)
self.assertEqual(test2, result2)
+ def test_invalidarray(self):
+ for i in ["<key>key inside an array</key>",
+ "<key>key inside an array2</key><real>3</real>",
+ "<true/><key>key inside an array3</key>"]:
+ self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ ("<plist><array>%s</array></plist>"%i).encode())
+
+ def test_invaliddict(self):
+ for i in ["<key><true/>k</key><string>compound key</string>",
+ "<key>single key</key>",
+ "<string>missing key</string>",
+ "<key>k1</key><string>v1</string><real>5.3</real>"
+ "<key>k1</key><key>k2</key><string>double key</string>"]:
+ self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ ("<plist><dict>%s</dict></plist>"%i).encode())
+ self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ ("<plist><array><dict>%s</dict></array></plist>"%i).encode())
+
+ def test_invalidinteger(self):
+ self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ b"<plist><integer>not integer</integer></plist>")
+
+ def test_invalidreal(self):
+ self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ b"<plist><integer>not real</integer></plist>")
+
def test_main():
support.run_unittest(TestPlistlib)
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 776e143..686c1b1 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -489,24 +489,21 @@
# This test will probably deadlock rather than fail, if
# communicate() does not work properly.
x, y = os.pipe()
- if mswindows:
- pipe_buf = 512
- else:
- pipe_buf = os.fpathconf(x, "PC_PIPE_BUF")
os.close(x)
os.close(y)
p = subprocess.Popen([sys.executable, "-c",
'import sys,os;'
'sys.stdout.write(sys.stdin.read(47));'
- 'sys.stderr.write("xyz"*%d);'
- 'sys.stdout.write(sys.stdin.read())' % pipe_buf],
+ 'sys.stderr.write("x" * %d);'
+ 'sys.stdout.write(sys.stdin.read())' %
+ support.PIPE_MAX_SIZE],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.addCleanup(p.stdout.close)
self.addCleanup(p.stderr.close)
self.addCleanup(p.stdin.close)
- string_to_write = b"abc"*pipe_buf
+ string_to_write = b"a" * support.PIPE_MAX_SIZE
(stdout, stderr) = p.communicate(string_to_write)
self.assertEqual(stdout, string_to_write)
@@ -1495,20 +1492,6 @@
ProcessTestCase.tearDown(self)
-@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False),
- "_posixsubprocess extension module not found.")
-class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase):
- def setUp(self):
- subprocess._posixsubprocess = None
- ProcessTestCase.setUp(self)
- POSIXProcessTestCase.setUp(self)
-
- def tearDown(self):
- subprocess._posixsubprocess = sys.modules['_posixsubprocess']
- POSIXProcessTestCase.tearDown(self)
- ProcessTestCase.tearDown(self)
-
-
class HelperFunctionTests(unittest.TestCase):
@unittest.skipIf(mswindows, "errno and EINTR make no sense on windows")
def test_eintr_retry_call(self):
@@ -1617,7 +1600,6 @@
unit_tests = (ProcessTestCase,
POSIXProcessTestCase,
Win32ProcessTestCase,
- ProcessTestCasePOSIXPurePython,
CommandTests,
ProcessTestCaseNoPoll,
HelperFunctionTests,
diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
index 2dc7773..c22d965 100644
--- a/Lib/test/test_threading.py
+++ b/Lib/test/test_threading.py
@@ -12,6 +12,7 @@
import weakref
import os
from test.script_helper import assert_python_ok, assert_python_failure
+import subprocess
from test import lock_tests
@@ -703,6 +704,37 @@
lock = threading.Lock()
self.assertRaises(RuntimeError, lock.release)
+ @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem')
+ def test_recursion_limit(self):
+ # Issue 9670
+ # test that excessive recursion within a non-main thread causes
+ # an exception rather than crashing the interpreter on platforms
+ # like Mac OS X or FreeBSD which have small default stack sizes
+ # for threads
+ script = """if True:
+ import threading
+
+ def recurse():
+ return recurse()
+
+ def outer():
+ try:
+ recurse()
+ except RuntimeError:
+ pass
+
+ w = threading.Thread(target=outer)
+ w.start()
+ w.join()
+ print('end of main thread')
+ """
+ expected_output = "end of main thread\n"
+ p = subprocess.Popen([sys.executable, "-c", script],
+ stdout=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ data = stdout.decode().replace('\r', '')
+ self.assertEqual(p.returncode, 0, "Unexpected error")
+ self.assertEqual(data, expected_output)
class LockTests(lock_tests.LockTests):
locktype = staticmethod(threading.Lock)
diff --git a/Mac/Makefile.in b/Mac/Makefile.in
index 5e57b57..8a62a90 100644
--- a/Mac/Makefile.in
+++ b/Mac/Makefile.in
@@ -76,6 +76,13 @@
do \
ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
done
+ifneq ($(LIPO_32BIT_FLAGS),)
+ for fn in python3-32 pythonw3-32 \
+ python$(VERSION)-32 pythonw$(VERSION)-32 ;\
+ do \
+ ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
+ done
+endif
#
@@ -90,6 +97,12 @@
do \
ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
done
+ifneq ($(LIPO_32BIT_FLAGS),)
+ for fn in python$(VERSION)-32 pythonw$(VERSION)-32 ;\
+ do \
+ ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
+ done
+endif
pythonw: $(srcdir)/Tools/pythonw.c Makefile
$(CC) $(LDFLAGS) -DPYTHONFRAMEWORK='"$(PYTHONFRAMEWORK)"' -o $@ $(srcdir)/Tools/pythonw.c -I.. -I$(srcdir)/../Include ../$(PYTHONFRAMEWORK).framework/Versions/$(VERSION)/$(PYTHONFRAMEWORK)
diff --git a/Misc/ACKS b/Misc/ACKS
index 68e7eef..350cc3c 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -615,6 +615,7 @@
Derek Morr
James A Morrison
Pablo Mouzo
+Mher Movsisyan
Sjoerd Mullender
Sape Mullender
Michael Muller
diff --git a/Misc/NEWS b/Misc/NEWS
index 2a916fd..dc00e69 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,14 @@
Core and Builtins
-----------------
+- Issue #9670: Increase the default stack size for secondary threads on
+ Mac OS X and FreeBSD to reduce the chances of a crash instead of a
+ "maximum recursion depth" RuntimeError exception.
+ (patch by Ronald Oussoren)
+
+- Issue #12106: The use of the multiple-with shorthand syntax is now reflected
+ in the AST.
+
- Issue #12190: Try to use the same filename object when compiling unmarshalling
a code objects in the same file.
@@ -164,6 +172,9 @@
Library
-------
+- Issue #985064: Make plistlib more resilient to faulty input plists.
+ Patch by Mher Movsisyan.
+
- Issue #1625: BZ2File and bz2.decompress() now support multi-stream files.
Initial patch by Nir Aides.
@@ -710,6 +721,10 @@
Build
-----
+- Issue #11217: For 64-bit/32-bit Mac OS X universal framework builds,
+ ensure "make install" creates symlinks in --prefix bin for the "-32"
+ files in the framework bin directory like the installer does.
+
- Issue #11347: Use --no-as-needed when linking libpython3.so.
- Issue #11411: Fix 'make DESTDIR=' with a relative destination.
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index 8e2e1ac..de48643 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -28,7 +28,7 @@
| For(expr target, expr iter, stmt* body, stmt* orelse)
| While(expr test, stmt* body, stmt* orelse)
| If(expr test, stmt* body, stmt* orelse)
- | With(expr context_expr, expr? optional_vars, stmt* body)
+ | With(withitem* items, stmt* body)
| Raise(expr? exc, expr? cause)
| TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
@@ -115,5 +115,7 @@
-- import name with optional 'as' alias.
alias = (identifier name, identifier? asname)
+
+ withitem = (expr context_expr, expr? optional_vars)
}
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 6b1ea3c..f076c7e 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -2,7 +2,7 @@
/*
- __version__ 0daa6ba25d9b.
+ __version__ 9b11cc4e2918.
This module must be committed separately after each AST grammar change;
The __version__ number is set to the revision number of the commit
@@ -95,8 +95,7 @@
};
static PyTypeObject *With_type;
static char *With_fields[]={
- "context_expr",
- "optional_vars",
+ "items",
"body",
};
static PyTypeObject *Raise_type;
@@ -392,6 +391,12 @@
"name",
"asname",
};
+static PyTypeObject *withitem_type;
+static PyObject* ast2obj_withitem(void*);
+static char *withitem_fields[]={
+ "context_expr",
+ "optional_vars",
+};
static int
@@ -680,7 +685,7 @@
if (!While_type) return 0;
If_type = make_type("If", stmt_type, If_fields, 3);
if (!If_type) return 0;
- With_type = make_type("With", stmt_type, With_fields, 3);
+ With_type = make_type("With", stmt_type, With_fields, 2);
if (!With_type) return 0;
Raise_type = make_type("Raise", stmt_type, Raise_fields, 2);
if (!Raise_type) return 0;
@@ -938,6 +943,8 @@
if (!keyword_type) return 0;
alias_type = make_type("alias", &AST_type, alias_fields, 2);
if (!alias_type) return 0;
+ withitem_type = make_type("withitem", &AST_type, withitem_fields, 2);
+ if (!withitem_type) return 0;
initialized = 1;
return 1;
}
@@ -960,6 +967,7 @@
static int obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena);
static int obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena);
static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena);
+static int obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena);
mod_ty
Module(asdl_seq * body, PyArena *arena)
@@ -1225,21 +1233,15 @@
}
stmt_ty
-With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno,
- int col_offset, PyArena *arena)
+With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena
+ *arena)
{
stmt_ty p;
- if (!context_expr) {
- PyErr_SetString(PyExc_ValueError,
- "field context_expr is required for With");
- return NULL;
- }
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = With_kind;
- p->v.With.context_expr = context_expr;
- p->v.With.optional_vars = optional_vars;
+ p->v.With.items = items;
p->v.With.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
@@ -2135,6 +2137,23 @@
return p;
}
+withitem_ty
+withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena)
+{
+ withitem_ty p;
+ if (!context_expr) {
+ PyErr_SetString(PyExc_ValueError,
+ "field context_expr is required for withitem");
+ return NULL;
+ }
+ p = (withitem_ty)PyArena_Malloc(arena, sizeof(*p));
+ if (!p)
+ return NULL;
+ p->context_expr = context_expr;
+ p->optional_vars = optional_vars;
+ return p;
+}
+
PyObject*
ast2obj_mod(void* _o)
@@ -2390,15 +2409,9 @@
case With_kind:
result = PyType_GenericNew(With_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.With.context_expr);
+ value = ast2obj_list(o->v.With.items, ast2obj_withitem);
if (!value) goto failed;
- if (PyObject_SetAttrString(result, "context_expr", value) == -1)
- goto failed;
- Py_DECREF(value);
- value = ast2obj_expr(o->v.With.optional_vars);
- if (!value) goto failed;
- if (PyObject_SetAttrString(result, "optional_vars", value) ==
- -1)
+ if (PyObject_SetAttrString(result, "items", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.With.body, ast2obj_stmt);
@@ -3370,6 +3383,35 @@
return NULL;
}
+PyObject*
+ast2obj_withitem(void* _o)
+{
+ withitem_ty o = (withitem_ty)_o;
+ PyObject *result = NULL, *value = NULL;
+ if (!o) {
+ Py_INCREF(Py_None);
+ return Py_None;
+ }
+
+ result = PyType_GenericNew(withitem_type, NULL, NULL);
+ if (!result) return NULL;
+ value = ast2obj_expr(o->context_expr);
+ if (!value) goto failed;
+ if (PyObject_SetAttrString(result, "context_expr", value) == -1)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_expr(o->optional_vars);
+ if (!value) goto failed;
+ if (PyObject_SetAttrString(result, "optional_vars", value) == -1)
+ goto failed;
+ Py_DECREF(value);
+ return result;
+failed:
+ Py_XDECREF(value);
+ Py_XDECREF(result);
+ return NULL;
+}
+
int
obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
@@ -4210,33 +4252,34 @@
return 1;
}
if (isinstance) {
- expr_ty context_expr;
- expr_ty optional_vars;
+ asdl_seq* items;
asdl_seq* body;
- if (PyObject_HasAttrString(obj, "context_expr")) {
+ if (PyObject_HasAttrString(obj, "items")) {
int res;
- tmp = PyObject_GetAttrString(obj, "context_expr");
+ Py_ssize_t len;
+ Py_ssize_t i;
+ tmp = PyObject_GetAttrString(obj, "items");
if (tmp == NULL) goto failed;
- res = obj2ast_expr(tmp, &context_expr, arena);
- if (res != 0) goto failed;
+ if (!PyList_Check(tmp)) {
+ PyErr_Format(PyExc_TypeError, "With field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+ goto failed;
+ }
+ len = PyList_GET_SIZE(tmp);
+ items = asdl_seq_new(len, arena);
+ if (items == NULL) goto failed;
+ for (i = 0; i < len; i++) {
+ withitem_ty value;
+ res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &value, arena);
+ if (res != 0) goto failed;
+ asdl_seq_SET(items, i, value);
+ }
Py_XDECREF(tmp);
tmp = NULL;
} else {
- PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from With");
+ PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from With");
return 1;
}
- if (PyObject_HasAttrString(obj, "optional_vars")) {
- int res;
- tmp = PyObject_GetAttrString(obj, "optional_vars");
- if (tmp == NULL) goto failed;
- res = obj2ast_expr(tmp, &optional_vars, arena);
- if (res != 0) goto failed;
- Py_XDECREF(tmp);
- tmp = NULL;
- } else {
- optional_vars = NULL;
- }
if (PyObject_HasAttrString(obj, "body")) {
int res;
Py_ssize_t len;
@@ -4262,8 +4305,7 @@
PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from With");
return 1;
}
- *out = With(context_expr, optional_vars, body, lineno,
- col_offset, arena);
+ *out = With(items, body, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6723,6 +6765,43 @@
return 1;
}
+int
+obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena)
+{
+ PyObject* tmp = NULL;
+ expr_ty context_expr;
+ expr_ty optional_vars;
+
+ if (PyObject_HasAttrString(obj, "context_expr")) {
+ int res;
+ tmp = PyObject_GetAttrString(obj, "context_expr");
+ if (tmp == NULL) goto failed;
+ res = obj2ast_expr(tmp, &context_expr, arena);
+ if (res != 0) goto failed;
+ Py_XDECREF(tmp);
+ tmp = NULL;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem");
+ return 1;
+ }
+ if (PyObject_HasAttrString(obj, "optional_vars")) {
+ int res;
+ tmp = PyObject_GetAttrString(obj, "optional_vars");
+ if (tmp == NULL) goto failed;
+ res = obj2ast_expr(tmp, &optional_vars, arena);
+ if (res != 0) goto failed;
+ Py_XDECREF(tmp);
+ tmp = NULL;
+ } else {
+ optional_vars = NULL;
+ }
+ *out = withitem(context_expr, optional_vars, arena);
+ return 0;
+failed:
+ Py_XDECREF(tmp);
+ return 1;
+}
+
static struct PyModuleDef _astmodule = {
PyModuleDef_HEAD_INIT, "_ast"
@@ -6739,7 +6818,7 @@
NULL;
if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)
return NULL;
- if (PyModule_AddStringConstant(m, "__version__", "0daa6ba25d9b") < 0)
+ if (PyModule_AddStringConstant(m, "__version__", "9b11cc4e2918") < 0)
return NULL;
if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return
NULL;
@@ -6940,6 +7019,8 @@
return NULL;
if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return
NULL;
+ if (PyDict_SetItemString(d, "withitem", (PyObject*)withitem_type) < 0)
+ return NULL;
return m;
}
diff --git a/Python/ast.c b/Python/ast.c
index 5b12da8..882452b 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -2967,8 +2967,8 @@
}
/* with_item: test ['as' expr] */
-static stmt_ty
-ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content)
+static withitem_ty
+ast_for_with_item(struct compiling *c, const node *n)
{
expr_ty context_expr, optional_vars = NULL;
@@ -2987,43 +2987,32 @@
}
}
- return With(context_expr, optional_vars, content, LINENO(n),
- n->n_col_offset, c->c_arena);
+ return withitem(context_expr, optional_vars, c->c_arena);
}
/* with_stmt: 'with' with_item (',' with_item)* ':' suite */
static stmt_ty
ast_for_with_stmt(struct compiling *c, const node *n)
{
- int i;
- stmt_ty ret;
- asdl_seq *inner;
+ int i, n_items;
+ asdl_seq *items, *body;
REQ(n, with_stmt);
- /* process the with items inside-out */
- i = NCH(n) - 1;
- /* the suite of the innermost with item is the suite of the with stmt */
- inner = ast_for_suite(c, CHILD(n, i));
- if (!inner)
- return NULL;
-
- for (;;) {
- i -= 2;
- ret = ast_for_with_item(c, CHILD(n, i), inner);
- if (!ret)
+ n_items = (NCH(n) - 2) / 2;
+ items = asdl_seq_new(n_items, c->c_arena);
+ for (i = 1; i < NCH(n) - 2; i += 2) {
+ withitem_ty item = ast_for_with_item(c, CHILD(n, i));
+ if (!item)
return NULL;
- /* was this the last item? */
- if (i == 1)
- break;
- /* if not, wrap the result so far in a new sequence */
- inner = asdl_seq_new(1, c->c_arena);
- if (!inner)
- return NULL;
- asdl_seq_SET(inner, 0, ret);
+ asdl_seq_SET(items, (i - 1) / 2, item);
}
- return ret;
+ body = ast_for_suite(c, CHILD(n, NCH(n) - 1));
+ if (!body)
+ return NULL;
+
+ return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
diff --git a/Python/compile.c b/Python/compile.c
index 34e7603..d24528b 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -179,7 +179,7 @@
static int inplace_binop(struct compiler *, operator_ty);
static int expr_constant(struct compiler *, expr_ty);
-static int compiler_with(struct compiler *, stmt_ty);
+static int compiler_with(struct compiler *, stmt_ty, int);
static int compiler_call_helper(struct compiler *c, int n,
asdl_seq *args,
asdl_seq *keywords,
@@ -1968,7 +1968,7 @@
compiler_use_next_block(c, except);
for (i = 0; i < n; i++) {
excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
- s->v.TryExcept.handlers, i);
+ s->v.TryExcept.handlers, i);
if (!handler->v.ExceptHandler.type && i < n-1)
return compiler_error(c, "default 'except:' must be last");
c->u->u_lineno_set = 0;
@@ -1985,70 +1985,70 @@
}
ADDOP(c, POP_TOP);
if (handler->v.ExceptHandler.name) {
- basicblock *cleanup_end, *cleanup_body;
+ basicblock *cleanup_end, *cleanup_body;
- cleanup_end = compiler_new_block(c);
- cleanup_body = compiler_new_block(c);
- if(!(cleanup_end || cleanup_body))
- return 0;
+ cleanup_end = compiler_new_block(c);
+ cleanup_body = compiler_new_block(c);
+ if (!(cleanup_end || cleanup_body))
+ return 0;
- compiler_nameop(c, handler->v.ExceptHandler.name, Store);
- ADDOP(c, POP_TOP);
+ compiler_nameop(c, handler->v.ExceptHandler.name, Store);
+ ADDOP(c, POP_TOP);
- /*
- try:
- # body
- except type as name:
- try:
- # body
- finally:
- name = None
- del name
- */
+ /*
+ try:
+ # body
+ except type as name:
+ try:
+ # body
+ finally:
+ name = None
+ del name
+ */
- /* second try: */
- ADDOP_JREL(c, SETUP_FINALLY, cleanup_end);
- compiler_use_next_block(c, cleanup_body);
- if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
- return 0;
+ /* second try: */
+ ADDOP_JREL(c, SETUP_FINALLY, cleanup_end);
+ compiler_use_next_block(c, cleanup_body);
+ if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
+ return 0;
- /* second # body */
- VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
- ADDOP(c, POP_BLOCK);
- ADDOP(c, POP_EXCEPT);
- compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
+ /* second # body */
+ VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
+ ADDOP(c, POP_BLOCK);
+ ADDOP(c, POP_EXCEPT);
+ compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
- /* finally: */
- ADDOP_O(c, LOAD_CONST, Py_None, consts);
- compiler_use_next_block(c, cleanup_end);
- if (!compiler_push_fblock(c, FINALLY_END, cleanup_end))
- return 0;
+ /* finally: */
+ ADDOP_O(c, LOAD_CONST, Py_None, consts);
+ compiler_use_next_block(c, cleanup_end);
+ if (!compiler_push_fblock(c, FINALLY_END, cleanup_end))
+ return 0;
- /* name = None */
- ADDOP_O(c, LOAD_CONST, Py_None, consts);
- compiler_nameop(c, handler->v.ExceptHandler.name, Store);
+ /* name = None */
+ ADDOP_O(c, LOAD_CONST, Py_None, consts);
+ compiler_nameop(c, handler->v.ExceptHandler.name, Store);
- /* del name */
- compiler_nameop(c, handler->v.ExceptHandler.name, Del);
+ /* del name */
+ compiler_nameop(c, handler->v.ExceptHandler.name, Del);
- ADDOP(c, END_FINALLY);
- compiler_pop_fblock(c, FINALLY_END, cleanup_end);
+ ADDOP(c, END_FINALLY);
+ compiler_pop_fblock(c, FINALLY_END, cleanup_end);
}
else {
- basicblock *cleanup_body;
+ basicblock *cleanup_body;
- cleanup_body = compiler_new_block(c);
- if(!cleanup_body)
- return 0;
+ cleanup_body = compiler_new_block(c);
+ if (!cleanup_body)
+ return 0;
ADDOP(c, POP_TOP);
- ADDOP(c, POP_TOP);
- compiler_use_next_block(c, cleanup_body);
- if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
- return 0;
+ ADDOP(c, POP_TOP);
+ compiler_use_next_block(c, cleanup_body);
+ if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
+ return 0;
VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
- ADDOP(c, POP_EXCEPT);
- compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
+ ADDOP(c, POP_EXCEPT);
+ compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
}
ADDOP_JREL(c, JUMP_FORWARD, end);
compiler_use_next_block(c, except);
@@ -2341,7 +2341,7 @@
case Continue_kind:
return compiler_continue(c);
case With_kind:
- return compiler_with(c, s);
+ return compiler_with(c, s, 0);
}
return 1;
}
@@ -3068,9 +3068,10 @@
exit(*exc)
*/
static int
-compiler_with(struct compiler *c, stmt_ty s)
+compiler_with(struct compiler *c, stmt_ty s, int pos)
{
basicblock *block, *finally;
+ withitem_ty item = asdl_seq_GET(s->v.With.items, pos);
assert(s->kind == With_kind);
@@ -3080,7 +3081,7 @@
return 0;
/* Evaluate EXPR */
- VISIT(c, expr, s->v.With.context_expr);
+ VISIT(c, expr, item->context_expr);
ADDOP_JREL(c, SETUP_WITH, finally);
/* SETUP_WITH pushes a finally block. */
@@ -3089,16 +3090,20 @@
return 0;
}
- if (s->v.With.optional_vars) {
- VISIT(c, expr, s->v.With.optional_vars);
+ if (item->optional_vars) {
+ VISIT(c, expr, item->optional_vars);
}
else {
/* Discard result from context.__enter__() */
ADDOP(c, POP_TOP);
}
- /* BLOCK code */
- VISIT_SEQ(c, stmt, s->v.With.body);
+ pos++;
+ if (pos == asdl_seq_LEN(s->v.With.items))
+ /* BLOCK code */
+ VISIT_SEQ(c, stmt, s->v.With.body)
+ else if (!compiler_with(c, s, pos))
+ return 0;
/* End of try block; start the finally block */
ADDOP(c, POP_BLOCK);
diff --git a/Python/symtable.c b/Python/symtable.c
index 8040665..d276254 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -185,6 +185,7 @@
static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args);
static int symtable_implicit_arg(struct symtable *st, int pos);
static int symtable_visit_annotations(struct symtable *st, stmt_ty s);
+static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
static identifier top = NULL, lambda = NULL, genexpr = NULL,
@@ -1305,10 +1306,7 @@
/* nothing to do here */
break;
case With_kind:
- VISIT(st, expr, s->v.With.context_expr);
- if (s->v.With.optional_vars) {
- VISIT(st, expr, s->v.With.optional_vars);
- }
+ VISIT_SEQ(st, withitem, s->v.With.items);
VISIT_SEQ(st, stmt, s->v.With.body);
break;
}
@@ -1540,6 +1538,16 @@
return 1;
}
+static int
+symtable_visit_withitem(struct symtable *st, withitem_ty item)
+{
+ VISIT(st, expr, item->context_expr);
+ if (item->optional_vars) {
+ VISIT(st, expr, item->optional_vars);
+ }
+ return 1;
+}
+
static int
symtable_visit_alias(struct symtable *st, alias_ty a)
diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h
index 40ebaf6..09a0887 100644
--- a/Python/thread_pthread.h
+++ b/Python/thread_pthread.h
@@ -18,6 +18,18 @@
#ifndef THREAD_STACK_SIZE
#define THREAD_STACK_SIZE 0 /* use default stack size */
#endif
+
+#if (defined(__APPLE__) || defined(__FreeBSD__)) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0
+ /* The default stack size for new threads on OSX is small enough that
+ * we'll get hard crashes instead of 'maximum recursion depth exceeded'
+ * exceptions.
+ *
+ * The default stack size below is the minimal stack size where a
+ * simple recursive function doesn't cause a hard crash.
+ */
+#undef THREAD_STACK_SIZE
+#define THREAD_STACK_SIZE 0x400000
+#endif
/* for safety, ensure a viable minimum stacksize */
#define THREAD_STACK_MIN 0x8000 /* 32kB */
#else /* !_POSIX_THREAD_ATTR_STACKSIZE */