Merge TP1A.220404.001
Change-Id: I358d48e925db2468cfcad391d16f9ab7e09e0abe
diff --git a/BUILD.bazel b/BUILD.bazel
index 3b2a41e..884e82a 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -1,3 +1,5 @@
+load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+
package(default_visibility = ["//visibility:public"])
[filegroup(
@@ -9,12 +11,28 @@
]),
) for platform in ("linux-x86", "darwin")]
+py_runtime(
+ name = "python2",
+ files = ["linux-x86/bin/py2-cmd"],
+ interpreter = "linux-x86/bin/py2-cmd",
+ python_version = "PY2",
+)
-# TODO(b/199038020): Temporary solution. Use py_runtime_pair and
-# python_toolchain as soon as Python rules are checked-in.
py_runtime(
name = "python3",
files = ["linux-x86/bin/py3-cmd"],
interpreter = "linux-x86/bin/py3-cmd",
python_version = "PY3",
)
+
+py_runtime_pair(
+ name = "py_runtime_pair",
+ py2_runtime = ":python2",
+ py3_runtime = ":python3",
+)
+
+toolchain(
+ name = "py_toolchain",
+ toolchain = ":py_runtime_pair",
+ toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)
diff --git a/common/bison/NOTICE b/common/bison/NOTICE
index 94a9ed0..f288702 100644
--- a/common/bison/NOTICE
+++ b/common/bison/NOTICE
@@ -1,7 +1,7 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
@@ -645,7 +645,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
@@ -664,11 +664,11 @@
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+<https://www.gnu.org/licenses/why-not-lgpl.html>.
diff --git a/common/bison/README.md b/common/bison/README.md
index 94e9891..09886ea 100644
--- a/common/bison/README.md
+++ b/common/bison/README.md
@@ -1,6 +1,6 @@
This directory contains data needed by Bison.
-# Directory content
+# Directory Content
## Skeletons
Bison skeletons: the general shapes of the different parser kinds, that are
specialized for specific grammars by the bison program.
@@ -48,7 +48,7 @@
- xml2xhtml.xsl
Conversion into XHTML.
-# Implementation note about the skeletons
+# Implementation Notes About the Skeletons
"Skeleton" in Bison parlance means "backend": a skeleton is fed by the bison
executable with LR tables, facts about the symbols, etc. and they generate
@@ -73,36 +73,63 @@
your language. Overall, be sure to follow the same patterns as the existing
skeletons.
+## Vocabulary
+
+We use "formal arguments", or "formals" for short, to denote the declared
+parameters of a function (e.g., `int argc, const char **argv`). Yes, this
+is somewhat contradictory with `param` in the `%param` directives.
+
+We use "effective arguments", or "args" for short, to denote the values
+passed in function calls (e.g., `argc, argv`).
+
## Symbols
### `b4_symbol(NUM, FIELD)`
In order to unify the handling of the various aspects of symbols (tag, type
name, whether terminal, etc.), bison.exe defines one macro per (token,
field), where field can `has_id`, `id`, etc.: see
-`prepare_symbols_definitions()` in `src/output.c`.
+`prepare_symbol_definitions()` in `src/output.c`.
-The macro `b4_symbol(NUM, FIELD)` gives access to the following FIELDS:
+NUM can be:
+- `empty` to denote the "empty" pseudo-symbol when it exists,
+- `eof`, `error`, or `undef`
+- a symbol number.
-- `has_id`: 0 or 1.
+FIELD can be:
- Whether the symbol has an id.
+- `has_id`: 0 or 1
+ Whether the symbol has an `id`.
-- `id`: string
- If has_id, the id (prefixed by api.token.prefix if defined), otherwise
- defined as empty. Guaranteed to be usable as a C identifier.
+- `id`: string (e.g., `exp`, `NUM`, or `TOK_NUM` with api.token.prefix)
+ If `has_id`, the name of the token kind (prefixed by api.token.prefix if
+ defined), otherwise empty. Guaranteed to be usable as a C identifier.
+ This is used to define the token kind (i.e., the enum used by the return
+ value of yylex). Should be named `token_kind`.
-- `tag`: string.
- A representation of the symbol. Can be 'foo', 'foo.id', '"foo"' etc.
+- `tag`: string
+ A human readable representation of the symbol. Can be `'foo'`,
+ `'foo.id'`, `'"foo"'` etc.
-- `user_number`: integer
+- `code`: integer
+ The token code associated to the token kind `id`.
The external number as used by yylex. Can be ASCII code when a character,
- some number chosen by bison, or some user number in the case of
- %token FOO <NUM>. Corresponds to yychar in yacc.c.
+ some number chosen by bison, or some user number in the case of `%token
+ FOO <NUM>`. Corresponds to `yychar` in `yacc.c`.
- `is_token`: 0 or 1
Whether this is a terminal symbol.
+- `kind_base`: string (e.g., `YYSYMBOL_exp`, `YYSYMBOL_NUM`)
+ The base of the symbol kind, i.e., the enumerator of this symbol (token or
+ nonterminal) which is mapped to its `number`.
+
+- `kind`: string
+ Same as `kind_base`, but possibly with a prefix in some languages. E.g.,
+ EOF's `kind_base` and `kind` are `YYSYMBOL_YYEOF` in C, but are
+ `S_YYEMPTY` and `symbol_kind::S_YYEMPTY` in C++.
+
- `number`: integer
+ The code associated to the `kind`.
The internal number (computed from the external number by yytranslate).
Corresponds to yytoken in yacc.c. This is the same number that serves as
key in b4_symbol(NUM, FIELD).
@@ -129,11 +156,17 @@
When api.value.type=union, the generated name for the union member.
yytype_INT etc. for symbols that has_id, otherwise yytype_1 etc.
-- `type`
+- `type`: string
If it has a semantic value, its type tag, or, if variant are used,
its type.
In the case of api.value.type=union, type is the real type (e.g. int).
+- `slot`: string
+ If it has a semantic value, the name of the union member (i.e., bounces to
+ either `type_tag` or `type`). It would be better to fix our mess and
+ always use `type` for the true type of the member, and `type_tag` for the
+ name of the union member.
+
- `has_printer`: 0, 1
- `printer`: string
- `printer_file`: string
@@ -166,7 +199,7 @@
Expansion of `$<TYPE>POS`, where the current rule has `RULE-LENGTH` symbols
on RHS.
------
+<!--
Local Variables:
mode: markdown
@@ -174,7 +207,7 @@
ispell-dictionary: "american"
End:
-Copyright (C) 2002, 2008-2015, 2018-2019 Free Software Foundation, Inc.
+Copyright (C) 2002, 2008-2015, 2018-2021 Free Software Foundation, Inc.
This file is part of GNU Bison.
@@ -189,4 +222,6 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+-->
diff --git a/common/bison/bison-default.css b/common/bison/bison-default.css
index 68df166..aadaba1 100644
--- a/common/bison/bison-default.css
+++ b/common/bison/bison-default.css
@@ -1,5 +1,5 @@
/* Default styling rules for Bison when doing terminal output.
- Copyright (C) 2019 Free Software Foundation, Inc.
+ Copyright (C) 2019-2021 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -29,3 +29,33 @@
/* "Sections" in traces (--trace). */
.trace0 { color: green; }
+
+/* Syntax error messages. */
+.expected { color: green; }
+.unexpected { color: red; }
+
+
+/* Counterexamples. */
+
+/* Cex: point in rule. */
+.cex-dot { color: red; }
+
+/* Cex: coloring various rules. */
+.cex-0 { color: yellow; }
+.cex-1 { color: green; }
+.cex-2 { color: blue; }
+.cex-3 { color: purple; }
+.cex-4 { color: violet; }
+.cex-5 { color: orange; }
+.cex-6 { color: brown; }
+.cex-7 { color: mauve; }
+.cex-8 { color: #013220; } /* Dark green. */
+.cex-9 { color: #e75480; } /* Dark pink. */
+.cex-10 { color: cyan; }
+.cex-11 { color: orange; }
+
+/* Cex: derivation rewriting steps. */
+.cex-step { font-style: italic; }
+
+/* Cex: leaves of a derivation. */
+.cex-leaf { font-weight: 600; }
diff --git a/common/bison/local.mk b/common/bison/local.mk
index f82aee7..c51d287 100644
--- a/common/bison/local.mk
+++ b/common/bison/local.mk
@@ -1,4 +1,4 @@
-## Copyright (C) 2002, 2005-2015, 2018-2019 Free Software Foundation,
+## Copyright (C) 2002, 2005-2015, 2018-2021 Free Software Foundation,
## Inc.
## This program is free software: you can redistribute it and/or modify
@@ -12,7 +12,7 @@
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
-## along with this program. If not, see <http://www.gnu.org/licenses/>.
+## along with this program. If not, see <https://www.gnu.org/licenses/>.
dist_pkgdata_DATA = \
data/README.md \
@@ -28,18 +28,19 @@
data/skeletons/c.m4 \
data/skeletons/glr.c \
data/skeletons/glr.cc \
+ data/skeletons/glr2.cc \
data/skeletons/java-skel.m4 \
data/skeletons/java.m4 \
data/skeletons/lalr1.cc \
data/skeletons/lalr1.java \
data/skeletons/location.cc \
data/skeletons/stack.hh \
+ data/skeletons/traceon.m4 \
data/skeletons/variant.hh \
data/skeletons/yacc.c
# Experimental support for the D language.
dist_skeletons_DATA += \
- data/skeletons/README-D.txt \
data/skeletons/d-skel.m4 \
data/skeletons/d.m4 \
data/skeletons/lalr1.d
diff --git a/common/bison/m4sugar/foreach.m4 b/common/bison/m4sugar/foreach.m4
index 7093d0f..2052d44 100644
--- a/common/bison/m4sugar/foreach.m4
+++ b/common/bison/m4sugar/foreach.m4
@@ -4,7 +4,7 @@
# Speeds up GNU M4 1.4.x by avoiding quadratic $@ recursion, but penalizes
# GNU M4 1.6 by requiring more memory and macro expansions.
#
-# Copyright (C) 2008-2017 Free Software Foundation, Inc.
+# Copyright (C) 2008-2017, 2020 Free Software Foundation, Inc.
# This file is part of Autoconf. This program is free
# software; you can redistribute it and/or modify it under the
diff --git a/common/bison/m4sugar/m4sugar.m4 b/common/bison/m4sugar/m4sugar.m4
index bbd6958..b42fc1a 100644
--- a/common/bison/m4sugar/m4sugar.m4
+++ b/common/bison/m4sugar/m4sugar.m4
@@ -3,7 +3,7 @@
# Base M4 layer.
# Requires GNU M4.
#
-# Copyright (C) 1999-2017 Free Software Foundation, Inc.
+# Copyright (C) 1999-2017, 2020 Free Software Foundation, Inc.
# This file is part of Autoconf. This program is free
# software; you can redistribute it and/or modify it under the
@@ -2412,6 +2412,27 @@
[m4_strip(m4_flatten([$1]))])
+# m4_validate_w(STRING)
+# ---------------------
+# Expands into m4_normalize(m4_expand([STRING])), but if that is not
+# the same as just m4_normalize([STRING]), issue a warning.
+#
+# This is used in several Autoconf macros that take a
+# whitespace-separated list of symbols as an argument. Ideally that
+# list would not be expanded before use, but several packages used
+# `dnl' to put comments inside those lists, so they must be expanded
+# for compatibility's sake.
+m4_define([m4_validate_w],
+[_m4_validate_w(m4_normalize([$1]), m4_normalize(m4_expand([$1])))])
+
+m4_define([_m4_validate_w],
+[m4_if([$1], [$2], [],
+ [m4_warn([obsolete], [whitespace-separated list contains macros;
+in a future version of Autoconf they will not be expanded]dnl
+m4_if(m4_bregexp([$1], [\bdn[l]\b]), -1, [], [
+note: `dn@&t@l' is a macro]))])dnl
+[$2]])
+
# m4_join(SEP, ARG1, ARG2...)
# ---------------------------
diff --git a/common/bison/skeletons/README-D.txt b/common/bison/skeletons/README-D.txt
deleted file mode 100644
index 214e309..0000000
--- a/common/bison/skeletons/README-D.txt
+++ /dev/null
@@ -1,60 +0,0 @@
-Some usage notes for the D Parser:
-
-- it is a port of the Java parser, so interface is very similar.
-
-- the lexer class needs to implement the interface 'Lexer' (similar to
- java). It typically (depending on options) looks like this:
-
-public interface Lexer
-{
- /**
- * Method to retrieve the beginning position of the last scanned token.
- * @return the position at which the last scanned token starts. */
- @property YYPosition startPos ();
-
- /**
- * Method to retrieve the ending position of the last scanned token.
- * @return the first position beyond the last scanned token. */
- @property YYPosition endPos ();
-
- /**
- * Method to retrieve the semantic value of the last scanned token.
- * @return the semantic value of the last scanned token. */
- @property YYSemanticType semanticVal ();
-
- /**
- * Entry point for the scanner. Returns the token identifier corresponding
- * to the next token and prepares to return the semantic value
- * and beginning/ending positions of the token.
- * @return the token identifier corresponding to the next token. */
- YYTokenType yylex ();
-
- /**
- * Entry point for error reporting. Emits an error
- * referring to the given location in a user-defined way.
- *
- * @param loc The location of the element to which the
- * error message is related
- * @param s The string for the error message. */
- void yyerror (YYLocation loc, string s);
-}
-
-- semantic types are handled by D usions (same as for C/C++ parsers)
-
-- the following (non-standard) %defines are supported:
-
- %define package "<package_name>"
- %define api.parser.class "my_class_name>"
- %define position_type "my_position_type"
- %define location_type "my_location_type"
-
-- the following declarations basically work like in C/C++:
-
- %locations
- %error-verbose
- %parse-param
- %initial-action
- %code
- %union
-
-- %destructor is not yet supported
diff --git a/common/bison/skeletons/bison.m4 b/common/bison/skeletons/bison.m4
index 7ca8497..b7bf5c5 100644
--- a/common/bison/skeletons/bison.m4
+++ b/common/bison/skeletons/bison.m4
@@ -2,7 +2,7 @@
# Language-independent M4 Macros for Bison.
-# Copyright (C) 2002, 2004-2015, 2018-2019 Free Software Foundation,
+# Copyright (C) 2002, 2004-2015, 2018-2021 Free Software Foundation,
# Inc.
# This program is free software: you can redistribute it and/or modify
@@ -16,7 +16,30 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+
+# m4_gsub(STRING, RE1, SUBST1, RE2, SUBST2, ...)
+# ----------------------------------------------
+# m4 equivalent of
+#
+# $_ = STRING;
+# s/RE1/SUBST1/g;
+# s/RE2/SUBST2/g;
+# ...
+#
+# Really similar to m4_bpatsubsts, but behaves properly with quotes.
+# See m4.at's "Generating Comments". Super inelegant, but so far, I
+# did not find any better solution.
+m4_define([b4_gsub],
+[m4_bpatsubst(m4_bpatsubst(m4_bpatsubst([[[[$1]]]],
+ [$2], [$3]),
+ [$4], [$5]),
+ [$6], [$7])])
+
+# m4_shift2 and m4_shift3 are provided by m4sugar.
+m4_define([m4_shift4], [m4_shift(m4_shift(m4_shift(m4_shift($@))))])
## ---------------- ##
@@ -26,7 +49,7 @@
# b4_generated_by
# ---------------
m4_define([b4_generated_by],
-[b4_comment([A Bison parser, made by GNU Bison b4_version.])
+[b4_comment([A Bison parser, made by GNU Bison b4_version_string.])
])
# b4_copyright(TITLE, [YEARS])
@@ -51,7 +74,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.])
+along with this program. If not, see <https://www.gnu.org/licenses/>.])
b4_comment([As a special exception, you may create a larger work that contains
part or all of the Bison parser skeleton and distribute that work
@@ -72,8 +95,9 @@
# -------------
# Issue a warning about private implementation details.
m4_define([b4_disclaimer],
-[b4_comment([Undocumented macros, especially those whose name start with YY_,
-are private implementation details. Do not rely on them.])
+[b4_comment([DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual,
+especially those whose name start with YY_ or yy_. They are
+private implementation details that can be changed or removed.])
])
@@ -161,7 +185,7 @@
[b4_cat([[@complain][(]$1[@,]$2[@,]$3[@,]$4[]]dnl
[m4_if([$#], [4], [],
[m4_foreach([b4_arg],
- m4_dquote(m4_shift(m4_shift(m4_shift(m4_shift($@))))),
+ m4_dquote(m4_shift4($@)),
[[@,]b4_arg])])[@)]])])
# b4_warn(FORMAT, [ARG1], [ARG2], ...)
@@ -237,6 +261,13 @@
[b4_error([[fatal]], $@)dnl
m4_exit(1)])
+# b4_canary(MSG)
+# --------------
+# Issue a warning on stderr and in the output. Used in the test suite
+# to catch spurious m4 evaluations.
+m4_define([b4_canary],
+[m4_errprintn([dead canary: $1])DEAD CANARY($1)])
+
## ------------ ##
## Data Types. ##
@@ -308,14 +339,14 @@
$1([check], [b4_check])
$1([stos], [b4_stos],
- [[YYSTOS[STATE-NUM] -- The (internal number of the) accessing
-symbol of state STATE-NUM.]])
+ [[YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of
+state STATE-NUM.]])
$1([r1], [b4_r1],
- [[YYR1[YYN] -- Symbol number of symbol that rule YYN derives.]])
+ [[YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM.]])
$1([r2], [b4_r2],
- [[YYR2[YYN] -- Number of symbols on the right hand side of rule YYN.]])
+ [[YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM.]])
])
@@ -367,8 +398,9 @@
# b4_FLAG_if(IF-TRUE, IF-FALSE)
# -----------------------------
# Expand IF-TRUE, if FLAG is true, IF-FALSE otherwise.
-b4_define_flag_if([defines]) # Whether headers are requested.
b4_define_flag_if([glr]) # Whether a GLR parser is requested.
+b4_define_flag_if([has_translations]) # Whether some tokens are internationalized.
+b4_define_flag_if([header]) # Whether a header is requested.
b4_define_flag_if([nondeterministic]) # Whether conflicts should be handled.
b4_define_flag_if([token_table]) # Whether yytoken_table is demanded.
b4_define_flag_if([yacc]) # Whether POSIX Yacc is emulated.
@@ -379,43 +411,100 @@
m4_define([b4_glr_cc_if],
[m4_if(b4_skeleton, ["glr.cc"], $@)])
+# b4_glr2_cc_if([IF-TRUE], [IF-FALSE])
+# ------------------------------------
+m4_define([b4_glr2_cc_if],
+ [m4_if(b4_skeleton, ["glr2.cc"], $@)])
## --------- ##
## Symbols. ##
## --------- ##
-# For a description of the Symbol handling, see README.
+# For a description of the Symbol handling, see README.md.
#
# The following macros provide access to symbol related values.
# __b4_symbol(NUM, FIELD)
# -----------------------
-# Recover a FIELD about symbol #NUM. Thanks to m4_indir, fails if
-# undefined.
+# Fetch FIELD of symbol #NUM. Fail if undefined.
m4_define([__b4_symbol],
[m4_indir([b4_symbol($1, $2)])])
# _b4_symbol(NUM, FIELD)
# ----------------------
-# Recover a FIELD about symbol #NUM (or "orig NUM"). Fails if
-# undefined.
+# Fetch FIELD of symbol #NUM (or "orig NUM", see README.md).
+# Fail if undefined.
m4_define([_b4_symbol],
[m4_ifdef([b4_symbol($1, number)],
[__b4_symbol(m4_indir([b4_symbol($1, number)]), $2)],
[__b4_symbol([$1], [$2])])])
+# b4_symbol_token_kind(NUM)
+# -------------------------
+# The token kind of this symbol.
+m4_define([b4_symbol_token_kind],
+[b4_percent_define_get([api.token.prefix])dnl
+_b4_symbol([$1], [id])])
+
+
+# b4_symbol_kind_base(NUM)
+# ------------------------
+# Build the name of the kind of this symbol. It must always exist,
+# otherwise some symbols might not be represented in the enum, which
+# might be compiled into too small a type to contain all the symbol
+# numbers.
+m4_define([b4_symbol_prefix], [b4_percent_define_get([api.symbol.prefix])])
+m4_define([b4_symbol_kind_base],
+[b4_percent_define_get([api.symbol.prefix])dnl
+m4_case([$1],
+ [-2], [[YYEMPTY]],
+ [0], [[YYEOF]],
+ [1], [[YYerror]],
+ [2], [[YYUNDEF]],
+ [m4_case(b4_symbol([$1], [tag]),
+ [$accept], [[YYACCEPT]],
+ [b4_symbol_if([$1], [has_id], _b4_symbol([$1], [id]),
+ [m4_bpatsubst([$1-][]_b4_symbol([$1], [tag]), [[^a-zA-Z_0-9]+], [_])])])])])
+
+
+# b4_symbol_kind(NUM)
+# -------------------
+# Same as b4_symbol_kind, but possibly with a prefix in some
+# languages. E.g., EOF's kind_base and kind are YYSYMBOL_YYEOF in C,
+# but are S_YYEMPTY and symbol_kind::S_YYEMPTY in C++.
+m4_copy([b4_symbol_kind_base], [b4_symbol_kind])
+
+
+# b4_symbol_slot(NUM)
+# -------------------
+# The name of union member that contains the value of these symbols.
+# Currently, we are messy, this should actually be type_tag, but type_tag
+# has several meanings.
+m4_define([b4_symbol_slot],
+[m4_case(b4_percent_define_get([[api.value.type]]),
+ [union], [b4_symbol([$1], [type_tag])],
+ [variant], [b4_symbol([$1], [type_tag])],
+ [b4_symbol([$1], [type])])])
+
# b4_symbol(NUM, FIELD)
# ---------------------
-# Recover a FIELD about symbol #NUM (or "orig NUM"). Fails if
-# undefined. If FIELD = id, prepend the token prefix.
+# Fetch FIELD of symbol #NUM (or "orig NUM", or "empty"). Fail if undefined.
+#
+# If FIELD = id, prepend the token prefix.
m4_define([b4_symbol],
-[m4_case([$2],
- [id], [m4_do([b4_percent_define_get([api.token.prefix])],
- [_b4_symbol([$1], [id])])],
- [_b4_symbol($@)])])
+[m4_if([$1], [empty], [b4_symbol([-2], [$2])],
+ [$1], [eof], [b4_symbol([0], [$2])],
+ [$1], [error], [b4_symbol([1], [$2])],
+ [$1], [undef], [b4_symbol([2], [$2])],
+ [m4_case([$2],
+ [id], [b4_symbol_token_kind([$1])],
+ [kind_base], [b4_symbol_kind_base([$1])],
+ [kind], [b4_symbol_kind([$1])],
+ [slot], [b4_symbol_slot([$1])],
+ [_b4_symbol($@)])])])
# b4_symbol_if(NUM, FIELD, IF-TRUE, IF-FALSE)
@@ -437,9 +526,9 @@
])
-# b4_symbol_action(SYMBOL-NUM, KIND)
-# ----------------------------------
-# Run the action KIND (destructor or printer) for SYMBOL-NUM.
+# b4_symbol_action(SYMBOL-NUM, ACTION)
+# ------------------------------------
+# Run the action ACTION ("destructor" or "printer") for SYMBOL-NUM.
m4_define([b4_symbol_action],
[b4_symbol_if([$1], [has_$2],
[b4_dollar_pushdef([(*yyvaluep)],
@@ -463,21 +552,21 @@
m4_define([b4_symbol_printer], [b4_symbol_action([$1], [printer])])
-# b4_symbol_actions(KIND, [TYPE = yytype])
-# ----------------------------------------
-# Emit the symbol actions for KIND ("printer" or "destructor").
-# Dispatch on TYPE.
+# b4_symbol_actions(ACTION, [KIND = yykind])
+# ------------------------------------------
+# Emit the symbol actions for ACTION ("destructor" or "printer").
+# Dispatch on KIND.
m4_define([b4_symbol_actions],
[m4_pushdef([b4_actions_], m4_expand([b4_symbol_foreach([b4_symbol_$1])]))dnl
m4_ifval(m4_defn([b4_actions_]),
-[switch (m4_default([$2], [yytype]))
+[switch (m4_default([$2], [yykind]))
{
m4_defn([b4_actions_])[]dnl
default:
break;
}dnl
],
-[YYUSE (m4_default([$2], [yytype]));])dnl
+[b4_use(m4_default([$2], [yykind]));])dnl
m4_popdef([b4_actions_])dnl
])
@@ -487,7 +576,7 @@
# easier to use with m4_map, but then, use []dnl to suppress the last
# one.
m4_define([_b4_symbol_case],
-[case b4_symbol([$1], [number]): b4_symbol_tag_comment([$1])])
+[case b4_symbol([$1], [kind]): b4_symbol_tag_comment([$1])])
])
@@ -507,13 +596,14 @@
# b4_token_visible_if(NUM, IF-TRUE, IF-FALSE)
# -------------------------------------------
-# Whether NUM denotes a token that has an exported definition (i.e.,
-# shows in enum yytokentype).
+# Whether NUM denotes a token kind that has an exported definition
+# (i.e., shows in enum yytokentype).
m4_define([b4_token_visible_if],
[b4_symbol_if([$1], [is_token],
[b4_symbol_if([$1], [has_id], [$2], [$3])],
[$3])])
+
# b4_token_has_definition(NUM)
# ----------------------------
# 1 if NUM is visible, nothing otherwise.
@@ -530,13 +620,27 @@
# b4_token_format(FORMAT, NUM)
# ----------------------------
+# If token NUM has a visible ID, format FORMAT with ID, USER_NUMBER.
m4_define([b4_token_format],
[b4_token_visible_if([$2],
-[m4_quote(m4_format([$1],
- [b4_symbol([$2], [id])],
- [b4_symbol([$2], b4_api_token_raw_if([[number]], [[user_number]]))]))])])
+[m4_format([[$1]],
+ b4_symbol([$2], [id]),
+ b4_symbol([$2], b4_api_token_raw_if([[number]], [[code]])))])])
+# b4_last_enum_token
+# ------------------
+# The code of the last token visible token.
+m4_define([_b4_last_enum_token],
+[b4_token_visible_if([$1],
+ [m4_define([b4_last_enum_token], [$1])])])
+b4_symbol_foreach([_b4_last_enum_token])
+
+# b4_last_symbol
+# --------------
+# The code of the last symbol.
+m4_define([b4_last_symbol], m4_eval(b4_tokens_number + b4_nterms_number - 1))
+
## ------- ##
## Types. ##
## ------- ##
@@ -558,11 +662,11 @@
])])
-# b4_type_foreach(MACRO)
-# ----------------------
+# b4_type_foreach(MACRO, [SEP])
+# -----------------------------
# Invoke MACRO(SYMBOL-NUMS) for each set of SYMBOL-NUMS for each type set.
m4_define([b4_type_foreach],
- [m4_map([$1], m4_defn([b4_type_names]))])
+ [m4_map_sep([$1], [$2], m4_defn([b4_type_names]))])
@@ -603,7 +707,7 @@
# This generates dependencies on the Bison skeletons hence lots of
# useless 'git diff'. This location is useless for the regular
# user (who does not care about the skeletons) and is actually not
-# useful for Bison developpers too (I, Akim, never used this to locate
+# useful for Bison developers too (I, Akim, never used this to locate
# the code in skeletons that generated output). So disable it
# completely. If someone thinks this was actually useful, a %define
# variable should be provided to control the level of verbosity of
@@ -683,7 +787,7 @@
[m4_pushdef([b4_occurrence], b4_occurrence)dnl
m4_pushdef([b4_user_name], m4_car(b4_occurrence))dnl
m4_pushdef([b4_start], m4_car(m4_shift(b4_occurrence)))dnl
-m4_pushdef([b4_end], m4_shift(m4_shift(b4_occurrence)))dnl
+m4_pushdef([b4_end], m4_shift2(b4_occurrence))dnl
m4_ifndef($3[(]m4_quote(b4_user_name)[)],
[b4_complain_at([b4_start], [b4_end],
[[%s '%s' is not used]],
@@ -878,15 +982,18 @@
# For example:
#
# b4_percent_define_default([[foo]], [[default value]])
+m4_define([_b4_percent_define_define],
+[m4_define([b4_percent_define(]$1[)], [$2])dnl
+m4_define([b4_percent_define_kind(]$1[)],
+ [m4_default([$3], [keyword])])dnl
+m4_define([b4_percent_define_loc(]$1[)],
+ [[[[<skeleton default value>:-1.-1]],
+ [[<skeleton default value>:-1.-1]]]])dnl
+m4_define([b4_percent_define_syncline(]$1[)], [[]])])
+
m4_define([b4_percent_define_default],
[_b4_percent_define_ifdef([$1], [],
- [m4_define([b4_percent_define(]$1[)], [$2])dnl
- m4_define([b4_percent_define_kind(]$1[)],
- [m4_default([$3], [keyword])])dnl
- m4_define([b4_percent_define_loc(]$1[)],
- [[[[<skeleton default value>:-1.-1]],
- [[<skeleton default value>:-1.-1]]]])dnl
- m4_define([b4_percent_define_syncline(]$1[)], [[]])])])
+ [_b4_percent_define_define($@)])])
# b4_percent_define_if_define(NAME, [VARIABLE = NAME])
@@ -896,11 +1003,12 @@
# to '_'.
m4_define([_b4_percent_define_if_define],
[m4_define(m4_bpatsubst([b4_$1_if], [[-.]], [_]),
- [b4_percent_define_flag_if(m4_default([$2], [$1]),
- [$3], [$4])])])
+ [b4_percent_define_default([m4_default([$2], [$1])], [[false]])dnl
+b4_percent_define_flag_if(m4_default([$2], [$1]),
+ [$3], [$4])])])
+
m4_define([b4_percent_define_if_define],
-[b4_percent_define_default([m4_default([$2], [$1])], [[false]])
-_b4_percent_define_if_define([$1], [$2], $[1], $[2])])
+[_b4_percent_define_if_define([$1], [$2], $[1], $[2])])
# b4_percent_define_check_kind(VARIABLE, KIND, [DIAGNOSTIC = complain])
@@ -1006,6 +1114,7 @@
b4_percent_define_if_define([locations]) # Whether locations are tracked.
b4_percent_define_if_define([parse.assert])
b4_percent_define_if_define([parse.trace])
+b4_percent_define_if_define([posix])
# b4_bison_locations_if([IF-TRUE])
@@ -1016,31 +1125,37 @@
[b4_locations_if([b4_percent_define_ifdef([[api.location.type]], [], [$1])])])
-# b4_error_verbose_if([IF-ERRORS-ARE-VERBOSE], [IF-NOT])
+
+# %define parse.error "(custom|detailed|simple|verbose)"
# ------------------------------------------------------
-# Map %define parse.error "(simple|verbose)" to b4_error_verbose_if and
-# b4_error_verbose_flag.
b4_percent_define_default([[parse.error]], [[simple]])
b4_percent_define_check_values([[[[parse.error]],
- [[simple]], [[verbose]]]])
-m4_define([b4_error_verbose_flag],
- [m4_case(b4_percent_define_get([[parse.error]]),
- [simple], [[0]],
- [verbose], [[1]])])
-b4_define_flag_if([error_verbose])
+ [[custom]], [[detailed]], [[simple]], [[verbose]]]])
-# yytoken_table is needed to support verbose errors.
-b4_error_verbose_if([m4_define([b4_token_table_flag], [1])])
+# b4_parse_error_case(CASE1, THEN1, CASE2, THEN2, ..., ELSE)
+# ----------------------------------------------------------
+m4_define([b4_parse_error_case],
+[m4_case(b4_percent_define_get([[parse.error]]), $@)])
+
+# b4_parse_error_bmatch(PATTERN1, THEN1, PATTERN2, THEN2, ..., ELSE)
+# ------------------------------------------------------------------
+m4_define([b4_parse_error_bmatch],
+[m4_bmatch(b4_percent_define_get([[parse.error]]), $@)])
+
+# b4_union_if([IF-UNION-ARE-USED], [IF-NOT])
# b4_variant_if([IF-VARIANT-ARE-USED], [IF-NOT])
# ----------------------------------------------
-b4_percent_define_if_define([variant])
+# Depend on whether api.value.type is union, or variant.
+m4_define([b4_union_flag], [[0]])
m4_define([b4_variant_flag], [[0]])
b4_percent_define_ifdef([[api.value.type]],
[m4_case(b4_percent_define_get_kind([[api.value.type]]), [keyword],
- [m4_case(b4_percent_define_get([[api.value.type]]), [variant],
- [m4_define([b4_variant_flag], [[1]])])])])
+ [m4_case(b4_percent_define_get([[api.value.type]]),
+ [union], [m4_define([b4_union_flag], [[1]])],
+ [variant], [m4_define([b4_variant_flag], [[1]])])])])
+b4_define_flag_if([union])
b4_define_flag_if([variant])
@@ -1116,3 +1231,11 @@
# api.value.union.name.
b4_percent_define_check_kind([api.value.union.name], [keyword])
+
+# parse.error (custom|detailed) >< token-table.
+b4_token_table_if(
+[b4_parse_error_bmatch([custom\|detailed],
+[b4_complain_at(b4_percent_define_get_loc([parse.error]),
+ [['%s' and '%s' cannot be used together]],
+ [%token-table],
+ [%define parse.error (custom|detailed)])])])
diff --git a/common/bison/skeletons/c++-skel.m4 b/common/bison/skeletons/c++-skel.m4
index 1c3721c..f22002b 100644
--- a/common/bison/skeletons/c++-skel.m4
+++ b/common/bison/skeletons/c++-skel.m4
@@ -2,7 +2,7 @@
# C++ skeleton dispatching for Bison.
-# Copyright (C) 2006-2007, 2009-2015, 2018-2019 Free Software
+# Copyright (C) 2006-2007, 2009-2015, 2018-2021 Free Software
# Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
b4_glr_if( [m4_define([b4_used_skeleton], [b4_skeletonsdir/[glr.cc]])])
b4_nondeterministic_if([m4_define([b4_used_skeleton], [b4_skeletonsdir/[glr.cc]])])
diff --git a/common/bison/skeletons/c++.m4 b/common/bison/skeletons/c++.m4
index 3e2e4df..2ae8423 100644
--- a/common/bison/skeletons/c++.m4
+++ b/common/bison/skeletons/c++.m4
@@ -2,7 +2,7 @@
# C++ skeleton for Bison
-# Copyright (C) 2002-2019 Free Software Foundation, Inc.
+# Copyright (C) 2002-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,13 +15,15 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
# Sanity checks, before defaults installed by c.m4.
b4_percent_define_ifdef([[api.value.union.name]],
[b4_complain_at(b4_percent_define_get_loc([[api.value.union.name]]),
[named %union is invalid in C++])])
+b4_percent_define_default([[api.symbol.prefix]], [[S_]])
+
m4_include(b4_skeletonsdir/[c.m4])
b4_percent_define_check_kind([api.namespace], [code], [deprecated])
@@ -103,11 +105,10 @@
#
# b4_percent_define_default([[api.location.type]], [[location]])
-b4_percent_define_default([[filename_type]], [[std::string]])
+b4_percent_define_default([[api.filename.type]], [[const std::string]])
# Make it a warning for those who used betas of Bison 3.0.
b4_percent_define_default([[api.namespace]], m4_defn([b4_prefix]))
-b4_percent_define_default([[global_tokens_and_yystype]], [[false]])
b4_percent_define_default([[define_location_comparison]],
[m4_if(b4_percent_define_get([[filename_type]]),
[std::string], [[true]], [[false]])])
@@ -160,20 +161,42 @@
[::\([^][:]\|:[^:]\)*], [} ])[} // ]b4_namespace_ref])])
+## ------------- ##
+## Token kinds. ##
+## ------------- ##
+
+
# b4_token_enums
# --------------
-# Output the definition of the tokens as enums.
+# Output the definition of the token kinds.
m4_define([b4_token_enums],
-[[enum yytokentype
+[[enum token_kind_type
{
- ]m4_join([,
- ],
- b4_symbol_map([b4_token_enum]))[
- };]dnl
+ ]b4_symbol([-2], [id])[ = -2,
+]b4_symbol_foreach([b4_token_enum])dnl
+[ };]dnl
])
+## -------------- ##
+## Symbol kinds. ##
+## -------------- ##
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+# Defining YYEMPTY here is important: it forces the compiler
+# to use a signed type, which matters for yytoken.
+m4_define([b4_declare_symbol_enum],
+[[enum symbol_kind_type
+ {
+ YYNTOKENS = ]b4_tokens_number[, ///< Number of tokens.
+ ]b4_symbol(empty, kind_base)[ = -2,
+]b4_symbol_foreach([ b4_symbol_enum])dnl
+[ };]])
+
+
## ----------------- ##
## Semantic Values. ##
@@ -183,16 +206,16 @@
# b4_value_type_declare
# ---------------------
-# Declare semantic_type.
+# Declare value_type.
m4_define([b4_value_type_declare],
[b4_value_type_setup[]dnl
[ /// Symbol semantic values.
]m4_bmatch(b4_percent_define_get_kind([[api.value.type]]),
[code],
-[[ typedef ]b4_percent_define_get([[api.value.type]])[ semantic_type;]],
+[[ typedef ]b4_percent_define_get([[api.value.type]])[ value_type;]],
[m4_bmatch(b4_percent_define_get([[api.value.type]]),
[union\|union-directive],
-[[ union semantic_type
+[[ union value_type
{
]b4_user_union_members[
};]])])dnl
@@ -204,11 +227,19 @@
# Define the public types: token, semantic value, location, and so forth.
# Depending on %define token_lex, may be output in the header or source file.
m4_define([b4_public_types_declare],
-[[#ifndef ]b4_api_PREFIX[STYPE
-]b4_value_type_declare[
+[b4_glr2_cc_if(
+[b4_value_type_declare],
+[[#ifdef ]b4_api_PREFIX[STYPE
+# ifdef __GNUC__
+# pragma GCC message "bison: do not #define ]b4_api_PREFIX[STYPE in C++, use %define api.value.type"
+# endif
+ typedef ]b4_api_PREFIX[STYPE value_type;
#else
- typedef ]b4_api_PREFIX[STYPE semantic_type;
-#endif]b4_locations_if([
+]b4_value_type_declare[
+#endif
+ /// Backward compatibility (Bison 3.8).
+ typedef value_type semantic_type;
+]])[]b4_locations_if([
/// Symbol locations.
typedef b4_percent_define_get([[api.location.type]],
[[location]]) location_type;])[
@@ -231,23 +262,31 @@
location_type location;])[
};
- /// Tokens.
+ /// Token kinds.
struct token
{
- ]b4_token_enums[
+ ]b4_token_enums[]b4_glr2_cc_if([], [[
+ /// Backward compatibility alias (Bison 3.6).
+ typedef token_kind_type yytokentype;]])[
};
- /// (External) token type, as returned by yylex.
- typedef token::yytokentype token_type;
+ /// Token kind, as returned by yylex.
+ typedef token::token_kind_type token_kind_type;]b4_glr2_cc_if([], [[
- /// Symbol type: an internal symbol number.
- typedef int symbol_number_type;
+ /// Backward compatibility alias (Bison 3.6).
+ typedef token_kind_type token_type;]])[
- /// The symbol type number to denote an empty symbol.
- enum { empty_symbol = -2 };
+ /// Symbol kinds.
+ struct symbol_kind
+ {
+ ]b4_declare_symbol_enum[
+ };
- /// Internal symbol number for tokens (subsumed by symbol_number_type).
- typedef ]b4_int_type_for([b4_translate])[ token_number_type;
+ /// (Internal) symbol kind.
+ typedef symbol_kind::symbol_kind_type symbol_kind_type;
+
+ /// The number of tokens.
+ static const symbol_kind_type YYNTOKENS = symbol_kind::YYNTOKENS;
]])
@@ -258,8 +297,8 @@
m4_define([b4_symbol_type_define],
[[ /// A complete symbol.
///
- /// Expects its Base type to provide access to the symbol type
- /// via type_get ().
+ /// Expects its Base type to provide access to the symbol kind
+ /// via kind ().
///
/// Provide access to semantic value]b4_locations_if([ and location])[.
template <typename Base>
@@ -269,28 +308,36 @@
typedef Base super_type;
/// Default constructor.
- basic_symbol ()
+ basic_symbol () YY_NOEXCEPT
: value ()]b4_locations_if([
, location ()])[
{}
#if 201103L <= YY_CPLUSPLUS
/// Move constructor.
- basic_symbol (basic_symbol&& that);
+ basic_symbol (basic_symbol&& that)
+ : Base (std::move (that))
+ , value (]b4_variant_if([], [std::move (that.value)]))b4_locations_if([
+ , location (std::move (that.location))])[
+ {]b4_variant_if([
+ b4_symbol_variant([this->kind ()], [value], [move],
+ [std::move (that.value)])
+ ])[}
#endif
/// Copy constructor.
basic_symbol (const basic_symbol& that);]b4_variant_if([[
- /// Constructor for valueless symbols, and symbols from each type.
-]b4_type_foreach([b4_basic_symbol_constructor_define])], [[
+ /// Constructors for typed symbols.
+]b4_type_foreach([b4_basic_symbol_constructor_define], [
+])], [[
/// Constructor for valueless symbols.
basic_symbol (typename Base::kind_type t]b4_locations_if([,
YY_MOVE_REF (location_type) l])[);
/// Constructor for symbols with semantic value.
basic_symbol (typename Base::kind_type t,
- YY_RVREF (semantic_type) v]b4_locations_if([,
+ YY_RVREF (value_type) v]b4_locations_if([,
YY_RVREF (location_type) l])[);
]])[
/// Destroy the symbol.
@@ -299,25 +346,75 @@
clear ();
}
+]b4_glr2_cc_if([[
+ /// Copy assignment.
+ basic_symbol& operator= (const basic_symbol& that)
+ {
+ Base::operator= (that);]b4_variant_if([[
+ ]b4_symbol_variant([this->kind ()], [value], [copy],
+ [that.value])], [[
+ value = that.value]])[;]b4_locations_if([[
+ location = that.location;]])[
+ return *this;
+ }
+
+ /// Move assignment.
+ basic_symbol& operator= (basic_symbol&& that)
+ {
+ Base::operator= (std::move (that));]b4_variant_if([[
+ ]b4_symbol_variant([this->kind ()], [value], [move],
+ [std::move (that.value)])], [[
+ value = std::move (that.value)]])[;]b4_locations_if([[
+ location = std::move (that.location);]])[
+ return *this;
+ }
+]])[
+
/// Destroy contents, and record that is empty.
- void clear ()
+ void clear () YY_NOEXCEPT
{]b4_variant_if([[
// User destructor.
- symbol_number_type yytype = this->type_get ();
+ symbol_kind_type yykind = this->kind ();
basic_symbol<Base>& yysym = *this;
(void) yysym;
- switch (yytype)
+ switch (yykind)
{
]b4_symbol_foreach([b4_symbol_destructor])dnl
[ default:
break;
}
- // Type destructor.
-]b4_symbol_variant([[yytype]], [[value]], [[template destroy]])])[
+ // Value type destructor.
+]b4_symbol_variant([[yykind]], [[value]], [[template destroy]])])[
Base::clear ();
}
+]b4_parse_error_bmatch(
+[custom\|detailed],
+[[ /// The user-facing name of this symbol.
+ const char *name () const YY_NOEXCEPT
+ {
+ return ]b4_parser_class[::symbol_name (this->kind ());
+ }]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+ /// The user-facing name of this symbol.
+ const char *name () const YY_NOEXCEPT
+ {
+ return ]b4_parser_class[::symbol_name (this->kind ());
+ }
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[ /// The user-facing name of this symbol.
+ std::string name () const YY_NOEXCEPT
+ {
+ return ]b4_parser_class[::symbol_name (this->kind ());
+ }]])[]b4_glr2_cc_if([], [[
+
+ /// Backward compatibility (Bison 3.6).
+ symbol_kind_type type_get () const YY_NOEXCEPT;]])[
+
/// Whether empty.
bool empty () const YY_NOEXCEPT;
@@ -325,7 +422,7 @@
void move (basic_symbol& s);
/// The semantic value.
- semantic_type value;]b4_locations_if([
+ value_type value;]b4_locations_if([
/// The location.
location_type location;])[
@@ -338,52 +435,65 @@
};
/// Type access provider for token (enum) based symbols.
- struct by_type
+ struct by_kind
{
+ /// The symbol kind as needed by the constructor.
+ typedef token_kind_type kind_type;
+
/// Default constructor.
- by_type ();
+ by_kind () YY_NOEXCEPT;
#if 201103L <= YY_CPLUSPLUS
/// Move constructor.
- by_type (by_type&& that);
+ by_kind (by_kind&& that) YY_NOEXCEPT;
#endif
/// Copy constructor.
- by_type (const by_type& that);
-
- /// The symbol type as needed by the constructor.
- typedef token_type kind_type;
+ by_kind (const by_kind& that) YY_NOEXCEPT;
/// Constructor from (external) token numbers.
- by_type (kind_type t);
+ by_kind (kind_type t) YY_NOEXCEPT;
+
+]b4_glr2_cc_if([[
+ /// Copy assignment.
+ by_kind& operator= (const by_kind& that);
+
+ /// Move assignment.
+ by_kind& operator= (by_kind&& that);
+]])[
/// Record that this symbol is empty.
- void clear ();
+ void clear () YY_NOEXCEPT;
- /// Steal the symbol type from \a that.
- void move (by_type& that);
+ /// Steal the symbol kind from \a that.
+ void move (by_kind& that);
/// The (internal) type number (corresponding to \a type).
/// \a empty when empty.
- symbol_number_type type_get () const YY_NOEXCEPT;
+ symbol_kind_type kind () const YY_NOEXCEPT;]b4_glr2_cc_if([], [[
- /// The symbol type.
- /// \a empty_symbol when empty.
- /// An int, not token_number_type, to be able to store empty_symbol.
- int type;
- };
+ /// Backward compatibility (Bison 3.6).
+ symbol_kind_type type_get () const YY_NOEXCEPT;]])[
+
+ /// The symbol kind.
+ /// \a ]b4_symbol_prefix[YYEMPTY when empty.
+ symbol_kind_type kind_;
+ };]b4_glr2_cc_if([], [[
+
+ /// Backward compatibility for a private implementation detail (Bison 3.6).
+ typedef by_kind by_type;]])[
/// "External" symbols: returned by the scanner.
- struct symbol_type : basic_symbol<by_type>
+ struct symbol_type : basic_symbol<by_kind>
{]b4_variant_if([[
/// Superclass.
- typedef basic_symbol<by_type> super_type;
+ typedef basic_symbol<by_kind> super_type;
/// Empty symbol.
- symbol_type () {}
+ symbol_type () YY_NOEXCEPT {}
/// Constructor for valueless symbols, and symbols from each type.
-]b4_type_foreach([_b4_token_constructor_define])dnl
+]b4_type_foreach([_b4_symbol_constructor_define])dnl
])[};
]])
@@ -393,25 +503,13 @@
# Provide the implementation needed by the public types.
m4_define([b4_public_types_define],
[[ // basic_symbol.
-#if 201103L <= YY_CPLUSPLUS
- template <typename Base>
- ]b4_parser_class[::basic_symbol<Base>::basic_symbol (basic_symbol&& that)
- : Base (std::move (that))
- , value (]b4_variant_if([], [std::move (that.value)]))b4_locations_if([
- , location (std::move (that.location))])[
- {]b4_variant_if([
- b4_symbol_variant([this->type_get ()], [value], [move],
- [std::move (that.value)])
- ])[}
-#endif
-
template <typename Base>
]b4_parser_class[::basic_symbol<Base>::basic_symbol (const basic_symbol& that)
: Base (that)
, value (]b4_variant_if([], [that.value]))b4_locations_if([
, location (that.location)])[
{]b4_variant_if([
- b4_symbol_variant([this->type_get ()], [value], [copy],
+ b4_symbol_variant([this->kind ()], [value], [copy],
[YY_MOVE (that.value)])
])[}
@@ -429,20 +527,29 @@
template <typename Base>
]b4_parser_class[::basic_symbol<Base>::basic_symbol (]b4_join(
[typename Base::kind_type t],
- [YY_RVREF (semantic_type) v],
+ [YY_RVREF (value_type) v],
b4_locations_if([YY_RVREF (location_type) l]))[)
: Base (t)
, value (]b4_variant_if([], [YY_MOVE (v)])[)]b4_locations_if([
, location (YY_MOVE (l))])[
{]b4_variant_if([[
(void) v;
- ]b4_symbol_variant([this->type_get ()], [value], [YY_MOVE_OR_COPY], [YY_MOVE (v)])])[}]])[
+ ]b4_symbol_variant([this->kind ()], [value], [YY_MOVE_OR_COPY], [YY_MOVE (v)])])[}]])[
+
+]b4_glr2_cc_if([], [[
+ template <typename Base>
+ ]b4_parser_class[::symbol_kind_type
+ ]b4_parser_class[::basic_symbol<Base>::type_get () const YY_NOEXCEPT
+ {
+ return this->kind ();
+ }
+]])[
template <typename Base>
bool
]b4_parser_class[::basic_symbol<Base>::empty () const YY_NOEXCEPT
{
- return Base::type_get () == empty_symbol;
+ return this->kind () == ]b4_symbol(empty, kind)[;
}
template <typename Base>
@@ -450,57 +557,82 @@
]b4_parser_class[::basic_symbol<Base>::move (basic_symbol& s)
{
super_type::move (s);
- ]b4_variant_if([b4_symbol_variant([this->type_get ()], [value], [move],
+ ]b4_variant_if([b4_symbol_variant([this->kind ()], [value], [move],
[YY_MOVE (s.value)])],
[value = YY_MOVE (s.value);])[]b4_locations_if([
location = YY_MOVE (s.location);])[
}
- // by_type.
- ]b4_inline([$1])b4_parser_class[::by_type::by_type ()
- : type (empty_symbol)
+ // by_kind.
+ ]b4_inline([$1])b4_parser_class[::by_kind::by_kind () YY_NOEXCEPT
+ : kind_ (]b4_symbol(empty, kind)[)
{}
#if 201103L <= YY_CPLUSPLUS
- ]b4_inline([$1])b4_parser_class[::by_type::by_type (by_type&& that)
- : type (that.type)
+ ]b4_inline([$1])b4_parser_class[::by_kind::by_kind (by_kind&& that) YY_NOEXCEPT
+ : kind_ (that.kind_)
{
that.clear ();
}
#endif
- ]b4_inline([$1])b4_parser_class[::by_type::by_type (const by_type& that)
- : type (that.type)
+ ]b4_inline([$1])b4_parser_class[::by_kind::by_kind (const by_kind& that) YY_NOEXCEPT
+ : kind_ (that.kind_)
{}
- ]b4_inline([$1])b4_parser_class[::by_type::by_type (token_type t)
- : type (yytranslate_ (t))
+ ]b4_inline([$1])b4_parser_class[::by_kind::by_kind (token_kind_type t) YY_NOEXCEPT
+ : kind_ (yytranslate_ (t))
{}
+]b4_glr2_cc_if([[
+ ]b4_inline([$1])]b4_parser_class[::by_kind&
+ b4_parser_class[::by_kind::by_kind::operator= (const by_kind& that)
+ {
+ kind_ = that.kind_;
+ return *this;
+ }
+
+ ]b4_inline([$1])]b4_parser_class[::by_kind&
+ b4_parser_class[::by_kind::by_kind::operator= (by_kind&& that)
+ {
+ kind_ = that.kind_;
+ that.clear ();
+ return *this;
+ }
+]])[
+
]b4_inline([$1])[void
- ]b4_parser_class[::by_type::clear ()
+ ]b4_parser_class[::by_kind::clear () YY_NOEXCEPT
{
- type = empty_symbol;
+ kind_ = ]b4_symbol(empty, kind)[;
}
]b4_inline([$1])[void
- ]b4_parser_class[::by_type::move (by_type& that)
+ ]b4_parser_class[::by_kind::move (by_kind& that)
{
- type = that.type;
+ kind_ = that.kind_;
that.clear ();
}
- ]b4_inline([$1])[int
- ]b4_parser_class[::by_type::type_get () const YY_NOEXCEPT
+ ]b4_inline([$1])[]b4_parser_class[::symbol_kind_type
+ ]b4_parser_class[::by_kind::kind () const YY_NOEXCEPT
{
- return type;
+ return kind_;
}
+
+]b4_glr2_cc_if([], [[
+ ]b4_inline([$1])[]b4_parser_class[::symbol_kind_type
+ ]b4_parser_class[::by_kind::type_get () const YY_NOEXCEPT
+ {
+ return this->kind ();
+ }
+]])[
]])
# b4_token_constructor_define
# ----------------------------
-# Define symbol constructors for all the value types.
+# Define make_FOO for all the token kinds.
# Use at class-level. Redefined in variant.hh.
m4_define([b4_token_constructor_define], [])
@@ -510,27 +642,28 @@
# Define yytranslate_. Sometimes used in the header file ($1=hh),
# sometimes in the cc file.
m4_define([b4_yytranslate_define],
-[ b4_inline([$1])b4_parser_class[::token_number_type
- ]b4_parser_class[::yytranslate_ (int t)
+[ b4_inline([$1])b4_parser_class[::symbol_kind_type
+ ]b4_parser_class[::yytranslate_ (int t) YY_NOEXCEPT
{
]b4_api_token_raw_if(
-[[ return static_cast<token_number_type> (t);]],
+[[ return static_cast<symbol_kind_type> (t);]],
[[ // YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to
// TOKEN-NUM as returned by yylex.
static
- const token_number_type
+ const ]b4_int_type_for([b4_translate])[
translate_table[] =
{
]b4_translate[
};
- const int user_token_number_max_ = ]b4_user_token_number_max[;
+ // Last valid token kind.
+ const int code_max = ]b4_code_max[;
if (t <= 0)
- return yyeof_;
- else if (t <= user_token_number_max_)
- return translate_table[t];
+ return symbol_kind::]b4_symbol_prefix[YYEOF;
+ else if (t <= code_max)
+ return static_cast <symbol_kind_type> (translate_table[t]);
else
- return yy_undef_token_;]])[
+ return symbol_kind::]b4_symbol_prefix[YYUNDEF;]])[
}
]])
diff --git a/common/bison/skeletons/c-like.m4 b/common/bison/skeletons/c-like.m4
index 8d891b6..a9bbc2e 100644
--- a/common/bison/skeletons/c-like.m4
+++ b/common/bison/skeletons/c-like.m4
@@ -2,7 +2,7 @@
# Common code for C-like languages (C, C++, Java, etc.)
-# Copyright (C) 2012-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2012-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,17 +15,23 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
# _b4_comment(TEXT, OPEN, CONTINUE, END)
# --------------------------------------
# Put TEXT in comment. Avoid trailing spaces: don't indent empty lines.
# Avoid adding indentation to the first line, as the indentation comes
# from OPEN. That's why we don't patsubst([$1], [^\(.\)], [ \1]).
+# Turn "*/" in TEXT into "* /" so that we don't unexpectedly close
+# the comments before its end.
#
# Prefix all the output lines with PREFIX.
m4_define([_b4_comment],
-[$2[]m4_bpatsubst(m4_expand([[$1]]), [
+[$2[]b4_gsub(m4_expand([$1]),
+ [[*]/], [*\\/],
+ [/[*]], [/\\*],
+ [
\(.\)], [
$3\1])$4])
diff --git a/common/bison/skeletons/c-skel.m4 b/common/bison/skeletons/c-skel.m4
index 2a21cfc..ac6ddd6 100644
--- a/common/bison/skeletons/c-skel.m4
+++ b/common/bison/skeletons/c-skel.m4
@@ -2,7 +2,7 @@
# C skeleton dispatching for Bison.
-# Copyright (C) 2006-2007, 2009-2015, 2018-2019 Free Software
+# Copyright (C) 2006-2007, 2009-2015, 2018-2021 Free Software
# Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
b4_glr_if( [m4_define([b4_used_skeleton], [b4_skeletonsdir/[glr.c]])])
b4_nondeterministic_if([m4_define([b4_used_skeleton], [b4_skeletonsdir/[glr.c]])])
diff --git a/common/bison/skeletons/c.m4 b/common/bison/skeletons/c.m4
index 4b2c094..2425b07 100644
--- a/common/bison/skeletons/c.m4
+++ b/common/bison/skeletons/c.m4
@@ -2,7 +2,7 @@
# C M4 Macros for Bison.
-# Copyright (C) 2002, 2004-2015, 2018-2019 Free Software Foundation,
+# Copyright (C) 2002, 2004-2015, 2018-2021 Free Software Foundation,
# Inc.
# This program is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
m4_include(b4_skeletonsdir/[c-like.m4])
@@ -58,11 +58,11 @@
# b4_pull_flag if they use the values of the %define variables api.pure or
# api.push-pull.
m4_define([b4_identification],
-[[/* Identify Bison output. */
-#define YYBISON 1
+[[/* Identify Bison output, and Bison version. */
+#define YYBISON ]b4_version[
-/* Bison version. */
-#define YYBISON_VERSION "]b4_version["
+/* Bison version string. */
+#define YYBISON_VERSION "]b4_version_string["
/* Skeleton name. */
#define YYSKELETON_NAME ]b4_skeleton[]m4_ifdef([b4_pure_flag], [[
@@ -101,32 +101,45 @@
b4_percent_define_default([[api.value.union.name]],
[b4_api_PREFIX[][STYPE]])
+b4_percent_define_default([[api.symbol.prefix]], [[YYSYMBOL_]])
## ------------------------ ##
## Pure/impure interfaces. ##
## ------------------------ ##
-# b4_lex_formals
-# --------------
+# b4_yylex_formals
+# ----------------
# All the yylex formal arguments.
# b4_lex_param arrives quoted twice, but we want to keep only one level.
-m4_define([b4_lex_formals],
-[b4_pure_if([[[[YYSTYPE *yylvalp]], [[&yylval]]][]dnl
-b4_locations_if([, [[YYLTYPE *yyllocp], [&yylloc]]])])dnl
+m4_define([b4_yylex_formals],
+[b4_pure_if([[[b4_api_PREFIX[STYPE *yylvalp]], [[&yylval]]][]dnl
+b4_locations_if([, [b4_api_PREFIX[LTYPE *yyllocp], [&yylloc]]])])dnl
m4_ifdef([b4_lex_param], [, ]b4_lex_param)])
-# b4_lex
-# ------
+# b4_yylex
+# --------
# Call yylex.
-m4_define([b4_lex],
-[b4_function_call([yylex], [int], b4_lex_formals)])
+m4_define([b4_yylex],
+[b4_function_call([yylex], [int], b4_yylex_formals)])
# b4_user_args
# ------------
m4_define([b4_user_args],
-[m4_ifset([b4_parse_param], [, b4_args(b4_parse_param)])])
+[m4_ifset([b4_parse_param], [, b4_user_args_no_comma])])
+
+# b4_user_args_no_comma
+# ---------------------
+m4_define([b4_user_args_no_comma],
+[m4_ifset([b4_parse_param], [b4_args(b4_parse_param)])])
+
+
+# b4_user_formals
+# ---------------
+# The possible parse-params formal arguments preceded by a comma.
+m4_define([b4_user_formals],
+[m4_ifset([b4_parse_param], [, b4_formals(b4_parse_param)])])
# b4_parse_param
@@ -150,13 +163,20 @@
m4_popdef([$1])dnl
])])
+
+# b4_use(EXPR)
+# ------------
+# Pacify the compiler about some maybe unused value.
+m4_define([b4_use],
+[YY_USE ($1)])
+
# b4_parse_param_use([VAL], [LOC])
# --------------------------------
-# 'YYUSE' VAL, LOC if locations are enabled, and all the parse-params.
+# 'YY_USE' VAL, LOC if locations are enabled, and all the parse-params.
m4_define([b4_parse_param_use],
-[m4_ifvaln([$1], [ YYUSE ([$1]);])dnl
-b4_locations_if([m4_ifvaln([$2], [ YYUSE ([$2]);])])dnl
-b4_parse_param_for([Decl], [Formal], [ YYUSE (Formal);
+[m4_ifvaln([$1], [ b4_use([$1]);])dnl
+b4_locations_if([m4_ifvaln([$2], [ b4_use([$2]);])])dnl
+b4_parse_param_for([Decl], [Formal], [ b4_use(Formal);
])dnl
])
@@ -197,7 +217,11 @@
# Define private types suitable for holding small integers in C99 or later.
m4_define([b4_c99_int_type_define],
[m4_copy_force([b4_c99_int_type], [b4_int_type])dnl
-[/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure
+[#ifdef short
+# undef short
+#endif
+
+/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure
<limits.h> and (if available) <stdint.h> are included
so that the code can choose integer types of a good width. */
@@ -230,6 +254,18 @@
typedef short yytype_int16;
#endif
+/* Work around bug in HP-UX 11.23, which defines these macros
+ incorrectly for preprocessor constants. This workaround can likely
+ be removed in 2023, as HPE has promised support for HP-UX 11.23
+ (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of
+ <https://h20195.www2.hpe.com/V2/getpdf.aspx/4AA4-7673ENW.pdf>. */
+#ifdef __hpux
+# undef UINT_LEAST8_MAX
+# undef UINT_LEAST16_MAX
+# define UINT_LEAST8_MAX 255
+# define UINT_LEAST16_MAX 65535
+#endif
+
#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__
typedef __UINT_LEAST8_TYPE__ yytype_uint8;
#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \
@@ -253,6 +289,50 @@
#endif]])
+# b4_sizes_types_define
+# ---------------------
+# Define YYPTRDIFF_T/YYPTRDIFF_MAXIMUM, YYSIZE_T/YYSIZE_MAXIMUM,
+# and YYSIZEOF.
+m4_define([b4_sizes_types_define],
+[[#ifndef YYPTRDIFF_T
+# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__
+# define YYPTRDIFF_T __PTRDIFF_TYPE__
+# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__
+# elif defined PTRDIFF_MAX
+# ifndef ptrdiff_t
+# include <stddef.h> /* INFRINGES ON USER NAME SPACE */
+# endif
+# define YYPTRDIFF_T ptrdiff_t
+# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX
+# else
+# define YYPTRDIFF_T long
+# define YYPTRDIFF_MAXIMUM LONG_MAX
+# endif
+#endif
+
+#ifndef YYSIZE_T
+# ifdef __SIZE_TYPE__
+# define YYSIZE_T __SIZE_TYPE__
+# elif defined size_t
+# define YYSIZE_T size_t
+# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__
+# include <stddef.h> /* INFRINGES ON USER NAME SPACE */
+# define YYSIZE_T size_t
+# else
+# define YYSIZE_T unsigned
+# endif
+#endif
+
+#define YYSIZE_MAXIMUM \
+ YY_CAST (YYPTRDIFF_T, \
+ (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \
+ ? YYPTRDIFF_MAXIMUM \
+ : YY_CAST (YYSIZE_T, -1)))
+
+#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X))
+]])
+
+
# b4_int_type_for(NAME)
# ---------------------
# Return a narrow int type able to handle numbers ranging from
@@ -299,17 +379,27 @@
#endif
]m4_bmatch([$1], [\bnoreturn\b], [[/* The _Noreturn keyword of C11. */
-]dnl This is an exact copy of lib/_Noreturn.h.
+]dnl This is close to lib/_Noreturn.h, except that we do enable
+dnl the use of [[noreturn]], because _Noreturn is used in places
+dnl where [[noreturn]] works in C++. We need this in particular
+dnl because of glr.cc which compiles code from glr.c in C++.
+dnl And the C++ compiler chokes on _Noreturn. Also, we do not
+dnl use C' _Noreturn in C++, to avoid -Wc11-extensions warnings.
[#ifndef _Noreturn
# if (defined __cplusplus \
&& ((201103 <= __cplusplus && !(__GNUC__ == 4 && __GNUC_MINOR__ == 7)) \
|| (defined _MSC_VER && 1900 <= _MSC_VER)))
# define _Noreturn [[noreturn]]
# elif ((!defined __cplusplus || defined __clang__) \
- && (201112 <= (defined __STDC_VERSION__ ? __STDC_VERSION__ : 0) \
- || 4 < __GNUC__ + (7 <= __GNUC_MINOR__)))
+ && (201112 <= (defined __STDC_VERSION__ ? __STDC_VERSION__ : 0) \
+ || (!defined __STRICT_ANSI__ \
+ && (4 < __GNUC__ + (7 <= __GNUC_MINOR__) \
+ || (defined __apple_build_version__ \
+ ? 6000000 <= __apple_build_version__ \
+ : 3 < __clang_major__ + (5 <= __clang_minor__))))))
/* _Noreturn works as-is. */
-# elif 2 < __GNUC__ + (8 <= __GNUC_MINOR__) || 0x5110 <= __SUNPRO_C
+# elif (2 < __GNUC__ + (8 <= __GNUC_MINOR__) || defined __clang__ \
+ || 0x5110 <= __SUNPRO_C)
# define _Noreturn __attribute__ ((__noreturn__))
# elif 1200 <= (defined _MSC_VER ? _MSC_VER : 0)
# define _Noreturn __declspec (noreturn)
@@ -320,17 +410,23 @@
]])[/* Suppress unused-variable warnings by "using" E. */
#if ! defined lint || defined __GNUC__
-# define YYUSE(E) ((void) (E))
+# define YY_USE(E) ((void) (E))
#else
-# define YYUSE(E) /* empty */
+# define YY_USE(E) /* empty */
#endif
-#if defined __GNUC__ && ! defined __ICC && 407 <= __GNUC__ * 100 + __GNUC_MINOR__
/* Suppress an incorrect diagnostic about yylval being uninitialized. */
-# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \
+#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__
+# if __GNUC__ * 100 + __GNUC_MINOR__ < 407
+# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \
+ _Pragma ("GCC diagnostic push") \
+ _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"")
+# else
+# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \
_Pragma ("GCC diagnostic push") \
_Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \
_Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"")
+# endif
# define YY_IGNORE_MAYBE_UNINITIALIZED_END \
_Pragma ("GCC diagnostic pop")
#else
@@ -410,7 +506,7 @@
# -------------------------------------------------------------
# Define "yy<TABLE-NAME>" whose contents is CONTENT.
m4_define([b4_integral_parser_table_define],
-[m4_ifvaln([$3], [b4_comment([$3], [ ])])dnl
+[m4_ifvaln([$3], [b4_comment([$3])])dnl
static const b4_int_type_for([$2]) yy$1[[]] =
{
$2
@@ -418,9 +514,22 @@
])
-## ------------------------- ##
-## Assigning token numbers. ##
-## ------------------------- ##
+## ------------- ##
+## Token kinds. ##
+## ------------- ##
+
+# Because C enums are not scoped, because tokens are exposed in the
+# header, and because these tokens are common to all the parsers, we
+# need to make sure their names don't collide: use the api.prefix.
+# YYEOF is special, since the user may give it a different name.
+m4_define([b4_symbol(-2, id)], [b4_api_PREFIX[][EMPTY]])
+m4_define([b4_symbol(-2, tag)], [[No symbol.]])
+
+m4_if(b4_symbol(eof, id), [YYEOF],
+ [m4_define([b4_symbol(0, id)], [b4_api_PREFIX[][EOF]])])
+m4_define([b4_symbol(1, id)], [b4_api_PREFIX[][error]])
+m4_define([b4_symbol(2, id)], [b4_api_PREFIX[][UNDEF]])
+
# b4_token_define(TOKEN-NUM)
# --------------------------
@@ -432,44 +541,92 @@
# ----------------
# Output the definition of the tokens.
m4_define([b4_token_defines],
-[b4_any_token_visible_if([/* Tokens. */
-m4_join([
+[[/* Token kinds. */
+#define ]b4_symbol(empty, [id])[ -2
+]m4_join([
], b4_symbol_map([b4_token_define]))
-])])
+])
# b4_token_enum(TOKEN-NUM)
# ------------------------
# Output the definition of this token as an enum.
m4_define([b4_token_enum],
-[b4_token_format([%s = %s], [$1])])
+[b4_token_visible_if([$1],
+ [m4_format([ %-30s %s],
+ m4_format([[%s = %s%s%s]],
+ b4_symbol([$1], [id]),
+ b4_symbol([$1], b4_api_token_raw_if([[number]], [[code]])),
+ m4_if([$1], b4_last_enum_token, [], [[,]])),
+ [b4_symbol_tag_comment([$1])])])])
# b4_token_enums
# --------------
-# Output the definition of the tokens (if there are) as enums.
+# The definition of the token kinds.
m4_define([b4_token_enums],
-[b4_any_token_visible_if([[/* Token type. */
+[b4_any_token_visible_if([[/* Token kinds. */
#ifndef ]b4_api_PREFIX[TOKENTYPE
# define ]b4_api_PREFIX[TOKENTYPE
enum ]b4_api_prefix[tokentype
{
- ]m4_join([,
- ],
- b4_symbol_map([b4_token_enum]))[
- };
+ ]b4_symbol(empty, [id])[ = -2,
+]b4_symbol_foreach([b4_token_enum])dnl
+[ };
+ typedef enum ]b4_api_prefix[tokentype ]b4_api_prefix[token_kind_t;
#endif
]])])
# b4_token_enums_defines
# ----------------------
-# Output the definition of the tokens (if there are any) as enums and,
+# The definition of the tokens (if there are any) as enums and,
# if POSIX Yacc is enabled, as #defines.
m4_define([b4_token_enums_defines],
[b4_token_enums[]b4_yacc_if([b4_token_defines])])
+# b4_symbol_translate(STRING)
+# ---------------------------
+# Used by "bison" in the array of symbol names to mark those that
+# require translation.
+m4_define([b4_symbol_translate],
+[[N_($1)]])
+
+
+
+## -------------- ##
+## Symbol kinds. ##
+## -------------- ##
+
+# b4_symbol_enum(SYMBOL-NUM)
+# --------------------------
+# Output the definition of this symbol as an enum.
+m4_define([b4_symbol_enum],
+[m4_format([ %-40s %s],
+ m4_format([[%s = %s%s%s]],
+ b4_symbol([$1], [kind_base]),
+ [$1],
+ m4_if([$1], b4_last_symbol, [], [[,]])),
+ [b4_symbol_tag_comment([$1])])])
+
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+# Defining YYEMPTY here is important: it forces the compiler
+# to use a signed type, which matters for yytoken.
+m4_define([b4_declare_symbol_enum],
+[[/* Symbol kind. */
+enum yysymbol_kind_t
+{
+ ]b4_symbol(empty, [kind_base])[ = -2,
+]b4_symbol_foreach([b4_symbol_enum])dnl
+[};
+typedef enum yysymbol_kind_t yysymbol_kind_t;
+]])])
+
+
## ----------------- ##
## Semantic Values. ##
## ----------------- ##
@@ -493,15 +650,6 @@
## ---------------------- ##
-# b4_function_define(NAME, RETURN-VALUE, [DECL1, NAME1], ...)
-# -----------------------------------------------------------
-# Declare the function NAME in C.
-m4_define([b4_function_define],
-[$2
-$1 (b4_formals(m4_shift2($@)))[]dnl
-])
-
-
# b4_formals([DECL1, NAME1], ...)
# -------------------------------
# The formal arguments of a C function definition.
@@ -514,12 +662,6 @@
[$1])
-
-## ----------------------- ##
-## Declaring C functions. ##
-## ----------------------- ##
-
-
# b4_function_declare(NAME, RETURN-VALUE, [DECL1, NAME1], ...)
# ------------------------------------------------------------
# Declare the function NAME.
@@ -529,7 +671,6 @@
-
## --------------------- ##
## Calling C functions. ##
## --------------------- ##
@@ -566,10 +707,10 @@
## User actions. ##
## -------------- ##
-# b4_case(LABEL, STATEMENTS)
-# --------------------------
+# b4_case(LABEL, STATEMENTS, [COMMENTS])
+# --------------------------------------
m4_define([b4_case],
-[ case $1:
+[ case $1:m4_ifval([$3], [ b4_comment([$3])])
$2
b4_syncline([@oline@], [@ofile@])dnl
break;])
@@ -593,18 +734,15 @@
| Release the memory associated to this symbol. |
`-----------------------------------------------*/
-]b4_function_define([yydestruct],
- [static void],
- [[const char *yymsg], [yymsg]],
- [[int yytype], [yytype]],
- [[YYSTYPE *yyvaluep], [yyvaluep]][]dnl
-b4_locations_if( [, [[YYLTYPE *yylocationp], [yylocationp]]])[]dnl
-m4_ifset([b4_parse_param], [, b4_parse_param]))[
+static void
+yydestruct (const char *yymsg,
+ yysymbol_kind_t yykind, YYSTYPE *yyvaluep]b4_locations_if(dnl
+[[, YYLTYPE *yylocationp]])[]b4_user_formals[)
{
]b4_parse_param_use([yyvaluep], [yylocationp])dnl
[ if (!yymsg)
yymsg = "Deleting";
- YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp);
+ YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp);
YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
]b4_symbol_actions([destructor])[
@@ -622,25 +760,15 @@
| Print this symbol's value on YYO. |
`-----------------------------------*/
-]b4_function_define([yy_symbol_value_print],
- [static void],
- [[FILE *yyo], [yyo]],
- [[int yytype], [yytype]],
- [[YYSTYPE const * const yyvaluep], [yyvaluep]][]dnl
-b4_locations_if([, [[YYLTYPE const * const yylocationp], [yylocationp]]])[]dnl
-m4_ifset([b4_parse_param], [, b4_parse_param]))[
+static void
+yy_symbol_value_print (FILE *yyo,
+ yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep]b4_locations_if(dnl
+[[, YYLTYPE const * const yylocationp]])[]b4_user_formals[)
{
FILE *yyoutput = yyo;
]b4_parse_param_use([yyoutput], [yylocationp])dnl
[ if (!yyvaluep)
return;]
-dnl glr.c does not feature yytoknum.
-m4_if(b4_skeleton, ["yacc.c"],
-[[# ifdef YYPRINT
- if (yytype < YYNTOKENS)
- YYPRINT (yyo, yytoknum[yytype], *yyvaluep);
-# endif
-]])dnl
b4_percent_code_get([[pre-printer]])dnl
YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
b4_symbol_actions([printer])
@@ -653,21 +781,18 @@
| Print this symbol on YYO. |
`---------------------------*/
-]b4_function_define([yy_symbol_print],
- [static void],
- [[FILE *yyo], [yyo]],
- [[int yytype], [yytype]],
- [[YYSTYPE const * const yyvaluep], [yyvaluep]][]dnl
-b4_locations_if([, [[YYLTYPE const * const yylocationp], [yylocationp]]])[]dnl
-m4_ifset([b4_parse_param], [, b4_parse_param]))[
+static void
+yy_symbol_print (FILE *yyo,
+ yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep]b4_locations_if(dnl
+[[, YYLTYPE const * const yylocationp]])[]b4_user_formals[)
{
YYFPRINTF (yyo, "%s %s (",
- yytype < YYNTOKENS ? "token" : "nterm", yytname[yytype]);
+ yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind));
-]b4_locations_if([ YY_LOCATION_PRINT (yyo, *yylocationp);
+]b4_locations_if([ YYLOCATION_PRINT (yyo, yylocationp);
YYFPRINTF (yyo, ": ");
])dnl
-[ yy_symbol_value_print (yyo, yytype, yyvaluep]dnl
+[ yy_symbol_value_print (yyo, yykind, yyvaluep]dnl
b4_locations_if([, yylocationp])[]b4_user_args[);
YYFPRINTF (yyo, ")");
}]dnl
@@ -693,11 +818,11 @@
[m4_define([b4_symbol($1, type_tag)],
[b4_symbol_if([$1], [has_id],
[b4_symbol([$1], [id])],
- [yytype_[]b4_symbol([$1], [number])])])dnl
+ [yykind_[]b4_symbol([$1], [number])])])dnl
m4_append([b4_union_members],
-m4_expand([
- b4_symbol_tag_comment([$1])dnl
- b4_symbol([$1], [type]) b4_symbol([$1], [type_tag]);]))
+m4_expand([m4_format([ %-40s %s],
+ m4_expand([b4_symbol([$1], [type]) b4_symbol([$1], [type_tag]);]),
+ [b4_symbol_tag_comment([$1])])]))
])
@@ -851,7 +976,7 @@
# b4_declare_yylstype
# -------------------
-# Declarations that might either go into the header (if --defines) or
+# Declarations that might either go into the header (if --header) or
# in the parser body. Declare YYSTYPE/YYLTYPE, and yylval/yylloc.
m4_define([b4_declare_yylstype],
[b4_value_type_define[]b4_locations_if([
@@ -921,25 +1046,30 @@
#endif
]])
-# b4_yy_location_print_define
-# ---------------------------
-# Define YY_LOCATION_PRINT.
-m4_define([b4_yy_location_print_define],
+# b4_yylocation_print_define
+# --------------------------
+# Define YYLOCATION_PRINT.
+m4_define([b4_yylocation_print_define],
[b4_locations_if([[
-/* YY_LOCATION_PRINT -- Print the location on the stream.
+/* YYLOCATION_PRINT -- Print the location on the stream.
This macro was not mandated originally: define only if we know
we won't break user code: when these are the locations we know. */
-#ifndef YY_LOCATION_PRINT
-# if defined ]b4_api_PREFIX[LTYPE_IS_TRIVIAL && ]b4_api_PREFIX[LTYPE_IS_TRIVIAL
+# ifndef YYLOCATION_PRINT
+
+# if defined YY_LOCATION_PRINT
+
+ /* Temporary convenience wrapper in case some people defined the
+ undocumented and private YY_LOCATION_PRINT macros. */
+# define YYLOCATION_PRINT(File, Loc) YY_LOCATION_PRINT(File, *(Loc))
+
+# elif defined ]b4_api_PREFIX[LTYPE_IS_TRIVIAL && ]b4_api_PREFIX[LTYPE_IS_TRIVIAL
/* Print *YYLOCP on YYO. Private, do not rely on its existence. */
YY_ATTRIBUTE_UNUSED
-]b4_function_define([yy_location_print_],
- [static int],
- [[FILE *yyo], [yyo]],
- [[YYLTYPE const * const yylocp], [yylocp]])[
+static int
+yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp)
{
int res = 0;
int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0;
@@ -961,19 +1091,23 @@
res += YYFPRINTF (yyo, "-%d", end_col);
}
return res;
- }
+}
-# define YY_LOCATION_PRINT(File, Loc) \
- yy_location_print_ (File, &(Loc))
+# define YYLOCATION_PRINT yy_location_print_
-# else
-# define YY_LOCATION_PRINT(File, Loc) ((void) 0)
-# endif
-#endif]],
-[[/* This macro is provided for backward compatibility. */
-#ifndef YY_LOCATION_PRINT
-# define YY_LOCATION_PRINT(File, Loc) ((void) 0)
-#endif]])
+ /* Temporary convenience wrapper in case some people defined the
+ undocumented and private YY_LOCATION_PRINT macros. */
+# define YY_LOCATION_PRINT(File, Loc) YYLOCATION_PRINT(File, &(Loc))
+
+# else
+
+# define YYLOCATION_PRINT(File, Loc) ((void) 0)
+ /* Temporary convenience wrapper in case some people defined the
+ undocumented and private YY_LOCATION_PRINT macros. */
+# define YY_LOCATION_PRINT YYLOCATION_PRINT
+
+# endif
+# endif /* !defined YYLOCATION_PRINT */]])
])
# b4_yyloc_default
diff --git a/common/bison/skeletons/d-skel.m4 b/common/bison/skeletons/d-skel.m4
index 1705d84..2a38f02 100644
--- a/common/bison/skeletons/d-skel.m4
+++ b/common/bison/skeletons/d-skel.m4
@@ -2,7 +2,7 @@
# D skeleton dispatching for Bison.
-# Copyright (C) 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,7 +15,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
b4_glr_if( [b4_complain([%%glr-parser not supported for D])])
b4_nondeterministic_if([b4_complain([%%nondeterministic-parser not supported for D])])
diff --git a/common/bison/skeletons/d.m4 b/common/bison/skeletons/d.m4
index 2d29127..c0632e4 100644
--- a/common/bison/skeletons/d.m4
+++ b/common/bison/skeletons/d.m4
@@ -2,7 +2,7 @@
# D language support for Bison
-# Copyright (C) 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,27 +15,36 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
-# _b4_comment(TEXT, OPEN, CONTINUE, END)
-# --------------------------------------
-# Put TEXT in comment. Avoid trailing spaces: don't indent empty lines.
-# Avoid adding indentation to the first line, as the indentation comes
-# from OPEN. That's why we don't patsubst([$1], [^\(.\)], [ \1]).
-#
-# Prefix all the output lines with PREFIX.
-m4_define([_b4_comment],
-[$2[]m4_bpatsubst(m4_expand([[$1]]), [
-\(.\)], [
-$3\1])$4])
+m4_include(b4_skeletonsdir/[c-like.m4])
-# b4_comment(TEXT, [PREFIX])
-# --------------------------
-# Put TEXT in comment. Prefix all the output lines with PREFIX.
-m4_define([b4_comment],
-[_b4_comment([$1], [$2/* ], [$2 ], [ */])])
+# b4_symbol_action(SYMBOL-NUM, ACTION)
+# ------------------------------------
+# Run the action ACTION ("destructor" or "printer") for SYMBOL-NUM.
+m4_define([b4_symbol_action],
+[b4_symbol_if([$1], [has_$2],
+[b4_dollar_pushdef([yyval],
+ [$1],
+ [],
+ [yyloc])dnl
+ _b4_symbol_case([$1])[]dnl
+b4_syncline([b4_symbol([$1], [$2_line])], [b4_symbol([$1], [$2_file])])dnl
+b4_symbol([$1], [$2])
+b4_syncline([@oline@], [@ofile@])dnl
+ break;
+
+b4_dollar_popdef[]dnl
+])])
+
+
+# b4_use(EXPR)
+# ------------
+# Pacify the compiler about some maybe unused value.
+m4_define([b4_use],
+[])
# b4_sync_start(LINE, FILE)
@@ -58,6 +67,23 @@
[m4_ifval(m4_quote(b4_percent_define_get([$1])),
[$2[]b4_percent_define_get([$1])[]$3], [$4])])
+# b4_percent_define_if_get2(ARG1, ARG2, DEF, NOT)
+# -----------------------------------------------
+# Expand to the value of DEF if ARG1 or ARG2 are %define'ed,
+# otherwise NOT.
+m4_define([b4_percent_define_if_get2],
+ [m4_ifval(m4_quote(b4_percent_define_get([$1])),
+ [$3], [m4_ifval(m4_quote(b4_percent_define_get([$2])),
+ [$3], [$4])])])
+
+# b4_percent_define_class_before_interface(CLASS, INTERFACE)
+# ----------------------------------------------------------
+# Expand to a ', ' if both a class and an interface have been %define'ed
+m4_define([b4_percent_define_class_before_interface],
+ [m4_ifval(m4_quote(b4_percent_define_get([$1])),
+ [m4_ifval(m4_quote(b4_percent_define_get([$2])),
+ [, ])])])
+
# b4_flag_value(BOOLEAN-FLAG)
# ---------------------------
@@ -78,8 +104,10 @@
b4_percent_define_flag_if([api.parser.abstract], [abstract ])dnl
b4_percent_define_flag_if([api.parser.final], [final ])dnl
[class ]b4_parser_class[]dnl
-b4_percent_define_get3([api.parser.extends], [ extends ])dnl
-b4_percent_define_get3([api.parser.implements], [ implements ])])
+b4_percent_define_if_get2([api.parser.extends], [api.parser.implements], [ : ])dnl
+b4_percent_define_get([api.parser.extends])dnl
+b4_percent_define_class_before_interface([api.parser.extends], [api.parser.implements])dnl
+b4_percent_define_get([api.parser.implements])])
# b4_lexer_if(TRUE, FALSE)
@@ -103,12 +131,12 @@
# b4_identification
# -----------------
m4_define([b4_identification],
-[/** Version number for the Bison executable that generated this parser. */
- public static immutable string yy_bison_version = "b4_version";
+[[/** Version number for the Bison executable that generated this parser. */
+ public static immutable string yy_bison_version = "]b4_version_string[";
/** Name of the skeleton that generated this parser. */
- public static immutable string yy_bison_skeleton = b4_skeleton;
-])
+ public static immutable string yy_bison_skeleton = ]b4_skeleton[;
+]])
## ------------ ##
@@ -148,9 +176,12 @@
])
-## ------------------------- ##
-## Assigning token numbers. ##
-## ------------------------- ##
+## ------------- ##
+## Token kinds. ##
+## ------------- ##
+
+m4_define([b4_symbol(-2, id)], [[YYEMPTY]])
+b4_percent_define_default([[api.token.raw]], [[true]])
# b4_token_enum(TOKEN-NAME, TOKEN-NUMBER)
# ---------------------------------------
@@ -159,22 +190,121 @@
[b4_token_format([ %s = %s,
], [$1])])
-# b4_token_enums(LIST-OF-PAIRS-TOKEN-NAME-TOKEN-NUMBER)
-# -----------------------------------------------------
+# b4_token_enums
+# --------------
# Output the definition of the tokens as enums.
m4_define([b4_token_enums],
-[/* Tokens. */
-public enum YYTokenType {
-
- /** Token returned by the scanner to signal the end of its input. */
- EOF = 0,
-b4_symbol_foreach([b4_token_enum])
+[/* Token kinds. */
+public enum TokenKind {
+ ]b4_symbol(empty, id)[ = -2,
+b4_symbol_foreach([b4_token_enum])dnl
}
])
-# b4-case(ID, CODE)
-# -----------------
-m4_define([b4_case], [ case $1:
+# b4_symbol_translate(STRING)
+# ---------------------------
+# Used by "bison" in the array of symbol names to mark those that
+# require translation.
+m4_define([b4_symbol_translate],
+[[_($1)]])
+
+
+# _b4_token_constructor_define(SYMBOL-NUM)
+# ----------------------------------------
+# Define Symbol.FOO for SYMBOL-NUM.
+m4_define([_b4_token_constructor_define],
+[b4_token_visible_if([$1],
+[[
+ static auto ]b4_symbol([$1], [id])[(]b4_symbol_if([$1], [has_type],
+[b4_union_if([b4_symbol([$1], [type]],
+[[typeof(YYSemanticType.]b4_symbol([$1], [type])[]])) [val]])dnl
+[]b4_locations_if([b4_symbol_if([$1], [has_type], [[, ]])[Location l]])[)
+ {
+ return Symbol(TokenKind.]b4_symbol([$1], [id])[]b4_symbol_if([$1], [has_type],
+ [[, val]])[]b4_locations_if([[, l]])[);
+ }]])])
+
+# b4_token_constructor_define
+# ---------------------------
+# Define Symbol.FOO for each token kind FOO.
+m4_define([b4_token_constructor_define],
+[[
+ /* Implementation of token constructors for each symbol type visible to
+ * the user. The code generates static methods that have the same names
+ * as the TokenKinds.
+ */]b4_symbol_foreach([_b4_token_constructor_define])dnl
+])
+
+## -------------- ##
+## Symbol kinds. ##
+## -------------- ##
+
+# b4_symbol_kind(NUM)
+# -------------------
+m4_define([b4_symbol_kind],
+[SymbolKind.b4_symbol_kind_base($@)])
+
+
+# b4_symbol_enum(SYMBOL-NUM)
+# --------------------------
+# Output the definition of this symbol as an enum.
+m4_define([b4_symbol_enum],
+[m4_format([ %-30s %s],
+ m4_format([[%s = %s,]],
+ b4_symbol([$1], [kind_base]),
+ [$1]),
+ [b4_symbol_tag_comment([$1])])])
+
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+# Defining YYEMPTY here is important: it forces the compiler
+# to use a signed type, which matters for yytoken.
+m4_define([b4_declare_symbol_enum],
+[[ /* Symbol kinds. */
+ struct SymbolKind
+ {
+ enum
+ {
+ ]b4_symbol(empty, kind_base)[ = -2, /* No symbol. */
+]b4_symbol_foreach([b4_symbol_enum])dnl
+[ }
+
+ private int yycode_;
+ alias yycode_ this;
+
+ this(int code)
+ {
+ yycode_ = code;
+ }
+
+ /* Return YYSTR after stripping away unnecessary quotes and
+ backslashes, so that it's suitable for yyerror. The heuristic is
+ that double-quoting is unnecessary unless the string contains an
+ apostrophe, a comma, or backslash (other than backslash-backslash).
+ YYSTR is taken from yytname. */
+ final void toString(W)(W sink) const
+ if (isOutputRange!(W, char))
+ {
+ immutable string[] yy_sname = @{
+ ]b4_symbol_names[
+ @};]b4_has_translations_if([[
+ /* YYTRANSLATABLE[SYMBOL-NUM] -- Whether YY_SNAME[SYMBOL-NUM] is
+ internationalizable. */
+ immutable ]b4_int_type_for([b4_translatable])[[] yytranslatable = @{
+ ]b4_translatable[
+ @};]])[
+
+ put(sink, yy_sname[yycode_]);
+ }
+ }
+]])
+
+
+# b4_case(ID, CODE, [COMMENTS])
+# -----------------------------
+m4_define([b4_case], [ case $1:m4_ifval([$3], [ b4_comment([$3])])
$2
break;])
@@ -199,6 +329,109 @@
m4_define([b4_position_type], b4_percent_define_ifdef([[position_type]],[b4_percent_define_get([[position_type]])],[YYPosition]))
+## ---------------- ##
+## api.value.type. ##
+## ---------------- ##
+
+
+# ---------------------- #
+# api.value.type=union. #
+# ---------------------- #
+
+# b4_symbol_type_register(SYMBOL-NUM)
+# -----------------------------------
+# Symbol SYMBOL-NUM has a type (for union) instead of a type-tag.
+# Extend the definition of %union's body (b4_union_members) with a
+# field of that type, and extend the symbol's "type" field to point to
+# the field name, instead of the type name.
+m4_define([b4_symbol_type_register],
+[m4_define([b4_symbol($1, type_tag)],
+ [b4_symbol_if([$1], [has_id],
+ [b4_symbol([$1], [id])],
+ [yykind_[]b4_symbol([$1], [number])])])dnl
+m4_append([b4_union_members],
+m4_expand([m4_format([ %-40s %s],
+ m4_expand([b4_symbol([$1], [type]) b4_symbol([$1], [type_tag]);]),
+ [b4_symbol_tag_comment([$1])])]))
+])
+
+
+# b4_type_define_tag(SYMBOL1-NUM, ...)
+# ------------------------------------
+# For the batch of symbols SYMBOL1-NUM... (which all have the same
+# type), enhance the %union definition for each of them, and set
+# there "type" field to the field tag name, instead of the type name.
+m4_define([b4_type_define_tag],
+[b4_symbol_if([$1], [has_type],
+ [m4_map([b4_symbol_type_register], [$@])])
+])
+
+
+# b4_symbol_value_union(VAL, SYMBOL-NUM, [TYPE])
+# ----------------------------------------------
+# Same of b4_symbol_value, but when api.value.type=union.
+m4_define([b4_symbol_value_union],
+[m4_ifval([$3],
+ [(*($3*)(&$1))],
+ [m4_ifval([$2],
+ [b4_symbol_if([$2], [has_type],
+ [($1.b4_symbol([$2], [type_tag]))],
+ [$1])],
+ [$1])])])
+
+
+# b4_value_type_setup_union
+# -------------------------
+# Setup support for api.value.type=union. Symbols are defined with a
+# type instead of a union member name: build the corresponding union,
+# and give the symbols their tag.
+m4_define([b4_value_type_setup_union],
+[m4_define([b4_union_members])
+b4_type_foreach([b4_type_define_tag])
+m4_copy_force([b4_symbol_value_union], [b4_symbol_value])
+])
+
+
+# _b4_value_type_setup_keyword
+# ----------------------------
+# api.value.type is defined with a keyword/string syntax. Check if
+# that is properly defined, and prepare its use.
+m4_define([_b4_value_type_setup_keyword],
+[b4_percent_define_check_values([[[[api.value.type]],
+ [[none]],
+ [[union]],
+ [[union-directive]],
+ [[yystype]]]])dnl
+m4_case(b4_percent_define_get([[api.value.type]]),
+ [union], [b4_value_type_setup_union])])
+
+
+# b4_value_type_setup
+# -------------------
+# Check if api.value.type is properly defined, and possibly prepare
+# its use.
+b4_define_silent([b4_value_type_setup],
+[
+# Define default value.
+b4_percent_define_ifdef([[api.value.type]], [],
+[# %union => api.value.type=union-directive
+m4_ifdef([b4_union_members],
+[m4_define([b4_percent_define_kind(api.value.type)], [keyword])
+m4_define([b4_percent_define(api.value.type)], [union-directive])],
+[# no tag seen => api.value.type={int}
+m4_if(b4_tag_seen_flag, 0,
+[m4_define([b4_percent_define_kind(api.value.type)], [code])
+m4_define([b4_percent_define(api.value.type)], [int])],
+[# otherwise api.value.type=yystype
+m4_define([b4_percent_define_kind(api.value.type)], [keyword])
+m4_define([b4_percent_define(api.value.type)], [yystype])])])])
+
+# Set up.
+m4_bmatch(b4_percent_define_get_kind([[api.value.type]]),
+ [keyword], [_b4_value_type_setup_keyword])
+])
+
+
## ----------------- ##
## Semantic Values. ##
## ----------------- ##
@@ -245,7 +478,7 @@
# Expansion of @POS, where the current rule has RULE-LENGTH symbols
# on RHS.
m4_define([b4_rhs_location],
-[yystack.locationAt ([$1], [$2])])
+[yystack.locationAt (b4_subtract($@))])
# b4_lex_param
@@ -330,3 +563,66 @@
], [$@])])
m4_define([b4_var_decl],
[ protected $1;])
+
+
+# b4_public_types_declare
+# -----------------------
+# Define the public types: token, semantic value, location, and so forth.
+# Depending on %define token_lex, may be output in the header or source file.
+m4_define([b4_public_types_declare],
+[[
+alias Symbol = ]b4_parser_class[.Symbol;
+alias Value = ]b4_yystype[;]b4_locations_if([[
+alias Location = ]b4_location_type[;
+alias Position = ]b4_position_type[;]b4_push_if([[
+alias PUSH_MORE = ]b4_parser_class[.YYPUSH_MORE;
+alias ABORT = ]b4_parser_class[.YYABORT;
+alias ACCEPT = ]b4_parser_class[.YYACCEPT;]])[]])[
+]])
+
+
+# b4_basic_symbol_constructor_define
+# ----------------------------------
+# Create Symbol struct constructors for all the visible types.
+m4_define([b4_basic_symbol_constructor_define],
+[b4_token_visible_if([$1],
+[[ this(TokenKind token]b4_symbol_if([$1], [has_type],
+[[, ]b4_union_if([], [[typeof(YYSemanticType.]])b4_symbol([$1], [type])dnl
+[]b4_union_if([], [[) ]])[ val]])[]b4_locations_if([[, Location loc]])[)
+ {
+ kind = yytranslate_(token);]b4_union_if([b4_symbol_if([$1], [has_type], [[
+ static foreach (member; __traits(allMembers, YYSemanticType))
+ {
+ static if (is(typeof(mixin("value_." ~ member)) == ]b4_symbol([$1], [type])[))
+ {
+ mixin("value_." ~ member ~ " = val;");
+ }
+ }]])], [b4_symbol_if([$1], [has_type], [[
+ value_.]b4_symbol([$1], [type])[ = val;]])])[]b4_locations_if([
+ location_ = loc;])[
+ }
+]])])
+
+
+# b4_symbol_type_define
+# ---------------------
+# Define symbol_type, the external type for symbols used for symbol
+# constructors.
+m4_define([b4_symbol_type_define],
+[[
+ /**
+ * A complete symbol
+ */
+ struct Symbol
+ {
+ private SymbolKind kind;
+ private Value value_;]b4_locations_if([[
+ private Location location_;]])[
+
+]b4_type_foreach([b4_basic_symbol_constructor_define])[
+ SymbolKind token() { return kind; }
+ Value value() { return value_; }]b4_locations_if([[
+ Location location() { return location_; }]])[
+]b4_token_ctor_if([b4_token_constructor_define])[
+ }
+]])
diff --git a/common/bison/skeletons/glr.c b/common/bison/skeletons/glr.c
index 945ff0a..fab3733 100644
--- a/common/bison/skeletons/glr.c
+++ b/common/bison/skeletons/glr.c
@@ -1,8 +1,8 @@
- -*- C -*-
+# -*- C -*-
# GLR skeleton for Bison
-# Copyright (C) 2002-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,7 +15,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
# If we are loaded by glr.cc, do not override c++.m4 definitions by
@@ -32,7 +32,9 @@
m4_define_default([b4_stack_depth_max], [10000])
m4_define_default([b4_stack_depth_init], [200])
-
+# Included header.
+b4_percent_define_default([[api.header.include]],
+ [["@basename(]b4_spec_header_file[@)"]])
## ------------------------ ##
## Pure/impure interfaces. ##
@@ -46,16 +48,6 @@
m4_define([b4_pure_flag],
[b4_percent_define_flag_if([[api.pure]], [[1]], [[0]])])])
-# b4_user_formals
-# ---------------
-# The possible parse-params formal arguments preceded by a comma.
-#
-# This is not shared with yacc.c in c.m4 because GLR relies on ISO C
-# formal argument declarations.
-m4_define([b4_user_formals],
-[m4_ifset([b4_parse_param], [, b4_formals(b4_parse_param)])])
-
-
# b4_yyerror_args
# ---------------
# Optional effective arguments passed to yyerror: user args plus yylloc, and
@@ -132,7 +124,7 @@
# --------------------------------------------------
# Expansion of $$ or $<TYPE>$, for symbol SYMBOL-NUM.
m4_define([b4_rhs_value],
-[b4_symbol_value([b4_rhs_data([$1], [$2]).yysemantics.yysval], [$3], [$4])])
+[b4_symbol_value([b4_rhs_data([$1], [$2]).yysemantics.yyval], [$3], [$4])])
@@ -155,13 +147,22 @@
[(b4_rhs_data([$1], [$2]).yyloc)])
+# b4_call_merger(MERGER-NUM, MERGER-NAME, SYMBOL-SUM)
+# ---------------------------------------------------
+m4_define([b4_call_merger],
+[b4_case([$1],
+ [ b4_symbol_if([$3], [has_type],
+ [yy0->b4_symbol($3, slot) = $2 (*yy0, *yy1);],
+ [*yy0 = $2 (*yy0, *yy1);])])])
+
+
## -------------- ##
## Declarations. ##
## -------------- ##
# b4_shared_declarations
# ----------------------
-# Declaration that might either go into the header (if --defines)
+# Declaration that might either go into the header (if --header)
# or open coded in the parser body. glr.cc has its own definition.
m4_if(b4_skeleton, ["glr.c"],
[m4_define([b4_shared_declarations],
@@ -169,7 +170,7 @@
]b4_percent_code_get([[requires]])[
]b4_token_enums[
]b4_declare_yylstype[
-]b4_function_declare(b4_prefix[parse], [int], b4_parse_param)[
+int ]b4_prefix[parse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[);
]b4_percent_code_get([[provides]])[]dnl
])
])
@@ -187,7 +188,7 @@
# members of this union.
#
# To avoid this issue, just generate the header before the
-# implementation file. But we should also make them more independant.
+# implementation file. But we should also make them more independent.
# ----------------- #
# The header file. #
@@ -195,13 +196,13 @@
# glr.cc produces its own header.
b4_glr_cc_if([],
-[b4_defines_if(
+[b4_header_if(
[b4_output_begin([b4_spec_header_file])
b4_copyright([Skeleton interface for Bison GLR parsers in C],
- [2002-2015, 2018-2019])[
-]b4_cpp_guard_open([b4_spec_header_file])[
+ [2002-2015, 2018-2021])[
+]b4_cpp_guard_open([b4_spec_mapped_header_file])[
]b4_shared_declarations[
-]b4_cpp_guard_close([b4_spec_header_file])[
+]b4_cpp_guard_close([b4_spec_mapped_header_file])[
]b4_output_end[
]])])
@@ -212,7 +213,7 @@
b4_output_begin([b4_parser_file_name])
b4_copyright([Skeleton implementation for Bison GLR parsers in C],
- [2002-2015, 2018-2019])[
+ [2002-2015, 2018-2021])[
/* C GLR parser skeleton written by Paul Hilfinger. */
]b4_disclaimer[
@@ -225,7 +226,7 @@
#define YYLTYPE ]b4_api_PREFIX[LTYPE]])])[
]m4_if(b4_prefix, [yy], [],
[[/* Substitute the variable and function names. */
-#define yyparse ]b4_prefix[parse
+#define ]b4_glr_cc_if([yy_parse_impl], [yyparse])[ ]b4_prefix[]b4_glr_cc_if([_parse_impl], [parse])[
#define yylex ]b4_prefix[lex
#define yyerror ]b4_prefix[error
#define yydebug ]b4_prefix[debug]]b4_pure_if([], [[
@@ -238,16 +239,11 @@
]b4_cast_define[
]b4_null_define[
-]b4_defines_if([[#include "@basename(]b4_spec_header_file[@)"]],
- [b4_shared_declarations])[
+]b4_header_if([[#include ]b4_percent_define_get([[api.header.include]])],
+ [b4_shared_declarations])[
-/* Enabling verbose error messages. */
-#ifdef YYERROR_VERBOSE
-# undef YYERROR_VERBOSE
-# define YYERROR_VERBOSE 1
-#else
-# define YYERROR_VERBOSE ]b4_error_verbose_if([1], [0])[
-#endif
+]b4_glr_cc_if([b4_glr_cc_setup],
+ [b4_declare_symbol_enum])[
/* Default (constant) value used for initialization for null
right-hand sides. Unlike the standard yacc.c template, here we set
@@ -266,6 +262,7 @@
#include <string.h>
]b4_c99_int_type_define[
+]b4_sizes_types_define[
#ifndef YY_
# if defined YYENABLE_NLS && YYENABLE_NLS
@@ -278,6 +275,11 @@
# define YY_(Msgid) Msgid
# endif
#endif
+]b4_has_translations_if([
+#ifndef N_
+# define N_(Msgid) Msgid
+#endif
+])[
#ifndef YYFREE
# define YYFREE free
@@ -289,9 +291,6 @@
# define YYREALLOC realloc
#endif
-#define YYSIZEMAX \
- (PTRDIFF_MAX < SIZE_MAX ? PTRDIFF_MAX : YY_CAST (ptrdiff_t, SIZE_MAX))
-
#ifdef __cplusplus
typedef bool yybool;
# define yytrue true
@@ -345,21 +344,17 @@
accessed by $0, $-1, etc., in any rule. */
#define YYMAXLEFT ]b4_max_left_semantic_context[
-/* YYMAXUTOK -- Last valid token number (for yychar). */
-#define YYMAXUTOK ]b4_user_token_number_max[]b4_glr_cc_if([[
-/* YYFAULTYTOK -- Token number (for yychar) that denotes a
- syntax_error thrown from the scanner. */
-#define YYFAULTYTOK (YYMAXUTOK + 1)]])[
-/* YYUNDEFTOK -- Symbol number (for yytoken) that denotes an unknown
- token. */
-#define YYUNDEFTOK ]b4_undef_token_number[
+/* YYMAXUTOK -- Last valid token kind. */
+#define YYMAXUTOK ]b4_code_max[
/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
as returned by yylex, with out-of-bounds checking. */
]b4_api_token_raw_if(dnl
-[[#define YYTRANSLATE(YYX) (YYX)]],
-[[#define YYTRANSLATE(YYX) \
- (0 <= (YYX) && (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
+[[#define YYTRANSLATE(YYX) YY_CAST (yysymbol_kind_t, YYX)]],
+[[#define YYTRANSLATE(YYX) \
+ (0 <= (YYX) && (YYX) <= YYMAXUTOK \
+ ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \
+ : ]b4_symbol_prefix[YYUNDEF)
/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM
as returned by yylex. */
@@ -376,15 +371,6 @@
};
#endif
-#if ]b4_api_PREFIX[DEBUG || YYERROR_VERBOSE || ]b4_token_table_flag[
-/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
- First, the terminals, then, starting at YYNTOKENS, nonterminals. */
-static const char *const yytname[] =
-{
- ]b4_tname[
-};
-#endif
-
#define YYPACT_NINF (]b4_pact_ninf[)
#define YYTABLE_NINF (]b4_table_ninf[)
@@ -428,9 +414,6 @@
]b4_conflicting_rules[
};
-/* Error token number */
-#define YYTERROR 1
-
]b4_locations_if([[
]b4_yylloc_default_define[
# define YYRHSLOC(Rhs, K) ((Rhs)[K].yystate.yyloc)
@@ -457,10 +440,9 @@
int yynerrs;
int yychar;])[
-static const int YYEOF = 0;
-static const int YYEMPTY = -2;
+enum { YYENOMEM = -2 };
-typedef enum { yyok, yyaccept, yyabort, yyerr } YYRESULTTAG;
+typedef enum { yyok, yyaccept, yyabort, yyerr, yynomem } YYRESULTTAG;
#define YYCHK(YYE) \
do { \
@@ -469,62 +451,6 @@
return yychk_flag; \
} while (0)
-#if ]b4_api_PREFIX[DEBUG
-
-# ifndef YYFPRINTF
-# define YYFPRINTF fprintf
-# endif
-
-# define YY_FPRINTF \
- YY_IGNORE_USELESS_CAST_BEGIN YY_FPRINTF_
-
-# define YY_FPRINTF_(Args) \
- do { \
- YYFPRINTF Args; \
- YY_IGNORE_USELESS_CAST_END \
- } while (0)
-
-# define YY_DPRINTF \
- YY_IGNORE_USELESS_CAST_BEGIN YY_DPRINTF_
-
-# define YY_DPRINTF_(Args) \
- do { \
- if (yydebug) \
- YYFPRINTF Args; \
- YY_IGNORE_USELESS_CAST_END \
- } while (0)
-
-]b4_yy_location_print_define[
-
-]b4_yy_symbol_print_define[
-
-# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \
- do { \
- if (yydebug) \
- { \
- YY_FPRINTF ((stderr, "%s ", Title)); \
- yy_symbol_print (stderr, Type, Value]b4_locuser_args([Location])[); \
- YY_FPRINTF ((stderr, "\n")); \
- } \
- } while (0)
-
-/* Nonzero means print parse trace. It is left uninitialized so that
- multiple parsers can coexist. */
-int yydebug;
-
-struct yyGLRStack;
-static void yypstack (struct yyGLRStack* yystackp, ptrdiff_t yyk)
- YY_ATTRIBUTE_UNUSED;
-static void yypdumpstack (struct yyGLRStack* yystackp)
- YY_ATTRIBUTE_UNUSED;
-
-#else /* !]b4_api_PREFIX[DEBUG */
-
-# define YY_DPRINTF(Args) do {} while (yyfalse)
-# define YY_SYMBOL_PRINT(Title, Type, Value, Location)
-
-#endif /* !]b4_api_PREFIX[DEBUG */
-
/* YYINITDEPTH -- initial size of the parser's stacks. */
#ifndef YYINITDEPTH
# define YYINITDEPTH ]b4_stack_depth_init[
@@ -566,91 +492,12 @@
} while (0)
#endif
-
-#if YYERROR_VERBOSE
-
-# ifndef yystpcpy
-# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE
-# define yystpcpy stpcpy
-# else
-/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in
- YYDEST. */
-static char *
-yystpcpy (char *yydest, const char *yysrc)
-{
- char *yyd = yydest;
- const char *yys = yysrc;
-
- while ((*yyd++ = *yys++) != '\0')
- continue;
-
- return yyd - 1;
-}
-# endif
-# endif
-
-# ifndef yytnamerr
-/* Copy to YYRES the contents of YYSTR after stripping away unnecessary
- quotes and backslashes, so that it's suitable for yyerror. The
- heuristic is that double-quoting is unnecessary unless the string
- contains an apostrophe, a comma, or backslash (other than
- backslash-backslash). YYSTR is taken from yytname. If YYRES is
- null, do not copy; instead, return the length of what the result
- would have been. */
-static ptrdiff_t
-yytnamerr (char *yyres, const char *yystr)
-{
- if (*yystr == '"')
- {
- ptrdiff_t yyn = 0;
- char const *yyp = yystr;
-
- for (;;)
- switch (*++yyp)
- {
- case '\'':
- case ',':
- goto do_not_strip_quotes;
-
- case '\\':
- if (*++yyp != '\\')
- goto do_not_strip_quotes;
- else
- goto append;
-
- append:
- default:
- if (yyres)
- yyres[yyn] = *yyp;
- yyn++;
- break;
-
- case '"':
- if (yyres)
- yyres[yyn] = '\0';
- return yyn;
- }
- do_not_strip_quotes: ;
- }
-
- if (yyres)
- return yystpcpy (yyres, yystr) - yyres;
- else
- return YY_CAST (ptrdiff_t, strlen (yystr));
-}
-# endif
-
-#endif /* !YYERROR_VERBOSE */
-
/** State numbers. */
-typedef int yyStateNum;
+typedef int yy_state_t;
/** Rule numbers. */
typedef int yyRuleNum;
-/** Grammar symbol. */
-typedef int yySymbol;
-
/** Item references. */
typedef short yyItemNum;
@@ -660,42 +507,45 @@
typedef union yyGLRStackItem yyGLRStackItem;
typedef struct yyGLRStack yyGLRStack;
-struct yyGLRState {
+struct yyGLRState
+{
/** Type tag: always true. */
yybool yyisState;
- /** Type tag for yysemantics. If true, yysval applies, otherwise
+ /** Type tag for yysemantics. If true, yyval applies, otherwise
* yyfirstVal applies. */
yybool yyresolved;
/** Number of corresponding LALR(1) machine state. */
- yyStateNum yylrState;
+ yy_state_t yylrState;
/** Preceding state in this stack */
yyGLRState* yypred;
/** Source position of the last token produced by my symbol */
- ptrdiff_t yyposn;
+ YYPTRDIFF_T yyposn;
union {
/** First in a chain of alternative reductions producing the
* nonterminal corresponding to this state, threaded through
* yynext. */
yySemanticOption* yyfirstVal;
/** Semantic value for this state. */
- YYSTYPE yysval;
+ YYSTYPE yyval;
} yysemantics;]b4_locations_if([[
/** Source location for this state. */
YYLTYPE yyloc;]])[
};
-struct yyGLRStateSet {
+struct yyGLRStateSet
+{
yyGLRState** yystates;
/** During nondeterministic operation, yylookaheadNeeds tracks which
* stacks have actually needed the current lookahead. During deterministic
* operation, yylookaheadNeeds[0] is not maintained since it would merely
- * duplicate yychar != YYEMPTY. */
+ * duplicate yychar != ]b4_symbol(empty, id)[. */
yybool* yylookaheadNeeds;
- ptrdiff_t yysize;
- ptrdiff_t yycapacity;
+ YYPTRDIFF_T yysize;
+ YYPTRDIFF_T yycapacity;
};
-struct yySemanticOption {
+struct yySemanticOption
+{
/** Type tag: always false. */
yybool yyisState;
/** Rule number for this reduction */
@@ -732,7 +582,7 @@
YYJMP_BUF yyexception_buffer;
yyGLRStackItem* yyitems;
yyGLRStackItem* yynextFree;
- ptrdiff_t yyspaceLeft;
+ YYPTRDIFF_T yyspaceLeft;
yyGLRState* yysplitPoint;
yyGLRState* yylastDeleted;
yyGLRStateSet yytops;
@@ -756,14 +606,206 @@
YYLONGJMP (yystackp->yyexception_buffer, 2);
}
-#if ]b4_api_PREFIX[DEBUG || YYERROR_VERBOSE
-/** A printable representation of TOKEN. */
-static inline const char*
-yytokenName (yySymbol yytoken)
+/** Accessing symbol of state YYSTATE. */
+static inline yysymbol_kind_t
+yy_accessing_symbol (yy_state_t yystate)
{
- return yytoken == YYEMPTY ? "" : yytname[yytoken];
+ return YY_CAST (yysymbol_kind_t, yystos[yystate]);
+}
+
+#if ]b4_parse_error_case([simple], [b4_api_PREFIX[DEBUG || ]b4_token_table_flag], [[1]])[
+/* The user-facing name of the symbol whose (internal) number is
+ YYSYMBOL. No bounds checking. */
+static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED;
+
+]b4_parse_error_bmatch([simple\|verbose],
+[[/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+ First, the terminals, then, starting at YYNTOKENS, nonterminals. */
+static const char *const yytname[] =
+{
+ ]b4_tname[
+};
+
+static const char *
+yysymbol_name (yysymbol_kind_t yysymbol)
+{
+ return yytname[yysymbol];
+}]],
+[[static const char *
+yysymbol_name (yysymbol_kind_t yysymbol)
+{
+ static const char *const yy_sname[] =
+ {
+ ]b4_symbol_names[
+ };]b4_has_translations_if([[
+ /* YYTRANSLATABLE[SYMBOL-NUM] -- Whether YY_SNAME[SYMBOL-NUM] is
+ internationalizable. */
+ static ]b4_int_type_for([b4_translatable])[ yytranslatable[] =
+ {
+ ]b4_translatable[
+ };
+ return (yysymbol < YYNTOKENS && yytranslatable[yysymbol]
+ ? _(yy_sname[yysymbol])
+ : yy_sname[yysymbol]);]], [[
+ return yy_sname[yysymbol];]])[
+}]])[
+#endif
+
+/** Left-hand-side symbol for rule #YYRULE. */
+static inline yysymbol_kind_t
+yylhsNonterm (yyRuleNum yyrule)
+{
+ return YY_CAST (yysymbol_kind_t, yyr1[yyrule]);
+}
+
+#if ]b4_api_PREFIX[DEBUG
+
+# ifndef YYFPRINTF
+# define YYFPRINTF fprintf
+# endif
+
+# define YY_FPRINTF \
+ YY_IGNORE_USELESS_CAST_BEGIN YY_FPRINTF_
+
+# define YY_FPRINTF_(Args) \
+ do { \
+ YYFPRINTF Args; \
+ YY_IGNORE_USELESS_CAST_END \
+ } while (0)
+
+# define YY_DPRINTF \
+ YY_IGNORE_USELESS_CAST_BEGIN YY_DPRINTF_
+
+# define YY_DPRINTF_(Args) \
+ do { \
+ if (yydebug) \
+ YYFPRINTF Args; \
+ YY_IGNORE_USELESS_CAST_END \
+ } while (0)
+
+]b4_yylocation_print_define[
+
+]b4_yy_symbol_print_define[
+
+# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) \
+ do { \
+ if (yydebug) \
+ { \
+ YY_FPRINTF ((stderr, "%s ", Title)); \
+ yy_symbol_print (stderr, Kind, Value]b4_locuser_args([Location])[); \
+ YY_FPRINTF ((stderr, "\n")); \
+ } \
+ } while (0)
+
+static inline void
+yy_reduce_print (yybool yynormal, yyGLRStackItem* yyvsp, YYPTRDIFF_T yyk,
+ yyRuleNum yyrule]b4_user_formals[);
+
+# define YY_REDUCE_PRINT(Args) \
+ do { \
+ if (yydebug) \
+ yy_reduce_print Args; \
+ } while (0)
+
+/* Nonzero means print parse trace. It is left uninitialized so that
+ multiple parsers can coexist. */
+int yydebug;
+
+static void yypstack (yyGLRStack* yystackp, YYPTRDIFF_T yyk)
+ YY_ATTRIBUTE_UNUSED;
+static void yypdumpstack (yyGLRStack* yystackp)
+ YY_ATTRIBUTE_UNUSED;
+
+#else /* !]b4_api_PREFIX[DEBUG */
+
+# define YY_DPRINTF(Args) do {} while (yyfalse)
+# define YY_SYMBOL_PRINT(Title, Kind, Value, Location)
+# define YY_REDUCE_PRINT(Args)
+
+#endif /* !]b4_api_PREFIX[DEBUG */
+
+]b4_parse_error_case(
+ [simple],
+[[]],
+[[#ifndef yystrlen
+# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S)))
+#endif
+
+]b4_parse_error_bmatch(
+ [detailed\|verbose],
+[[#ifndef yystpcpy
+# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE
+# define yystpcpy stpcpy
+# else
+/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in
+ YYDEST. */
+static char *
+yystpcpy (char *yydest, const char *yysrc)
+{
+ char *yyd = yydest;
+ const char *yys = yysrc;
+
+ while ((*yyd++ = *yys++) != '\0')
+ continue;
+
+ return yyd - 1;
+}
+# endif
+#endif]])[
+
+]b4_parse_error_case(
+ [verbose],
+[[#ifndef yytnamerr
+/* Copy to YYRES the contents of YYSTR after stripping away unnecessary
+ quotes and backslashes, so that it's suitable for yyerror. The
+ heuristic is that double-quoting is unnecessary unless the string
+ contains an apostrophe, a comma, or backslash (other than
+ backslash-backslash). YYSTR is taken from yytname. If YYRES is
+ null, do not copy; instead, return the length of what the result
+ would have been. */
+static YYPTRDIFF_T
+yytnamerr (char *yyres, const char *yystr)
+{
+ if (*yystr == '"')
+ {
+ YYPTRDIFF_T yyn = 0;
+ char const *yyp = yystr;
+
+ for (;;)
+ switch (*++yyp)
+ {
+ case '\'':
+ case ',':
+ goto do_not_strip_quotes;
+
+ case '\\':
+ if (*++yyp != '\\')
+ goto do_not_strip_quotes;
+ else
+ goto append;
+
+ append:
+ default:
+ if (yyres)
+ yyres[yyn] = *yyp;
+ yyn++;
+ break;
+
+ case '"':
+ if (yyres)
+ yyres[yyn] = '\0';
+ return yyn;
+ }
+ do_not_strip_quotes: ;
+ }
+
+ if (yyres)
+ return yystpcpy (yyres, yystr) - yyres;
+ else
+ return yystrlen (yystr);
}
#endif
+]])])[
/** Fill in YYVSP[YYLOW1 .. YYLOW0-1] from the chain of states starting
* at YYVSP[YYLOW0].yystate.yypred. Leaves YYVSP[YYLOW1].yystate.yypred
@@ -781,9 +823,9 @@
#endif
yyvsp[i].yystate.yyresolved = s->yyresolved;
if (s->yyresolved)
- yyvsp[i].yystate.yysemantics.yysval = s->yysemantics.yysval;
+ yyvsp[i].yystate.yysemantics.yyval = s->yysemantics.yyval;
else
- /* The effect of using yysval or yyloc (in an immediate rule) is
+ /* The effect of using yyval or yyloc (in an immediate rule) is
* undefined. */
yyvsp[i].yystate.yysemantics.yyfirstVal = YY_NULLPTR;]b4_locations_if([[
yyvsp[i].yystate.yyloc = s->yyloc;]])[
@@ -794,19 +836,19 @@
]m4_define([b4_yygetToken_call],
[[yygetToken (&yychar][]b4_pure_if([, yystackp])[]b4_user_args[)]])[
/** If yychar is empty, fetch the next token. */
-static inline yySymbol
+static inline yysymbol_kind_t
yygetToken (int *yycharp][]b4_pure_if([, yyGLRStack* yystackp])[]b4_user_formals[)
{
- yySymbol yytoken;
+ yysymbol_kind_t yytoken;
]b4_parse_param_use()dnl
-[ if (*yycharp == YYEMPTY)
+[ if (*yycharp == ]b4_symbol(empty, id)[)
{
- YY_DPRINTF ((stderr, "Reading a token: "));]b4_glr_cc_if([[
+ YY_DPRINTF ((stderr, "Reading a token\n"));]b4_glr_cc_if([[
#if YY_EXCEPTIONS
try
{
#endif // YY_EXCEPTIONS
- *yycharp = ]b4_lex[;
+ *yycharp = ]b4_yylex[;
#if YY_EXCEPTIONS
}
catch (const ]b4_namespace_ref[::]b4_parser_class[::syntax_error& yyexc)
@@ -814,17 +856,18 @@
YY_DPRINTF ((stderr, "Caught exception: %s\n", yyexc.what()));]b4_locations_if([
yylloc = yyexc.location;])[
yyerror (]b4_lyyerror_args[yyexc.what ());
- // Map errors caught in the scanner to the undefined token
- // (YYUNDEFTOK), so that error handling is started.
- // However, record this with this special value of yychar.
- *yycharp = YYFAULTYTOK;
+ // Map errors caught in the scanner to the undefined token,
+ // so that error handling is started. However, record this
+ // with this special value of yychar.
+ *yycharp = ]b4_symbol(error, id)[;
}
#endif // YY_EXCEPTIONS]], [[
- *yycharp = ]b4_lex[;]])[
+ *yycharp = ]b4_yylex[;]])[
}
- if (*yycharp <= YYEOF)
+ if (*yycharp <= ]b4_symbol(eof, [id])[)
{
- *yycharp = yytoken = YYEOF;
+ *yycharp = ]b4_symbol(eof, [id])[;
+ yytoken = ]b4_symbol_prefix[YYEOF;
YY_DPRINTF ((stderr, "Now at end of input.\n"));
}
else
@@ -855,28 +898,31 @@
* and top stack item YYVSP. YYLVALP points to place to put semantic
* value ($$), and yylocp points to place for location information
* (@@$). Returns yyok for normal return, yyaccept for YYACCEPT,
- * yyerr for YYERROR, yyabort for YYABORT. */
+ * yyerr for YYERROR, yyabort for YYABORT, yynomem for YYNOMEM. */
static YYRESULTTAG
-yyuserAction (yyRuleNum yyn, int yyrhslen, yyGLRStackItem* yyvsp,
- yyGLRStack* yystackp,
+yyuserAction (yyRuleNum yyrule, int yyrhslen, yyGLRStackItem* yyvsp,
+ yyGLRStack* yystackp, YYPTRDIFF_T yyk,
YYSTYPE* yyvalp]b4_locuser_formals[)
{
- yybool yynormal YY_ATTRIBUTE_UNUSED = yystackp->yysplitPoint == YY_NULLPTR;
- int yylow;
+ const yybool yynormal YY_ATTRIBUTE_UNUSED = yystackp->yysplitPoint == YY_NULLPTR;
+ int yylow = 1;
]b4_parse_param_use([yyvalp], [yylocp])dnl
-[ YYUSE (yyrhslen);
+[ YY_USE (yyk);
+ YY_USE (yyrhslen);
# undef yyerrok
# define yyerrok (yystackp->yyerrState = 0)
# undef YYACCEPT
# define YYACCEPT return yyaccept
# undef YYABORT
# define YYABORT return yyabort
+# undef YYNOMEM
+# define YYNOMEM return yynomem
# undef YYERROR
# define YYERROR return yyerrok, yyerr
# undef YYRECOVERING
# define YYRECOVERING() (yystackp->yyerrState != 0)
# undef yyclearin
-# define yyclearin (yychar = YYEMPTY)
+# define yyclearin (yychar = ]b4_symbol(empty, id)[)
# undef YYFILL
# define YYFILL(N) yyfill (yyvsp, &yylow, (N), yynormal)
# undef YYBACKUP
@@ -884,21 +930,23 @@
return yyerror (]b4_yyerror_args[YY_("syntax error: cannot back up")), \
yyerrok, yyerr
- yylow = 1;
if (yyrhslen == 0)
*yyvalp = yyval_default;
else
- *yyvalp = yyvsp[YYFILL (1-yyrhslen)].yystate.yysemantics.yysval;]b4_locations_if([[
+ *yyvalp = yyvsp[YYFILL (1-yyrhslen)].yystate.yysemantics.yyval;]b4_locations_if([[
/* Default location. */
YYLLOC_DEFAULT ((*yylocp), (yyvsp - yyrhslen), yyrhslen);
- yystackp->yyerror_range[1].yystate.yyloc = *yylocp;
-]])[]b4_glr_cc_if([[
+ yystackp->yyerror_range[1].yystate.yyloc = *yylocp;]])[
+ /* If yyk == -1, we are running a deferred action on a temporary
+ stack. In that case, YY_REDUCE_PRINT must not play with YYFILL,
+ so pretend the stack is "normal". */
+ YY_REDUCE_PRINT ((yynormal || yyk == -1, yyvsp, yyk, yyrule]b4_user_args[));]b4_glr_cc_if([[
#if YY_EXCEPTIONS
typedef ]b4_namespace_ref[::]b4_parser_class[::syntax_error syntax_error;
try
{
#endif // YY_EXCEPTIONS]])[
- switch (yyn)
+ switch (yyrule)
{
]b4_user_actions[
default: break;
@@ -913,11 +961,13 @@
YYERROR;
}
#endif // YY_EXCEPTIONS]])[
+ YY_SYMBOL_PRINT ("-> $$ =", yylhsNonterm (yyrule), yyvalp, yylocp);
return yyok;
# undef yyerrok
# undef YYABORT
# undef YYACCEPT
+# undef YYNOMEM
# undef YYERROR
# undef YYBACKUP
# undef yyclearin
@@ -928,8 +978,8 @@
static void
yyuserMerge (int yyn, YYSTYPE* yy0, YYSTYPE* yy1)
{
- YYUSE (yy0);
- YYUSE (yy1);
+ YY_USE (yy0);
+ YY_USE (yy1);
switch (yyn)
{
@@ -953,8 +1003,8 @@
yydestroyGLRState (char const *yymsg, yyGLRState *yys]b4_user_formals[)
{
if (yys->yyresolved)
- yydestruct (yymsg, yystos[yys->yylrState],
- &yys->yysemantics.yysval]b4_locuser_args([&yys->yyloc])[);
+ yydestruct (yymsg, yy_accessing_symbol (yys->yylrState),
+ &yys->yysemantics.yyval]b4_locuser_args([&yys->yyloc])[);
else
{
#if ]b4_api_PREFIX[DEBUG
@@ -964,7 +1014,7 @@
YY_FPRINTF ((stderr, "%s unresolved", yymsg));
else
YY_FPRINTF ((stderr, "%s incomplete", yymsg));
- YY_SYMBOL_PRINT ("", yystos[yys->yylrState], YY_NULLPTR, &yys->yyloc);
+ YY_SYMBOL_PRINT ("", yy_accessing_symbol (yys->yylrState), YY_NULLPTR, &yys->yyloc);
}
#endif
@@ -981,27 +1031,20 @@
}
}
-/** Left-hand-side symbol for rule #YYRULE. */
-static inline yySymbol
-yylhsNonterm (yyRuleNum yyrule)
-{
- return yyr1[yyrule];
-}
-
#define yypact_value_is_default(Yyn) \
]b4_table_value_equals([[pact]], [[Yyn]], [b4_pact_ninf], [YYPACT_NINF])[
/** True iff LR state YYSTATE has only a default reduction (regardless
* of token). */
static inline yybool
-yyisDefaultedState (yyStateNum yystate)
+yyisDefaultedState (yy_state_t yystate)
{
return yypact_value_is_default (yypact[yystate]);
}
/** The default reduction for YYSTATE, assuming it has one. */
static inline yyRuleNum
-yydefaultAction (yyStateNum yystate)
+yydefaultAction (yy_state_t yystate)
{
return yydefact[yystate];
}
@@ -1018,11 +1061,17 @@
* of conflicting reductions.
*/
static inline int
-yygetLRActions (yyStateNum yystate, yySymbol yytoken, const short** yyconflicts)
+yygetLRActions (yy_state_t yystate, yysymbol_kind_t yytoken, const short** yyconflicts)
{
int yyindex = yypact[yystate] + yytoken;
- if (yyisDefaultedState (yystate)
- || yyindex < 0 || YYLAST < yyindex || yycheck[yyindex] != yytoken)
+ if (yytoken == ]b4_symbol(error, kind)[)
+ {
+ // This is the error token.
+ *yyconflicts = yyconfl;
+ return 0;
+ }
+ else if (yyisDefaultedState (yystate)
+ || yyindex < 0 || YYLAST < yyindex || yycheck[yyindex] != yytoken)
{
*yyconflicts = yyconfl;
return -yydefact[yystate];
@@ -1043,8 +1092,8 @@
* \param yystate the current state
* \param yysym the nonterminal to push on the stack
*/
-static inline yyStateNum
-yyLRgotoState (yyStateNum yystate, yySymbol yysym)
+static inline yy_state_t
+yyLRgotoState (yy_state_t yystate, yysymbol_kind_t yysym)
{
int yyr = yypgoto[yysym - YYNTOKENS] + yystate;
if (0 <= yyr && yyr <= YYLAST && yycheck[yyr] == yystate)
@@ -1087,7 +1136,7 @@
* alternative actions for YYSTATE. Assumes that YYRHS comes from
* stack #YYK of *YYSTACKP. */
static void
-yyaddDeferredAction (yyGLRStack* yystackp, ptrdiff_t yyk, yyGLRState* yystate,
+yyaddDeferredAction (yyGLRStack* yystackp, YYPTRDIFF_T yyk, yyGLRState* yystate,
yyGLRState* yyrhs, yyRuleNum yyrule)
{
yySemanticOption* yynewOption =
@@ -1102,7 +1151,7 @@
yynewOption->yyloc = yylloc;])[
}
else
- yynewOption->yyrawchar = YYEMPTY;
+ yynewOption->yyrawchar = ]b4_symbol(empty, id)[;
yynewOption->yynext = yystate->yysemantics.yyfirstVal;
yystate->yysemantics.yyfirstVal = yynewOption;
@@ -1119,14 +1168,14 @@
yyset->yycapacity = 16;
yyset->yystates
= YY_CAST (yyGLRState**,
- YYMALLOC (YY_CAST (size_t, yyset->yycapacity)
+ YYMALLOC (YY_CAST (YYSIZE_T, yyset->yycapacity)
* sizeof yyset->yystates[0]));
if (! yyset->yystates)
return yyfalse;
yyset->yystates[0] = YY_NULLPTR;
yyset->yylookaheadNeeds
= YY_CAST (yybool*,
- YYMALLOC (YY_CAST (size_t, yyset->yycapacity)
+ YYMALLOC (YY_CAST (YYSIZE_T, yyset->yycapacity)
* sizeof yyset->yylookaheadNeeds[0]));
if (! yyset->yylookaheadNeeds)
{
@@ -1135,7 +1184,7 @@
}
memset (yyset->yylookaheadNeeds,
0,
- YY_CAST (size_t, yyset->yycapacity) * sizeof yyset->yylookaheadNeeds[0]);
+ YY_CAST (YYSIZE_T, yyset->yycapacity) * sizeof yyset->yylookaheadNeeds[0]);
return yytrue;
}
@@ -1148,14 +1197,14 @@
/** Initialize *YYSTACKP to a single empty stack, with total maximum
* capacity for all stacks of YYSIZE. */
static yybool
-yyinitGLRStack (yyGLRStack* yystackp, ptrdiff_t yysize)
+yyinitGLRStack (yyGLRStack* yystackp, YYPTRDIFF_T yysize)
{
yystackp->yyerrState = 0;
yynerrs = 0;
yystackp->yyspaceLeft = yysize;
yystackp->yyitems
= YY_CAST (yyGLRStackItem*,
- YYMALLOC (YY_CAST (size_t, yysize)
+ YYMALLOC (YY_CAST (YYSIZE_T, yysize)
* sizeof yystackp->yynextFree[0]));
if (!yystackp->yyitems)
return yyfalse;
@@ -1181,9 +1230,9 @@
{
yyGLRStackItem* yynewItems;
yyGLRStackItem* yyp0, *yyp1;
- ptrdiff_t yynewSize;
- ptrdiff_t yyn;
- ptrdiff_t yysize = yystackp->yynextFree - yystackp->yyitems;
+ YYPTRDIFF_T yynewSize;
+ YYPTRDIFF_T yyn;
+ YYPTRDIFF_T yysize = yystackp->yynextFree - yystackp->yyitems;
if (YYMAXDEPTH - YYHEADROOM < yysize)
yyMemoryExhausted (yystackp);
yynewSize = 2*yysize;
@@ -1191,7 +1240,7 @@
yynewSize = YYMAXDEPTH;
yynewItems
= YY_CAST (yyGLRStackItem*,
- YYMALLOC (YY_CAST (size_t, yynewSize)
+ YYMALLOC (YY_CAST (YYSIZE_T, yynewSize)
* sizeof yynewItems[0]));
if (! yynewItems)
yyMemoryExhausted (yystackp);
@@ -1256,7 +1305,7 @@
/** Invalidate stack #YYK in *YYSTACKP. */
static inline void
-yymarkStackDeleted (yyGLRStack* yystackp, ptrdiff_t yyk)
+yymarkStackDeleted (yyGLRStack* yystackp, YYPTRDIFF_T yyk)
{
if (yystackp->yytops.yystates[yyk] != YY_NULLPTR)
yystackp->yylastDeleted = yystackp->yytops.yystates[yyk];
@@ -1280,7 +1329,7 @@
static inline void
yyremoveDeletes (yyGLRStack* yystackp)
{
- ptrdiff_t yyi, yyj;
+ YYPTRDIFF_T yyi, yyj;
yyi = yyj = 0;
while (yyj < yystackp->yytops.yysize)
{
@@ -1313,8 +1362,8 @@
* state YYLRSTATE, at input position YYPOSN, with (resolved) semantic
* value *YYVALP and source location *YYLOCP. */
static inline void
-yyglrShift (yyGLRStack* yystackp, ptrdiff_t yyk, yyStateNum yylrState,
- ptrdiff_t yyposn,
+yyglrShift (yyGLRStack* yystackp, YYPTRDIFF_T yyk, yy_state_t yylrState,
+ YYPTRDIFF_T yyposn,
YYSTYPE* yyvalp]b4_locations_if([, YYLTYPE* yylocp])[)
{
yyGLRState* yynewState = &yynewGLRStackItem (yystackp, yytrue)->yystate;
@@ -1323,7 +1372,7 @@
yynewState->yyposn = yyposn;
yynewState->yyresolved = yytrue;
yynewState->yypred = yystackp->yytops.yystates[yyk];
- yynewState->yysemantics.yysval = *yyvalp;]b4_locations_if([
+ yynewState->yysemantics.yyval = *yyvalp;]b4_locations_if([
yynewState->yyloc = *yylocp;])[
yystackp->yytops.yystates[yyk] = yynewState;
@@ -1334,8 +1383,8 @@
* state YYLRSTATE, at input position YYPOSN, with the (unresolved)
* semantic value of YYRHS under the action for YYRULE. */
static inline void
-yyglrShiftDefer (yyGLRStack* yystackp, ptrdiff_t yyk, yyStateNum yylrState,
- ptrdiff_t yyposn, yyGLRState* yyrhs, yyRuleNum yyrule)
+yyglrShiftDefer (yyGLRStack* yystackp, YYPTRDIFF_T yyk, yy_state_t yylrState,
+ YYPTRDIFF_T yyposn, yyGLRState* yyrhs, yyRuleNum yyrule)
{
yyGLRState* yynewState = &yynewGLRStackItem (yystackp, yytrue)->yystate;
YY_ASSERT (yynewState->yyisState);
@@ -1351,21 +1400,14 @@
yyaddDeferredAction (yystackp, yyk, yynewState, yyrhs, yyrule);
}
-#if !]b4_api_PREFIX[DEBUG
-# define YY_REDUCE_PRINT(Args)
-#else
-# define YY_REDUCE_PRINT(Args) \
- do { \
- if (yydebug) \
- yy_reduce_print Args; \
- } while (0)
+#if ]b4_api_PREFIX[DEBUG
/*----------------------------------------------------------------------.
| Report that stack #YYK of *YYSTACKP is going to be reduced by YYRULE. |
`----------------------------------------------------------------------*/
static inline void
-yy_reduce_print (yybool yynormal, yyGLRStackItem* yyvsp, ptrdiff_t yyk,
+yy_reduce_print (yybool yynormal, yyGLRStackItem* yyvsp, YYPTRDIFF_T yyk,
yyRuleNum yyrule]b4_user_formals[)
{
int yynrhs = yyrhsLength (yyrule);]b4_locations_if([
@@ -1380,8 +1422,8 @@
{
YY_FPRINTF ((stderr, " $%d = ", yyi + 1));
yy_symbol_print (stderr,
- yystos[yyvsp[yyi - yynrhs + 1].yystate.yylrState],
- &yyvsp[yyi - yynrhs + 1].yystate.yysemantics.yysval]b4_locations_if([,
+ yy_accessing_symbol (yyvsp[yyi - yynrhs + 1].yystate.yylrState),
+ &yyvsp[yyi - yynrhs + 1].yystate.yysemantics.yyval]b4_locations_if([,
&]b4_rhs_location(yynrhs, yyi + 1))[]dnl
b4_user_args[);
if (!yyvsp[yyi - yynrhs + 1].yystate.yyresolved)
@@ -1398,7 +1440,7 @@
* and *YYLOCP to the computed location (if any). Return value is as
* for userAction. */
static inline YYRESULTTAG
-yydoAction (yyGLRStack* yystackp, ptrdiff_t yyk, yyRuleNum yyrule,
+yydoAction (yyGLRStack* yystackp, YYPTRDIFF_T yyk, yyRuleNum yyrule,
YYSTYPE* yyvalp]b4_locuser_formals[)
{
int yynrhs = yyrhsLength (yyrule);
@@ -1412,8 +1454,7 @@
yystackp->yynextFree -= yynrhs;
yystackp->yyspaceLeft += yynrhs;
yystackp->yytops.yystates[0] = & yystackp->yynextFree[-1].yystate;
- YY_REDUCE_PRINT ((yytrue, yyrhs, yyk, yyrule]b4_user_args[));
- return yyuserAction (yyrule, yynrhs, yyrhs, yystackp,
+ return yyuserAction (yyrule, yynrhs, yyrhs, yystackp, yyk,
yyvalp]b4_locuser_args[);
}
else
@@ -1432,9 +1473,8 @@
}
yyupdateSplit (yystackp, yys);
yystackp->yytops.yystates[yyk] = yys;
- YY_REDUCE_PRINT ((yyfalse, yyrhsVals + YYMAXRHS + YYMAXLEFT - 1, yyk, yyrule]b4_user_args[));
return yyuserAction (yyrule, yynrhs, yyrhsVals + YYMAXRHS + YYMAXLEFT - 1,
- yystackp, yyvalp]b4_locuser_args[);
+ yystackp, yyk, yyvalp]b4_locuser_args[);
}
}
@@ -1450,35 +1490,34 @@
* added to the options for the existing state's semantic value.
*/
static inline YYRESULTTAG
-yyglrReduce (yyGLRStack* yystackp, ptrdiff_t yyk, yyRuleNum yyrule,
+yyglrReduce (yyGLRStack* yystackp, YYPTRDIFF_T yyk, yyRuleNum yyrule,
yybool yyforceEval]b4_user_formals[)
{
- ptrdiff_t yyposn = yystackp->yytops.yystates[yyk]->yyposn;
+ YYPTRDIFF_T yyposn = yystackp->yytops.yystates[yyk]->yyposn;
if (yyforceEval || yystackp->yysplitPoint == YY_NULLPTR)
{
- YYSTYPE yysval;]b4_locations_if([[
+ YYSTYPE yyval;]b4_locations_if([[
YYLTYPE yyloc;]])[
- YYRESULTTAG yyflag = yydoAction (yystackp, yyk, yyrule, &yysval]b4_locuser_args([&yyloc])[);
+ YYRESULTTAG yyflag = yydoAction (yystackp, yyk, yyrule, &yyval]b4_locuser_args([&yyloc])[);
if (yyflag == yyerr && yystackp->yysplitPoint != YY_NULLPTR)
YY_DPRINTF ((stderr,
"Parse on stack %ld rejected by rule %d (line %d).\n",
- YY_CAST (long, yyk), yyrule - 1, yyrline[yyrule - 1]));
+ YY_CAST (long, yyk), yyrule - 1, yyrline[yyrule]));
if (yyflag != yyok)
return yyflag;
- YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyrule], &yysval, &yyloc);
yyglrShift (yystackp, yyk,
yyLRgotoState (yystackp->yytops.yystates[yyk]->yylrState,
yylhsNonterm (yyrule)),
- yyposn, &yysval]b4_locations_if([, &yyloc])[);
+ yyposn, &yyval]b4_locations_if([, &yyloc])[);
}
else
{
- ptrdiff_t yyi;
+ YYPTRDIFF_T yyi;
int yyn;
yyGLRState* yys, *yys0 = yystackp->yytops.yystates[yyk];
- yyStateNum yynewLRState;
+ yy_state_t yynewLRState;
for (yys = yystackp->yytops.yystates[yyk], yyn = yyrhsLength (yyrule);
0 < yyn; yyn -= 1)
@@ -1491,7 +1530,7 @@
YY_DPRINTF ((stderr,
"Reduced stack %ld by rule %d (line %d); action deferred. "
"Now in state %d.\n",
- YY_CAST (long, yyk), yyrule - 1, yyrline[yyrule - 1],
+ YY_CAST (long, yyk), yyrule - 1, yyrline[yyrule],
yynewLRState));
for (yyi = 0; yyi < yystackp->yytops.yysize; yyi += 1)
if (yyi != yyk && yystackp->yytops.yystates[yyi] != YY_NULLPTR)
@@ -1517,8 +1556,8 @@
return yyok;
}
-static ptrdiff_t
-yysplitStack (yyGLRStack* yystackp, ptrdiff_t yyk)
+static YYPTRDIFF_T
+yysplitStack (yyGLRStack* yystackp, YYPTRDIFF_T yyk)
{
if (yystackp->yysplitPoint == YY_NULLPTR)
{
@@ -1527,8 +1566,8 @@
}
if (yystackp->yytops.yycapacity <= yystackp->yytops.yysize)
{
- ptrdiff_t state_size = sizeof yystackp->yytops.yystates[0];
- ptrdiff_t half_max_capacity = YYSIZEMAX / 2 / state_size;
+ YYPTRDIFF_T state_size = YYSIZEOF (yystackp->yytops.yystates[0]);
+ YYPTRDIFF_T half_max_capacity = YYSIZE_MAXIMUM / 2 / state_size;
if (half_max_capacity < yystackp->yytops.yycapacity)
yyMemoryExhausted (yystackp);
yystackp->yytops.yycapacity *= 2;
@@ -1537,7 +1576,7 @@
yyGLRState** yynewStates
= YY_CAST (yyGLRState**,
YYREALLOC (yystackp->yytops.yystates,
- (YY_CAST (size_t, yystackp->yytops.yycapacity)
+ (YY_CAST (YYSIZE_T, yystackp->yytops.yycapacity)
* sizeof yynewStates[0])));
if (yynewStates == YY_NULLPTR)
yyMemoryExhausted (yystackp);
@@ -1548,7 +1587,7 @@
yybool* yynewLookaheadNeeds
= YY_CAST (yybool*,
YYREALLOC (yystackp->yytops.yylookaheadNeeds,
- (YY_CAST (size_t, yystackp->yytops.yycapacity)
+ (YY_CAST (YYSIZE_T, yystackp->yytops.yycapacity)
* sizeof yynewLookaheadNeeds[0])));
if (yynewLookaheadNeeds == YY_NULLPTR)
yyMemoryExhausted (yystackp);
@@ -1602,12 +1641,12 @@
else if (yys0->yyresolved)
{
yys1->yyresolved = yytrue;
- yys1->yysemantics.yysval = yys0->yysemantics.yysval;
+ yys1->yysemantics.yyval = yys0->yysemantics.yyval;
}
else if (yys1->yyresolved)
{
yys0->yyresolved = yytrue;
- yys0->yysemantics.yysval = yys1->yysemantics.yysval;
+ yys0->yysemantics.yyval = yys1->yysemantics.yyval;
}
else
{
@@ -1661,8 +1700,8 @@
return 0;
}
-static YYRESULTTAG yyresolveValue (yyGLRState* yys,
- yyGLRStack* yystackp]b4_user_formals[);
+static YYRESULTTAG
+yyresolveValue (yyGLRState* yys, yyGLRStack* yystackp]b4_user_formals[);
/** Resolve the previous YYN states starting at and including state YYS
@@ -1718,7 +1757,7 @@
yylloc = yyopt->yyloc;])[
yyflag = yyuserAction (yyopt->yyrule, yynrhs,
yyrhsVals + YYMAXRHS + YYMAXLEFT - 1,
- yystackp, yyvalp]b4_locuser_args[);
+ yystackp, -1, yyvalp]b4_locuser_args[);
yychar = yychar_current;
yylval = yylval_current;]b4_locations_if([
yylloc = yylloc_current;])[
@@ -1748,11 +1787,11 @@
if (yyx->yystate->yyposn < yys->yyposn + 1)
YY_FPRINTF ((stderr, "%*s%s -> <Rule %d, empty>\n",
- yyindent, "", yytokenName (yylhsNonterm (yyx->yyrule)),
+ yyindent, "", yysymbol_name (yylhsNonterm (yyx->yyrule)),
yyx->yyrule - 1));
else
YY_FPRINTF ((stderr, "%*s%s -> <Rule %d, tokens %ld .. %ld>\n",
- yyindent, "", yytokenName (yylhsNonterm (yyx->yyrule)),
+ yyindent, "", yysymbol_name (yylhsNonterm (yyx->yyrule)),
yyx->yyrule - 1, YY_CAST (long, yys->yyposn + 1),
YY_CAST (long, yyx->yystate->yyposn)));
for (yyi = 1; yyi <= yynrhs; yyi += 1)
@@ -1761,10 +1800,10 @@
{
if (yystates[yyi-1]->yyposn+1 > yystates[yyi]->yyposn)
YY_FPRINTF ((stderr, "%*s%s <empty>\n", yyindent+2, "",
- yytokenName (yystos[yystates[yyi]->yylrState])));
+ yysymbol_name (yy_accessing_symbol (yystates[yyi]->yylrState))));
else
YY_FPRINTF ((stderr, "%*s%s <tokens %ld .. %ld>\n", yyindent+2, "",
- yytokenName (yystos[yystates[yyi]->yylrState]),
+ yysymbol_name (yy_accessing_symbol (yystates[yyi]->yylrState)),
YY_CAST (long, yystates[yyi-1]->yyposn + 1),
YY_CAST (long, yystates[yyi]->yyposn)));
}
@@ -1778,8 +1817,8 @@
yyreportAmbiguity (yySemanticOption* yyx0,
yySemanticOption* yyx1]b4_pure_formals[)
{
- YYUSE (yyx0);
- YYUSE (yyx1);
+ YY_USE (yyx0);
+ YY_USE (yyx1);
#if ]b4_api_PREFIX[DEBUG
YY_FPRINTF ((stderr, "Ambiguity detected.\n"));
@@ -1853,7 +1892,7 @@
yySemanticOption* yybest = yyoptionList;
yySemanticOption** yypp;
yybool yymerge = yyfalse;
- YYSTYPE yysval;
+ YYSTYPE yyval;
YYRESULTTAG yyflag;]b4_locations_if([
YYLTYPE *yylocp = &yys->yyloc;])[
@@ -1897,33 +1936,33 @@
{
yySemanticOption* yyp;
int yyprec = yydprec[yybest->yyrule];
- yyflag = yyresolveAction (yybest, yystackp, &yysval]b4_locuser_args[);
+ yyflag = yyresolveAction (yybest, yystackp, &yyval]b4_locuser_args[);
if (yyflag == yyok)
for (yyp = yybest->yynext; yyp != YY_NULLPTR; yyp = yyp->yynext)
{
if (yyprec == yydprec[yyp->yyrule])
{
- YYSTYPE yysval_other;]b4_locations_if([
+ YYSTYPE yyval_other;]b4_locations_if([
YYLTYPE yydummy;])[
- yyflag = yyresolveAction (yyp, yystackp, &yysval_other]b4_locuser_args([&yydummy])[);
+ yyflag = yyresolveAction (yyp, yystackp, &yyval_other]b4_locuser_args([&yydummy])[);
if (yyflag != yyok)
{
yydestruct ("Cleanup: discarding incompletely merged value for",
- yystos[yys->yylrState],
- &yysval]b4_locuser_args[);
+ yy_accessing_symbol (yys->yylrState),
+ &yyval]b4_locuser_args[);
break;
}
- yyuserMerge (yymerger[yyp->yyrule], &yysval, &yysval_other);
+ yyuserMerge (yymerger[yyp->yyrule], &yyval, &yyval_other);
}
}
}
else
- yyflag = yyresolveAction (yybest, yystackp, &yysval]b4_locuser_args([yylocp])[);
+ yyflag = yyresolveAction (yybest, yystackp, &yyval]b4_locuser_args([yylocp])[);
if (yyflag == yyok)
{
yys->yyresolved = yytrue;
- yys->yysemantics.yysval = yysval;
+ yys->yysemantics.yyval = yyval;
}
else
yys->yysemantics.yyfirstVal = YY_NULLPTR;
@@ -1948,18 +1987,24 @@
return yyok;
}
+/** Called when returning to deterministic operation to clean up the extra
+ * stacks. */
static void
yycompressStack (yyGLRStack* yystackp)
{
- yyGLRState* yyp, *yyq, *yyr;
+ /* yyr is the state after the split point. */
+ yyGLRState *yyr;
if (yystackp->yytops.yysize != 1 || yystackp->yysplitPoint == YY_NULLPTR)
return;
- for (yyp = yystackp->yytops.yystates[0], yyq = yyp->yypred, yyr = YY_NULLPTR;
- yyp != yystackp->yysplitPoint;
- yyr = yyp, yyp = yyq, yyq = yyp->yypred)
- yyp->yypred = yyr;
+ {
+ yyGLRState *yyp, *yyq;
+ for (yyp = yystackp->yytops.yystates[0], yyq = yyp->yypred, yyr = YY_NULLPTR;
+ yyp != yystackp->yysplitPoint;
+ yyr = yyp, yyp = yyq, yyq = yyp->yypred)
+ yyp->yypred = yyr;
+ }
yystackp->yyspaceLeft += yystackp->yynextFree - yystackp->yyitems;
yystackp->yynextFree = YY_REINTERPRET_CAST (yyGLRStackItem*, yystackp->yysplitPoint) + 1;
@@ -1979,13 +2024,14 @@
}
static YYRESULTTAG
-yyprocessOneStack (yyGLRStack* yystackp, ptrdiff_t yyk,
- ptrdiff_t yyposn]b4_pure_formals[)
+yyprocessOneStack (yyGLRStack* yystackp, YYPTRDIFF_T yyk,
+ YYPTRDIFF_T yyposn]b4_pure_formals[)
{
while (yystackp->yytops.yystates[yyk] != YY_NULLPTR)
{
- yyStateNum yystate = yystackp->yytops.yystates[yyk]->yylrState;
- YY_DPRINTF ((stderr, "Stack %ld Entering state %d\n", YY_CAST (long, yyk), yystate));
+ yy_state_t yystate = yystackp->yytops.yystates[yyk]->yylrState;
+ YY_DPRINTF ((stderr, "Stack %ld Entering state %d\n",
+ YY_CAST (long, yyk), yystate));
YY_ASSERT (yystate != YYFINAL);
@@ -2014,15 +2060,15 @@
}
else
{
- yySymbol yytoken = ]b4_yygetToken_call[;
+ yysymbol_kind_t yytoken = ]b4_yygetToken_call[;
const short* yyconflicts;
const int yyaction = yygetLRActions (yystate, yytoken, &yyconflicts);
yystackp->yytops.yylookaheadNeeds[yyk] = yytrue;
- while (*yyconflicts != 0)
+ for (/* nothing */; *yyconflicts; yyconflicts += 1)
{
YYRESULTTAG yyflag;
- ptrdiff_t yynewStack = yysplitStack (yystackp, yyk);
+ YYPTRDIFF_T yynewStack = yysplitStack (yystackp, yyk);
YY_DPRINTF ((stderr, "Splitting off stack %ld from %ld.\n",
YY_CAST (long, yynewStack), YY_CAST (long, yyk)));
yyflag = yyglrReduce (yystackp, yynewStack,
@@ -2038,7 +2084,6 @@
}
else
return yyflag;
- yyconflicts += 1;
}
if (yyisShiftAction (yyaction))
@@ -2070,29 +2115,82 @@
return yyok;
}
-static void
-yyreportSyntaxError (yyGLRStack* yystackp]b4_user_formals[)
+]b4_parse_error_case([simple], [],
+[[/* Put in YYARG at most YYARGN of the expected tokens given the
+ current YYSTACKP, and return the number of tokens stored in YYARG. If
+ YYARG is null, return the number of expected tokens (guaranteed to
+ be less than YYNTOKENS). */
+static int
+yypcontext_expected_tokens (const yyGLRStack* yystackp,
+ yysymbol_kind_t yyarg[], int yyargn)
{
- if (yystackp->yyerrState != 0)
- return;
-#if ! YYERROR_VERBOSE
- yyerror (]b4_lyyerror_args[YY_("syntax error"));
-#else
- {
- yySymbol yytoken = yychar == YYEMPTY ? YYEMPTY : YYTRANSLATE (yychar);
- yybool yysize_overflow = yyfalse;
- char* yymsg = YY_NULLPTR;
- enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 };
- /* Internationalized format string. */
- const char *yyformat = YY_NULLPTR;
- /* Arguments of yyformat: reported tokens (one for the "unexpected",
- one per "expected"). */
- char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
/* Actual size of YYARG. */
int yycount = 0;
- /* Cumulated lengths of YYARG. */
- ptrdiff_t yysize = 0;
+ int yyn = yypact[yystackp->yytops.yystates[0]->yylrState];
+ if (!yypact_value_is_default (yyn))
+ {
+ /* Start YYX at -YYN if negative to avoid negative indexes in
+ YYCHECK. In other words, skip the first -YYN actions for
+ this state because they are default actions. */
+ int yyxbegin = yyn < 0 ? -yyn : 0;
+ /* Stay within bounds of both yycheck and yytname. */
+ int yychecklim = YYLAST - yyn + 1;
+ int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
+ int yyx;
+ for (yyx = yyxbegin; yyx < yyxend; ++yyx)
+ if (yycheck[yyx + yyn] == yyx && yyx != ]b4_symbol(error, kind)[
+ && !yytable_value_is_error (yytable[yyx + yyn]))
+ {
+ if (!yyarg)
+ ++yycount;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx);
+ }
+ }
+ if (yyarg && yycount == 0 && 0 < yyargn)
+ yyarg[0] = ]b4_symbol(empty, kind)[;
+ return yycount;
+}]])[
+]b4_parse_error_bmatch(
+ [custom],
+[[/* User defined function to report a syntax error. */
+typedef yyGLRStack yypcontext_t;
+static int
+yyreport_syntax_error (const yyGLRStack* yystackp]b4_user_formals[);
+
+/* The kind of the lookahead of this context. */
+static yysymbol_kind_t
+yypcontext_token (const yyGLRStack *yystackp) YY_ATTRIBUTE_UNUSED;
+
+static yysymbol_kind_t
+yypcontext_token (const yyGLRStack *yystackp)
+{
+ YY_USE (yystackp);
+ yysymbol_kind_t yytoken = yychar == ]b4_symbol(empty, id)[ ? ]b4_symbol(empty, kind)[ : YYTRANSLATE (yychar);
+ return yytoken;
+}
+
+]b4_locations_if([[/* The location of the lookahead of this context. */
+static const YYLTYPE *
+yypcontext_location (const yyGLRStack *yystackp) YY_ATTRIBUTE_UNUSED;
+
+static const YYLTYPE *
+yypcontext_location (const yyGLRStack *yystackp)
+{
+ YY_USE (yystackp);
+ return &yylloc;
+}]])],
+ [detailed\|verbose],
+[[static int
+yy_syntax_error_arguments (const yyGLRStack* yystackp,
+ yysymbol_kind_t yyarg[], int yyargn)
+{
+ yysymbol_kind_t yytoken = yychar == ]b4_symbol(empty, id)[ ? ]b4_symbol(empty, kind)[ : YYTRANSLATE (yychar);
+ /* Actual size of YYARG. */
+ int yycount = 0;
/* There are many possibilities here to consider:
- If this state is a consistent state with a default action, then
the only way this function was invoked is if the default action
@@ -2116,43 +2214,52 @@
one exception: it will still contain any token that will not be
accepted due to an error action in a later state.
*/
- if (yytoken != YYEMPTY)
+ if (yytoken != ]b4_symbol(empty, kind)[)
{
- int yyn = yypact[yystackp->yytops.yystates[0]->yylrState];
- ptrdiff_t yysize0 = yytnamerr (YY_NULLPTR, yytokenName (yytoken));
- yysize = yysize0;
- yyarg[yycount++] = yytokenName (yytoken);
- if (!yypact_value_is_default (yyn))
- {
- /* Start YYX at -YYN if negative to avoid negative indexes in
- YYCHECK. In other words, skip the first -YYN actions for this
- state because they are default actions. */
- int yyxbegin = yyn < 0 ? -yyn : 0;
- /* Stay within bounds of both yycheck and yytname. */
- int yychecklim = YYLAST - yyn + 1;
- int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
- int yyx;
- for (yyx = yyxbegin; yyx < yyxend; ++yyx)
- if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR
- && !yytable_value_is_error (yytable[yyx + yyn]))
- {
- if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
- {
- yycount = 1;
- yysize = yysize0;
- break;
- }
- yyarg[yycount++] = yytokenName (yyx);
- {
- ptrdiff_t yysz = yytnamerr (YY_NULLPTR, yytokenName (yyx));
- if (YYSIZEMAX - yysize < yysz)
- yysize_overflow = yytrue;
- else
- yysize += yysz;
- }
- }
- }
+ int yyn;
+ if (yyarg)
+ yyarg[yycount] = yytoken;
+ ++yycount;
+ yyn = yypcontext_expected_tokens (yystackp,
+ yyarg ? yyarg + 1 : yyarg, yyargn - 1);
+ if (yyn == YYENOMEM)
+ return YYENOMEM;
+ else
+ yycount += yyn;
}
+ return yycount;
+}
+]])[
+
+
+static void
+yyreportSyntaxError (yyGLRStack* yystackp]b4_user_formals[)
+{
+ if (yystackp->yyerrState != 0)
+ return;
+]b4_parse_error_case(
+ [custom],
+[[ if (yyreport_syntax_error (yystackp]b4_user_args[))
+ yyMemoryExhausted (yystackp);]],
+ [simple],
+[[ yyerror (]b4_lyyerror_args[YY_("syntax error"));]],
+[[ {
+ yybool yysize_overflow = yyfalse;
+ char* yymsg = YY_NULLPTR;
+ enum { YYARGS_MAX = 5 };
+ /* Internationalized format string. */
+ const char *yyformat = YY_NULLPTR;
+ /* Arguments of yyformat: reported tokens (one for the "unexpected",
+ one per "expected"). */
+ yysymbol_kind_t yyarg[YYARGS_MAX];
+ /* Cumulated lengths of YYARG. */
+ YYPTRDIFF_T yysize = 0;
+
+ /* Actual size of YYARG. */
+ int yycount
+ = yy_syntax_error_arguments (yystackp, yyarg, YYARGS_MAX);
+ if (yycount == YYENOMEM)
+ yyMemoryExhausted (yystackp);
switch (yycount)
{
@@ -2170,18 +2277,26 @@
#undef YYCASE_
}
+ /* Compute error message size. Don't count the "%s"s, but reserve
+ room for the terminator. */
+ yysize = yystrlen (yyformat) - 2 * yycount + 1;
{
- /* Don't count the "%s"s in the final size, but reserve room for
- the terminator. */
- ptrdiff_t yysz = YY_CAST (ptrdiff_t, strlen (yyformat)) - 2 * yycount + 1;
- if (YYSIZEMAX - yysize < yysz)
- yysize_overflow = yytrue;
- else
- yysize += yysz;
+ int yyi;
+ for (yyi = 0; yyi < yycount; ++yyi)
+ {
+ YYPTRDIFF_T yysz
+ = ]b4_parse_error_case(
+ [verbose], [[yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]])]],
+ [[yystrlen (yysymbol_name (yyarg[yyi]))]]);[
+ if (YYSIZE_MAXIMUM - yysize < yysz)
+ yysize_overflow = yytrue;
+ else
+ yysize += yysz;
+ }
}
if (!yysize_overflow)
- yymsg = YY_CAST (char *, YYMALLOC (YY_CAST (size_t, yysize)));
+ yymsg = YY_CAST (char *, YYMALLOC (YY_CAST (YYSIZE_T, yysize)));
if (yymsg)
{
@@ -2190,8 +2305,9 @@
while ((*yyp = *yyformat))
{
if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount)
- {
- yyp += yytnamerr (yyp, yyarg[yyi++]);
+ {]b4_parse_error_case([verbose], [[
+ yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]);]], [[
+ yyp = yystpcpy (yyp, yysymbol_name (yyarg[yyi++]));]])[
yyformat += 2;
}
else
@@ -2208,8 +2324,7 @@
yyerror (]b4_lyyerror_args[YY_("syntax error"));
yyMemoryExhausted (yystackp);
}
- }
-#endif /* YYERROR_VERBOSE */
+ }]])[
yynerrs += 1;
}
@@ -2224,11 +2339,11 @@
reductions. Skip tokens until we can proceed. */
while (yytrue)
{
- yySymbol yytoken;
+ yysymbol_kind_t yytoken;
int yyj;
- if (yychar == YYEOF)
+ if (yychar == ]b4_symbol(eof, [id])[)
yyFail (yystackp][]b4_lpure_args[, YY_NULLPTR);
- if (yychar != YYEMPTY)
+ if (yychar != ]b4_symbol(empty, id)[)
{]b4_locations_if([[
/* We throw away the lookahead, but the error range
of the shifted error token must take it into account. */
@@ -2240,7 +2355,7 @@
yytoken = YYTRANSLATE (yychar);
yydestruct ("Error: discarding",
yytoken, &yylval]b4_locuser_args([&yylloc])[);
- yychar = YYEMPTY;
+ yychar = ]b4_symbol(empty, id)[;
}
yytoken = ]b4_yygetToken_call[;
yyj = yypact[yystackp->yytops.yystates[0]->yylrState];
@@ -2258,7 +2373,7 @@
/* Reduce to one stack. */
{
- ptrdiff_t yyk;
+ YYPTRDIFF_T yyk;
for (yyk = 0; yyk < yystackp->yytops.yysize; yyk += 1)
if (yystackp->yytops.yystates[yyk] != YY_NULLPTR)
break;
@@ -2270,7 +2385,7 @@
yycompressStack (yystackp);
}
- /* Now pop stack until we find a state that shifts the error token. */
+ /* Pop stack until we find a state that shifts the error token. */
yystackp->yyerrState = 3;
while (yystackp->yytops.yystates[0] != YY_NULLPTR)
{
@@ -2278,8 +2393,8 @@
int yyj = yypact[yys->yylrState];
if (! yypact_value_is_default (yyj))
{
- yyj += YYTERROR;
- if (0 <= yyj && yyj <= YYLAST && yycheck[yyj] == YYTERROR
+ yyj += ]b4_symbol(error, kind)[;
+ if (0 <= yyj && yyj <= YYLAST && yycheck[yyj] == ]b4_symbol(error, kind)[
&& yyisShiftAction (yytable[yyj]))
{
/* Shift the error token. */
@@ -2288,7 +2403,7 @@
YYLTYPE yyerrloc;
yystackp->yyerror_range[2].yystate.yyloc = yylloc;
YYLLOC_DEFAULT (yyerrloc, (yystackp->yyerror_range), 2);]])[
- YY_SYMBOL_PRINT ("Shifting", yystos[yyaction],
+ YY_SYMBOL_PRINT ("Shifting", yy_accessing_symbol (yyaction),
&yylval, &yyerrloc);
yyglrShift (yystackp, 0, yyaction,
yys->yyposn, &yylval]b4_locations_if([, &yyerrloc])[);
@@ -2307,36 +2422,33 @@
yyFail (yystackp][]b4_lpure_args[, YY_NULLPTR);
}
-#define YYCHK1(YYE) \
- do { \
- switch (YYE) { \
- case yyok: \
- break; \
- case yyabort: \
- goto yyabortlab; \
- case yyaccept: \
- goto yyacceptlab; \
- case yyerr: \
- goto yyuser_error; \
- default: \
- goto yybuglab; \
- } \
+#define YYCHK1(YYE) \
+ do { \
+ switch (YYE) { \
+ case yyok: break; \
+ case yyabort: goto yyabortlab; \
+ case yyaccept: goto yyacceptlab; \
+ case yyerr: goto yyuser_error; \
+ case yynomem: goto yyexhaustedlab; \
+ default: goto yybuglab; \
+ } \
} while (0)
/*----------.
| yyparse. |
`----------*/
-]b4_function_define([yyparse], [int], b4_parse_param)[
+int
+]b4_glr_cc_if([yy_parse_impl], [yyparse])[ (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)
{
int yyresult;
yyGLRStack yystack;
yyGLRStack* const yystackp = &yystack;
- ptrdiff_t yyposn;
+ YYPTRDIFF_T yyposn;
YY_DPRINTF ((stderr, "Starting parse\n"));
- yychar = YYEMPTY;
+ yychar = ]b4_symbol(empty, id)[;
yylval = yyval_default;]b4_locations_if([
yylloc = yyloc_default;])[
]m4_ifdef([b4_initial_action], [
@@ -2361,10 +2473,10 @@
/* For efficiency, we have two loops, the first of which is
specialized to deterministic operation (single stack, no
potential ambiguity). */
- /* Standard mode */
+ /* Standard mode. */
while (yytrue)
{
- yyStateNum yystate = yystack.yytops.yystates[0]->yylrState;
+ yy_state_t yystate = yystack.yytops.yystates[0]->yylrState;
YY_DPRINTF ((stderr, "Entering state %d\n", yystate));
if (yystate == YYFINAL)
goto yyacceptlab;
@@ -2381,15 +2493,16 @@
}
else
{
- yySymbol yytoken = ]b4_yygetToken_call;[
+ yysymbol_kind_t yytoken = ]b4_yygetToken_call;[
const short* yyconflicts;
int yyaction = yygetLRActions (yystate, yytoken, &yyconflicts);
- if (*yyconflicts != 0)
+ if (*yyconflicts)
+ /* Enter nondeterministic mode. */
break;
if (yyisShiftAction (yyaction))
{
YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc);
- yychar = YYEMPTY;
+ yychar = ]b4_symbol(empty, id)[;
yyposn += 1;
yyglrShift (&yystack, 0, yyaction, yyposn, &yylval]b4_locations_if([, &yylloc])[);
if (0 < yystack.yyerrState)
@@ -2397,11 +2510,11 @@
}
else if (yyisErrorAction (yyaction))
{]b4_locations_if([[
- yystack.yyerror_range[1].yystate.yyloc = yylloc;]])[]b4_glr_cc_if([[
- /* Don't issue an error message again for exceptions
- thrown from the scanner. */
- if (yychar != YYFAULTYTOK)
- ]])[ yyreportSyntaxError (&yystack]b4_user_args[);
+ yystack.yyerror_range[1].yystate.yyloc = yylloc;]])[
+ /* Issue an error message unless the scanner already
+ did. */
+ if (yychar != ]b4_symbol(error, id)[)
+ yyreportSyntaxError (&yystack]b4_user_args[);
goto yyuser_error;
}
else
@@ -2409,13 +2522,14 @@
}
}
+ /* Nondeterministic mode. */
while (yytrue)
{
- yySymbol yytoken_to_shift;
- ptrdiff_t yys;
+ yysymbol_kind_t yytoken_to_shift;
+ YYPTRDIFF_T yys;
for (yys = 0; yys < yystack.yytops.yysize; yys += 1)
- yystackp->yytops.yylookaheadNeeds[yys] = yychar != YYEMPTY;
+ yystackp->yytops.yylookaheadNeeds[yys] = yychar != ]b4_symbol(empty, id)[;
/* yyprocessOneStack returns one of three things:
@@ -2453,15 +2567,15 @@
/* If any yyglrShift call fails, it will fail after shifting. Thus,
a copy of yylval will already be on stack 0 in the event of a
- failure in the following loop. Thus, yychar is set to YYEMPTY
+ failure in the following loop. Thus, yychar is set to ]b4_symbol(empty, id)[
before the loop to make sure the user destructor for yylval isn't
called twice. */
yytoken_to_shift = YYTRANSLATE (yychar);
- yychar = YYEMPTY;
+ yychar = ]b4_symbol(empty, id)[;
yyposn += 1;
for (yys = 0; yys < yystack.yytops.yysize; yys += 1)
{
- yyStateNum yystate = yystack.yytops.yystates[yys]->yylrState;
+ yy_state_t yystate = yystack.yytops.yystates[yys]->yylrState;
const short* yyconflicts;
int yyaction = yygetLRActions (yystate, yytoken_to_shift,
&yyconflicts);
@@ -2470,7 +2584,7 @@
YY_SYMBOL_PRINT ("shifting", yytoken_to_shift, &yylval, &yylloc);
yyglrShift (&yystack, yys, yyaction, yyposn,
&yylval]b4_locations_if([, &yylloc])[);
- YY_DPRINTF ((stderr, "Stack %ld now in state #%d\n",
+ YY_DPRINTF ((stderr, "Stack %ld now in state %d\n",
YY_CAST (long, yys),
yystack.yytops.yystates[yys]->yylrState));
}
@@ -2491,7 +2605,7 @@
yyacceptlab:
yyresult = 0;
- goto yyreturn;
+ goto yyreturnlab;
yybuglab:
YY_ASSERT (yyfalse);
@@ -2499,15 +2613,15 @@
yyabortlab:
yyresult = 1;
- goto yyreturn;
+ goto yyreturnlab;
yyexhaustedlab:
yyerror (]b4_lyyerror_args[YY_("memory exhausted"));
yyresult = 2;
- goto yyreturn;
+ goto yyreturnlab;
- yyreturn:
- if (yychar != YYEMPTY)
+ yyreturnlab:
+ if (yychar != ]b4_symbol(empty, id)[)
yydestruct ("Cleanup: discarding lookahead",
YYTRANSLATE (yychar), &yylval]b4_locuser_args([&yylloc])[);
@@ -2519,8 +2633,8 @@
yyGLRState** yystates = yystack.yytops.yystates;
if (yystates)
{
- ptrdiff_t yysize = yystack.yytops.yysize;
- ptrdiff_t yyk;
+ YYPTRDIFF_T yysize = yystack.yytops.yysize;
+ YYPTRDIFF_T yyk;
for (yyk = 0; yyk < yysize; yyk += 1)
if (yystates[yyk])
{
@@ -2545,6 +2659,7 @@
/* DEBUGGING ONLY */
#if ]b4_api_PREFIX[DEBUG
+/* Print *YYS and its predecessors. */
static void
yy_yypstack (yyGLRState* yys)
{
@@ -2556,22 +2671,25 @@
YY_FPRINTF ((stderr, "%d@@%ld", yys->yylrState, YY_CAST (long, yys->yyposn)));
}
+/* Print YYS (possibly NULL) and its predecessors. */
static void
-yypstates (yyGLRState* yyst)
+yypstates (yyGLRState* yys)
{
- if (yyst == YY_NULLPTR)
+ if (yys == YY_NULLPTR)
YY_FPRINTF ((stderr, "<null>"));
else
- yy_yypstack (yyst);
+ yy_yypstack (yys);
YY_FPRINTF ((stderr, "\n"));
}
+/* Print the stack #YYK. */
static void
-yypstack (yyGLRStack* yystackp, ptrdiff_t yyk)
+yypstack (yyGLRStack* yystackp, YYPTRDIFF_T yyk)
{
yypstates (yystackp->yytops.yystates[yyk]);
}
+/* Print all the stacks. */
static void
yypdumpstack (yyGLRStack* yystackp)
{
@@ -2612,7 +2730,7 @@
YY_FPRINTF ((stderr, "Tops:"));
{
- ptrdiff_t yyi;
+ YYPTRDIFF_T yyi;
for (yyi = 0; yyi < yystackp->yytops.yysize; yyi += 1)
YY_FPRINTF ((stderr, "%ld: %ld; ", YY_CAST (long, yyi),
YYINDEX (yystackp->yytops.yystates[yyi])));
@@ -2638,5 +2756,8 @@
#define yynerrs ]b4_prefix[nerrs]b4_locations_if([[
#define yylloc ]b4_prefix[lloc]])])[
-]b4_epilogue[]dnl
+]b4_glr_cc_if([b4_glr_cc_pre_epilogue
+b4_glr_cc_cleanup])[
+]b4_percent_code_get([[epilogue]])[]dnl
+b4_epilogue[]dnl
b4_output_end
diff --git a/common/bison/skeletons/glr.cc b/common/bison/skeletons/glr.cc
index 343b52e..7181402 100644
--- a/common/bison/skeletons/glr.cc
+++ b/common/bison/skeletons/glr.cc
@@ -1,6 +1,6 @@
# C++ GLR skeleton for Bison
-# Copyright (C) 2002-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
# This skeleton produces a C++ class that encapsulates a C glr parser.
@@ -72,24 +72,17 @@
# Bypass the default implementation to generate the "yy_symbol_print"
# and "yy_symbol_value_print" functions.
m4_define([b4_yy_symbol_print_define],
-[[
-/*--------------------.
+[[/*--------------------.
| Print this symbol. |
`--------------------*/
-]b4_function_define([yy_symbol_print],
- [static void],
- [[FILE *], []],
- [[int yytype], [yytype]],
- [[const ]b4_namespace_ref::b4_parser_class[::semantic_type *yyvaluep],
- [yyvaluep]][]dnl
-b4_locations_if([,
- [[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp],
- [yylocationp]]]),
- b4_parse_param)[
+static void
+yy_symbol_print (FILE *, ]b4_namespace_ref::b4_parser_class[::symbol_kind_type yytoken,
+ const ]b4_namespace_ref::b4_parser_class[::value_type *yyvaluep]b4_locations_if([[,
+ const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp]])[]b4_user_formals[)
{
]b4_parse_param_use[]dnl
-[ yyparser.yy_symbol_print_ (yytype, yyvaluep]b4_locations_if([, yylocationp])[);
+[ yyparser.yy_symbol_print_ (yytoken, yyvaluep]b4_locations_if([, yylocationp])[);
}
]])[
@@ -101,45 +94,24 @@
# Hijack the post prologue to declare yyerror.
]m4_append([b4_post_prologue],
[b4_syncline([@oline@], [@ofile@])dnl
-b4_function_declare([yyerror],
- [static void],b4_locations_if([
- [[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp],
- [yylocationp]],])
- b4_parse_param,
- [[const char* msg], [msg]])])[
+[static void
+yyerror (]b4_locations_if([[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp,
+ ]])[]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param),
+ ])[const char* msg);]])[
-
-#undef yynerrs
-#undef yychar
-#undef yylval]b4_locations_if([
-#undef yylloc])
-
-m4_if(b4_prefix, [yy], [],
-[[/* Substitute the variable and function names. */
-#define yyparse ]b4_prefix[parse
-#define yylex ]b4_prefix[lex
-#define yyerror ]b4_prefix[error
-#define yydebug ]b4_prefix[debug]]b4_pure_if([], [[
-#define yylval ]b4_prefix[lval
-#define yychar ]b4_prefix[char
-#define yynerrs ]b4_prefix[nerrs]b4_locations_if([[
-#define yylloc ]b4_prefix[lloc]])]))
-
-# Hijack the epilogue to define implementations (yyerror, parser member
+# Inserted before the epilogue to define implementations (yyerror, parser member
# functions etc.).
-m4_append([b4_epilogue],
+]m4_define([b4_glr_cc_pre_epilogue],
[b4_syncline([@oline@], [@ofile@])dnl
[
/*------------------.
| Report an error. |
`------------------*/
-]b4_function_define([yyerror],
- [static void],b4_locations_if([
- [[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp],
- [yylocationp]],])
- b4_parse_param,
- [[const char* msg], [msg]])[
+static void
+yyerror (]b4_locations_if([[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp,
+ ]])[]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param),
+ ])[const char* msg)
{
]b4_parse_param_use[]dnl
[ yyparser.error (]b4_locations_if([[*yylocationp, ]])[msg);
@@ -172,7 +144,7 @@
int
]b4_parser_class[::parse ()
{
- return ::yyparse (*this]b4_user_args[);
+ return ::yy_parse_impl (*this]b4_user_args[);
}
#if ]b4_api_PREFIX[DEBUG
@@ -181,28 +153,28 @@
`--------------------*/
void
- ]b4_parser_class[::yy_symbol_value_print_ (int yytype,
- const semantic_type* yyvaluep]b4_locations_if([[,
- const location_type* yylocationp]])[)
+ ]b4_parser_class[::yy_symbol_value_print_ (symbol_kind_type yykind,
+ const value_type* yyvaluep]b4_locations_if([[,
+ const location_type* yylocationp]])[) const
{]b4_locations_if([[
- YYUSE (yylocationp);]])[
- YYUSE (yyvaluep);
+ YY_USE (yylocationp);]])[
+ YY_USE (yyvaluep);
std::ostream& yyo = debug_stream ();
std::ostream& yyoutput = yyo;
- YYUSE (yyoutput);
+ YY_USE (yyoutput);
]b4_symbol_actions([printer])[
}
void
- ]b4_parser_class[::yy_symbol_print_ (int yytype,
- const semantic_type* yyvaluep]b4_locations_if([[,
- const location_type* yylocationp]])[)
+ ]b4_parser_class[::yy_symbol_print_ (symbol_kind_type yykind,
+ const value_type* yyvaluep]b4_locations_if([[,
+ const location_type* yylocationp]])[) const
{
- *yycdebug_ << (yytype < YYNTOKENS ? "token" : "nterm")
- << ' ' << yytname[yytype] << " ("]b4_locations_if([[
+ *yycdebug_ << (yykind < YYNTOKENS ? "token" : "nterm")
+ << ' ' << yysymbol_name (yykind) << " ("]b4_locations_if([[
<< *yylocationp << ": "]])[;
- yy_symbol_value_print_ (yytype, yyvaluep]b4_locations_if([[, yylocationp]])[);
+ yy_symbol_value_print_ (yykind, yyvaluep]b4_locations_if([[, yylocationp]])[);
*yycdebug_ << ')';
}
@@ -238,23 +210,18 @@
])
-# b4_shared_declarations(hh|cc)
-# -----------------------------
-# Declaration that might either go into the header (if --defines, $1 = hh)
-# or in the implementation file.
-m4_define([b4_shared_declarations],
-[m4_pushdef([b4_parse_param], m4_defn([b4_parse_param_orig]))dnl
-b4_percent_code_get([[requires]])[
-#include <iostream>
-#include <stdexcept>
-#include <string>
+m4_define([b4_define_symbol_kind],
+[m4_format([#define %-15s %s],
+ b4_symbol($][1, kind_base),
+ b4_namespace_ref[::]b4_parser_class[::symbol_kind::]b4_symbol($1, kind_base))
+])
-]b4_cxx_portability[
-]m4_ifdef([b4_location_include],
- [[# include ]b4_location_include])[
-]b4_variant_if([b4_variant_includes])[
-
-]b4_attribute_define[
+# b4_glr_cc_setup
+# ---------------
+# Setup redirections for glr.c: Map the names used in c.m4 to the ones used
+# in c++.m4.
+m4_define([b4_glr_cc_setup],
+[[]b4_attribute_define[
]b4_null_define[
// This skeleton is based on C, yet compiles it as C++.
@@ -267,10 +234,67 @@
// On MacOS, PTRDIFF_MAX is defined as long long, which Clang's
// -pedantic reports as being a C++11 extension.
-#if defined __APPLE__ && YY_CPLUSPLUS < 201103L && 4 <= __clang_major__
+#if defined __APPLE__ && YY_CPLUSPLUS < 201103L \
+ && defined __clang__ && 4 <= __clang_major__
# pragma clang diagnostic ignored "-Wc++11-long-long"
#endif
+#undef ]b4_symbol(empty, [id])[
+#define ]b4_symbol(empty, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(empty, [id])[
+#undef ]b4_symbol(eof, [id])[
+#define ]b4_symbol(eof, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(eof, [id])[
+#undef ]b4_symbol(error, [id])[
+#define ]b4_symbol(error, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(error, [id])[
+
+#ifndef ]b4_api_PREFIX[STYPE
+# define ]b4_api_PREFIX[STYPE ]b4_namespace_ref[::]b4_parser_class[::value_type
+#endif
+#ifndef ]b4_api_PREFIX[LTYPE
+# define ]b4_api_PREFIX[LTYPE ]b4_namespace_ref[::]b4_parser_class[::location_type
+#endif
+
+typedef ]b4_namespace_ref[::]b4_parser_class[::symbol_kind_type yysymbol_kind_t;
+
+// Expose C++ symbol kinds to C.
+]b4_define_symbol_kind(-2)dnl
+b4_symbol_foreach([b4_define_symbol_kind])])[
+]])
+
+
+m4_define([b4_undef_symbol_kind],
+[[#undef ]b4_symbol($1, kind_base)[
+]])
+
+
+# b4_glr_cc_cleanup
+# -----------------
+# Remove redirections for glr.c.
+m4_define([b4_glr_cc_cleanup],
+[[#undef ]b4_symbol(empty, [id])[
+#undef ]b4_symbol(eof, [id])[
+#undef ]b4_symbol(error, [id])[
+
+]b4_undef_symbol_kind(-2)dnl
+b4_symbol_foreach([b4_undef_symbol_kind])dnl
+])
+
+
+# b4_shared_declarations(hh|cc)
+# -----------------------------
+# Declaration that might either go into the header (if --header, $1 = hh)
+# or in the implementation file.
+m4_define([b4_shared_declarations],
+[m4_pushdef([b4_parse_param], m4_defn([b4_parse_param_orig]))dnl
+b4_percent_code_get([[requires]])[
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+]b4_cxx_portability[
+]m4_ifdef([b4_location_include],
+ [[# include ]b4_location_include])[
+]b4_variant_if([b4_variant_includes])[
+
// Whether we are compiled with exception support.
#ifndef YY_EXCEPTIONS
# if defined __GNUC__ && !defined __EXCEPTIONS
@@ -327,53 +351,43 @@
# if ]b4_api_PREFIX[DEBUG
public:
/// \brief Report a symbol value on the debug stream.
- /// \param yytype The token type.
+ /// \param yykind The symbol kind.
/// \param yyvaluep Its semantic value.]b4_locations_if([[
/// \param yylocationp Its location.]])[
- virtual void yy_symbol_value_print_ (int yytype,
- const semantic_type* yyvaluep]b4_locations_if([[,
- const location_type* yylocationp]])[);
+ virtual void yy_symbol_value_print_ (symbol_kind_type yykind,
+ const value_type* yyvaluep]b4_locations_if([[,
+ const location_type* yylocationp]])[) const;
/// \brief Report a symbol on the debug stream.
- /// \param yytype The token type.
+ /// \param yykind The symbol kind.
/// \param yyvaluep Its semantic value.]b4_locations_if([[
/// \param yylocationp Its location.]])[
- virtual void yy_symbol_print_ (int yytype,
- const semantic_type* yyvaluep]b4_locations_if([[,
- const location_type* yylocationp]])[);
+ virtual void yy_symbol_print_ (symbol_kind_type yykind,
+ const value_type* yyvaluep]b4_locations_if([[,
+ const location_type* yylocationp]])[) const;
private:
- // Debugging.
+ /// Debug stream.
std::ostream* yycdebug_;
#endif
]b4_parse_param_vars[
};
-]dnl Redirections for glr.c.
-b4_percent_define_flag_if([[global_tokens_and_yystype]],
-[b4_token_defines])
-[
-#ifndef ]b4_api_PREFIX[STYPE
-# define ]b4_api_PREFIX[STYPE ]b4_namespace_ref[::]b4_parser_class[::semantic_type
-#endif
-#ifndef ]b4_api_PREFIX[LTYPE
-# define ]b4_api_PREFIX[LTYPE ]b4_namespace_ref[::]b4_parser_class[::location_type
-#endif
-
]b4_namespace_close[
+
]b4_percent_code_get([[provides]])[
]m4_popdef([b4_parse_param])dnl
-])
+])[
-b4_defines_if(
+]b4_header_if(
[b4_output_begin([b4_spec_header_file])
b4_copyright([Skeleton interface for Bison GLR parsers in C++],
- [2002-2015, 2018-2019])[
+ [2002-2015, 2018-2021])[
// C++ GLR parser skeleton written by Akim Demaille.
]b4_disclaimer[
-]b4_cpp_guard_open([b4_spec_header_file])[
+]b4_cpp_guard_open([b4_spec_mapped_header_file])[
]b4_shared_declarations[
-]b4_cpp_guard_close([b4_spec_header_file])[
+]b4_cpp_guard_close([b4_spec_mapped_header_file])[
]b4_output_end])
# Let glr.c (and b4_shared_declarations) believe that the user
diff --git a/common/bison/skeletons/glr2.cc b/common/bison/skeletons/glr2.cc
new file mode 100644
index 0000000..757d68d
--- /dev/null
+++ b/common/bison/skeletons/glr2.cc
@@ -0,0 +1,3533 @@
+# C++ GLR skeleton for Bison
+
+# Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+m4_include(b4_skeletonsdir/[c++.m4])
+
+# api.value.type=variant is valid.
+m4_define([b4_value_type_setup_variant])
+
+# b4_tname_if(TNAME-NEEDED, TNAME-NOT-NEEDED)
+# -------------------------------------------
+m4_define([b4_tname_if],
+[m4_case(b4_percent_define_get([[parse.error]]),
+ [verbose], [$1],
+ [b4_token_table_if([$1],
+ [$2])])])
+
+b4_bison_locations_if([
+ m4_define([b4_location_constructors])
+ m4_include(b4_skeletonsdir/[location.cc])])
+b4_variant_if([m4_include(b4_skeletonsdir/[variant.hh])])
+
+m4_define([b4_parser_class],
+ [b4_percent_define_get([[api.parser.class]])])
+
+]m4_define([b4_define_symbol_kind],
+[m4_format([#define %-15s %s],
+ b4_symbol($][1, kind_base),
+ b4_namespace_ref[::]b4_parser_class[::symbol_kind::]b4_symbol($1, kind_base))
+])
+
+
+# b4_integral_parser_table_define(TABLE-NAME, CONTENT, COMMENT)
+# -------------------------------------------------------------
+# Define "yy<TABLE-NAME>" whose contents is CONTENT. Does not use "static",
+# should be in unnamed namespace.
+m4_define([b4_integral_parser_table_define],
+[m4_ifvaln([$3], [ b4_comment([$3])])dnl
+ const b4_int_type_for([$2]) yy$1[[]] =
+ {
+ $2
+ };dnl
+])
+
+
+## ---------------- ##
+## Default values. ##
+## ---------------- ##
+
+# Stack parameters.
+m4_define_default([b4_stack_depth_max], [10000])
+m4_define_default([b4_stack_depth_init], [200])
+
+
+
+## ------------ ##
+## Interfaces. ##
+## ------------ ##
+
+# b4_user_formals
+# ---------------
+# The possible parse-params formal arguments preceded by a comma.
+#
+# This is not shared with yacc.c in c.m4 because GLR relies on ISO C
+# formal argument declarations.
+m4_define([b4_user_formals],
+[m4_ifset([b4_parse_param], [, b4_formals(b4_parse_param)])])
+
+
+# b4_symbol_kind(NUM)
+# -------------------
+m4_define([b4_symbol_kind],
+[symbol_kind::b4_symbol_kind_base($@)])
+
+
+## ----------------- ##
+## Semantic Values. ##
+## ----------------- ##
+
+
+# b4_lhs_value(SYMBOL-NUM, [TYPE])
+# --------------------------------
+# See README.
+m4_define([b4_lhs_value],
+[b4_symbol_value([(*yyvalp)], [$1], [$2])])
+
+
+# b4_rhs_data(RULE-LENGTH, POS)
+# -----------------------------
+# See README.
+m4_define([b4_rhs_data],
+[(static_cast<glr_stack_item const *>(yyvsp))@{YYFILL (b4_subtract([$2], [$1]))@}.getState()])
+
+
+# b4_rhs_value(RULE-LENGTH, POS, SYMBOL-NUM, [TYPE])
+# --------------------------------------------------
+# Expansion of $$ or $<TYPE>$, for symbol SYMBOL-NUM.
+m4_define([b4_rhs_value],
+[b4_symbol_value([b4_rhs_data([$1], [$2]).value ()], [$3], [$4])])
+
+
+
+## ----------- ##
+## Locations. ##
+## ----------- ##
+
+# b4_lhs_location()
+# -----------------
+# Expansion of @$.
+m4_define([b4_lhs_location],
+[(*yylocp)])
+
+
+# b4_rhs_location(RULE-LENGTH, NUM)
+# ---------------------------------
+# Expansion of @NUM, where the current rule has RULE-LENGTH symbols
+# on RHS.
+m4_define([b4_rhs_location],
+[(b4_rhs_data([$1], [$2]).yyloc)])
+
+
+# b4_symbol_action(SYMBOL-NUM, KIND)
+# ----------------------------------
+# Run the action KIND (destructor or printer) for SYMBOL-NUM.
+# Same as in C, but using references instead of pointers.
+#
+# Currently we need two different b4_symbol_action: once for the
+# self-contained symbols, and another time for yy_destroy_ and
+# yy_symbol_value_print_, which don't use genuine symbols yet.
+m4_define([b4_symbol_action],
+[b4_symbol_if([$1], [has_$2],
+[m4_pushdef([b4_symbol_value], m4_defn([b4_symbol_value_template]))[]dnl
+b4_dollar_pushdef([yysym.value],
+ [$1],
+ [],
+ [yysym.location])dnl
+ _b4_symbol_case([$1])[]dnl
+b4_syncline([b4_symbol([$1], [$2_line])], [b4_symbol([$1], [$2_file])])dnl
+ b4_symbol([$1], [$2])
+b4_syncline([@oline@], [@ofile@])dnl
+ break;
+
+m4_popdef([b4_symbol_value])[]dnl
+b4_dollar_popdef[]dnl
+])])
+
+
+# b4_symbol_action_for_yyval(SYMBOL-NUM, KIND)
+# --------------------------------------------
+# Run the action KIND (destructor or printer) for SYMBOL-NUM.
+# Same as in C, but using references instead of pointers.
+m4_define([b4_symbol_action_for_yyval],
+[b4_symbol_if([$1], [has_$2],
+[b4_dollar_pushdef([yyval],
+ [$1],
+ [],
+ [yyloc])dnl
+ _b4_symbol_case([$1])[]dnl
+b4_syncline([b4_symbol([$1], [$2_line])], [b4_symbol([$1], [$2_file])])dnl
+ b4_symbol([$1], [$2])
+b4_syncline([@oline@], [@ofile@])dnl
+ break;
+
+b4_dollar_popdef[]dnl
+])])
+
+
+# b4_call_merger(MERGER-NUM, MERGER-NAME, SYMBOL-SUM)
+# ---------------------------------------------------
+m4_define([b4_call_merger],
+[b4_case([$1],
+ [ b4_symbol_if([$3], [has_type],
+ [b4_variant_if([yy0.as< b4_symbol($3, type) > () = $2 (yy0.as< b4_symbol($3, type) >(), yy1.as< b4_symbol($3, type) >());],
+ [yy0.b4_symbol($3, slot) = $2 (yy0, yy1);])],
+ [yy0 = $2 (yy0, yy1);])])])
+
+# b4_yylex
+# --------
+# Call yylex.
+m4_define([b4_yylex],
+[b4_token_ctor_if(
+[b4_function_call([yylex],
+ [symbol_type], m4_ifdef([b4_lex_param], b4_lex_param))],
+[b4_function_call([yylex], [int],
+ [[value_type *], [&this->yyla.value]][]dnl
+b4_locations_if([, [[location_type *], [&this->yyla.location]]])dnl
+m4_ifdef([b4_lex_param], [, ]b4_lex_param))])])
+
+
+# b4_shared_declarations(hh|cc)
+# -----------------------------
+# Declaration that might either go into the header (if --header, $1 = hh)
+# or in the implementation file.
+m4_define([b4_shared_declarations],
+[b4_percent_code_get([[requires]])[
+#include <algorithm>
+#include <cstddef> // ptrdiff_t
+#include <cstring> // memcpy
+#include <iostream>
+#include <iomanip>
+#include <limits>
+#include <stdexcept>
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+]b4_cxx_portability[
+]m4_ifdef([b4_location_include],
+ [[# include ]b4_location_include])[
+]b4_variant_if([b4_variant_includes])[
+
+]b4_YYDEBUG_define[
+
+]b4_namespace_open[
+
+]b4_bison_locations_if([m4_ifndef([b4_location_file],
+ [b4_location_define])])[
+
+ /// A Bison parser.
+ class ]b4_parser_class[
+ {
+ public:
+]b4_public_types_declare[
+]b4_symbol_type_define[
+
+ // FIXME: should be private eventually.
+ class glr_stack;
+ class glr_state;
+
+ /// Build a parser object.
+ ]b4_parser_class[ (]b4_parse_param_decl[);
+ ~]b4_parser_class[ ();
+
+ /// Parse. An alias for parse ().
+ /// \returns 0 iff parsing succeeded.
+ int operator() ();
+
+ /// Parse.
+ /// \returns 0 iff parsing succeeded.
+ int parse ();
+
+#if ]b4_api_PREFIX[DEBUG
+ /// The current debugging stream.
+ std::ostream& debug_stream () const;
+ /// Set the current debugging stream.
+ void set_debug_stream (std::ostream &);
+
+ /// Type for debugging levels.
+ using debug_level_type = int;
+ /// The current debugging level.
+ debug_level_type debug_level () const;
+ /// Set the current debugging level.
+ void set_debug_level (debug_level_type l);
+#endif
+
+ /// Report a syntax error.]b4_locations_if([[
+ /// \param loc where the syntax error is found.]])[
+ /// \param msg a description of the syntax error.
+ void error (]b4_locations_if([[const location_type& loc, ]])[const std::string& msg);
+
+]b4_parse_error_bmatch(
+[custom\|detailed],
+[[ /// The user-facing name of the symbol whose (internal) number is
+ /// YYSYMBOL. No bounds checking.
+ static const char *symbol_name (symbol_kind_type yysymbol);]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+ /// The user-facing name of the symbol whose (internal) number is
+ /// YYSYMBOL. No bounds checking.
+ static const char *symbol_name (symbol_kind_type yysymbol);
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[ /// The user-facing name of the symbol whose (internal) number is
+ /// YYSYMBOL. No bounds checking.
+ static std::string symbol_name (symbol_kind_type yysymbol);]])[
+
+]b4_token_constructor_define[
+]b4_parse_error_bmatch([custom\|detailed\|verbose], [[
+ class context
+ {
+ public:
+ context (glr_stack& yystack, const symbol_type& yyla);
+ const symbol_type& lookahead () const YY_NOEXCEPT { return yyla_; }
+ symbol_kind_type token () const YY_NOEXCEPT { return yyla_.kind (); }]b4_locations_if([[
+ const location_type& location () const YY_NOEXCEPT { return yyla_.location; }
+]])[
+ /// Put in YYARG at most YYARGN of the expected tokens, and return the
+ /// number of tokens stored in YYARG. If YYARG is null, return the
+ /// number of expected tokens (guaranteed to be less than YYNTOKENS).
+ int expected_tokens (symbol_kind_type yyarg[], int yyargn) const;
+
+ private:
+ glr_stack& yystack_;
+ const symbol_type& yyla_;
+ };
+]])[
+# if ]b4_api_PREFIX[DEBUG
+ public:
+ /// \brief Report a symbol value on the debug stream.
+ /// \param yykind The symbol kind.
+ /// \param yyval Its semantic value.]b4_locations_if([[
+ /// \param yyloc Its location.]])[
+ void yy_symbol_value_print_ (symbol_kind_type yykind,
+ const value_type& yyval]b4_locations_if([[,
+ const location_type& yyloc]])[) const;
+ /// \brief Report a symbol on the debug stream.
+ /// \param yykind The symbol kind.
+ /// \param yyval Its semantic value.]b4_locations_if([[
+ /// \param yyloc Its location.]])[
+ void yy_symbol_print_ (symbol_kind_type yykind,
+ const value_type& yyval]b4_locations_if([[,
+ const location_type& yyloc]])[) const;
+ private:
+ /// Debug stream.
+ std::ostream* yycdebug_;
+#endif
+
+]b4_parse_error_bmatch(
+[custom], [[
+ private:
+ /// Report a syntax error
+ /// \param yyctx the context in which the error occurred.
+ void report_syntax_error (const context& yyctx) const;]],
+[detailed\|verbose], [[
+ private:
+ /// The arguments of the error message.
+ int yy_syntax_error_arguments_ (const context& yyctx,
+ symbol_kind_type yyarg[], int yyargn) const;
+
+ /// Generate an error message.
+ /// \param yyctx the context in which the error occurred.
+ virtual std::string yysyntax_error_ (const context& yyctx) const;]])[
+
+ /// Convert a scanner token kind \a t to a symbol kind.
+ /// In theory \a t should be a token_kind_type, but character literals
+ /// are valid, yet not members of the token_kind_type enum.
+ static symbol_kind_type yytranslate_ (int t) YY_NOEXCEPT;
+
+]b4_parse_error_bmatch(
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+ /// For a symbol, its name in clear.
+ static const char* const yytname_[];
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[ /// Convert the symbol name \a n to a form suitable for a diagnostic.
+ static std::string yytnamerr_ (const char *yystr);
+
+ /// For a symbol, its name in clear.
+ static const char* const yytname_[];
+]])[
+
+ /// \brief Reclaim the memory associated to a symbol.
+ /// \param yymsg Why this token is reclaimed.
+ /// If null, print nothing.
+ /// \param yykind The symbol kind.
+ void yy_destroy_ (const char* yymsg, symbol_kind_type yykind,
+ value_type& yyval]b4_locations_if([[,
+ location_type& yyloc]])[);
+
+]b4_parse_param_vars[
+ // Needs access to yy_destroy_, report_syntax_error, etc.
+ friend glr_stack;
+ };
+
+]b4_token_ctor_if([b4_yytranslate_define([$1])[
+]b4_public_types_define([$1])])[
+]b4_namespace_close[
+
+]b4_percent_code_get([[provides]])[
+]])[
+
+
+## -------------- ##
+## Output files. ##
+## -------------- ##
+
+
+# ------------- #
+# Header file. #
+# ------------- #
+
+]b4_header_if([[
+]b4_output_begin([b4_spec_header_file])[
+]b4_copyright([Skeleton interface for Bison GLR parsers in C++],
+ [2002-2015, 2018-2021])[
+// C++ GLR parser skeleton written by Valentin Tolmer.
+
+]b4_disclaimer[
+]b4_cpp_guard_open([b4_spec_mapped_header_file])[
+]b4_shared_declarations([hh])[
+]b4_cpp_guard_close([b4_spec_mapped_header_file])[
+]b4_output_end])[
+
+
+# --------------------- #
+# Implementation file. #
+# --------------------- #
+
+]b4_output_begin([b4_parser_file_name])[
+]b4_copyright([Skeleton implementation for Bison GLR parsers in C],
+ [2002-2015, 2018-2021])[
+// C++ GLR parser skeleton written by Valentin Tolmer.
+
+]b4_disclaimer[
+]b4_identification[
+
+]b4_percent_code_get([[top]])[
+]m4_if(b4_prefix, [yy], [],
+[[/* Substitute the variable and function names. */
+#define yyparse ]b4_prefix[parse
+#define yylex ]b4_prefix[lex
+#define yyerror ]b4_prefix[error
+#define yydebug ]b4_prefix[debug]])[
+
+]b4_user_pre_prologue[
+
+]b4_null_define[
+
+]b4_header_if([[#include "@basename(]b4_spec_header_file[@)"]],
+ [b4_shared_declarations([cc])])[
+
+namespace
+{
+ /* Default (constant) value used for initialization for null
+ right-hand sides. Unlike the standard yacc.c template, here we set
+ the default value of $$ to a zeroed-out value. Since the default
+ value is undefined, this behavior is technically correct. */
+ ]b4_namespace_ref[::]b4_parser_class[::value_type yyval_default;
+}
+
+]b4_user_post_prologue[
+]b4_percent_code_get[
+
+#include <cstdio>
+#include <cstdlib>
+
+#ifndef YY_
+# if defined YYENABLE_NLS && YYENABLE_NLS
+# if ENABLE_NLS
+# include <libintl.h> /* INFRINGES ON USER NAME SPACE */
+# define YY_(Msgid) dgettext ("bison-runtime", Msgid)
+# endif
+# endif
+# ifndef YY_
+# define YY_(Msgid) Msgid
+# endif
+#endif
+
+// Whether we are compiled with exception support.
+#ifndef YY_EXCEPTIONS
+# if defined __GNUC__ && !defined __EXCEPTIONS
+# define YY_EXCEPTIONS 0
+# else
+# define YY_EXCEPTIONS 1
+# endif
+#endif
+
+#ifndef YYFREE
+# define YYFREE free
+#endif
+#ifndef YYMALLOC
+# define YYMALLOC malloc
+#endif
+
+#ifndef YYSETJMP
+# include <setjmp.h>
+# define YYJMP_BUF jmp_buf
+# define YYSETJMP(Env) setjmp (Env)
+/* Pacify Clang and ICC. */
+# define YYLONGJMP(Env, Val) \
+ do { \
+ longjmp (Env, Val); \
+ YYASSERT (0); \
+ } while (false)
+#endif
+
+]b4_attribute_define([noreturn])[
+
+#if defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__
+# define YY_IGNORE_NULL_DEREFERENCE_BEGIN \
+ _Pragma ("GCC diagnostic push") \
+ _Pragma ("GCC diagnostic ignored \"-Wnull-dereference\"")
+# define YY_IGNORE_NULL_DEREFERENCE_END \
+ _Pragma ("GCC diagnostic pop")
+#else
+# define YY_IGNORE_NULL_DEREFERENCE_BEGIN
+# define YY_IGNORE_NULL_DEREFERENCE_END
+#endif
+
+]b4_null_define[
+]b4_cast_define[
+
+// FIXME: Use the same conventions as lalr1.cc.
+]b4_parse_assert_if[
+#ifndef YYASSERT
+# define YYASSERT(Condition) ((void) ((Condition) || (abort (), 0)))
+#endif
+
+#ifdef YYDEBUG
+# define YYDASSERT(Condition) YYASSERT(Condition)
+#else
+# define YYDASSERT(Condition)
+#endif
+
+/* YYFINAL -- State number of the termination state. */
+#define YYFINAL ]b4_final_state_number[
+/* YYLAST -- Last index in YYTABLE. */
+#define YYLAST ]b4_last[
+
+/* YYNTOKENS -- Number of terminals. */
+#define YYNTOKENS ]b4_tokens_number[
+/* YYNNTS -- Number of nonterminals. */
+#define YYNNTS ]b4_nterms_number[
+/* YYNRULES -- Number of rules. */
+#define YYNRULES ]b4_rules_number[
+/* YYNSTATES -- Number of states. */
+#define YYNSTATES ]b4_states_number[
+/* YYMAXRHS -- Maximum number of symbols on right-hand side of rule. */
+#define YYMAXRHS ]b4_r2_max[
+/* YYMAXLEFT -- Maximum number of symbols to the left of a handle
+ accessed by $0, $-1, etc., in any rule. */
+#define YYMAXLEFT ]b4_max_left_semantic_context[
+
+namespace
+{
+#if ]b4_api_PREFIX[DEBUG
+ /* YYRLINE[YYN] -- source line where rule number YYN was defined. */
+ const ]b4_int_type_for([b4_rline])[ yyrline[] =
+ {
+ ]b4_rline[
+ };
+#endif
+
+#define YYPACT_NINF ]b4_pact_ninf[
+#define YYTABLE_NINF ]b4_table_ninf[
+
+]b4_parser_tables_define[
+
+ /* YYDPREC[RULE-NUM] -- Dynamic precedence of rule #RULE-NUM (0 if none). */
+ const ]b4_int_type_for([b4_dprec])[ yydprec[] =
+ {
+ ]b4_dprec[
+ };
+
+ /* YYMERGER[RULE-NUM] -- Index of merging function for rule #RULE-NUM. */
+ const ]b4_int_type_for([b4_merger])[ yymerger[] =
+ {
+ ]b4_merger[
+ };
+
+ /* YYIMMEDIATE[RULE-NUM] -- True iff rule #RULE-NUM is not to be deferred, as
+ in the case of predicates. */
+ const bool yyimmediate[] =
+ {
+ ]b4_immediate[
+ };
+
+ /* YYCONFLP[YYPACT[STATE-NUM]] -- Pointer into YYCONFL of start of
+ list of conflicting reductions corresponding to action entry for
+ state STATE-NUM in yytable. 0 means no conflicts. The list in
+ yyconfl is terminated by a rule number of 0. */
+ const ]b4_int_type_for([b4_conflict_list_heads])[ yyconflp[] =
+ {
+ ]b4_conflict_list_heads[
+ };
+
+ /* YYCONFL[I] -- lists of conflicting rule numbers, each terminated by
+ 0, pointed into by YYCONFLP. */
+ ]dnl Do not use b4_int_type_for here, since there are places where
+ dnl pointers onto yyconfl are taken, whose type is "short*".
+ dnl We probably ought to introduce a type for confl.
+ [const short yyconfl[] =
+ {
+ ]b4_conflicting_rules[
+ };
+} // namespace
+
+
+/* Error token number */
+#define YYTERROR 1
+
+]b4_locations_if([[
+]b4_yylloc_default_define[
+# define YYRHSLOC(Rhs, K) ((Rhs)[K].getState().yyloc)
+]])[
+
+enum YYRESULTTAG { yyok, yyaccept, yyabort, yyerr };
+
+#define YYCHK(YYE) \
+ do { \
+ YYRESULTTAG yychk_flag = YYE; \
+ if (yychk_flag != yyok) \
+ return yychk_flag; \
+ } while (false)
+
+#if ]b4_api_PREFIX[DEBUG
+
+#define YYCDEBUG if (!yydebug) {} else std::cerr
+
+# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) \
+ do { \
+ if (yydebug) \
+ { \
+ std::cerr << Title << ' '; \
+ yyparser.yy_symbol_print_ (Kind, Value]b4_locations_if([, Location])[); \
+ std::cerr << '\n'; \
+ } \
+ } while (false)
+
+# define YY_REDUCE_PRINT(Args) \
+ do { \
+ if (yydebug) \
+ yystateStack.yy_reduce_print Args; \
+ } while (false)
+
+/* Nonzero means print parse trace. It is left uninitialized so that
+ multiple parsers can coexist. */
+int yydebug;
+
+namespace
+{
+ using glr_stack = ]b4_namespace_ref[::]b4_parser_class[::glr_stack;
+ using glr_state = ]b4_namespace_ref[::]b4_parser_class[::glr_state;
+
+ void yypstack (const glr_stack& yystack, size_t yyk)
+ YY_ATTRIBUTE_UNUSED;
+ void yypdumpstack (const glr_stack& yystack)
+ YY_ATTRIBUTE_UNUSED;
+}
+
+#else /* !]b4_api_PREFIX[DEBUG */
+
+# define YYCDEBUG if (true) {} else std::cerr
+# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) {}
+# define YY_REDUCE_PRINT(Args) {}
+
+#endif /* !]b4_api_PREFIX[DEBUG */
+
+/* YYINITDEPTH -- initial size of the parser's stacks. */
+#ifndef YYINITDEPTH
+# define YYINITDEPTH ]b4_stack_depth_init[
+#endif
+
+/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only
+ if the built-in stack extension method is used).
+
+ Do not make this value too large; the results are undefined if
+ SIZE_MAX < YYMAXDEPTH * sizeof (GLRStackItem)
+ evaluated with infinite-precision integer arithmetic. */
+
+#ifndef YYMAXDEPTH
+# define YYMAXDEPTH ]b4_stack_depth_max[
+#endif
+
+/* Minimum number of free items on the stack allowed after an
+ allocation. This is to allow allocation and initialization
+ to be completed by functions that call yyexpandGLRStack before the
+ stack is expanded, thus insuring that all necessary pointers get
+ properly redirected to new data. */
+#define YYHEADROOM 2
+
+#ifndef YYSTACKEXPANDABLE
+# define YYSTACKEXPANDABLE 1
+#endif
+
+namespace
+{
+ template <typename Parameter>
+ class strong_index_alias
+ {
+ public:
+ static strong_index_alias create (std::ptrdiff_t value)
+ {
+ strong_index_alias result;
+ result.value_ = value;
+ return result;
+ }
+
+ std::ptrdiff_t const& get () const { return value_; }
+
+ size_t uget () const { return static_cast<size_t> (value_); }
+
+ strong_index_alias operator+ (std::ptrdiff_t other) const
+ {
+ return strong_index_alias (get () + other);
+ }
+
+ void operator+= (std::ptrdiff_t other)
+ {
+ value_ += other;
+ }
+
+ strong_index_alias operator- (std::ptrdiff_t other)
+ {
+ return strong_index_alias (get () - other);
+ }
+
+ void operator-= (std::ptrdiff_t other)
+ {
+ value_ -= other;
+ }
+
+ size_t operator- (strong_index_alias other)
+ {
+ return strong_index_alias (get () - other.get ());
+ }
+
+ strong_index_alias& operator++ ()
+ {
+ ++value_;
+ return *this;
+ }
+
+ bool isValid () const
+ {
+ return value_ != INVALID_INDEX;
+ }
+
+ void setInvalid()
+ {
+ value_ = INVALID_INDEX;
+ }
+
+ bool operator== (strong_index_alias other)
+ {
+ return get () == other.get ();
+ }
+
+ bool operator!= (strong_index_alias other)
+ {
+ return get () != other.get ();
+ }
+
+ bool operator< (strong_index_alias other)
+ {
+ return get () < other.get ();
+ }
+
+ private:
+ static const std::ptrdiff_t INVALID_INDEX;
+
+ // WARNING: 0-initialized.
+ std::ptrdiff_t value_;
+ }; // class strong_index_alias
+
+ template<typename T>
+ const std::ptrdiff_t strong_index_alias<T>::INVALID_INDEX =
+ std::numeric_limits<std::ptrdiff_t>::max ();
+
+ using state_set_index = strong_index_alias<struct glr_state_set_tag>;
+
+ state_set_index create_state_set_index (std::ptrdiff_t value)
+ {
+ return state_set_index::create (value);
+ }
+
+ /** State numbers, as in LALR(1) machine */
+ using state_num = int;
+
+ /** Rule numbers, as in LALR(1) machine */
+ using rule_num = int;
+
+ using parser_type = ]b4_namespace_ref[::]b4_parser_class[;
+ using glr_state = parser_type::glr_state;
+ using symbol_kind = parser_type::symbol_kind;
+ using symbol_kind_type = parser_type::symbol_kind_type;
+ using symbol_type = parser_type::symbol_type;
+ using value_type = parser_type::value_type;]b4_locations_if([[
+ using location_type = parser_type::location_type;]])[
+
+ // Forward declarations.
+ class glr_stack_item;
+ class semantic_option;
+} // namespace
+
+namespace
+{
+ /** Accessing symbol of state YYSTATE. */
+ inline symbol_kind_type
+ yy_accessing_symbol (state_num yystate)
+ {
+ return YY_CAST (symbol_kind_type, yystos[yystate]);
+ }
+
+ /** Left-hand-side symbol for rule #YYRULE. */
+ inline symbol_kind_type
+ yylhsNonterm (rule_num yyrule)
+ {
+ return static_cast<symbol_kind_type>(yyr1[yyrule]);
+ }
+
+ /** Number of symbols composing the right hand side of rule #RULE. */
+ inline int
+ yyrhsLength (rule_num yyrule)
+ {
+ return yyr2[yyrule];
+ }
+}
+
+namespace ]b4_namespace_ref[
+{
+ class ]b4_parser_class[::glr_state
+ {
+ public:
+ glr_state ()
+ : yyresolved (false)
+ , yylrState (0)
+ , yyposn (0)
+ , yypred (0)
+ , yyfirstVal (0)]b4_locations_if([[
+ , yyloc ()]])[]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {}
+
+ /// Build with a semantic value.
+ glr_state (state_num lrState, size_t posn, const value_type& val]b4_locations_if([[, const location_type& loc]])[)
+ : yyresolved (true)
+ , yylrState (lrState)
+ , yyposn (posn)
+ , yypred (0)
+ , yyval (]b4_variant_if([], [[val]])[)]b4_locations_if([[
+ , yyloc (loc)]])[]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {]b4_variant_if([[
+ ]b4_symbol_variant([yy_accessing_symbol (lrState)],
+ [yyval], [copy], [val])])[}
+
+ /// Build with a semantic option.
+ glr_state (state_num lrState, size_t posn)
+ : yyresolved (false)
+ , yylrState (lrState)
+ , yyposn (posn)
+ , yypred (0)
+ , yyfirstVal (0)]b4_locations_if([[
+ , yyloc ()]])[]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {}
+
+ glr_state (const glr_state& other)
+ : yyresolved (other.yyresolved)
+ , yylrState (other.yylrState)
+ , yyposn (other.yyposn)
+ , yypred (0)]b4_locations_if([[
+ , yyloc (other.yyloc)]])[]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {
+ setPred (other.pred ());
+ if (other.yyresolved)]b4_variant_if([[
+ {
+ new (&yyval) value_type ();
+ ]b4_symbol_variant([yy_accessing_symbol (other.yylrState)],
+ [yyval], [copy], [other.value ()])[
+ }]], [[
+ new (&yyval) value_type (other.value ());]])[
+ else
+ {
+ yyfirstVal = 0;
+ setFirstVal (other.firstVal ());
+ }]b4_parse_assert_if([[
+ check_();]])[
+ }
+
+ ~glr_state ()
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ if (yyresolved)
+ {]b4_variant_if([[
+ symbol_kind_type yykind = yy_accessing_symbol (yylrState);
+ // FIXME: User destructors.
+ // Value type destructor.
+ ]b4_symbol_variant([[yykind]], [[yyval]], [[template destroy]])])[
+ yyval.~value_type ();
+ }]b4_parse_assert_if([[
+ magic_ = 0;]])[
+ }
+
+ glr_state& operator= (const glr_state& other)
+ {]b4_parse_assert_if([[
+ check_ ();
+ other.check_ ();]])[
+ if (!yyresolved && other.yyresolved)
+ new (&yyval) value_type;
+ yyresolved = other.yyresolved;
+ yylrState = other.yylrState;
+ yyposn = other.yyposn;
+ setPred (other.pred ());
+ if (other.yyresolved)]b4_variant_if([[
+ ]b4_symbol_variant([yy_accessing_symbol (other.yylrState)],
+ [yyval], [copy], [other.value ()])], [[
+ value () = other.value ();]])[
+ else
+ setFirstVal (other.firstVal ());]b4_locations_if([[
+ yyloc = other.yyloc;]])[
+ return *this;
+ }
+
+ /** Type tag for the semantic value. If true, yyval applies, otherwise
+ * yyfirstVal applies. */
+ bool yyresolved;
+ /** Number of corresponding LALR(1) machine state. */
+ state_num yylrState;
+ /** Source position of the last token produced by my symbol */
+ size_t yyposn;
+
+ /// Only call pred() and setPred() on objects in yyitems, not temporaries.
+ glr_state* pred ();
+ const glr_state* pred () const;
+ void setPred (const glr_state* state);
+
+ /// Only call firstVal() and setFirstVal() on objects in yyitems, not
+ /// temporaries.
+ semantic_option* firstVal ();
+ const semantic_option* firstVal () const;
+ void setFirstVal (const semantic_option* option);
+
+ value_type& value ()
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ return yyval;
+ }
+
+ const value_type& value () const
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ return yyval;
+ }
+
+ void
+ destroy (char const *yymsg, ]b4_namespace_ref[::]b4_parser_class[& yyparser);
+
+ /* DEBUGGING ONLY */
+ #if ]b4_api_PREFIX[DEBUG
+ void yy_yypstack () const
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ if (pred () != YY_NULLPTR)
+ {
+ pred ()->yy_yypstack ();
+ std::cerr << " -> ";
+ }
+ std::cerr << yylrState << "@@" << yyposn;
+ }
+ #endif
+
+ std::ptrdiff_t indexIn (const glr_stack_item* array) const YY_ATTRIBUTE_UNUSED;
+
+ glr_stack_item* asItem ()
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ return asItem(this);
+ }
+
+ const glr_stack_item* asItem () const
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ return asItem (this);
+ }
+
+ private:
+ template <typename T>
+ static const glr_stack_item* asItem (const T* state)
+ {
+ return reinterpret_cast<const glr_stack_item*>(state);
+ }
+ template <typename T>
+ static glr_stack_item* asItem (T* state)
+ {
+ return reinterpret_cast<glr_stack_item*> (state);
+ }
+ static const char *as_pointer_ (const glr_state *state)
+ {
+ return reinterpret_cast<const char *> (state);
+ }
+ static char *as_pointer_ (glr_state *state)
+ {
+ return reinterpret_cast<char *> (state);
+ }
+ /** Preceding state in this stack */
+ std::ptrdiff_t yypred;
+ union {
+ /** First in a chain of alternative reductions producing the
+ * nonterminal corresponding to this state, threaded through
+ * yyfirstVal. Value "0" means empty. */
+ std::ptrdiff_t yyfirstVal;
+ /** Semantic value for this state. */
+ value_type yyval;
+ };]b4_locations_if([[
+ // FIXME: Why public?
+ public:
+ /** Source location for this state. */
+ location_type yyloc;]])[
+
+]b4_parse_assert_if([[
+ public:
+ // Check invariants.
+ void check_ () const
+ {
+ YY_IGNORE_NULL_DEREFERENCE_BEGIN
+ YYASSERT (this->magic_ == MAGIC);
+ YY_IGNORE_NULL_DEREFERENCE_END
+ }
+
+ // A magic number to check our pointer arithmetic is sane.
+ enum { MAGIC = 713705 };
+ unsigned int magic_;]])[
+ }; // class ]b4_parser_class[::glr_state
+} // namespace ]b4_namespace_ref[
+
+
+namespace
+{
+ /** A stack of GLRState representing the different heads during
+ * nondeterministic evaluation. */
+ class glr_state_set
+ {
+ public:
+ /** Initialize YYSET to a singleton set containing an empty stack. */
+ glr_state_set ()
+ : yylastDeleted (YY_NULLPTR)
+ {
+ yystates.push_back (YY_NULLPTR);
+ yylookaheadNeeds.push_back (false);
+ }
+
+ // Behave like a vector of states.
+ glr_state*& operator[] (state_set_index index)
+ {
+ return yystates[index.uget()];
+ }
+
+ glr_state* operator[] (state_set_index index) const
+ {
+ return yystates[index.uget()];
+ }
+
+ size_t size () const
+ {
+ return yystates.size ();
+ }
+
+ std::vector<glr_state*>::iterator begin ()
+ {
+ return yystates.begin ();
+ }
+
+ std::vector<glr_state*>::iterator end ()
+ {
+ return yystates.end ();
+ }
+
+ bool lookaheadNeeds (state_set_index index) const
+ {
+ return yylookaheadNeeds[index.uget ()];
+ }
+
+ bool setLookaheadNeeds (state_set_index index, bool value)
+ {
+ return yylookaheadNeeds[index.uget ()] = value;
+ }
+
+ /** Invalidate stack #YYK. */
+ void
+ yymarkStackDeleted (state_set_index yyk)
+ {
+ size_t k = yyk.uget ();
+ if (yystates[k] != YY_NULLPTR)
+ yylastDeleted = yystates[k];
+ yystates[k] = YY_NULLPTR;
+ }
+
+ /** Undelete the last stack in *this that was marked as deleted. Can
+ only be done once after a deletion, and only when all other stacks have
+ been deleted. */
+ void
+ yyundeleteLastStack ()
+ {
+ if (yylastDeleted == YY_NULLPTR || !yystates.empty ())
+ return;
+ yystates.push_back (yylastDeleted);
+ YYCDEBUG << "Restoring last deleted stack as stack #0.\n";
+ clearLastDeleted ();
+ }
+
+ /** Remove the dead stacks (yystates[i] == YY_NULLPTR) and shift the later
+ * ones. */
+ void
+ yyremoveDeletes ()
+ {
+ size_t newsize = yystates.size ();
+ /* j is the number of live stacks we have seen. */
+ for (size_t i = 0, j = 0; j < newsize; ++i)
+ {
+ if (yystates[i] == YY_NULLPTR)
+ {
+ if (i == j)
+ {
+ YYCDEBUG << "Removing dead stacks.\n";
+ }
+ newsize -= 1;
+ }
+ else
+ {
+ yystates[j] = yystates[i];
+ /* In the current implementation, it's unnecessary to copy
+ yylookaheadNeeds[i] since, after
+ yyremoveDeletes returns, the parser immediately either enters
+ deterministic operation or shifts a token. However, it doesn't
+ hurt, and the code might evolve to need it. */
+ yylookaheadNeeds[j] = yylookaheadNeeds[i];
+ if (j != i)
+ {
+ YYCDEBUG << "Rename stack " << i << " -> " << j << ".\n";
+ }
+ j += 1;
+ }
+ }
+ yystates.resize (newsize);
+ yylookaheadNeeds.resize (newsize);
+ }
+
+
+ state_set_index
+ yysplitStack (state_set_index yyk)
+ {
+ const size_t k = yyk.uget ();
+ yystates.push_back (yystates[k]);
+ yylookaheadNeeds.push_back (yylookaheadNeeds[k]);
+ return create_state_set_index (static_cast<std::ptrdiff_t> (yystates.size () - 1));
+ }
+
+ void clearLastDeleted ()
+ {
+ yylastDeleted = YY_NULLPTR;
+ }
+
+ private:
+
+ std::vector<glr_state*> yystates;
+ /** During nondeterministic operation, yylookaheadNeeds tracks which
+ * stacks have actually needed the current lookahead. During deterministic
+ * operation, yylookaheadNeeds[0] is not maintained since it would merely
+ * duplicate !yyla.empty (). */
+ std::vector<bool> yylookaheadNeeds;
+
+ /** The last stack we invalidated. */
+ glr_state* yylastDeleted;
+ }; // class glr_state_set
+} // namespace
+
+namespace
+{
+ class semantic_option
+ {
+ public:
+ semantic_option ()
+ : yyrule (0)
+ , yystate (0)
+ , yynext (0)
+ , yyla ()]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {}
+
+ semantic_option (rule_num rule)
+ : yyrule (rule)
+ , yystate (0)
+ , yynext (0)
+ , yyla ()]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {}
+
+ semantic_option (const semantic_option& that)
+ : yyrule (that.yyrule)
+ , yystate (that.yystate)
+ , yynext (that.yynext)
+ , yyla (that.yyla)]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {]b4_parse_assert_if([[
+ that.check_ ();]])[
+ }
+
+ // Needed for the assignment in yynewSemanticOption.
+ semantic_option& operator= (const semantic_option& that)
+ {]b4_parse_assert_if([[
+ check_ ();
+ that.check_ ();]])[
+ yyrule = that.yyrule;
+ yystate = that.yystate;
+ yynext = that.yynext;
+ yyla = that.yyla;
+ return *this;
+ }
+
+ /// Only call state() and setState() on objects in yyitems, not temporaries.
+ glr_state* state();
+ const glr_state* state() const;
+ void setState(const glr_state* s);
+
+ const semantic_option* next () const YY_ATTRIBUTE_UNUSED;
+ semantic_option* next ();
+ void setNext (const semantic_option* s);
+
+ std::ptrdiff_t indexIn (const glr_stack_item* array) const YY_ATTRIBUTE_UNUSED;
+
+ /** True iff YYY0 and YYY1 represent identical options at the top level.
+ * That is, they represent the same rule applied to RHS symbols
+ * that produce the same terminal symbols. */
+ bool
+ isIdenticalTo (const semantic_option& yyy1) const
+ {]b4_parse_assert_if([[
+ check_ ();
+ yyy1.check_ ();]])[
+ if (this->yyrule == yyy1.yyrule)
+ {
+ const glr_state *yys0, *yys1;
+ int yyn;
+ for (yys0 = this->state(),
+ yys1 = yyy1.state(),
+ yyn = yyrhsLength (this->yyrule);
+ yyn > 0;
+ yys0 = yys0->pred(),
+ yys1 = yys1->pred(), yyn -= 1)
+ if (yys0->yyposn != yys1->yyposn)
+ return false;
+ return true;
+ }
+ else
+ return false;
+ }
+
+ /** Assuming identicalOptions (YYY0,YYY1), destructively merge the
+ * alternative semantic values for the RHS-symbols of YYY1 and YYY0. */
+ void
+ mergeWith (semantic_option& yyy1)
+ {]b4_parse_assert_if([[
+ check_ ();
+ yyy1.check_ ();]])[
+ glr_state *yys0 = this->state ();
+ glr_state *yys1 = yyy1.state ();
+ for (int yyn = yyrhsLength (this->yyrule);
+ yyn > 0;
+ yyn -= 1, yys0 = yys0->pred (), yys1 = yys1->pred ())
+ {
+ if (yys0 == yys1)
+ break;
+ else if (yys0->yyresolved)
+ {
+ yys1->yyresolved = true;]b4_variant_if([[
+ YYASSERT (yys1->yylrState == yys0->yylrState);
+ ]b4_symbol_variant([yy_accessing_symbol (yys0->yylrState)],
+ [yys1->value ()], [copy], [yys0->value ()])], [[
+ yys1->value () = yys0->value ();]])[
+ }
+ else if (yys1->yyresolved)
+ {
+ yys0->yyresolved = true;]b4_variant_if([[
+ YYASSERT (yys0->yylrState == yys1->yylrState);
+ ]b4_symbol_variant([yy_accessing_symbol (yys1->yylrState)],
+ [yys0->value ()], [copy], [yys1->value ()])], [[
+ yys0->value () = yys1->value ();]])[
+ }
+ else
+ {
+ semantic_option* yyz0prev = YY_NULLPTR;
+ semantic_option* yyz0 = yys0->firstVal();
+ semantic_option* yyz1 = yys1->firstVal();
+ while (true)
+ {
+ if (yyz1 == yyz0 || yyz1 == YY_NULLPTR)
+ break;
+ else if (yyz0 == YY_NULLPTR)
+ {
+ if (yyz0prev != YY_NULLPTR)
+ yyz0prev->setNext (yyz1);
+ else
+ yys0->setFirstVal (yyz1);
+ break;
+ }
+ else if (yyz0 < yyz1)
+ {
+ semantic_option* yyz = yyz0;
+ if (yyz0prev != YY_NULLPTR)
+ yyz0prev->setNext(yyz1);
+ else
+ yys0->setFirstVal(yyz1);
+ yyz1 = yyz1->next();
+ yyz0->setNext(yyz);
+ }
+ yyz0prev = yyz0;
+ yyz0 = yyz0->next();
+ }
+ yys1->setFirstVal(yys0->firstVal());
+ }
+ }
+ }
+
+#if ]b4_api_PREFIX[DEBUG
+ void yyreportTree (size_t yyindent = 2) const
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ int yynrhs = yyrhsLength (this->yyrule);
+ const glr_state* yystates[1 + YYMAXRHS];
+ glr_state yyleftmost_state;
+
+ {
+ const glr_state* yys = this->state();
+ for (int yyi = yynrhs; 0 < yyi; yyi -= 1)
+ {
+ yystates[yyi] = yys;
+ yys = yys->pred();
+ }
+ if (yys == YY_NULLPTR)
+ {
+ yyleftmost_state.yyposn = 0;
+ yystates[0] = &yyleftmost_state;
+ }
+ else
+ yystates[0] = yys;
+ }
+
+ std::string yylhs = ]b4_namespace_ref[::]b4_parser_class[::symbol_name (yylhsNonterm (this->yyrule));
+ YYASSERT(this->state());
+ if (this->state()->yyposn < yystates[0]->yyposn + 1)
+ std::cerr << std::string(yyindent, ' ') << yylhs << " -> <Rule "
+ << this->yyrule - 1 << ", empty>\n";
+ else
+ std::cerr << std::string(yyindent, ' ') << yylhs << " -> <Rule "
+ << this->yyrule - 1 << ", tokens "
+ << yystates[0]->yyposn + 1 << " .. "
+ << this->state()->yyposn << ">\n";
+ for (int yyi = 1; yyi <= yynrhs; yyi += 1)
+ {
+ if (yystates[yyi]->yyresolved)
+ {
+ std::string yysym = ]b4_namespace_ref[::]b4_parser_class[::symbol_name (yy_accessing_symbol (yystates[yyi]->yylrState));
+ if (yystates[yyi-1]->yyposn+1 > yystates[yyi]->yyposn)
+ std::cerr << std::string(yyindent + 2, ' ') << yysym
+ << " <empty>\n";
+ else
+ std::cerr << std::string(yyindent + 2, ' ') << yysym
+ << " <tokens " << yystates[yyi-1]->yyposn + 1
+ << " .. " << yystates[yyi]->yyposn << ">\n";
+ }
+ else
+ yystates[yyi]->firstVal ()->yyreportTree (yyindent+2);
+ }
+ }
+#endif
+
+ /** Rule number for this reduction */
+ rule_num yyrule;
+
+ private:
+ template <typename T>
+ static const glr_stack_item* asItem(const T* state)
+ {
+ return reinterpret_cast<const glr_stack_item*>(state);
+ }
+ template <typename T>
+ static glr_stack_item* asItem(T* state)
+ {
+ return reinterpret_cast<glr_stack_item*>(state);
+ }
+ /** The last RHS state in the list of states to be reduced. */
+ std::ptrdiff_t yystate;
+ /** Next sibling in chain of options. To facilitate merging,
+ * options are chained in decreasing order by address. */
+ std::ptrdiff_t yynext;
+
+ public:
+ /** The lookahead for this reduction. */
+ symbol_type yyla;
+
+]b4_parse_assert_if([[
+ public:
+ // Check invariants.
+ void check_ () const
+ {
+ YY_IGNORE_NULL_DEREFERENCE_BEGIN
+ YYASSERT (this->magic_ == MAGIC);
+ YY_IGNORE_NULL_DEREFERENCE_END
+ }
+
+ // A magic number to check our pointer arithmetic is sane.
+ enum { MAGIC = 0xeff1cace };
+ unsigned int magic_;]])[
+ }; // class semantic_option
+} // namespace
+
+namespace
+{
+ /** Type of the items in the GLR stack.
+ * It can be either a glr_state or a semantic_option. The is_state_ field
+ * indicates which item of the union is valid. */
+ class glr_stack_item
+ {
+ public:
+ glr_stack_item (bool state = true)
+ : is_state_ (state)]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {
+ if (is_state_)
+ new (&raw_) glr_state;
+ else
+ new (&raw_) semantic_option;
+ }
+
+ glr_stack_item (const glr_stack_item& other) YY_NOEXCEPT YY_NOTHROW
+ : is_state_ (other.is_state_)]b4_parse_assert_if([[
+ , magic_ (MAGIC)]])[
+ {]b4_parse_assert_if([[
+ other.check_ ();]])[
+ std::memcpy (raw_, other.raw_, union_size);
+ }
+
+ glr_stack_item& operator= (glr_stack_item other)
+ {]b4_parse_assert_if([[
+ check_ ();
+ other.check_ ();]])[
+ std::swap (is_state_, other.is_state_);
+ std::swap (raw_, other.raw_);
+ return *this;
+ }
+
+ ~glr_stack_item ()
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ if (is_state ())
+ getState ().~glr_state ();
+ else
+ getOption ().~semantic_option ();
+ }
+
+ void setState (const glr_state &state)
+ {]b4_parse_assert_if([[
+ check_ ();
+ state.check_ ();]])[
+ if (this != state.asItem ())
+ {
+ if (is_state_)
+ getState ().~glr_state ();
+ else
+ getOption ().~semantic_option ();
+ new (&raw_) glr_state (state);
+ is_state_ = true;
+ }
+ }
+
+ glr_state& getState ()
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ YYDASSERT (is_state ());
+ void *yyp = raw_;
+ glr_state& res = *static_cast<glr_state*> (yyp);]b4_parse_assert_if([[
+ res.check_ ();]])[
+ return res;
+ }
+
+ const glr_state& getState () const
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ YYDASSERT (is_state ());
+ const void *yyp = raw_;
+ const glr_state& res = *static_cast<const glr_state*> (yyp);]b4_parse_assert_if([[
+ res.check_ ();]])[
+ return res;
+ }
+
+ semantic_option& getOption ()
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ YYDASSERT (!is_state ());
+ void *yyp = raw_;
+ return *static_cast<semantic_option*> (yyp);
+ }
+ const semantic_option& getOption () const
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ YYDASSERT (!is_state ());
+ const void *yyp = raw_;
+ return *static_cast<const semantic_option*> (yyp);
+ }
+ bool is_state () const
+ {]b4_parse_assert_if([[
+ check_ ();]])[
+ return is_state_;
+ }
+
+ private:
+ /// The possible contents of raw_. Since they have constructors, they cannot
+ /// be directly included in the union.
+ union contents
+ {
+ char yystate[sizeof (glr_state)];
+ char yyoption[sizeof (semantic_option)];
+ };
+ enum { union_size = sizeof (contents) };
+ union {
+ /// Strongest alignment constraints.
+ long double yyalign_me;
+ /// A buffer large enough to store the contents.
+ char raw_[union_size];
+ };
+ /** Type tag for the union. */
+ bool is_state_;
+]b4_parse_assert_if([[
+ public:
+ // Check invariants.
+ void check_ () const
+ {
+ YYASSERT (this->magic_ == MAGIC);
+ YYASSERT (this->is_state_ == false || this->is_state_ == true);
+ }
+ // A magic number to check our pointer arithmetic is sane.
+ enum { MAGIC = 0xDEAD1ACC }; // 3735886540.
+ const unsigned int magic_;]])[
+ }; // class glr_stack_item
+} // namespace
+
+glr_state* glr_state::pred ()
+{]b4_parse_assert_if([[
+ check_ ();]])[
+ YY_IGNORE_NULL_DEREFERENCE_BEGIN
+ return yypred ? &asItem (as_pointer_ (this) - yypred)->getState () : YY_NULLPTR;
+ YY_IGNORE_NULL_DEREFERENCE_END
+}
+
+const glr_state* glr_state::pred () const
+{]b4_parse_assert_if([[
+ check_ ();]])[
+ YY_IGNORE_NULL_DEREFERENCE_BEGIN
+ return yypred ? &asItem (as_pointer_ (this) - yypred)->getState () : YY_NULLPTR;
+ YY_IGNORE_NULL_DEREFERENCE_END
+}
+
+void glr_state::setPred (const glr_state* state)
+{]b4_parse_assert_if([[
+ check_ ();
+ if (state)
+ state->check_ ();]])[
+ yypred = state ? as_pointer_ (this) - as_pointer_ (state) : 0;
+}
+
+semantic_option* glr_state::firstVal ()
+{]b4_parse_assert_if([[
+ check_ ();]])[
+ return yyfirstVal ? &(asItem(this) - yyfirstVal)->getOption() : YY_NULLPTR;
+}
+
+const semantic_option* glr_state::firstVal () const
+{]b4_parse_assert_if([[
+ check_ ();]])[
+ return yyfirstVal ? &(asItem(this) - yyfirstVal)->getOption() : YY_NULLPTR;
+}
+
+void glr_state::setFirstVal (const semantic_option* option)
+{]b4_parse_assert_if([[
+ check_ ();]])[
+ yyfirstVal = option ? asItem(this) - asItem(option) : 0;
+}
+
+std::ptrdiff_t glr_state::indexIn (const glr_stack_item* array) const
+{]b4_parse_assert_if([[
+ check_ ();]])[
+ return asItem(this) - array;
+}
+
+std::ptrdiff_t semantic_option::indexIn (const glr_stack_item* array) const
+{
+ return asItem(this) - array;
+}
+
+glr_state* semantic_option::state ()
+{
+ YY_IGNORE_NULL_DEREFERENCE_BEGIN
+ return yystate ? &(asItem(this) - yystate)->getState() : YY_NULLPTR;
+ YY_IGNORE_NULL_DEREFERENCE_END
+}
+
+const glr_state* semantic_option::state () const
+{
+ return yystate ? &(asItem(this) - yystate)->getState() : YY_NULLPTR;
+}
+
+void semantic_option::setState (const glr_state* s)
+{
+ yystate = s ? asItem(this) - asItem(s) : 0;
+}
+
+const semantic_option* semantic_option::next () const
+{
+ return yynext ? &(asItem(this) - yynext)->getOption() : YY_NULLPTR;
+}
+
+semantic_option* semantic_option::next ()
+{
+ return yynext ? &(asItem(this) - yynext)->getOption() : YY_NULLPTR;
+}
+
+void semantic_option::setNext (const semantic_option* s)
+{
+ yynext = s ? asItem(this) - asItem(s) : 0;
+}
+
+void glr_state::destroy (char const* yymsg, ]b4_namespace_ref[::]b4_parser_class[& yyparser)
+{]b4_parse_assert_if([[
+ check_ ();]])[
+ if (yyresolved)
+ yyparser.yy_destroy_ (yymsg, yy_accessing_symbol(yylrState),
+ value ()]b4_locations_if([, yyloc])[);
+ else
+ {
+#if ]b4_api_PREFIX[DEBUG
+ YYCDEBUG << yymsg
+ << (firstVal() ? " unresolved " : " incomplete ")
+ << (yy_accessing_symbol (yylrState) < YYNTOKENS ? "token" : "nterm")
+ << ' ' << yyparser.symbol_name (yy_accessing_symbol (yylrState))
+ << " ("]b4_locations_if([[
+ << yyloc << ": "]])[
+ << ")\n";
+#endif
+ if (firstVal() != YY_NULLPTR)
+ {
+ semantic_option& yyoption = *firstVal ();
+ glr_state *yyrh = yyoption.state ();
+ for (int yyn = yyrhsLength (yyoption.yyrule); yyn > 0; yyn -= 1)
+ {
+ yyrh->destroy (yymsg, yyparser);
+ yyrh = yyrh->pred();
+ }
+ }
+ }
+}
+
+
+#undef YYFILL
+#define YYFILL(N) yyfill (yyvsp, yylow, (N), yynormal)
+
+namespace
+{
+ class state_stack
+ {
+ public:
+ using parser_type = ]b4_namespace_ref[::]b4_parser_class[;
+ using symbol_kind = parser_type::symbol_kind;
+ using value_type = parser_type::value_type;]b4_locations_if([[
+ using location_type = parser_type::location_type;]])[
+
+ /** Initialize to a single empty stack, with total maximum
+ * capacity for all stacks of YYSIZE. */
+ state_stack (size_t yysize)
+ : yysplitPoint (YY_NULLPTR)
+ {
+ yyitems.reserve (yysize);
+ }
+
+#if YYSTACKEXPANDABLE
+ /** Returns false if it tried to expand but could not. */
+ bool
+ yyexpandGLRStackIfNeeded ()
+ {
+ return YYHEADROOM <= spaceLeft () || yyexpandGLRStack ();
+ }
+
+ private:
+ /** If *this is expandable, extend it. WARNING: Pointers into the
+ stack from outside should be considered invalid after this call.
+ We always expand when there are 1 or fewer items left AFTER an
+ allocation, so that we can avoid having external pointers exist
+ across an allocation. */
+ bool
+ yyexpandGLRStack ()
+ {
+ const size_t oldsize = yyitems.size();
+ if (YYMAXDEPTH - YYHEADROOM < oldsize)
+ return false;
+ const size_t yynewSize = YYMAXDEPTH < 2 * oldsize ? YYMAXDEPTH : 2 * oldsize;
+ const glr_stack_item *oldbase = &yyitems[0];
+
+ yyitems.reserve (yynewSize);
+ const glr_stack_item *newbase = &yyitems[0];
+
+ // Adjust the pointers. Perform raw pointer arithmetic, as there
+ // is no reason for objects to be aligned on their size.
+ const ptrdiff_t disp
+ = reinterpret_cast<const char*> (newbase) - reinterpret_cast<const char*> (oldbase);
+ if (yysplitPoint)
+ const_cast<glr_state*&> (yysplitPoint)
+ = reinterpret_cast<glr_state*> (reinterpret_cast<char*> (const_cast<glr_state*> (yysplitPoint)) + disp);
+
+ for (std::vector<glr_state*>::iterator
+ i = yytops.begin (),
+ yyend = yytops.end ();
+ i != yyend; ++i)
+ if (glr_state_not_null (*i))
+ *i = reinterpret_cast<glr_state*>(reinterpret_cast<char*>(*i) + disp);
+
+ return true;
+ }
+
+ public:
+#else
+ bool yyexpandGLRStackIfNeeded ()
+ {
+ return YYHEADROOM <= spaceLeft ();
+ }
+#endif
+#undef YYSTACKEXPANDABLE
+
+ static bool glr_state_not_null (glr_state* s)
+ {
+ return s != YY_NULLPTR;
+ }
+
+ bool
+ reduceToOneStack ()
+ {
+ using iterator = std::vector<glr_state*>::iterator;
+ const iterator yybegin = yytops.begin();
+ const iterator yyend = yytops.end();
+ const iterator yyit = std::find_if(yybegin, yyend, glr_state_not_null);
+ if (yyit == yyend)
+ return false;
+ for (state_set_index yyk = create_state_set_index(yyit + 1 - yybegin);
+ yyk.uget() != numTops(); ++yyk)
+ yytops.yymarkStackDeleted (yyk);
+ yytops.yyremoveDeletes ();
+ yycompressStack ();
+ return true;
+ }
+
+ /** Called when returning to deterministic operation to clean up the extra
+ * stacks. */
+ void
+ yycompressStack ()
+ {
+ if (yytops.size() != 1 || !isSplit())
+ return;
+
+ // yyr is the state after the split point.
+ glr_state* yyr = YY_NULLPTR;
+ for (glr_state *yyp = firstTop(), *yyq = yyp->pred();
+ yyp != yysplitPoint;
+ yyr = yyp, yyp = yyq, yyq = yyp->pred())
+ yyp->setPred(yyr);
+
+ // This const_cast is okay, since anyway we have access to the mutable
+ // yyitems into which yysplitPoint points.
+ glr_stack_item* nextFreeItem
+ = const_cast<glr_state*> (yysplitPoint)->asItem () + 1;
+ yysplitPoint = YY_NULLPTR;
+ yytops.clearLastDeleted ();
+
+ while (yyr != YY_NULLPTR)
+ {
+ nextFreeItem->setState (*yyr);
+ glr_state& nextFreeState = nextFreeItem->getState();
+ yyr = yyr->pred();
+ nextFreeState.setPred(&(nextFreeItem - 1)->getState());
+ setFirstTop (&nextFreeState);
+ ++nextFreeItem;
+ }
+ yyitems.resize(static_cast<size_t>(nextFreeItem - yyitems.data()));
+ }
+
+ bool isSplit() const {
+ return yysplitPoint != YY_NULLPTR;
+ }
+
+ // Present the interface of a vector of glr_stack_item.
+ std::vector<glr_stack_item>::const_iterator begin () const
+ {
+ return yyitems.begin ();
+ }
+
+ std::vector<glr_stack_item>::const_iterator end () const
+ {
+ return yyitems.end ();
+ }
+
+ size_t size() const
+ {
+ return yyitems.size ();
+ }
+
+ glr_stack_item& operator[] (size_t i)
+ {
+ return yyitems[i];
+ }
+
+ glr_stack_item& stackItemAt (size_t index)
+ {
+ return yyitems[index];
+ }
+
+ size_t numTops () const
+ {
+ return yytops.size ();
+ }
+
+ glr_state* firstTop () const
+ {
+ return yytops[create_state_set_index (0)];
+ }
+
+ glr_state* topAt (state_set_index i) const
+ {
+ return yytops[i];
+ }
+
+ void setFirstTop (glr_state* value)
+ {
+ yytops[create_state_set_index (0)] = value;
+ }
+
+ void setTopAt (state_set_index i, glr_state* value)
+ {
+ yytops[i] = value;
+ }
+
+ void pop_back ()
+ {
+ yyitems.pop_back ();
+ }
+
+ void pop_back (size_t n)
+ {
+ yyitems.resize (yyitems.size () - n);
+ }
+
+ state_set_index
+ yysplitStack (state_set_index yyk)
+ {
+ if (!isSplit ())
+ {
+ YYASSERT (yyk.get () == 0);
+ yysplitPoint = topAt (yyk);
+ }
+ return yytops.yysplitStack (yyk);
+ }
+
+ /** Assuming that YYS is a GLRState somewhere on *this, update the
+ * splitpoint of *this, if needed, so that it is at least as deep as
+ * YYS. */
+ void
+ yyupdateSplit (glr_state& yys)
+ {
+ if (isSplit() && &yys < yysplitPoint)
+ yysplitPoint = &yys;
+ }
+
+ /** Return a fresh GLRState.
+ * Callers should call yyreserveStack afterwards to make sure there is
+ * sufficient headroom. */
+ glr_state& yynewGLRState (const glr_state& newState)
+ {
+ glr_state& state = yyitems[yynewGLRStackItem (true)].getState ();
+#if false && 201103L <= YY_CPLUSPLUS
+ state = std::move (newState);
+#else
+ state = newState;
+#endif
+ return state;
+ }
+
+ /** Return a fresh SemanticOption.
+ * Callers should call yyreserveStack afterwards to make sure there is
+ * sufficient headroom. */
+ semantic_option& yynewSemanticOption (semantic_option newOption)
+ {
+ semantic_option& option = yyitems[yynewGLRStackItem (false)].getOption ();
+ option = std::move (newOption);
+ return option;
+ }
+
+ /* Do nothing if YYNORMAL or if *YYLOW <= YYLOW1. Otherwise, fill in
+ * YYVSP[YYLOW1 .. *YYLOW-1] as in yyfillin and set *YYLOW = YYLOW1.
+ * For convenience, always return YYLOW1. */
+ int
+ yyfill (glr_stack_item *yyvsp, int &yylow, int yylow1, bool yynormal)
+ {
+ if (!yynormal && yylow1 < yylow)
+ {
+ yyfillin (yyvsp, yylow, yylow1);
+ yylow = yylow1;
+ }
+ return yylow1;
+ }
+
+ /** Fill in YYVSP[YYLOW1 .. YYLOW0-1] from the chain of states starting
+ * at YYVSP[YYLOW0].getState().pred(). Leaves YYVSP[YYLOW1].getState().pred()
+ * containing the pointer to the next state in the chain. */
+ void
+ yyfillin (glr_stack_item *yyvsp, int yylow0, int yylow1)
+ {
+ glr_state* s = yyvsp[yylow0].getState().pred();
+ YYASSERT(s != YY_NULLPTR);
+ for (int i = yylow0-1; i >= yylow1; i -= 1, s = s->pred())
+ {
+ glr_state& yys = yyvsp[i].getState();
+#if ]b4_api_PREFIX[DEBUG
+ yys.yylrState = s->yylrState;
+#endif
+ yys.yyresolved = s->yyresolved;
+ if (s->yyresolved)
+ {]b4_variant_if([[
+ new (&yys.value ()) value_type ();
+ ]b4_symbol_variant([yy_accessing_symbol (s->yylrState)],
+ [yys.value ()], [copy], [s->value ()])], [[
+ new (&yys.value ()) value_type (s->value ());]])[
+ }
+ else
+ /* The effect of using yyval or yyloc (in an immediate
+ * rule) is undefined. */
+ yys.setFirstVal (YY_NULLPTR);]b4_locations_if([[
+ yys.yyloc = s->yyloc;]])[
+ yys.setPred(s->pred());
+ }
+ }
+
+#if ]b4_api_PREFIX[DEBUG
+
+ /*----------------------------------------------------------------------.
+ | Report that stack #YYK of *YYSTACKP is going to be reduced by YYRULE. |
+ `----------------------------------------------------------------------*/
+
+ void
+ yy_reduce_print (bool yynormal, glr_stack_item* yyvsp, state_set_index yyk,
+ rule_num yyrule, parser_type& yyparser)
+ {
+ int yynrhs = yyrhsLength (yyrule);]b4_locations_if([
+ int yylow = 1;])[
+ int yyi;
+ std::cerr << "Reducing stack " << yyk.get() << " by rule " << yyrule - 1
+ << " (line " << int (yyrline[yyrule]) << "):\n";
+ if (! yynormal)
+ yyfillin (yyvsp, 1, -yynrhs);
+ /* The symbols being reduced. */
+ for (yyi = 0; yyi < yynrhs; yyi++)
+ {
+ std::cerr << " $" << yyi + 1 << " = ";
+ yyparser.yy_symbol_print_
+ (yy_accessing_symbol (yyvsp[yyi - yynrhs + 1].getState().yylrState),
+ yyvsp[yyi - yynrhs + 1].getState().value ()]b4_locations_if([[,
+ ]b4_rhs_location(yynrhs, yyi + 1)])[);
+ if (!yyvsp[yyi - yynrhs + 1].getState().yyresolved)
+ std::cerr << " (unresolved)";
+ std::cerr << '\n';
+ }
+ }
+
+
+#define YYINDEX(YYX) \
+ ((YYX) == YY_NULLPTR ? -1 : (YYX)->indexIn (yyitems.data ()))
+
+ void
+ dumpStack () const
+ {
+ for (size_t yyi = 0; yyi < size(); ++yyi)
+ {
+ const glr_stack_item& item = yyitems[yyi];
+ std::cerr << std::setw(3) << yyi << ". ";
+ if (item.is_state())
+ {
+ std::cerr << "Res: " << item.getState().yyresolved
+ << ", LR State: " << item.getState().yylrState
+ << ", posn: " << item.getState().yyposn
+ << ", pred: " << YYINDEX(item.getState().pred());
+ if (! item.getState().yyresolved)
+ std::cerr << ", firstVal: "
+ << YYINDEX(item.getState().firstVal());
+ }
+ else
+ {
+ std::cerr << "Option. rule: " << item.getOption().yyrule - 1
+ << ", state: " << YYINDEX(item.getOption().state())
+ << ", next: " << YYINDEX(item.getOption().next());
+ }
+ std::cerr << '\n';
+ }
+ std::cerr << "Tops:";
+ for (state_set_index yyi = create_state_set_index(0); yyi.uget() < numTops(); ++yyi) {
+ std::cerr << yyi.get() << ": " << YYINDEX(topAt(yyi)) << "; ";
+ }
+ std::cerr << '\n';
+ }
+
+#undef YYINDEX
+#endif
+
+ YYRESULTTAG
+ yyreportAmbiguity (const semantic_option& yyx0,
+ const semantic_option& yyx1, parser_type& yyparser]b4_locations_if([, const location_type& yyloc])[)
+ {
+ YY_USE (yyx0);
+ YY_USE (yyx1);
+
+#if ]b4_api_PREFIX[DEBUG
+ std::cerr << "Ambiguity detected.\n"
+ "Option 1,\n";
+ yyx0.yyreportTree ();
+ std::cerr << "\nOption 2,\n";
+ yyx1.yyreportTree ();
+ std::cerr << '\n';
+#endif
+
+ yyparser.error (]b4_locations_if([yyloc, ])[YY_("syntax is ambiguous"));
+ return yyabort;
+ }
+
+#if ]b4_api_PREFIX[DEBUG
+ /* Print YYS (possibly NULL) and its predecessors. */
+ void
+ yypstates (const glr_state* yys) const
+ {
+ if (yys != YY_NULLPTR)
+ yys->yy_yypstack();
+ else
+ std::cerr << "<null>";
+ std::cerr << '\n';
+ }
+#endif
+
+ private:
+ size_t spaceLeft() const
+ {
+ return yyitems.capacity() - yyitems.size();
+ }
+
+ /** Return a fresh GLRStackItem in this. The item is an LR state
+ * if YYIS_STATE, and otherwise a semantic option. Callers should call
+ * yyreserveStack afterwards to make sure there is sufficient
+ * headroom. */
+ size_t
+ yynewGLRStackItem (bool yyis_state)
+ {
+ YYDASSERT(yyitems.size() < yyitems.capacity());
+ yyitems.push_back(glr_stack_item(yyis_state));
+ return yyitems.size() - 1;
+ }
+
+
+ public:
+ std::vector<glr_stack_item> yyitems;
+ // Where the stack splits. Anything below this address is deterministic.
+ const glr_state* yysplitPoint;
+ glr_state_set yytops;
+ }; // class state_stack
+} // namespace
+
+#undef YYFILL
+#define YYFILL(N) yystateStack.yyfill (yyvsp, yylow, (N), yynormal)
+
+namespace ]b4_namespace_ref[
+{
+ class ]b4_parser_class[::glr_stack
+ {
+ public:
+]b4_parse_error_bmatch([custom\|detailed\|verbose], [[
+ // Needs access to yypact_value_is_default, etc.
+ friend context;
+]])[
+
+ glr_stack (size_t yysize, parser_type& yyparser_yyarg]m4_ifset([b4_parse_param], [, b4_parse_param_decl])[)
+ : yyerrState (0)
+ , yystateStack (yysize)
+ , yyerrcnt (0)
+ , yyla ()
+ , yyparser (yyparser_yyarg)]m4_ifset([b4_parse_param], [,b4_parse_param_cons])[
+ {}
+
+ ~glr_stack ()
+ {
+ if (!this->yyla.empty ())
+ yyparser.yy_destroy_ ("Cleanup: discarding lookahead",
+ this->yyla.kind (), this->yyla.value]b4_locations_if([, this->yyla.location])[);
+ popall_ ();
+ }
+
+ int yyerrState;
+]b4_locations_if([[ /* To compute the location of the error token. */
+ glr_stack_item yyerror_range[3];]])[
+ state_stack yystateStack;
+ int yyerrcnt;
+ symbol_type yyla;
+ YYJMP_BUF yyexception_buffer;
+ parser_type& yyparser;
+
+ #define YYCHK1(YYE) \
+ do { \
+ switch (YYE) { \
+ case yyok: \
+ break; \
+ case yyabort: \
+ goto yyabortlab; \
+ case yyaccept: \
+ goto yyacceptlab; \
+ case yyerr: \
+ goto yyuser_error; \
+ default: \
+ goto yybuglab; \
+ } \
+ } while (false)
+
+ int
+ parse ()
+ {
+ int yyresult;
+ size_t yyposn;
+
+ YYCDEBUG << "Starting parse\n";
+
+ this->yyla.clear ();
+]m4_ifdef([b4_initial_action], [
+b4_dollar_pushdef([yyla.value], [], [], [yyla.location])dnl
+ b4_user_initial_action
+b4_dollar_popdef])[]dnl
+[
+ switch (YYSETJMP (this->yyexception_buffer))
+ {
+ case 0: break;
+ case 1: goto yyabortlab;
+ case 2: goto yyexhaustedlab;
+ default: goto yybuglab;
+ }
+ this->yyglrShift (create_state_set_index(0), 0, 0, this->yyla.value]b4_locations_if([, this->yyla.location])[);
+ yyposn = 0;
+
+ while (true)
+ {
+ /* For efficiency, we have two loops, the first of which is
+ specialized to deterministic operation (single stack, no
+ potential ambiguity). */
+ /* Standard mode */
+ while (true)
+ {
+ const state_num yystate = this->firstTopState()->yylrState;
+ YYCDEBUG << "Entering state " << yystate << '\n';
+ if (yystate == YYFINAL)
+ goto yyacceptlab;
+ if (yy_is_defaulted_state (yystate))
+ {
+ const rule_num yyrule = yy_default_action (yystate);
+ if (yyrule == 0)
+ {]b4_locations_if([[
+ this->yyerror_range[1].getState().yyloc = this->yyla.location;]])[
+ this->yyreportSyntaxError ();
+ goto yyuser_error;
+ }
+ YYCHK1 (this->yyglrReduce (create_state_set_index(0), yyrule, true));
+ }
+ else
+ {
+ yyget_token ();
+ const short* yyconflicts;
+ const int yyaction = yygetLRActions (yystate, this->yyla.kind (), yyconflicts);
+ if (*yyconflicts != 0)
+ break;
+ if (yy_is_shift_action (yyaction))
+ {
+ YY_SYMBOL_PRINT ("Shifting", this->yyla.kind (), this->yyla.value, this->yyla.location);
+ yyposn += 1;
+ // FIXME: we should move yylval.
+ this->yyglrShift (create_state_set_index(0), yyaction, yyposn, this->yyla.value]b4_locations_if([, this->yyla.location])[);
+ yyla.clear ();
+ if (0 < this->yyerrState)
+ this->yyerrState -= 1;
+ }
+ else if (yy_is_error_action (yyaction))
+ {]b4_locations_if([[
+ this->yyerror_range[1].getState().yyloc = this->yyla.location;]])[
+ /* Don't issue an error message again for exceptions
+ thrown from the scanner. */
+ if (this->yyla.kind () != ]b4_symbol(error, kind)[)
+ this->yyreportSyntaxError ();
+ goto yyuser_error;
+ }
+ else
+ YYCHK1 (this->yyglrReduce (create_state_set_index(0), -yyaction, true));
+ }
+ }
+
+ while (true)
+ {
+ for (state_set_index yys = create_state_set_index(0); yys.uget() < this->yystateStack.numTops(); ++yys)
+ this->yystateStack.yytops.setLookaheadNeeds(yys, !this->yyla.empty ());
+
+ /* yyprocessOneStack returns one of three things:
+
+ - An error flag. If the caller is yyprocessOneStack, it
+ immediately returns as well. When the caller is finally
+ yyparse, it jumps to an error label via YYCHK1.
+
+ - yyok, but yyprocessOneStack has invoked yymarkStackDeleted
+ (yys), which sets the top state of yys to NULL. Thus,
+ yyparse's following invocation of yyremoveDeletes will remove
+ the stack.
+
+ - yyok, when ready to shift a token.
+
+ Except in the first case, yyparse will invoke yyremoveDeletes and
+ then shift the next token onto all remaining stacks. This
+ synchronization of the shift (that is, after all preceding
+ reductions on all stacks) helps prevent double destructor calls
+ on yylval in the event of memory exhaustion. */
+
+ for (state_set_index yys = create_state_set_index (0); yys.uget () < this->yystateStack.numTops (); ++yys)
+ YYCHK1 (this->yyprocessOneStack (yys, yyposn]b4_locations_if([, &this->yyla.location])[));
+ this->yystateStack.yytops.yyremoveDeletes ();
+ if (this->yystateStack.yytops.size() == 0)
+ {
+ this->yystateStack.yytops.yyundeleteLastStack ();
+ if (this->yystateStack.yytops.size() == 0)
+ this->yyFail (]b4_locations_if([&this->yyla.location, ])[YY_("syntax error"));
+ YYCHK1 (this->yyresolveStack ());
+ YYCDEBUG << "Returning to deterministic operation.\n";]b4_locations_if([[
+ this->yyerror_range[1].getState ().yyloc = this->yyla.location;]])[
+ this->yyreportSyntaxError ();
+ goto yyuser_error;
+ }
+
+ /* If any yyglrShift call fails, it will fail after shifting. Thus,
+ a copy of yylval will already be on stack 0 in the event of a
+ failure in the following loop. Thus, yyla is emptied
+ before the loop to make sure the user destructor for yylval isn't
+ called twice. */
+ symbol_kind_type yytoken_to_shift = this->yyla.kind ();
+ this->yyla.kind_ = ]b4_symbol(empty, kind)[;
+ yyposn += 1;
+ for (state_set_index yys = create_state_set_index (0); yys.uget () < this->yystateStack.numTops (); ++yys)
+ {
+ const state_num yystate = this->topState (yys)->yylrState;
+ const short* yyconflicts;
+ const int yyaction = yygetLRActions (yystate, yytoken_to_shift, yyconflicts);
+ /* Note that yyconflicts were handled by yyprocessOneStack. */
+ YYCDEBUG << "On stack " << yys.get() << ", ";
+ YY_SYMBOL_PRINT ("shifting", yytoken_to_shift, this->yyla.value, this->yyla.location);
+ this->yyglrShift (yys, yyaction, yyposn, this->yyla.value]b4_locations_if([, this->yyla.location])[);
+ YYCDEBUG << "Stack " << yys.get() << " now in state "
+ << this->topState(yys)->yylrState << '\n';
+ }
+]b4_variant_if([[
+ // FIXME: User destructors.
+ // Value type destructor.
+ ]b4_symbol_variant([[yytoken_to_shift]], [[this->yyla.value]], [[template destroy]])])[
+
+ if (this->yystateStack.yytops.size () == 1)
+ {
+ YYCHK1 (this->yyresolveStack ());
+ YYCDEBUG << "Returning to deterministic operation.\n";
+ this->yystateStack.yycompressStack ();
+ break;
+ }
+ }
+ continue;
+ yyuser_error:
+ this->yyrecoverSyntaxError (]b4_locations_if([&this->yyla.location])[);
+ yyposn = this->firstTopState()->yyposn;
+ }
+
+ yyacceptlab:
+ yyresult = 0;
+ goto yyreturn;
+
+ yybuglab:
+ YYASSERT (false);
+ goto yyabortlab;
+
+ yyabortlab:
+ yyresult = 1;
+ goto yyreturn;
+
+ yyexhaustedlab:
+ yyparser.error (]b4_locations_if([this->yyla.location, ])[YY_("memory exhausted"));
+ yyresult = 2;
+ goto yyreturn;
+
+ yyreturn:
+ return yyresult;
+ }
+ #undef YYCHK1
+
+ void yyreserveGlrStack ()
+ {
+ if (!yystateStack.yyexpandGLRStackIfNeeded ())
+ yyMemoryExhausted ();
+ }
+
+ _Noreturn void
+ yyMemoryExhausted ()
+ {
+ YYLONGJMP (yyexception_buffer, 2);
+ }
+
+ _Noreturn void
+ yyFail (]b4_locations_if([location_type* yylocp, ])[const char* yymsg)
+ {
+ if (yymsg != YY_NULLPTR)
+ yyparser.error (]b4_locations_if([*yylocp, ])[yymsg);
+ YYLONGJMP (yyexception_buffer, 1);
+ }
+
+ /* GLRStates */
+
+
+ /** Add a new semantic action that will execute the action for rule
+ * YYRULE on the semantic values in YYRHS to the list of
+ * alternative actions for YYSTATE. Assumes that YYRHS comes from
+ * stack #YYK of *this. */
+ void
+ yyaddDeferredAction (state_set_index yyk, glr_state* yystate,
+ glr_state* yyrhs, rule_num yyrule)
+ {
+ semantic_option& yyopt = yystateStack.yynewSemanticOption (semantic_option (yyrule));
+ yyopt.setState (yyrhs);
+ yyopt.setNext (yystate->firstVal ());
+ if (yystateStack.yytops.lookaheadNeeds (yyk))
+ yyopt.yyla = this->yyla;
+ yystate->setFirstVal (&yyopt);
+
+ yyreserveGlrStack ();
+ }
+
+ #if ]b4_api_PREFIX[DEBUG
+ void yypdumpstack () const
+ {
+ yystateStack.dumpStack();
+ }
+ #endif
+
+ void
+ yyreportSyntaxError ()
+ {
+ if (yyerrState != 0)
+ return;
+]b4_parse_error_case(
+[simple], [[
+ std::string msg = YY_("syntax error");
+ yyparser.error (]b4_join(b4_locations_if([yyla.location]), [[YY_MOVE (msg)]])[);]],
+[custom], [[
+ context yyctx (*this, yyla);
+ yyparser.report_syntax_error (yyctx);]],
+[[
+ context yyctx (*this, yyla);
+ std::string msg = yyparser.yysyntax_error_ (yyctx);
+ yyparser.error (]b4_join(b4_locations_if([yyla.location]), [[YY_MOVE (msg)]])[);]])[
+ yyerrcnt += 1;
+ }
+
+ /* Recover from a syntax error on this, assuming that yytoken,
+ yylval, and yylloc are the syntactic category, semantic value, and location
+ of the lookahead. */
+ void
+ yyrecoverSyntaxError (]b4_locations_if([location_type* yylocp])[)
+ {
+ if (yyerrState == 3)
+ /* We just shifted the error token and (perhaps) took some
+ reductions. Skip tokens until we can proceed. */
+ while (true)
+ {
+ if (this->yyla.kind () == ]b4_symbol(eof, kind)[)
+ yyFail (]b4_locations_if([yylocp, ])[YY_NULLPTR);
+ if (this->yyla.kind () != ]b4_symbol(empty, kind)[)
+ {]b4_locations_if([[
+ /* We throw away the lookahead, but the error range
+ of the shifted error token must take it into account. */
+ glr_state *yys = firstTopState();
+ yyerror_range[1].getState().yyloc = yys->yyloc;
+ yyerror_range[2].getState().yyloc = this->yyla.location;
+ YYLLOC_DEFAULT ((yys->yyloc), yyerror_range, 2);]])[
+ yyparser.yy_destroy_ ("Error: discarding",
+ this->yyla.kind (), this->yyla.value]b4_locations_if([, this->yyla.location])[);]b4_variant_if([[
+ // Value type destructor.
+ ]b4_symbol_variant([[this->yyla.kind ()]], [[this->yyla.value]], [[template destroy]])])[
+ this->yyla.kind_ = ]b4_symbol(empty, kind)[;
+ }
+ yyget_token ();
+ int yyj = yypact[firstTopState()->yylrState];
+ if (yypact_value_is_default (yyj))
+ return;
+ yyj += this->yyla.kind ();
+ if (yyj < 0 || YYLAST < yyj || yycheck[yyj] != this->yyla.kind ())
+ {
+ if (yydefact[firstTopState()->yylrState] != 0)
+ return;
+ }
+ else if (! yytable_value_is_error (yytable[yyj]))
+ return;
+ }
+
+ if (!yystateStack.reduceToOneStack())
+ yyFail (]b4_locations_if([yylocp, ])[YY_NULLPTR);
+
+ /* Now pop stack until we find a state that shifts the error token. */
+ yyerrState = 3;
+ while (firstTopState () != YY_NULLPTR)
+ {
+ glr_state *yys = firstTopState ();
+ int yyj = yypact[yys->yylrState];
+ if (! yypact_value_is_default (yyj))
+ {
+ yyj += YYTERROR;
+ if (0 <= yyj && yyj <= YYLAST && yycheck[yyj] == YYTERROR
+ && yy_is_shift_action (yytable[yyj]))
+ {
+ /* Shift the error token. */]b4_locations_if([[
+ /* First adjust its location.*/
+ location_type yyerrloc;
+ yyerror_range[2].getState().yyloc = this->yyla.location;
+ YYLLOC_DEFAULT (yyerrloc, (yyerror_range), 2);]])[
+ YY_SYMBOL_PRINT ("Shifting", yy_accessing_symbol (yytable[yyj]),
+ this->yyla.value, yyerrloc);
+ yyglrShift (create_state_set_index(0), yytable[yyj],
+ yys->yyposn, yyla.value]b4_locations_if([, yyerrloc])[);
+ yys = firstTopState();
+ break;
+ }
+ }]b4_locations_if([[
+ yyerror_range[1].getState().yyloc = yys->yyloc;]])[
+ if (yys->pred() != YY_NULLPTR)
+ yys->destroy ("Error: popping", yyparser);
+ yystateStack.setFirstTop(yys->pred());
+ yystateStack.pop_back();
+ }
+ if (firstTopState() == YY_NULLPTR)
+ yyFail (]b4_locations_if([yylocp, ])[YY_NULLPTR);
+ }
+
+ YYRESULTTAG
+ yyprocessOneStack (state_set_index yyk,
+ size_t yyposn]b4_locations_if([, location_type* yylocp])[)
+ {
+ while (yystateStack.topAt(yyk) != YY_NULLPTR)
+ {
+ const state_num yystate = topState(yyk)->yylrState;
+ YYCDEBUG << "Stack " << yyk.get()
+ << " Entering state " << yystate << '\n';
+
+ YYASSERT (yystate != YYFINAL);
+
+ if (yy_is_defaulted_state (yystate))
+ {
+ const rule_num yyrule = yy_default_action (yystate);
+ if (yyrule == 0)
+ {
+ YYCDEBUG << "Stack " << yyk.get() << " dies.\n";
+ yystateStack.yytops.yymarkStackDeleted (yyk);
+ return yyok;
+ }
+ const YYRESULTTAG yyflag
+ = yyglrReduce (yyk, yyrule, yyimmediate[yyrule]);
+ if (yyflag == yyerr)
+ {
+ YYCDEBUG << "Stack " << yyk.get() << " dies"
+ " (predicate failure or explicit user error).\n";
+ yystateStack.yytops.yymarkStackDeleted (yyk);
+ return yyok;
+ }
+ if (yyflag != yyok)
+ return yyflag;
+ }
+ else
+ {
+ yystateStack.yytops.setLookaheadNeeds(yyk, true);
+ yyget_token ();
+ const short* yyconflicts;
+ const int yyaction = yygetLRActions (yystate, this->yyla.kind (), yyconflicts);
+
+ for (; *yyconflicts != 0; ++yyconflicts)
+ {
+ state_set_index yynewStack = yystateStack.yysplitStack (yyk);
+ YYCDEBUG << "Splitting off stack " << yynewStack.get()
+ << " from " << yyk.get() << ".\n";
+ YYRESULTTAG yyflag =
+ yyglrReduce (yynewStack, *yyconflicts, yyimmediate[*yyconflicts]);
+ if (yyflag == yyok)
+ YYCHK (yyprocessOneStack (yynewStack,
+ yyposn]b4_locations_if([, yylocp])[));
+ else if (yyflag == yyerr)
+ {
+ YYCDEBUG << "Stack " << yynewStack.get() << " dies.\n";
+ yystateStack.yytops.yymarkStackDeleted (yynewStack);
+ }
+ else
+ return yyflag;
+ }
+
+ if (yy_is_shift_action (yyaction))
+ break;
+ else if (yy_is_error_action (yyaction))
+ {
+ YYCDEBUG << "Stack " << yyk.get() << " dies.\n";
+ yystateStack.yytops.yymarkStackDeleted (yyk);
+ break;
+ }
+ else
+ {
+ YYRESULTTAG yyflag
+ = yyglrReduce (yyk, -yyaction, yyimmediate[-yyaction]);
+ if (yyflag == yyerr)
+ {
+ YYCDEBUG << "Stack " << yyk.get() << " dies"
+ " (predicate failure or explicit user error).\n";
+ yystateStack.yytops.yymarkStackDeleted (yyk);
+ break;
+ }
+ else if (yyflag != yyok)
+ return yyflag;
+ }
+ }
+ }
+ return yyok;
+ }
+
+ /** Perform user action for rule number YYN, with RHS length YYRHSLEN,
+ * and top stack item YYVSP. YYVALP points to place to put semantic
+ * value ($$), and yylocp points to place for location information
+ * (@@$). Returns yyok for normal return, yyaccept for YYACCEPT,
+ * yyerr for YYERROR, yyabort for YYABORT. */
+ YYRESULTTAG
+ yyuserAction (rule_num yyrule, int yyrhslen, glr_stack_item* yyvsp, state_set_index yyk,
+ value_type* yyvalp]b4_locations_if([, location_type* yylocp])[)
+ {
+ bool yynormal YY_ATTRIBUTE_UNUSED = !yystateStack.isSplit();
+ int yylow = 1;
+]b4_parse_param_use([yyvalp], [yylocp])dnl
+[ YY_USE (yyk);
+ YY_USE (yyrhslen);
+ # undef yyerrok
+ # define yyerrok (yyerrState = 0)
+ # undef YYACCEPT
+ # define YYACCEPT return yyaccept
+ # undef YYABORT
+ # define YYABORT return yyabort
+ # undef YYERROR
+ # define YYERROR return yyerrok, yyerr
+ # undef YYRECOVERING
+ # define YYRECOVERING() (yyerrState != 0)
+ # undef yytoken
+ # define yytoken this->yyla.kind_
+ # undef yyclearin
+ # define yyclearin (yytoken = ]b4_symbol(empty, kind)[)
+ # undef YYBACKUP
+ # define YYBACKUP(Token, Value) \
+ return yyparser.error (]b4_locations_if([*yylocp, ])[YY_("syntax error: cannot back up")), \
+ yyerrok, yyerr
+
+]b4_variant_if([[
+ /* Variants are always initialized to an empty instance of the
+ correct type. The default '$$ = $1' action is NOT applied
+ when using variants. */
+ // However we really need to prepare yyvsp now if we want to get
+ // correct locations, so invoke YYFILL for $1 anyway.
+ (void) YYFILL (1-yyrhslen);
+ ]b4_symbol_variant([[yylhsNonterm (yyrule)]], [(*yyvalp)], [emplace])], [[
+ if (yyrhslen == 0)
+ *yyvalp = yyval_default;
+ else
+ *yyvalp = yyvsp[YYFILL (1-yyrhslen)].getState().value ();]])[]b4_locations_if([[
+ /* Default location. */
+ YYLLOC_DEFAULT ((*yylocp), (yyvsp - yyrhslen), yyrhslen);
+ yyerror_range[1].getState().yyloc = *yylocp;
+]])[
+ /* If yyk == -1, we are running a deferred action on a temporary
+ stack. In that case, YY_REDUCE_PRINT must not play with YYFILL,
+ so pretend the stack is "normal". */
+ YY_REDUCE_PRINT ((yynormal || yyk == create_state_set_index (-1), yyvsp, yyk, yyrule, yyparser));
+ #if YY_EXCEPTIONS
+ try
+ {
+ #endif // YY_EXCEPTIONS
+ switch (yyrule)
+ {
+ ]b4_user_actions[
+ default: break;
+ }
+ #if YY_EXCEPTIONS
+ }
+ catch (const syntax_error& yyexc)
+ {
+ YYCDEBUG << "Caught exception: " << yyexc.what() << '\n';]b4_locations_if([
+ *yylocp = yyexc.location;])[
+ yyparser.error (]b4_locations_if([*yylocp, ])[yyexc.what ());
+ YYERROR;
+ }
+ #endif // YY_EXCEPTIONS
+ YY_SYMBOL_PRINT ("-> $$ =", yylhsNonterm (yyrule), *yyvalp, *yylocp);
+
+ return yyok;
+ # undef yyerrok
+ # undef YYABORT
+ # undef YYACCEPT
+ # undef YYERROR
+ # undef YYBACKUP
+ # undef yytoken
+ # undef yyclearin
+ # undef YYRECOVERING
+ }
+
+ YYRESULTTAG
+ yyresolveStack ()
+ {
+ if (yystateStack.isSplit ())
+ {
+ int yyn = 0;
+ for (glr_state* yys = firstTopState ();
+ yys != yystateStack.yysplitPoint;
+ yys = yys->pred ())
+ yyn += 1;
+ YYCHK (yyresolveStates (*firstTopState (), yyn));
+ }
+ return yyok;
+ }
+
+ /** Pop the symbols consumed by reduction #YYRULE from the top of stack
+ * #YYK of *YYSTACKP, and perform the appropriate semantic action on their
+ * semantic values. Assumes that all ambiguities in semantic values
+ * have been previously resolved. Set *YYVALP to the resulting value,
+ * and *YYLOCP to the computed location (if any). Return value is as
+ * for userAction. */
+ YYRESULTTAG
+ yydoAction (state_set_index yyk, rule_num yyrule,
+ value_type* yyvalp]b4_locations_if([, location_type* yylocp])[)
+ {
+ const int yynrhs = yyrhsLength (yyrule);
+
+ if (!yystateStack.isSplit())
+ {
+ /* Standard special case: single stack. */
+ YYASSERT (yyk.get() == 0);
+ glr_stack_item* yyrhs = yystateStack.firstTop()->asItem();
+ const YYRESULTTAG res
+ = yyuserAction (yyrule, yynrhs, yyrhs, yyk, yyvalp]b4_locations_if([, yylocp])[);
+ yystateStack.pop_back(static_cast<size_t>(yynrhs));
+ yystateStack.setFirstTop(&yystateStack[yystateStack.size() - 1].getState());
+ return res;
+ }
+ else
+ {
+ glr_stack_item yyrhsVals[YYMAXRHS + YYMAXLEFT + 1];
+ glr_state* yys = yystateStack.topAt(yyk);
+ yyrhsVals[YYMAXRHS + YYMAXLEFT].getState().setPred(yys);]b4_locations_if([[
+ if (yynrhs == 0)
+ /* Set default location. */
+ yyrhsVals[YYMAXRHS + YYMAXLEFT - 1].getState().yyloc = yys->yyloc;]])[
+ for (int yyi = 0; yyi < yynrhs; yyi += 1)
+ {
+ yys = yys->pred();
+ YYASSERT (yys != YY_NULLPTR);
+ }
+ yystateStack.yyupdateSplit (*yys);
+ yystateStack.setTopAt(yyk, yys);
+ return yyuserAction (yyrule, yynrhs, yyrhsVals + YYMAXRHS + YYMAXLEFT - 1,
+ yyk,
+ yyvalp]b4_locations_if([, yylocp])[);
+ }
+ }
+
+ /** Pop items off stack #YYK of *YYSTACKP according to grammar rule YYRULE,
+ * and push back on the resulting nonterminal symbol. Perform the
+ * semantic action associated with YYRULE and store its value with the
+ * newly pushed state, if YYFORCEEVAL or if *YYSTACKP is currently
+ * unambiguous. Otherwise, store the deferred semantic action with
+ * the new state. If the new state would have an identical input
+ * position, LR state, and predecessor to an existing state on the stack,
+ * it is identified with that existing state, eliminating stack #YYK from
+ * *YYSTACKP. In this case, the semantic value is
+ * added to the options for the existing state's semantic value.
+ */
+ YYRESULTTAG
+ yyglrReduce (state_set_index yyk, rule_num yyrule, bool yyforceEval)
+ {
+ size_t yyposn = topState(yyk)->yyposn;
+
+ if (yyforceEval || !yystateStack.isSplit())
+ {
+ value_type val;]b4_locations_if([[
+ location_type loc;]])[
+
+ YYRESULTTAG yyflag = yydoAction (yyk, yyrule, &val]b4_locations_if([, &loc])[);
+ if (yyflag == yyerr && yystateStack.isSplit())
+ {]b4_parse_trace_if([[
+ YYCDEBUG << "Parse on stack " << yyk.get ()
+ << " rejected by rule " << yyrule - 1
+ << " (line " << int (yyrline[yyrule]) << ").\n";
+ ]])[}
+ if (yyflag != yyok)
+ return yyflag;
+ yyglrShift (yyk,
+ yyLRgotoState (topState(yyk)->yylrState,
+ yylhsNonterm (yyrule)),
+ yyposn, val]b4_locations_if([, loc])[);]b4_variant_if([[
+ // FIXME: User destructors.
+ // Value type destructor.
+ ]b4_symbol_variant([[yylhsNonterm (yyrule)]], [[val]], [[template destroy]])])[
+ }
+ else
+ {
+ glr_state *yys = yystateStack.topAt(yyk);
+ glr_state *yys0 = yys;
+ for (int yyn = yyrhsLength (yyrule); 0 < yyn; yyn -= 1)
+ {
+ yys = yys->pred();
+ YYASSERT (yys != YY_NULLPTR);
+ }
+ yystateStack.yyupdateSplit (*yys);
+ state_num yynewLRState = yyLRgotoState (yys->yylrState, yylhsNonterm (yyrule));]b4_parse_trace_if([[
+ YYCDEBUG << "Reduced stack " << yyk.get ()
+ << " by rule " << yyrule - 1 << " (line " << int (yyrline[yyrule])
+ << "); action deferred. Now in state " << yynewLRState
+ << ".\n";]])[
+ for (state_set_index yyi = create_state_set_index(0); yyi.uget() < yystateStack.numTops(); ++yyi)
+ if (yyi != yyk && yystateStack.topAt(yyi) != YY_NULLPTR)
+ {
+ const glr_state* yysplit = yystateStack.yysplitPoint;
+ glr_state* yyp = yystateStack.topAt(yyi);
+ while (yyp != yys && yyp != yysplit
+ && yyp->yyposn >= yyposn)
+ {
+ if (yyp->yylrState == yynewLRState
+ && yyp->pred() == yys)
+ {
+ yyaddDeferredAction (yyk, yyp, yys0, yyrule);
+ yystateStack.yytops.yymarkStackDeleted (yyk);
+ YYCDEBUG << "Merging stack " << yyk.get ()
+ << " into stack " << yyi.get () << ".\n";
+ return yyok;
+ }
+ yyp = yyp->pred();
+ }
+ }
+ yystateStack.setTopAt(yyk, yys);
+ yyglrShiftDefer (yyk, yynewLRState, yyposn, yys0, yyrule);
+ }
+ return yyok;
+ }
+
+ /** Shift stack #YYK of *YYSTACKP, to a new state corresponding to LR
+ * state YYLRSTATE, at input position YYPOSN, with the (unresolved)
+ * semantic value of YYRHS under the action for YYRULE. */
+ void
+ yyglrShiftDefer (state_set_index yyk, state_num yylrState,
+ size_t yyposn, glr_state* yyrhs, rule_num yyrule)
+ {
+ glr_state& yynewState = yystateStack.yynewGLRState (
+ glr_state (yylrState, yyposn));
+ yynewState.setPred (yystateStack.topAt (yyk));
+ yystateStack.setTopAt (yyk, &yynewState);
+
+ /* Invokes yyreserveStack. */
+ yyaddDeferredAction (yyk, &yynewState, yyrhs, yyrule);
+ }
+
+ /** Shift to a new state on stack #YYK of *YYSTACKP, corresponding to LR
+ * state YYLRSTATE, at input position YYPOSN, with (resolved) semantic
+ * value YYVAL_ARG and source location YYLOC_ARG. */
+ void
+ yyglrShift (state_set_index yyk, state_num yylrState,
+ size_t yyposn,
+ const value_type& yyval_arg]b4_locations_if([, const location_type& yyloc_arg])[)
+ {
+ glr_state& yynewState = yystateStack.yynewGLRState (
+ glr_state (yylrState, yyposn, yyval_arg]b4_locations_if([, yyloc_arg])[));
+ yynewState.setPred (yystateStack.topAt(yyk));
+ yystateStack.setTopAt (yyk, &yynewState);
+ yyreserveGlrStack ();
+ }
+
+#if ]b4_api_PREFIX[DEBUG
+ void
+ yypstack (state_set_index yyk) const
+ {
+ yystateStack.yypstates (yystateStack.topAt (yyk));
+ }
+#endif
+
+ glr_state* topState(state_set_index i) {
+ return yystateStack.topAt(i);
+ }
+
+ glr_state* firstTopState() {
+ return yystateStack.firstTop();
+ }
+
+ private:
+
+ void popall_ ()
+ {
+ /* If the stack is well-formed, pop the stack until it is empty,
+ destroying its entries as we go. But free the stack regardless
+ of whether it is well-formed. */
+ for (state_set_index k = create_state_set_index(0); k.uget() < yystateStack.numTops(); k += 1)
+ if (yystateStack.topAt(k) != YY_NULLPTR)
+ {
+ while (yystateStack.topAt(k) != YY_NULLPTR)
+ {
+ glr_state* state = topState(k);]b4_locations_if([[
+ yyerror_range[1].getState().yyloc = state->yyloc;]])[
+ if (state->pred() != YY_NULLPTR)
+ state->destroy ("Cleanup: popping", yyparser);
+ yystateStack.setTopAt(k, state->pred());
+ yystateStack.pop_back();
+ }
+ break;
+ }
+ }
+
+ /** Resolve the previous YYN states starting at and including state YYS
+ * on *YYSTACKP. If result != yyok, some states may have been left
+ * unresolved possibly with empty semantic option chains. Regardless
+ * of whether result = yyok, each state has been left with consistent
+ * data so that destroy can be invoked if necessary. */
+ YYRESULTTAG
+ yyresolveStates (glr_state& yys, int yyn)
+ {
+ if (0 < yyn)
+ {
+ YYASSERT (yys.pred() != YY_NULLPTR);
+ YYCHK (yyresolveStates (*yys.pred(), yyn-1));
+ if (! yys.yyresolved)
+ YYCHK (yyresolveValue (yys));
+ }
+ return yyok;
+ }
+
+ static void
+ yyuserMerge (int yyn, value_type& yy0, value_type& yy1)
+ {
+ YY_USE (yy0);
+ YY_USE (yy1);
+
+ switch (yyn)
+ {
+]b4_mergers[
+ default: break;
+ }
+ }
+
+ /** Resolve the ambiguity represented in state YYS in *YYSTACKP,
+ * perform the indicated actions, and set the semantic value of YYS.
+ * If result != yyok, the chain of semantic options in YYS has been
+ * cleared instead or it has been left unmodified except that
+ * redundant options may have been removed. Regardless of whether
+ * result = yyok, YYS has been left with consistent data so that
+ * destroy can be invoked if necessary. */
+ YYRESULTTAG
+ yyresolveValue (glr_state& yys)
+ {
+ semantic_option* yybest = yys.firstVal();
+ YYASSERT(yybest != YY_NULLPTR);
+ bool yymerge = false;
+ YYRESULTTAG yyflag;]b4_locations_if([
+ location_type *yylocp = &yys.yyloc;])[
+
+ semantic_option* yypPrev = yybest;
+ for (semantic_option* yyp = yybest->next();
+ yyp != YY_NULLPTR; )
+ {
+ if (yybest->isIdenticalTo (*yyp))
+ {
+ yybest->mergeWith (*yyp);
+ yypPrev->setNext(yyp->next());
+ yyp = yypPrev->next();
+ }
+ else
+ {
+ switch (yypreference (*yybest, *yyp))
+ {
+ case 0:]b4_locations_if([[
+ yyresolveLocations (yys, 1);]])[
+ return yystateStack.yyreportAmbiguity (*yybest, *yyp, yyparser]b4_locations_if([, *yylocp])[);
+ break;
+ case 1:
+ yymerge = true;
+ break;
+ case 2:
+ break;
+ case 3:
+ yybest = yyp;
+ yymerge = false;
+ break;
+ default:
+ /* This cannot happen so it is not worth a YYASSERT (false),
+ but some compilers complain if the default case is
+ omitted. */
+ break;
+ }
+ yypPrev = yyp;
+ yyp = yyp->next();
+ }
+ }
+
+ value_type val;
+ if (yymerge)
+ {
+ int yyprec = yydprec[yybest->yyrule];
+ yyflag = yyresolveAction (*yybest, &val]b4_locations_if([, yylocp])[);
+ if (yyflag == yyok)
+ for (semantic_option* yyp = yybest->next();
+ yyp != YY_NULLPTR;
+ yyp = yyp->next())
+ {
+ if (yyprec == yydprec[yyp->yyrule])
+ {
+ value_type yyval_other;]b4_locations_if([
+ location_type yydummy;])[
+ yyflag = yyresolveAction (*yyp, &yyval_other]b4_locations_if([, &yydummy])[);
+ if (yyflag != yyok)
+ {
+ yyparser.yy_destroy_ ("Cleanup: discarding incompletely merged value for",
+ yy_accessing_symbol (yys.yylrState),
+ this->yyla.value]b4_locations_if([, *yylocp])[);
+ break;
+ }
+ yyuserMerge (yymerger[yyp->yyrule], val, yyval_other);]b4_variant_if([[
+ // FIXME: User destructors.
+ // Value type destructor.
+ ]b4_symbol_variant([[yy_accessing_symbol (yys.yylrState)]], [[yyval_other]], [[template destroy]])])[
+ }
+ }
+ }
+ else
+ yyflag = yyresolveAction (*yybest, &val]b4_locations_if([, yylocp])[);
+
+ if (yyflag == yyok)
+ {
+ yys.yyresolved = true;
+ YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN]b4_variant_if([[
+ new (&yys.value ()) value_type ();
+ ]b4_symbol_variant([yy_accessing_symbol (yys.yylrState)],
+ [yys.value ()], [copy], [val])], [[
+ new (&yys.value ()) value_type (val);]])[
+
+ YY_IGNORE_MAYBE_UNINITIALIZED_END
+ }
+ else
+ yys.setFirstVal(YY_NULLPTR);
+]b4_variant_if([[
+ // FIXME: User destructors.
+ // Value type destructor.
+ ]b4_symbol_variant([[yy_accessing_symbol (yys.yylrState)]], [[val]], [[template destroy]])])[
+ return yyflag;
+ }
+
+ /** Resolve the states for the RHS of YYOPT on *YYSTACKP, perform its
+ * user action, and return the semantic value and location in *YYVALP
+ * and *YYLOCP. Regardless of whether result = yyok, all RHS states
+ * have been destroyed (assuming the user action destroys all RHS
+ * semantic values if invoked). */
+ YYRESULTTAG
+ yyresolveAction (semantic_option& yyopt, value_type* yyvalp]b4_locations_if([, location_type* yylocp])[)
+ {
+ glr_state* yyoptState = yyopt.state();
+ YYASSERT(yyoptState != YY_NULLPTR);
+ int yynrhs = yyrhsLength (yyopt.yyrule);
+ YYRESULTTAG yyflag = yyresolveStates (*yyoptState, yynrhs);
+ if (yyflag != yyok)
+ {
+ for (glr_state *yys = yyoptState; yynrhs > 0; yys = yys->pred(), yynrhs -= 1)
+ yys->destroy ("Cleanup: popping", yyparser);
+ return yyflag;
+ }
+
+ glr_stack_item yyrhsVals[YYMAXRHS + YYMAXLEFT + 1];
+ yyrhsVals[YYMAXRHS + YYMAXLEFT].getState().setPred(yyopt.state());]b4_locations_if([[
+ if (yynrhs == 0)
+ /* Set default location. */
+ yyrhsVals[YYMAXRHS + YYMAXLEFT - 1].getState().yyloc = yyoptState->yyloc;]])[
+ {
+ symbol_type yyla_current = std::move (this->yyla);
+ this->yyla = std::move (yyopt.yyla);
+ yyflag = yyuserAction (yyopt.yyrule, yynrhs,
+ yyrhsVals + YYMAXRHS + YYMAXLEFT - 1,
+ create_state_set_index (-1),
+ yyvalp]b4_locations_if([, yylocp])[);
+ this->yyla = std::move (yyla_current);
+ }
+ return yyflag;
+ }]b4_locations_if([[
+
+ /** Resolve the locations for each of the YYN1 states in *YYSTACKP,
+ * ending at YYS1. Has no effect on previously resolved states.
+ * The first semantic option of a state is always chosen. */
+ void
+ yyresolveLocations (glr_state &yys1, int yyn1)
+ {
+ if (0 < yyn1)
+ {
+ yyresolveLocations (*yys1.pred(), yyn1 - 1);
+ if (!yys1.yyresolved)
+ {
+ glr_stack_item yyrhsloc[1 + YYMAXRHS];
+ YYASSERT (yys1.firstVal() != YY_NULLPTR);
+ semantic_option& yyoption = *yys1.firstVal();
+ const int yynrhs = yyrhsLength (yyoption.yyrule);
+ if (0 < yynrhs)
+ {
+ yyresolveLocations (*yyoption.state(), yynrhs);
+ const glr_state *yys = yyoption.state();
+ for (int yyn = yynrhs; yyn > 0; yyn -= 1)
+ {
+ yyrhsloc[yyn].getState().yyloc = yys->yyloc;
+ yys = yys->pred();
+ }
+ }
+ else
+ {
+ /* Both yyresolveAction and yyresolveLocations traverse the GSS
+ in reverse rightmost order. It is only necessary to invoke
+ yyresolveLocations on a subforest for which yyresolveAction
+ would have been invoked next had an ambiguity not been
+ detected. Thus the location of the previous state (but not
+ necessarily the previous state itself) is guaranteed to be
+ resolved already. */
+ YY_IGNORE_NULL_DEREFERENCE_BEGIN
+ yyrhsloc[0].getState().yyloc = yyoption.state()->yyloc;
+ YY_IGNORE_NULL_DEREFERENCE_END
+ }
+ YYLLOC_DEFAULT ((yys1.yyloc), yyrhsloc, yynrhs);
+ }
+ }
+ }]])[
+
+ /** If yytoken is empty, fetch the next token. */
+ void
+ yyget_token ()
+ {
+]b4_parse_param_use()dnl
+[ if (this->yyla.empty ())
+ {
+ YYCDEBUG << "Reading a token\n";
+#if YY_EXCEPTIONS
+ try
+#endif // YY_EXCEPTIONS
+ {]b4_token_ctor_if([[
+ symbol_type yylookahead (]b4_yylex[);
+ yyla.move (yylookahead);]], [[
+ yyla.kind_ = yyparser.yytranslate_ (]b4_yylex[);]])[
+ }
+#if YY_EXCEPTIONS
+ catch (const parser_type::syntax_error& yyexc)
+ {
+ YYCDEBUG << "Caught exception: " << yyexc.what () << '\n';]b4_locations_if([
+ this->yyla.location = yyexc.location;])[
+ yyparser.error (]b4_locations_if([this->yyla.location, ])[yyexc.what ());
+ // Map errors caught in the scanner to the error token, so that error
+ // handling is started.
+ this->yyla.kind_ = ]b4_symbol(error, kind)[;
+ }
+ }
+#endif // YY_EXCEPTIONS
+ if (this->yyla.kind () == ]b4_symbol(eof, kind)[)
+ YYCDEBUG << "Now at end of input.\n";
+ else
+ YY_SYMBOL_PRINT ("Next token is", this->yyla.kind (), this->yyla.value, this->yyla.location);
+ }
+
+
+ /* Bison grammar-table manipulation. */
+
+ /** The action to take in YYSTATE on seeing YYTOKEN.
+ * Result R means
+ * R < 0: Reduce on rule -R.
+ * R = 0: Error.
+ * R > 0: Shift to state R.
+ * Set *YYCONFLICTS to a pointer into yyconfl to a 0-terminated list
+ * of conflicting reductions.
+ */
+ static int
+ yygetLRActions (state_num yystate, symbol_kind_type yytoken, const short*& yyconflicts)
+ {
+ int yyindex = yypact[yystate] + yytoken;
+ if (yytoken == ]b4_symbol(error, kind)[)
+ {
+ // This is the error token.
+ yyconflicts = yyconfl;
+ return 0;
+ }
+ else if (yy_is_defaulted_state (yystate)
+ || yyindex < 0 || YYLAST < yyindex || yycheck[yyindex] != yytoken)
+ {
+ yyconflicts = yyconfl;
+ return -yydefact[yystate];
+ }
+ else if (! yytable_value_is_error (yytable[yyindex]))
+ {
+ yyconflicts = yyconfl + yyconflp[yyindex];
+ return yytable[yyindex];
+ }
+ else
+ {
+ yyconflicts = yyconfl + yyconflp[yyindex];
+ return 0;
+ }
+ }
+
+ /** Compute post-reduction state.
+ * \param yystate the current state
+ * \param yysym the nonterminal to push on the stack
+ */
+ static state_num
+ yyLRgotoState (state_num yystate, symbol_kind_type yysym)
+ {
+ const int yyr = yypgoto[yysym - YYNTOKENS] + yystate;
+ if (0 <= yyr && yyr <= YYLAST && yycheck[yyr] == yystate)
+ return yytable[yyr];
+ else
+ return yydefgoto[yysym - YYNTOKENS];
+ }
+
+ static bool
+ yypact_value_is_default (state_num yystate)
+ {
+ return ]b4_table_value_equals([[pact]], [[yystate]], [b4_pact_ninf], [YYPACT_NINF])[;
+ }
+
+ static bool
+ yytable_value_is_error (int yytable_value YY_ATTRIBUTE_UNUSED)
+ {
+ return ]b4_table_value_equals([[table]], [[yytable_value]], [b4_table_ninf], [YYTABLE_NINF])[;
+ }
+
+ static bool
+ yy_is_shift_action (int yyaction) YY_NOEXCEPT
+ {
+ return 0 < yyaction;
+ }
+
+ static bool
+ yy_is_error_action (int yyaction) YY_NOEXCEPT
+ {
+ return yyaction == 0;
+ }
+
+ /** Whether LR state YYSTATE has only a default reduction
+ * (regardless of token). */
+ static bool
+ yy_is_defaulted_state (state_num yystate)
+ {
+ return yypact_value_is_default (yypact[yystate]);
+ }
+
+ /** The default reduction for YYSTATE, assuming it has one. */
+ static rule_num
+ yy_default_action (state_num yystate)
+ {
+ return yydefact[yystate];
+ }
+
+ /* GLRStacks */
+
+ /** Y0 and Y1 represent two possible actions to take in a given
+ * parsing state; return 0 if no combination is possible,
+ * 1 if user-mergeable, 2 if Y0 is preferred, 3 if Y1 is preferred. */
+ static int
+ yypreference (const semantic_option& y0, const semantic_option& y1)
+ {
+ const rule_num r0 = y0.yyrule, r1 = y1.yyrule;
+ const int p0 = yydprec[r0], p1 = yydprec[r1];
+
+ if (p0 == p1)
+ {
+ if (yymerger[r0] == 0 || yymerger[r0] != yymerger[r1])
+ return 0;
+ else
+ return 1;
+ }
+ else if (p0 == 0 || p1 == 0)
+ return 0;
+ else if (p0 < p1)
+ return 3;
+ else if (p1 < p0)
+ return 2;
+ else
+ return 0;
+ }
+
+]b4_parse_param_vars[
+ }; // class ]b4_parser_class[::glr_stack
+} // namespace ]b4_namespace_ref[
+
+
+#if ]b4_api_PREFIX[DEBUG
+namespace
+{
+ void
+ yypstack (const glr_stack& yystack, size_t yyk)
+ {
+ yystack.yypstack (create_state_set_index (static_cast<std::ptrdiff_t> (yyk)));
+ }
+
+ void
+ yypdumpstack (const glr_stack& yystack)
+ {
+ yystack.yypdumpstack ();
+ }
+}
+#endif
+
+]b4_namespace_open[
+ /// Build a parser object.
+ ]b4_parser_class::b4_parser_class[ (]b4_parse_param_decl[)]m4_ifset([b4_parse_param], [
+ :])[
+#if ]b4_api_PREFIX[DEBUG
+ ]m4_ifset([b4_parse_param], [ ], [ :])[yycdebug_ (&std::cerr)]m4_ifset([b4_parse_param], [,])[
+#endif]b4_parse_param_cons[
+ {}
+
+ ]b4_parser_class::~b4_parser_class[ ()
+ {}
+
+ ]b4_parser_class[::syntax_error::~syntax_error () YY_NOEXCEPT YY_NOTHROW
+ {}
+
+ int
+ ]b4_parser_class[::operator() ()
+ {
+ return parse ();
+ }
+
+ int
+ ]b4_parser_class[::parse ()
+ {
+ glr_stack yystack(YYINITDEPTH, *this]b4_user_args[);
+ return yystack.parse ();
+ }
+
+]b4_parse_error_bmatch([custom\|detailed],
+[[ const char *
+ ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+ {
+ static const char *const yy_sname[] =
+ {
+ ]b4_symbol_names[
+ };]b4_has_translations_if([[
+ /* YYTRANSLATABLE[SYMBOL-NUM] -- Whether YY_SNAME[SYMBOL-NUM] is
+ internationalizable. */
+ static ]b4_int_type_for([b4_translatable])[ yytranslatable[] =
+ {
+ ]b4_translatable[
+ };
+ return (yysymbol < YYNTOKENS && yytranslatable[yysymbol]
+ ? _(yy_sname[yysymbol])
+ : yy_sname[yysymbol]);]], [[
+ return yy_sname[yysymbol];]])[
+ }
+]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+ const char *
+ ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+ {
+ return yytname_[yysymbol];
+ }
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[ /* Return YYSTR after stripping away unnecessary quotes and
+ backslashes, so that it's suitable for yyerror. The heuristic is
+ that double-quoting is unnecessary unless the string contains an
+ apostrophe, a comma, or backslash (other than backslash-backslash).
+ YYSTR is taken from yytname. */
+ std::string
+ ]b4_parser_class[::yytnamerr_ (const char *yystr)
+ {
+ if (*yystr == '"')
+ {
+ std::string yyr;
+ char const *yyp = yystr;
+
+ for (;;)
+ switch (*++yyp)
+ {
+ case '\'':
+ case ',':
+ goto do_not_strip_quotes;
+
+ case '\\':
+ if (*++yyp != '\\')
+ goto do_not_strip_quotes;
+ else
+ goto append;
+
+ append:
+ default:
+ yyr += *yyp;
+ break;
+
+ case '"':
+ return yyr;
+ }
+ do_not_strip_quotes: ;
+ }
+
+ return yystr;
+ }
+
+ std::string
+ ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+ {
+ return yytnamerr_ (yytname_[yysymbol]);
+ }
+]])[
+
+]b4_parse_error_bmatch([simple\|verbose],
+[[#if ]b4_api_PREFIX[DEBUG]b4_tname_if([[ || 1]])[
+ // YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+ // First, the terminals, then, starting at \a YYNTOKENS, nonterminals.
+ const char*
+ const ]b4_parser_class[::yytname_[] =
+ {
+ ]b4_tname[
+ };
+#endif
+]])[
+
+]b4_parse_error_bmatch([custom\|detailed\|verbose], [[
+ // ]b4_parser_class[::context.
+ ]b4_parser_class[::context::context (glr_stack& yystack, const symbol_type& yyla)
+ : yystack_ (yystack)
+ , yyla_ (yyla)
+ {}
+
+ int
+ ]b4_parser_class[::context::expected_tokens (symbol_kind_type yyarg[], int yyargn) const
+ {
+ // Actual number of expected tokens
+ int yycount = 0;
+ const int yyn = yypact[yystack_.firstTopState()->yylrState];
+ if (!yystack_.yypact_value_is_default (yyn))
+ {
+ /* Start YYX at -YYN if negative to avoid negative indexes in
+ YYCHECK. In other words, skip the first -YYN actions for this
+ state because they are default actions. */
+ const int yyxbegin = yyn < 0 ? -yyn : 0;
+ /* Stay within bounds of both yycheck and yytname. */
+ const int yychecklim = YYLAST - yyn + 1;
+ const int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
+ for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
+ if (yycheck[yyx + yyn] == yyx && yyx != ]b4_symbol(error, kind)[
+ && !yystack_.yytable_value_is_error (yytable[yyx + yyn]))
+ {
+ if (!yyarg)
+ ++yycount;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = YY_CAST (symbol_kind_type, yyx);
+ }
+ }
+ if (yyarg && yycount == 0 && 0 < yyargn)
+ yyarg[0] = ]b4_symbol(empty, kind)[;
+ return yycount;
+ }
+
+]])[
+
+]b4_parse_error_bmatch([detailed\|verbose], [[
+ int
+ ]b4_parser_class[::yy_syntax_error_arguments_ (const context& yyctx,
+ symbol_kind_type yyarg[], int yyargn) const
+ {
+ /* There are many possibilities here to consider:
+ - If this state is a consistent state with a default action, then
+ the only way this function was invoked is if the default action
+ is an error action. In that case, don't check for expected
+ tokens because there are none.
+ - The only way there can be no lookahead present (in yyla) is
+ if this state is a consistent state with a default action.
+ Thus, detecting the absence of a lookahead is sufficient to
+ determine that there is no unexpected or expected token to
+ report. In that case, just report a simple "syntax error".
+ - Don't assume there isn't a lookahead just because this state is
+ a consistent state with a default action. There might have
+ been a previous inconsistent state, consistent state with a
+ non-default action, or user semantic action that manipulated
+ yyla. (However, yyla is currently not documented for users.)
+ */
+
+ if (!yyctx.lookahead ().empty ())
+ {
+ if (yyarg)
+ yyarg[0] = yyctx.token ();
+ int yyn = yyctx.expected_tokens (yyarg ? yyarg + 1 : yyarg, yyargn - 1);
+ return yyn + 1;
+ }
+ return 0;
+ }
+
+ // Generate an error message.
+ std::string
+ ]b4_parser_class[::yysyntax_error_ (const context& yyctx) const
+ {
+ // Its maximum.
+ enum { YYARGS_MAX = 5 };
+ // Arguments of yyformat.
+ symbol_kind_type yyarg[YYARGS_MAX];
+ int yycount = yy_syntax_error_arguments_ (yyctx, yyarg, YYARGS_MAX);
+
+ char const* yyformat = YY_NULLPTR;
+ switch (yycount)
+ {
+#define YYCASE_(N, S) \
+ case N: \
+ yyformat = S; \
+ break
+ default: // Avoid compiler warnings.
+ YYCASE_ (0, YY_("syntax error"));
+ YYCASE_ (1, YY_("syntax error, unexpected %s"));
+ YYCASE_ (2, YY_("syntax error, unexpected %s, expecting %s"));
+ YYCASE_ (3, YY_("syntax error, unexpected %s, expecting %s or %s"));
+ YYCASE_ (4, YY_("syntax error, unexpected %s, expecting %s or %s or %s"));
+ YYCASE_ (5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"));
+#undef YYCASE_
+ }
+
+ std::string yyres;
+ // Argument number.
+ std::ptrdiff_t yyi = 0;
+ for (char const* yyp = yyformat; *yyp; ++yyp)
+ if (yyp[0] == '%' && yyp[1] == 's' && yyi < yycount)
+ {
+ yyres += symbol_name (yyarg[yyi++]);
+ ++yyp;
+ }
+ else
+ yyres += *yyp;
+ return yyres;
+ }]])[
+
+ void
+ ]b4_parser_class[::yy_destroy_ (const char* yymsg, symbol_kind_type yykind,
+ value_type& yyval]b4_locations_if([[,
+ location_type& yyloc]])[)
+ {
+ YY_USE (yyval);]b4_locations_if([[
+ YY_USE (yyloc);]])[
+ if (!yymsg)
+ yymsg = "Deleting";
+ ]b4_parser_class[& yyparser = *this;
+ YY_USE (yyparser);
+ YY_SYMBOL_PRINT (yymsg, yykind, yyval, yyloc);
+
+ YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
+ ]m4_do([m4_pushdef([b4_symbol_action], m4_defn([b4_symbol_action_for_yyval]))],
+ [b4_symbol_actions([destructor])],
+ [m4_popdef([b4_symbol_action])])[
+ YY_IGNORE_MAYBE_UNINITIALIZED_END
+ }
+
+#if ]b4_api_PREFIX[DEBUG
+ /*--------------------.
+ | Print this symbol. |
+ `--------------------*/
+
+ void
+ ]b4_parser_class[::yy_symbol_value_print_ (symbol_kind_type yykind,
+ const value_type& yyval]b4_locations_if([[,
+ const location_type& yyloc]])[) const
+ {]b4_locations_if([[
+ YY_USE (yyloc);]])[
+ YY_USE (yyval);
+ std::ostream& yyo = debug_stream ();
+ YY_USE (yyo);
+ ]m4_do([m4_pushdef([b4_symbol_action], m4_defn([b4_symbol_action_for_yyval]))],
+ [b4_symbol_actions([printer])],
+ [m4_popdef([b4_symbol_action])])[
+ }
+
+ void
+ ]b4_parser_class[::yy_symbol_print_ (symbol_kind_type yykind,
+ const value_type& yyval]b4_locations_if([[,
+ const location_type& yyloc]])[) const
+ {
+ *yycdebug_ << (yykind < YYNTOKENS ? "token" : "nterm")
+ << ' ' << symbol_name (yykind) << " ("]b4_locations_if([[
+ << yyloc << ": "]])[;
+ yy_symbol_value_print_ (yykind, yyval]b4_locations_if([[, yyloc]])[);
+ *yycdebug_ << ')';
+ }
+
+ std::ostream&
+ ]b4_parser_class[::debug_stream () const
+ {
+ return *yycdebug_;
+ }
+
+ void
+ ]b4_parser_class[::set_debug_stream (std::ostream& o)
+ {
+ yycdebug_ = &o;
+ }
+
+
+ ]b4_parser_class[::debug_level_type
+ ]b4_parser_class[::debug_level () const
+ {
+ return yydebug;
+ }
+
+ void
+ ]b4_parser_class[::set_debug_level (debug_level_type l)
+ {
+ // Actually, it is yydebug which is really used.
+ yydebug = l;
+ }
+#endif // ]b4_api_PREFIX[DEBUG
+
+]b4_token_ctor_if([], [b4_yytranslate_define([cc])])[
+
+]b4_token_ctor_if([], [[
+ /*---------.
+ | symbol. |
+ `---------*/
+]b4_public_types_define([cc])])[
+]b4_namespace_close[]dnl
+b4_epilogue[]dnl
+b4_output_end
diff --git a/common/bison/skeletons/java-skel.m4 b/common/bison/skeletons/java-skel.m4
index 922f1a8..11cbc49 100644
--- a/common/bison/skeletons/java-skel.m4
+++ b/common/bison/skeletons/java-skel.m4
@@ -2,7 +2,7 @@
# Java skeleton dispatching for Bison.
-# Copyright (C) 2007, 2009-2015, 2018-2019 Free Software Foundation,
+# Copyright (C) 2007, 2009-2015, 2018-2021 Free Software Foundation,
# Inc.
# This program is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
b4_glr_if( [b4_complain([%%glr-parser not supported for Java])])
b4_nondeterministic_if([b4_complain([%%nondeterministic-parser not supported for Java])])
diff --git a/common/bison/skeletons/java.m4 b/common/bison/skeletons/java.m4
index edf23a9..8b0828b 100644
--- a/common/bison/skeletons/java.m4
+++ b/common/bison/skeletons/java.m4
@@ -2,7 +2,7 @@
# Java language support for Bison
-# Copyright (C) 2007-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2007-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,7 +15,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
m4_include(b4_skeletonsdir/[c-like.m4])
@@ -71,12 +71,12 @@
# b4_identification
# -----------------
m4_define([b4_identification],
-[ /** Version number for the Bison executable that generated this parser. */
- public static final String bisonVersion = "b4_version";
+[[ /** Version number for the Bison executable that generated this parser. */
+ public static final String bisonVersion = "]b4_version_string[";
/** Name of the skeleton that generated this parser. */
- public static final String bisonSkeleton = b4_skeleton;
-])
+ public static final String bisonSkeleton = ]b4_skeleton[;
+]])
## ------------ ##
@@ -106,10 +106,13 @@
# b4_typed_parser_table_define(TYPE, NAME, DATA, COMMENT)
# -------------------------------------------------------
+# We use intermediate functions (e.g., yypact_init) to work around the
+# 64KB limit for JVM methods. See
+# https://lists.gnu.org/r/help-bison/2008-11/msg00004.html.
m4_define([b4_typed_parser_table_define],
[m4_ifval([$4], [b4_comment([$4])
])dnl
-[private static final ]$1[ yy$2_[] = yy$2_init();
+[private static final ]$1[[] yy$2_ = yy$2_init();
private static final ]$1[[] yy$2_init()
{
return new ]$1[[]
@@ -125,40 +128,158 @@
[b4_typed_parser_table_define([b4_int_type_for([$2])], [$1], [$2], [$3])])
-## ------------------------- ##
-## Assigning token numbers. ##
-## ------------------------- ##
+## ------------- ##
+## Token kinds. ##
+## ------------- ##
+
# b4_token_enum(TOKEN-NUM)
# ------------------------
# Output the definition of this token as an enum.
m4_define([b4_token_enum],
-[b4_token_format([ /** Token number, to be returned by the scanner. */
- static final int %s = %s;
-], [$1])])
+[b4_token_visible_if([$1],
+ [m4_format([[ /** Token %s, to be returned by the scanner. */
+ static final int %s = %s%s;
+]],
+ b4_symbol([$1], [tag]),
+ b4_symbol([$1], [id]),
+ b4_symbol([$1], b4_api_token_raw_if([[number]], [[code]])))])])
+
# b4_token_enums
# --------------
# Output the definition of the tokens (if there are) as enums.
m4_define([b4_token_enums],
-[b4_any_token_visible_if([/* Tokens. */
+[b4_any_token_visible_if([ /* Token kinds. */
b4_symbol_foreach([b4_token_enum])])])
-# b4-case(ID, CODE)
-# -----------------
+
+
+## -------------- ##
+## Symbol kinds. ##
+## -------------- ##
+
+
+# b4_symbol_kind(NUM)
+# -------------------
+m4_define([b4_symbol_kind],
+[SymbolKind.b4_symbol_kind_base($@)])
+
+
+# b4_symbol_enum(SYMBOL-NUM)
+# --------------------------
+# Output the definition of this symbol as an enum.
+m4_define([b4_symbol_enum],
+[m4_format([ %-30s %s],
+ m4_format([[%s(%s)%s]],
+ b4_symbol([$1], [kind_base]),
+ [$1],
+ m4_if([$1], b4_last_symbol, [[;]], [[,]])),
+ [b4_symbol_tag_comment([$1])])])
+
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+m4_define([b4_declare_symbol_enum],
+[[ public enum SymbolKind
+ {
+]b4_symbol_foreach([b4_symbol_enum])[
+
+ private final int yycode_;
+
+ SymbolKind (int n) {
+ this.yycode_ = n;
+ }
+
+ private static final SymbolKind[] values_ = {
+ ]m4_map_args_sep([b4_symbol_kind(], [)], [,
+ ], b4_symbol_numbers)[
+ };
+
+ static final SymbolKind get(int code) {
+ return values_[code];
+ }
+
+ public final int getCode() {
+ return this.yycode_;
+ }
+
+]b4_parse_error_bmatch(
+[simple\|verbose],
+[[ /* Return YYSTR after stripping away unnecessary quotes and
+ backslashes, so that it's suitable for yyerror. The heuristic is
+ that double-quoting is unnecessary unless the string contains an
+ apostrophe, a comma, or backslash (other than backslash-backslash).
+ YYSTR is taken from yytname. */
+ private static String yytnamerr_(String yystr)
+ {
+ if (yystr.charAt (0) == '"')
+ {
+ StringBuffer yyr = new StringBuffer();
+ strip_quotes: for (int i = 1; i < yystr.length(); i++)
+ switch (yystr.charAt(i))
+ {
+ case '\'':
+ case ',':
+ break strip_quotes;
+
+ case '\\':
+ if (yystr.charAt(++i) != '\\')
+ break strip_quotes;
+ /* Fall through. */
+ default:
+ yyr.append(yystr.charAt(i));
+ break;
+
+ case '"':
+ return yyr.toString();
+ }
+ }
+ return yystr;
+ }
+
+ /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+ First, the terminals, then, starting at \a YYNTOKENS_, nonterminals. */
+ ]b4_typed_parser_table_define([String], [tname], [b4_tname])[
+
+ /* The user-facing name of this symbol. */
+ public final String getName() {
+ return yytnamerr_(yytname_[yycode_]);
+ }
+]],
+[custom\|detailed],
+[[ /* YYNAMES_[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+ First, the terminals, then, starting at \a YYNTOKENS_, nonterminals. */
+ ]b4_typed_parser_table_define([String], [names], [b4_symbol_names])[
+
+ /* The user-facing name of this symbol. */
+ public final String getName() {
+ return yynames_[yycode_];
+ }]])[
+ };
+]])])
+
+
+
+# b4_case(ID, CODE, [COMMENTS])
+# -----------------------------
# We need to fool Java's stupid unreachable code detection.
-m4_define([b4_case], [ case $1:
+m4_define([b4_case],
+[ case $1:m4_ifval([$3], [ b4_comment([$3])])
if (yyn == $1)
$2;
break;
- ])
+])
+
# b4_predicate_case(LABEL, CONDITIONS)
# ------------------------------------
-m4_define([b4_predicate_case], [ case $1:
+m4_define([b4_predicate_case],
+[ case $1:
if (! ($2)) YYERROR;
break;
- ])
+])
## -------- ##
@@ -183,6 +304,7 @@
m4_define([b4_yystype], [b4_percent_define_get([[api.value.type]])])
b4_percent_define_default([[api.value.type]], [[Object]])
+b4_percent_define_default([[api.symbol.prefix]], [[S_]])
# b4_api_prefix, b4_api_PREFIX
# ----------------------------
@@ -222,6 +344,22 @@
## ----------------- ##
+# b4_symbol_translate(STRING)
+# ---------------------------
+# Used by "bison" in the array of symbol names to mark those that
+# require translation.
+m4_define([b4_symbol_translate],
+[[i18n($1)]])
+
+
+# b4_trans(STRING)
+# ----------------
+# Translate a string if i18n is enabled. Avoid collision with b4_translate.
+m4_define([b4_trans],
+[b4_has_translations_if([i18n($1)], [$1])])
+
+
+
# b4_symbol_value(VAL, [SYMBOL-NUM], [TYPE-TAG])
# ----------------------------------------------
# See README.
@@ -361,4 +499,4 @@
# -----------------------
# Expand to either an empty string or "throws THROWS".
m4_define([b4_maybe_throws],
- [m4_ifval($1, [throws $1])])
+ [m4_ifval($1, [ throws $1])])
diff --git a/common/bison/skeletons/lalr1.cc b/common/bison/skeletons/lalr1.cc
index d245796..7cb69d3 100644
--- a/common/bison/skeletons/lalr1.cc
+++ b/common/bison/skeletons/lalr1.cc
@@ -1,6 +1,6 @@
# C++ skeleton for Bison
-# Copyright (C) 2002-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,22 +13,31 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
m4_include(b4_skeletonsdir/[c++.m4])
# api.value.type=variant is valid.
m4_define([b4_value_type_setup_variant])
-# Check the value of %define parse.lac, where LAC stands for lookahead
-# correction.
+# parse.lac
b4_percent_define_default([[parse.lac]], [[none]])
+b4_percent_define_check_values([[[[parse.lac]], [[full]], [[none]]]])
b4_define_flag_if([lac])
m4_define([b4_lac_flag],
[m4_if(b4_percent_define_get([[parse.lac]]),
[none], [[0]], [[1]])])
+# b4_tname_if(TNAME-NEEDED, TNAME-NOT-NEEDED)
+# -------------------------------------------
+m4_define([b4_tname_if],
+[m4_case(b4_percent_define_get([[parse.error]]),
+ [verbose], [$1],
+ [b4_token_table_if([$1],
+ [$2])])])
+
+
# b4_integral_parser_table_declare(TABLE-NAME, CONTENT, COMMENT)
# --------------------------------------------------------------
# Declare "parser::yy<TABLE-NAME>_" whose contents is CONTENT.
@@ -49,6 +58,13 @@
};dnl
])
+
+# b4_symbol_kind(NUM)
+# -------------------
+m4_define([b4_symbol_kind],
+[symbol_kind::b4_symbol_kind_base($@)])
+
+
# b4_symbol_value_template(VAL, SYMBOL-NUM, [TYPE])
# -------------------------------------------------
# Same as b4_symbol_value, but used in a template method. It makes
@@ -129,21 +145,21 @@
])])
-# b4_lex
-# ------
+# b4_yylex
+# --------
# Call yylex.
-m4_define([b4_lex],
+m4_define([b4_yylex],
[b4_token_ctor_if(
[b4_function_call([yylex],
[symbol_type], m4_ifdef([b4_lex_param], b4_lex_param))],
[b4_function_call([yylex], [int],
- [b4_api_PREFIX[STYPE*], [&yyla.value]][]dnl
-b4_locations_if([, [[location*], [&yyla.location]]])dnl
+ [[value_type *], [&yyla.value]][]dnl
+b4_locations_if([, [[location_type *], [&yyla.location]]])dnl
m4_ifdef([b4_lex_param], [, ]b4_lex_param))])])
m4_pushdef([b4_copyright_years],
- [2002-2015, 2018-2019])
+ [2002-2015, 2018-2021])
m4_define([b4_parser_class],
[b4_percent_define_get([[api.parser.class]])])
@@ -157,7 +173,7 @@
# b4_shared_declarations(hh|cc)
# -----------------------------
-# Declaration that might either go into the header (if --defines, $1 = hh)
+# Declaration that might either go into the header (if --header, $1 = hh)
# or in the implementation file.
m4_define([b4_shared_declarations],
[b4_percent_code_get([[requires]])[
@@ -194,6 +210,13 @@
]b4_parser_class[ (]b4_parse_param_decl[);
virtual ~]b4_parser_class[ ();
+#if 201103L <= YY_CPLUSPLUS
+ /// Non copyable.
+ ]b4_parser_class[ (const ]b4_parser_class[&) = delete;
+ /// Non copyable.
+ ]b4_parser_class[& operator= (const ]b4_parser_class[&) = delete;
+#endif
+
/// Parse. An alias for parse ().
/// \returns 0 iff parsing succeeded.
int operator() ();
@@ -224,19 +247,57 @@
/// Report a syntax error.
void error (const syntax_error& err);
+]b4_parse_error_bmatch(
+[custom\|detailed],
+[[ /// The user-facing name of the symbol whose (internal) number is
+ /// YYSYMBOL. No bounds checking.
+ static const char *symbol_name (symbol_kind_type yysymbol);]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+ /// The user-facing name of the symbol whose (internal) number is
+ /// YYSYMBOL. No bounds checking.
+ static const char *symbol_name (symbol_kind_type yysymbol);
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[ /// The user-facing name of the symbol whose (internal) number is
+ /// YYSYMBOL. No bounds checking.
+ static std::string symbol_name (symbol_kind_type yysymbol);]])[
+
]b4_token_constructor_define[
+]b4_parse_error_bmatch([custom\|detailed\|verbose], [[
+ class context
+ {
+ public:
+ context (const ]b4_parser_class[& yyparser, const symbol_type& yyla);
+ const symbol_type& lookahead () const YY_NOEXCEPT { return yyla_; }
+ symbol_kind_type token () const YY_NOEXCEPT { return yyla_.kind (); }]b4_locations_if([[
+ const location_type& location () const YY_NOEXCEPT { return yyla_.location; }
+]])[
+ /// Put in YYARG at most YYARGN of the expected tokens, and return the
+ /// number of tokens stored in YYARG. If YYARG is null, return the
+ /// number of expected tokens (guaranteed to be less than YYNTOKENS).
+ int expected_tokens (symbol_kind_type yyarg[], int yyargn) const;
+ private:
+ const ]b4_parser_class[& yyparser_;
+ const symbol_type& yyla_;
+ };
+]])[
private:
- /// This class is not copyable.
+#if YY_CPLUSPLUS < 201103L
+ /// Non copyable.
]b4_parser_class[ (const ]b4_parser_class[&);
- ]b4_parser_class[& operator= (const ]b4_parser_class[&);]b4_lac_if([[
-
+ /// Non copyable.
+ ]b4_parser_class[& operator= (const ]b4_parser_class[&);
+#endif
+]b4_lac_if([[
/// Check the lookahead yytoken.
/// \returns true iff the token will be eventually shifted.
- bool yy_lac_check_ (int yytoken) const;
+ bool yy_lac_check_ (symbol_kind_type yytoken) const;
/// Establish the initial context if no initial context currently exists.
/// \returns true iff the token will be eventually shifted.
- bool yy_lac_establish_ (int yytoken);
+ bool yy_lac_establish_ (symbol_kind_type yytoken);
/// Discard any previous initial lookahead context because of event.
/// \param event the event which caused the lookahead to be discarded.
/// Only used for debbuging output.
@@ -244,13 +305,19 @@
/// Stored state numbers (used for stacks).
typedef ]b4_int_type(0, m4_eval(b4_states_number - 1))[ state_type;
+]b4_parse_error_bmatch(
+[custom], [[
+ /// Report a syntax error
+ /// \param yyctx the context in which the error occurred.
+ void report_syntax_error (const context& yyctx) const;]],
+[detailed\|verbose], [[
+ /// The arguments of the error message.
+ int yy_syntax_error_arguments_ (const context& yyctx,
+ symbol_kind_type yyarg[], int yyargn) const;
/// Generate an error message.
- /// \param yystate the state where the error occurred.
- /// \param yyla the lookahead token.
- virtual std::string yysyntax_error_ (state_type yystate,
- const symbol_type& yyla) const;
-
+ /// \param yyctx the context in which the error occurred.
+ virtual std::string yysyntax_error_ (const context& yyctx) const;]])[
/// Compute post-reduction state.
/// \param yystate the current state
/// \param yysym the nonterminal to push on the stack
@@ -258,43 +325,52 @@
/// Whether the given \c yypact_ value indicates a defaulted state.
/// \param yyvalue the value to check
- static bool yy_pact_value_is_default_ (int yyvalue);
+ static bool yy_pact_value_is_default_ (int yyvalue) YY_NOEXCEPT;
/// Whether the given \c yytable_ value indicates a syntax error.
/// \param yyvalue the value to check
- static bool yy_table_value_is_error_ (int yyvalue);
+ static bool yy_table_value_is_error_ (int yyvalue) YY_NOEXCEPT;
static const ]b4_int_type(b4_pact_ninf, b4_pact_ninf)[ yypact_ninf_;
static const ]b4_int_type(b4_table_ninf, b4_table_ninf)[ yytable_ninf_;
- /// Convert a scanner token number \a t to a symbol number.
- /// In theory \a t should be a token_type, but character literals
- /// are valid, yet not members of the token_type enum.
- static token_number_type yytranslate_ (int t);
+ /// Convert a scanner token kind \a t to a symbol kind.
+ /// In theory \a t should be a token_kind_type, but character literals
+ /// are valid, yet not members of the token_kind_type enum.
+ static symbol_kind_type yytranslate_ (int t) YY_NOEXCEPT;
- // Tables.
-]b4_parser_tables_declare[]b4_error_verbose_if([
-
- /// Convert the symbol name \a n to a form suitable for a diagnostic.
- static std::string yytnamerr_ (const char *n);])[
-
-]b4_token_table_if([], [[#if ]b4_api_PREFIX[DEBUG]])[
+]b4_parse_error_bmatch(
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
/// For a symbol, its name in clear.
static const char* const yytname_[];
-]b4_token_table_if([[#if ]b4_api_PREFIX[DEBUG]])[
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[ /// Convert the symbol name \a n to a form suitable for a diagnostic.
+ static std::string yytnamerr_ (const char *yystr);
+
+ /// For a symbol, its name in clear.
+ static const char* const yytname_[];
+]])[
+
+ // Tables.
+]b4_parser_tables_declare[
+
+#if ]b4_api_PREFIX[DEBUG
]b4_integral_parser_table_declare([rline], [b4_rline],
[[YYRLINE[YYN] -- Source line where rule number YYN was defined.]])[
/// Report on the debug stream that the rule \a r is going to be reduced.
- virtual void yy_reduce_print_ (int r);
+ virtual void yy_reduce_print_ (int r) const;
/// Print the state stack on the debug stream.
- virtual void yystack_print_ ();
+ virtual void yy_stack_print_ () const;
/// Debugging level.
int yydebug_;
/// Debug stream.
std::ostream* yycdebug_;
- /// \brief Display a symbol type, value and location.
+ /// \brief Display a symbol kind, value and location.
/// \param yyo The output stream.
/// \param yysym The symbol.
template <typename Base>
@@ -315,7 +391,7 @@
/// Default constructor.
by_state () YY_NOEXCEPT;
- /// The symbol type as needed by the constructor.
+ /// The symbol kind as needed by the constructor.
typedef state_type kind_type;
/// Constructor.
@@ -327,12 +403,12 @@
/// Record that this symbol is empty.
void clear () YY_NOEXCEPT;
- /// Steal the symbol type from \a that.
+ /// Steal the symbol kind from \a that.
void move (by_state& that);
- /// The (internal) type number (corresponding to \a state).
- /// \a empty_symbol when empty.
- symbol_number_type type_get () const YY_NOEXCEPT;
+ /// The symbol kind (corresponding to \a state).
+ /// \a ]b4_symbol(empty, kind)[ when empty.
+ symbol_kind_type kind () const YY_NOEXCEPT;
/// The state number used to denote an empty symbol.
/// We use the initial state, as it does not have a value.
@@ -398,64 +474,60 @@
void yypush_ (const char* m, state_type s, YY_MOVE_REF (symbol_type) sym);
/// Pop \a n symbols from the stack.
- void yypop_ (int n = 1);
-
- /// Some specific tokens.
- static const token_number_type yy_error_token_ = 1;
- static const token_number_type yy_undef_token_ = ]b4_undef_token_number[;
+ void yypop_ (int n = 1) YY_NOEXCEPT;
/// Constants.
enum
{
- yyeof_ = 0,
yylast_ = ]b4_last[, ///< Last index in yytable_.
yynnts_ = ]b4_nterms_number[, ///< Number of nonterminal symbols.
- yyfinal_ = ]b4_final_state_number[, ///< Termination state number.
- yyntokens_ = ]b4_tokens_number[ ///< Number of tokens.
+ yyfinal_ = ]b4_final_state_number[ ///< Termination state number.
};
]b4_parse_param_vars[
+]b4_percent_code_get([[yy_bison_internal_hook]])[
};
]b4_token_ctor_if([b4_yytranslate_define([$1])[
]b4_public_types_define([$1])])[
]b4_namespace_close[
-]b4_percent_define_flag_if([[global_tokens_and_yystype]],
-[b4_token_defines
-
-#ifndef ]b4_api_PREFIX[STYPE
- // Redirection for backward compatibility.
-# define ]b4_api_PREFIX[STYPE b4_namespace_ref::b4_parser_class::semantic_type
-#endif
-])[
]b4_percent_code_get([[provides]])[
-]])
+]])[
+
## -------------- ##
## Output files. ##
## -------------- ##
-b4_defines_if(
-[b4_output_begin([b4_spec_header_file])
-b4_copyright([Skeleton interface for Bison LALR(1) parsers in C++])
-[
+# ------------- #
+# Header file. #
+# ------------- #
+
+]b4_header_if([[
+]b4_output_begin([b4_spec_header_file])[
+]b4_copyright([Skeleton interface for Bison LALR(1) parsers in C++])[
+
/**
- ** \file ]b4_spec_header_file[
+ ** \file ]b4_spec_mapped_header_file[
** Define the ]b4_namespace_ref[::parser class.
*/
// C++ LALR(1) parser skeleton written by Akim Demaille.
]b4_disclaimer[
-]b4_cpp_guard_open([b4_spec_header_file])[
+]b4_cpp_guard_open([b4_spec_mapped_header_file])[
]b4_shared_declarations(hh)[
-]b4_cpp_guard_close([b4_spec_header_file])[
+]b4_cpp_guard_close([b4_spec_mapped_header_file])[
]b4_output_end[
-]])
+]])[
-b4_output_begin([b4_parser_file_name])[
+# --------------------- #
+# Implementation file. #
+# --------------------- #
+
+]b4_output_begin([b4_parser_file_name])[
]b4_copyright([Skeleton implementation for Bison LALR(1) parsers in C++])[
]b4_disclaimer[
]b4_percent_code_get([[top]])[]dnl
@@ -466,7 +538,7 @@
]b4_user_pre_prologue[
-]b4_defines_if([[#include "@basename(]b4_spec_header_file[@)"]],
+]b4_header_if([[#include "@basename(]b4_spec_header_file[@)"]],
[b4_shared_declarations([cc])])[
]b4_user_post_prologue[
@@ -483,6 +555,11 @@
# define YY_(msgid) msgid
# endif
#endif
+]b4_has_translations_if([
+#ifndef N_
+# define N_(Msgid) Msgid
+#endif
+])[
// Whether we are compiled with exception support.
#ifndef YY_EXCEPTIONS
@@ -522,13 +599,13 @@
# define YY_STACK_PRINT() \
do { \
if (yydebug_) \
- yystack_print_ (); \
+ yy_stack_print_ (); \
} while (false)
#else // !]b4_api_PREFIX[DEBUG
# define YYCDEBUG if (false) std::cerr
-# define YY_SYMBOL_PRINT(Title, Symbol) YYUSE (Symbol)
+# define YY_SYMBOL_PRINT(Title, Symbol) YY_USE (Symbol)
# define YY_REDUCE_PRINT(Rule) static_cast<void> (0)
# define YY_STACK_PRINT() static_cast<void> (0)
@@ -542,9 +619,623 @@
#define YYERROR goto yyerrorlab
#define YYRECOVERING() (!!yyerrstatus_)
-]b4_namespace_open[]b4_error_verbose_if([[
+]b4_namespace_open[
+ /// Build a parser object.
+ ]b4_parser_class::b4_parser_class[ (]b4_parse_param_decl[)
+#if ]b4_api_PREFIX[DEBUG
+ : yydebug_ (false),
+ yycdebug_ (&std::cerr)]b4_lac_if([,], [m4_ifset([b4_parse_param], [,])])[
+#else
+]b4_lac_if([ :], [m4_ifset([b4_parse_param], [ :])])[
+#endif]b4_lac_if([[
+ yy_lac_established_ (false)]m4_ifset([b4_parse_param], [,])])[]b4_parse_param_cons[
+ {}
- /* Return YYSTR after stripping away unnecessary quotes and
+ ]b4_parser_class::~b4_parser_class[ ()
+ {}
+
+ ]b4_parser_class[::syntax_error::~syntax_error () YY_NOEXCEPT YY_NOTHROW
+ {}
+
+ /*---------.
+ | symbol. |
+ `---------*/
+
+]b4_token_ctor_if([], [b4_public_types_define([cc])])[
+
+ // by_state.
+ ]b4_parser_class[::by_state::by_state () YY_NOEXCEPT
+ : state (empty_state)
+ {}
+
+ ]b4_parser_class[::by_state::by_state (const by_state& that) YY_NOEXCEPT
+ : state (that.state)
+ {}
+
+ void
+ ]b4_parser_class[::by_state::clear () YY_NOEXCEPT
+ {
+ state = empty_state;
+ }
+
+ void
+ ]b4_parser_class[::by_state::move (by_state& that)
+ {
+ state = that.state;
+ that.clear ();
+ }
+
+ ]b4_parser_class[::by_state::by_state (state_type s) YY_NOEXCEPT
+ : state (s)
+ {}
+
+ ]b4_parser_class[::symbol_kind_type
+ ]b4_parser_class[::by_state::kind () const YY_NOEXCEPT
+ {
+ if (state == empty_state)
+ return ]b4_symbol(empty, kind)[;
+ else
+ return YY_CAST (symbol_kind_type, yystos_[+state]);
+ }
+
+ ]b4_parser_class[::stack_symbol_type::stack_symbol_type ()
+ {}
+
+ ]b4_parser_class[::stack_symbol_type::stack_symbol_type (YY_RVREF (stack_symbol_type) that)
+ : super_type (YY_MOVE (that.state)]b4_variant_if([], [, YY_MOVE (that.value)])b4_locations_if([, YY_MOVE (that.location)])[)
+ {]b4_variant_if([
+ b4_symbol_variant([that.kind ()],
+ [value], [YY_MOVE_OR_COPY], [YY_MOVE (that.value)])])[
+#if 201103L <= YY_CPLUSPLUS
+ // that is emptied.
+ that.state = empty_state;
+#endif
+ }
+
+ ]b4_parser_class[::stack_symbol_type::stack_symbol_type (state_type s, YY_MOVE_REF (symbol_type) that)
+ : super_type (s]b4_variant_if([], [, YY_MOVE (that.value)])[]b4_locations_if([, YY_MOVE (that.location)])[)
+ {]b4_variant_if([
+ b4_symbol_variant([that.kind ()],
+ [value], [move], [YY_MOVE (that.value)])])[
+ // that is emptied.
+ that.kind_ = ]b4_symbol(empty, kind)[;
+ }
+
+#if YY_CPLUSPLUS < 201103L
+ ]b4_parser_class[::stack_symbol_type&
+ ]b4_parser_class[::stack_symbol_type::operator= (const stack_symbol_type& that)
+ {
+ state = that.state;
+ ]b4_variant_if([b4_symbol_variant([that.kind ()],
+ [value], [copy], [that.value])],
+ [[value = that.value;]])[]b4_locations_if([
+ location = that.location;])[
+ return *this;
+ }
+
+ ]b4_parser_class[::stack_symbol_type&
+ ]b4_parser_class[::stack_symbol_type::operator= (stack_symbol_type& that)
+ {
+ state = that.state;
+ ]b4_variant_if([b4_symbol_variant([that.kind ()],
+ [value], [move], [that.value])],
+ [[value = that.value;]])[]b4_locations_if([
+ location = that.location;])[
+ // that is emptied.
+ that.state = empty_state;
+ return *this;
+ }
+#endif
+
+ template <typename Base>
+ void
+ ]b4_parser_class[::yy_destroy_ (const char* yymsg, basic_symbol<Base>& yysym) const
+ {
+ if (yymsg)
+ YY_SYMBOL_PRINT (yymsg, yysym);]b4_variant_if([], [
+
+ // User destructor.
+ b4_symbol_actions([destructor], [yysym.kind ()])])[
+ }
+
+#if ]b4_api_PREFIX[DEBUG
+ template <typename Base>
+ void
+ ]b4_parser_class[::yy_print_ (std::ostream& yyo, const basic_symbol<Base>& yysym) const
+ {
+ std::ostream& yyoutput = yyo;
+ YY_USE (yyoutput);
+ if (yysym.empty ())
+ yyo << "empty symbol";
+ else
+ {
+ symbol_kind_type yykind = yysym.kind ();
+ yyo << (yykind < YYNTOKENS ? "token" : "nterm")
+ << ' ' << yysym.name () << " ("]b4_locations_if([
+ << yysym.location << ": "])[;
+ ]b4_symbol_actions([printer])[
+ yyo << ')';
+ }
+ }
+#endif
+
+ void
+ ]b4_parser_class[::yypush_ (const char* m, YY_MOVE_REF (stack_symbol_type) sym)
+ {
+ if (m)
+ YY_SYMBOL_PRINT (m, sym);
+ yystack_.push (YY_MOVE (sym));
+ }
+
+ void
+ ]b4_parser_class[::yypush_ (const char* m, state_type s, YY_MOVE_REF (symbol_type) sym)
+ {
+#if 201103L <= YY_CPLUSPLUS
+ yypush_ (m, stack_symbol_type (s, std::move (sym)));
+#else
+ stack_symbol_type ss (s, sym);
+ yypush_ (m, ss);
+#endif
+ }
+
+ void
+ ]b4_parser_class[::yypop_ (int n) YY_NOEXCEPT
+ {
+ yystack_.pop (n);
+ }
+
+#if ]b4_api_PREFIX[DEBUG
+ std::ostream&
+ ]b4_parser_class[::debug_stream () const
+ {
+ return *yycdebug_;
+ }
+
+ void
+ ]b4_parser_class[::set_debug_stream (std::ostream& o)
+ {
+ yycdebug_ = &o;
+ }
+
+
+ ]b4_parser_class[::debug_level_type
+ ]b4_parser_class[::debug_level () const
+ {
+ return yydebug_;
+ }
+
+ void
+ ]b4_parser_class[::set_debug_level (debug_level_type l)
+ {
+ yydebug_ = l;
+ }
+#endif // ]b4_api_PREFIX[DEBUG
+
+ ]b4_parser_class[::state_type
+ ]b4_parser_class[::yy_lr_goto_state_ (state_type yystate, int yysym)
+ {
+ int yyr = yypgoto_[yysym - YYNTOKENS] + yystate;
+ if (0 <= yyr && yyr <= yylast_ && yycheck_[yyr] == yystate)
+ return yytable_[yyr];
+ else
+ return yydefgoto_[yysym - YYNTOKENS];
+ }
+
+ bool
+ ]b4_parser_class[::yy_pact_value_is_default_ (int yyvalue) YY_NOEXCEPT
+ {
+ return yyvalue == yypact_ninf_;
+ }
+
+ bool
+ ]b4_parser_class[::yy_table_value_is_error_ (int yyvalue) YY_NOEXCEPT
+ {
+ return yyvalue == yytable_ninf_;
+ }
+
+ int
+ ]b4_parser_class[::operator() ()
+ {
+ return parse ();
+ }
+
+ int
+ ]b4_parser_class[::parse ()
+ {
+ int yyn;
+ /// Length of the RHS of the rule being reduced.
+ int yylen = 0;
+
+ // Error handling.
+ int yynerrs_ = 0;
+ int yyerrstatus_ = 0;
+
+ /// The lookahead symbol.
+ symbol_type yyla;]b4_locations_if([[
+
+ /// The locations where the error started and ended.
+ stack_symbol_type yyerror_range[3];]])[
+
+ /// The return value of parse ().
+ int yyresult;]b4_lac_if([[
+
+ // Discard the LAC context in case there still is one left from a
+ // previous invocation.
+ yy_lac_discard_ ("init");]])[
+
+#if YY_EXCEPTIONS
+ try
+#endif // YY_EXCEPTIONS
+ {
+ YYCDEBUG << "Starting parse\n";
+
+]m4_ifdef([b4_initial_action], [
+b4_dollar_pushdef([yyla.value], [], [], [yyla.location])dnl
+ b4_user_initial_action
+b4_dollar_popdef])[]dnl
+
+ [ /* Initialize the stack. The initial state will be set in
+ yynewstate, since the latter expects the semantical and the
+ location values to have been already stored, initialize these
+ stacks with a primary value. */
+ yystack_.clear ();
+ yypush_ (YY_NULLPTR, 0, YY_MOVE (yyla));
+
+ /*-----------------------------------------------.
+ | yynewstate -- push a new symbol on the stack. |
+ `-----------------------------------------------*/
+ yynewstate:
+ YYCDEBUG << "Entering state " << int (yystack_[0].state) << '\n';
+ YY_STACK_PRINT ();
+
+ // Accept?
+ if (yystack_[0].state == yyfinal_)
+ YYACCEPT;
+
+ goto yybackup;
+
+
+ /*-----------.
+ | yybackup. |
+ `-----------*/
+ yybackup:
+ // Try to take a decision without lookahead.
+ yyn = yypact_[+yystack_[0].state];
+ if (yy_pact_value_is_default_ (yyn))
+ goto yydefault;
+
+ // Read a lookahead token.
+ if (yyla.empty ())
+ {
+ YYCDEBUG << "Reading a token\n";
+#if YY_EXCEPTIONS
+ try
+#endif // YY_EXCEPTIONS
+ {]b4_token_ctor_if([[
+ symbol_type yylookahead (]b4_yylex[);
+ yyla.move (yylookahead);]], [[
+ yyla.kind_ = yytranslate_ (]b4_yylex[);]])[
+ }
+#if YY_EXCEPTIONS
+ catch (const syntax_error& yyexc)
+ {
+ YYCDEBUG << "Caught exception: " << yyexc.what() << '\n';
+ error (yyexc);
+ goto yyerrlab1;
+ }
+#endif // YY_EXCEPTIONS
+ }
+ YY_SYMBOL_PRINT ("Next token is", yyla);
+
+ if (yyla.kind () == ]b4_symbol(error, kind)[)
+ {
+ // The scanner already issued an error message, process directly
+ // to error recovery. But do not keep the error token as
+ // lookahead, it is too special and may lead us to an endless
+ // loop in error recovery. */
+ yyla.kind_ = ]b4_symbol(undef, kind)[;
+ goto yyerrlab1;
+ }
+
+ /* If the proper action on seeing token YYLA.TYPE is to reduce or
+ to detect an error, take that action. */
+ yyn += yyla.kind ();
+ if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yyla.kind ())
+ {]b4_lac_if([[
+ if (!yy_lac_establish_ (yyla.kind ()))
+ goto yyerrlab;]])[
+ goto yydefault;
+ }
+
+ // Reduce or error.
+ yyn = yytable_[yyn];
+ if (yyn <= 0)
+ {
+ if (yy_table_value_is_error_ (yyn))
+ goto yyerrlab;]b4_lac_if([[
+ if (!yy_lac_establish_ (yyla.kind ()))
+ goto yyerrlab;
+]])[
+ yyn = -yyn;
+ goto yyreduce;
+ }
+
+ // Count tokens shifted since error; after three, turn off error status.
+ if (yyerrstatus_)
+ --yyerrstatus_;
+
+ // Shift the lookahead token.
+ yypush_ ("Shifting", state_type (yyn), YY_MOVE (yyla));]b4_lac_if([[
+ yy_lac_discard_ ("shift");]])[
+ goto yynewstate;
+
+
+ /*-----------------------------------------------------------.
+ | yydefault -- do the default action for the current state. |
+ `-----------------------------------------------------------*/
+ yydefault:
+ yyn = yydefact_[+yystack_[0].state];
+ if (yyn == 0)
+ goto yyerrlab;
+ goto yyreduce;
+
+
+ /*-----------------------------.
+ | yyreduce -- do a reduction. |
+ `-----------------------------*/
+ yyreduce:
+ yylen = yyr2_[yyn];
+ {
+ stack_symbol_type yylhs;
+ yylhs.state = yy_lr_goto_state_ (yystack_[yylen].state, yyr1_[yyn]);]b4_variant_if([[
+ /* Variants are always initialized to an empty instance of the
+ correct type. The default '$$ = $1' action is NOT applied
+ when using variants. */
+ ]b4_symbol_variant([[yyr1_@{yyn@}]], [yylhs.value], [emplace])], [[
+ /* If YYLEN is nonzero, implement the default value of the
+ action: '$$ = $1'. Otherwise, use the top of the stack.
+
+ Otherwise, the following line sets YYLHS.VALUE to garbage.
+ This behavior is undocumented and Bison users should not rely
+ upon it. */
+ if (yylen)
+ yylhs.value = yystack_@{yylen - 1@}.value;
+ else
+ yylhs.value = yystack_@{0@}.value;]])[
+]b4_locations_if([dnl
+[
+ // Default location.
+ {
+ stack_type::slice range (yystack_, yylen);
+ YYLLOC_DEFAULT (yylhs.location, range, yylen);
+ yyerror_range[1].location = yylhs.location;
+ }]])[
+
+ // Perform the reduction.
+ YY_REDUCE_PRINT (yyn);
+#if YY_EXCEPTIONS
+ try
+#endif // YY_EXCEPTIONS
+ {
+ switch (yyn)
+ {
+]b4_user_actions[
+ default:
+ break;
+ }
+ }
+#if YY_EXCEPTIONS
+ catch (const syntax_error& yyexc)
+ {
+ YYCDEBUG << "Caught exception: " << yyexc.what() << '\n';
+ error (yyexc);
+ YYERROR;
+ }
+#endif // YY_EXCEPTIONS
+ YY_SYMBOL_PRINT ("-> $$ =", yylhs);
+ yypop_ (yylen);
+ yylen = 0;
+
+ // Shift the result of the reduction.
+ yypush_ (YY_NULLPTR, YY_MOVE (yylhs));
+ }
+ goto yynewstate;
+
+
+ /*--------------------------------------.
+ | yyerrlab -- here on detecting error. |
+ `--------------------------------------*/
+ yyerrlab:
+ // If not already recovering from an error, report this error.
+ if (!yyerrstatus_)
+ {
+ ++yynerrs_;]b4_parse_error_case(
+ [simple], [[
+ std::string msg = YY_("syntax error");
+ error (]b4_join(b4_locations_if([yyla.location]), [[YY_MOVE (msg)]])[);]],
+ [custom], [[
+ context yyctx (*this, yyla);
+ report_syntax_error (yyctx);]],
+ [[
+ context yyctx (*this, yyla);
+ std::string msg = yysyntax_error_ (yyctx);
+ error (]b4_join(b4_locations_if([yyla.location]), [[YY_MOVE (msg)]])[);]])[
+ }
+
+]b4_locations_if([[
+ yyerror_range[1].location = yyla.location;]])[
+ if (yyerrstatus_ == 3)
+ {
+ /* If just tried and failed to reuse lookahead token after an
+ error, discard it. */
+
+ // Return failure if at end of input.
+ if (yyla.kind () == ]b4_symbol(eof, kind)[)
+ YYABORT;
+ else if (!yyla.empty ())
+ {
+ yy_destroy_ ("Error: discarding", yyla);
+ yyla.clear ();
+ }
+ }
+
+ // Else will try to reuse lookahead token after shifting the error token.
+ goto yyerrlab1;
+
+
+ /*---------------------------------------------------.
+ | yyerrorlab -- error raised explicitly by YYERROR. |
+ `---------------------------------------------------*/
+ yyerrorlab:
+ /* Pacify compilers when the user code never invokes YYERROR and
+ the label yyerrorlab therefore never appears in user code. */
+ if (false)
+ YYERROR;
+
+ /* Do not reclaim the symbols of the rule whose action triggered
+ this YYERROR. */
+ yypop_ (yylen);
+ yylen = 0;
+ YY_STACK_PRINT ();
+ goto yyerrlab1;
+
+
+ /*-------------------------------------------------------------.
+ | yyerrlab1 -- common code for both syntax error and YYERROR. |
+ `-------------------------------------------------------------*/
+ yyerrlab1:
+ yyerrstatus_ = 3; // Each real token shifted decrements this.
+ // Pop stack until we find a state that shifts the error token.
+ for (;;)
+ {
+ yyn = yypact_[+yystack_[0].state];
+ if (!yy_pact_value_is_default_ (yyn))
+ {
+ yyn += ]b4_symbol(error, kind)[;
+ if (0 <= yyn && yyn <= yylast_
+ && yycheck_[yyn] == ]b4_symbol(error, kind)[)
+ {
+ yyn = yytable_[yyn];
+ if (0 < yyn)
+ break;
+ }
+ }
+
+ // Pop the current state because it cannot handle the error token.
+ if (yystack_.size () == 1)
+ YYABORT;
+]b4_locations_if([[
+ yyerror_range[1].location = yystack_[0].location;]])[
+ yy_destroy_ ("Error: popping", yystack_[0]);
+ yypop_ ();
+ YY_STACK_PRINT ();
+ }
+ {
+ stack_symbol_type error_token;
+]b4_locations_if([[
+ yyerror_range[2].location = yyla.location;
+ YYLLOC_DEFAULT (error_token.location, yyerror_range, 2);]])[
+
+ // Shift the error token.]b4_lac_if([[
+ yy_lac_discard_ ("error recovery");]])[
+ error_token.state = state_type (yyn);
+ yypush_ ("Shifting", YY_MOVE (error_token));
+ }
+ goto yynewstate;
+
+
+ /*-------------------------------------.
+ | yyacceptlab -- YYACCEPT comes here. |
+ `-------------------------------------*/
+ yyacceptlab:
+ yyresult = 0;
+ goto yyreturn;
+
+
+ /*-----------------------------------.
+ | yyabortlab -- YYABORT comes here. |
+ `-----------------------------------*/
+ yyabortlab:
+ yyresult = 1;
+ goto yyreturn;
+
+
+ /*-----------------------------------------------------.
+ | yyreturn -- parsing is finished, return the result. |
+ `-----------------------------------------------------*/
+ yyreturn:
+ if (!yyla.empty ())
+ yy_destroy_ ("Cleanup: discarding lookahead", yyla);
+
+ /* Do not reclaim the symbols of the rule whose action triggered
+ this YYABORT or YYACCEPT. */
+ yypop_ (yylen);
+ YY_STACK_PRINT ();
+ while (1 < yystack_.size ())
+ {
+ yy_destroy_ ("Cleanup: popping", yystack_[0]);
+ yypop_ ();
+ }
+
+ return yyresult;
+ }
+#if YY_EXCEPTIONS
+ catch (...)
+ {
+ YYCDEBUG << "Exception caught: cleaning lookahead and stack\n";
+ // Do not try to display the values of the reclaimed symbols,
+ // as their printers might throw an exception.
+ if (!yyla.empty ())
+ yy_destroy_ (YY_NULLPTR, yyla);
+
+ while (1 < yystack_.size ())
+ {
+ yy_destroy_ (YY_NULLPTR, yystack_[0]);
+ yypop_ ();
+ }
+ throw;
+ }
+#endif // YY_EXCEPTIONS
+ }
+
+ void
+ ]b4_parser_class[::error (const syntax_error& yyexc)
+ {
+ error (]b4_join(b4_locations_if([yyexc.location]),
+ [[yyexc.what ()]])[);
+ }
+
+]b4_parse_error_bmatch([custom\|detailed],
+[[ const char *
+ ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+ {
+ static const char *const yy_sname[] =
+ {
+ ]b4_symbol_names[
+ };]b4_has_translations_if([[
+ /* YYTRANSLATABLE[SYMBOL-NUM] -- Whether YY_SNAME[SYMBOL-NUM] is
+ internationalizable. */
+ static ]b4_int_type_for([b4_translatable])[ yytranslatable[] =
+ {
+ ]b4_translatable[
+ };
+ return (yysymbol < YYNTOKENS && yytranslatable[yysymbol]
+ ? _(yy_sname[yysymbol])
+ : yy_sname[yysymbol]);]], [[
+ return yy_sname[yysymbol];]])[
+ }
+]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+ const char *
+ ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+ {
+ return yytname_[yysymbol];
+ }
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[ /* Return YYSTR after stripping away unnecessary quotes and
backslashes, so that it's suitable for yyerror. The heuristic is
that double-quoting is unnecessary unless the string contains an
apostrophe, a comma, or backslash (other than backslash-backslash).
@@ -583,582 +1274,89 @@
return yystr;
}
+
+ std::string
+ ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+ {
+ return yytnamerr_ (yytname_[yysymbol]);
+ }
]])[
- /// Build a parser object.
- ]b4_parser_class::b4_parser_class[ (]b4_parse_param_decl[)
-#if ]b4_api_PREFIX[DEBUG
- : yydebug_ (false),
- yycdebug_ (&std::cerr)]b4_lac_if([,], [m4_ifset([b4_parse_param], [,])])[
-#else
-]b4_lac_if([ :], [m4_ifset([b4_parse_param], [ :])])[
-#endif]b4_lac_if([[
- yy_lac_established_ (false)]m4_ifset([b4_parse_param], [,])])[]b4_parse_param_cons[
+]b4_parse_error_bmatch([custom\|detailed\|verbose], [[
+ // ]b4_parser_class[::context.
+ ]b4_parser_class[::context::context (const ]b4_parser_class[& yyparser, const symbol_type& yyla)
+ : yyparser_ (yyparser)
+ , yyla_ (yyla)
{}
- ]b4_parser_class::~b4_parser_class[ ()
- {}
-
- ]b4_parser_class[::syntax_error::~syntax_error () YY_NOEXCEPT YY_NOTHROW
- {}
-
- /*---------------.
- | Symbol types. |
- `---------------*/
-
-]b4_token_ctor_if([], [b4_public_types_define([cc])])[
-
- // by_state.
- ]b4_parser_class[::by_state::by_state () YY_NOEXCEPT
- : state (empty_state)
- {}
-
- ]b4_parser_class[::by_state::by_state (const by_state& that) YY_NOEXCEPT
- : state (that.state)
- {}
-
- void
- ]b4_parser_class[::by_state::clear () YY_NOEXCEPT
- {
- state = empty_state;
- }
-
- void
- ]b4_parser_class[::by_state::move (by_state& that)
- {
- state = that.state;
- that.clear ();
- }
-
- ]b4_parser_class[::by_state::by_state (state_type s) YY_NOEXCEPT
- : state (s)
- {}
-
- ]b4_parser_class[::symbol_number_type
- ]b4_parser_class[::by_state::type_get () const YY_NOEXCEPT
- {
- if (state == empty_state)
- return empty_symbol;
- else
- return yystos_[state];
- }
-
- ]b4_parser_class[::stack_symbol_type::stack_symbol_type ()
- {}
-
- ]b4_parser_class[::stack_symbol_type::stack_symbol_type (YY_RVREF (stack_symbol_type) that)
- : super_type (YY_MOVE (that.state)]b4_variant_if([], [, YY_MOVE (that.value)])b4_locations_if([, YY_MOVE (that.location)])[)
- {]b4_variant_if([
- b4_symbol_variant([that.type_get ()],
- [value], [YY_MOVE_OR_COPY], [YY_MOVE (that.value)])])[
-#if 201103L <= YY_CPLUSPLUS
- // that is emptied.
- that.state = empty_state;
-#endif
- }
-
- ]b4_parser_class[::stack_symbol_type::stack_symbol_type (state_type s, YY_MOVE_REF (symbol_type) that)
- : super_type (s]b4_variant_if([], [, YY_MOVE (that.value)])[]b4_locations_if([, YY_MOVE (that.location)])[)
- {]b4_variant_if([
- b4_symbol_variant([that.type_get ()],
- [value], [move], [YY_MOVE (that.value)])])[
- // that is emptied.
- that.type = empty_symbol;
- }
-
-#if YY_CPLUSPLUS < 201103L
- ]b4_parser_class[::stack_symbol_type&
- ]b4_parser_class[::stack_symbol_type::operator= (const stack_symbol_type& that)
- {
- state = that.state;
- ]b4_variant_if([b4_symbol_variant([that.type_get ()],
- [value], [copy], [that.value])],
- [[value = that.value;]])[]b4_locations_if([
- location = that.location;])[
- return *this;
- }
-
- ]b4_parser_class[::stack_symbol_type&
- ]b4_parser_class[::stack_symbol_type::operator= (stack_symbol_type& that)
- {
- state = that.state;
- ]b4_variant_if([b4_symbol_variant([that.type_get ()],
- [value], [move], [that.value])],
- [[value = that.value;]])[]b4_locations_if([
- location = that.location;])[
- // that is emptied.
- that.state = empty_state;
- return *this;
- }
-#endif
-
- template <typename Base>
- void
- ]b4_parser_class[::yy_destroy_ (const char* yymsg, basic_symbol<Base>& yysym) const
- {
- if (yymsg)
- YY_SYMBOL_PRINT (yymsg, yysym);]b4_variant_if([], [
-
- // User destructor.
- b4_symbol_actions([destructor], [yysym.type_get ()])])[
- }
-
-#if ]b4_api_PREFIX[DEBUG
- template <typename Base>
- void
- ]b4_parser_class[::yy_print_ (std::ostream& yyo,
- const basic_symbol<Base>& yysym) const
- {
- std::ostream& yyoutput = yyo;
- YYUSE (yyoutput);
- symbol_number_type yytype = yysym.type_get ();
-#if defined __GNUC__ && ! defined __clang__ && ! defined __ICC && __GNUC__ * 100 + __GNUC_MINOR__ <= 408
- // Avoid a (spurious) G++ 4.8 warning about "array subscript is
- // below array bounds".
- if (yysym.empty ())
- std::abort ();
-#endif
- yyo << (yytype < yyntokens_ ? "token" : "nterm")
- << ' ' << yytname_[yytype] << " ("]b4_locations_if([
- << yysym.location << ": "])[;
- ]b4_symbol_actions([printer])[
- yyo << ')';
- }
-#endif
-
- void
- ]b4_parser_class[::yypush_ (const char* m, YY_MOVE_REF (stack_symbol_type) sym)
- {
- if (m)
- YY_SYMBOL_PRINT (m, sym);
- yystack_.push (YY_MOVE (sym));
- }
-
- void
- ]b4_parser_class[::yypush_ (const char* m, state_type s, YY_MOVE_REF (symbol_type) sym)
- {
-#if 201103L <= YY_CPLUSPLUS
- yypush_ (m, stack_symbol_type (s, std::move (sym)));
-#else
- stack_symbol_type ss (s, sym);
- yypush_ (m, ss);
-#endif
- }
-
- void
- ]b4_parser_class[::yypop_ (int n)
- {
- yystack_.pop (n);
- }
-
-#if ]b4_api_PREFIX[DEBUG
- std::ostream&
- ]b4_parser_class[::debug_stream () const
- {
- return *yycdebug_;
- }
-
- void
- ]b4_parser_class[::set_debug_stream (std::ostream& o)
- {
- yycdebug_ = &o;
- }
-
-
- ]b4_parser_class[::debug_level_type
- ]b4_parser_class[::debug_level () const
- {
- return yydebug_;
- }
-
- void
- ]b4_parser_class[::set_debug_level (debug_level_type l)
- {
- yydebug_ = l;
- }
-#endif // ]b4_api_PREFIX[DEBUG
-
- ]b4_parser_class[::state_type
- ]b4_parser_class[::yy_lr_goto_state_ (state_type yystate, int yysym)
- {
- int yyr = yypgoto_[yysym - yyntokens_] + yystate;
- if (0 <= yyr && yyr <= yylast_ && yycheck_[yyr] == yystate)
- return yytable_[yyr];
- else
- return yydefgoto_[yysym - yyntokens_];
- }
-
- bool
- ]b4_parser_class[::yy_pact_value_is_default_ (int yyvalue)
- {
- return yyvalue == yypact_ninf_;
- }
-
- bool
- ]b4_parser_class[::yy_table_value_is_error_ (int yyvalue)
- {
- return yyvalue == yytable_ninf_;
- }
-
int
- ]b4_parser_class[::operator() ()
+ ]b4_parser_class[::context::expected_tokens (symbol_kind_type yyarg[], int yyargn) const
{
- return parse ();
- }
+ // Actual number of expected tokens
+ int yycount = 0;
+]b4_lac_if([[
+#if ]b4_api_PREFIX[DEBUG
+ // Execute LAC once. We don't care if it is successful, we
+ // only do it for the sake of debugging output.
+ if (!yyparser_.yy_lac_established_)
+ yyparser_.yy_lac_check_ (yyla_.kind ());
+#endif
- int
- ]b4_parser_class[::parse ()
- {
- int yyn;
- /// Length of the RHS of the rule being reduced.
- int yylen = 0;
-
- // Error handling.
- int yynerrs_ = 0;
- int yyerrstatus_ = 0;
-
- /// The lookahead symbol.
- symbol_type yyla;]b4_locations_if([[
-
- /// The locations where the error started and ended.
- stack_symbol_type yyerror_range[3];]])[
-
- /// The return value of parse ().
- int yyresult;]b4_lac_if([[
-
- /// Discard the LAC context in case there still is one left from a
- /// previous invocation.
- yy_lac_discard_ ("init");]])[
-
-#if YY_EXCEPTIONS
- try
-#endif // YY_EXCEPTIONS
+ for (int yyx = 0; yyx < YYNTOKENS; ++yyx)
{
- YYCDEBUG << "Starting parse\n";
-
-]m4_ifdef([b4_initial_action], [
-b4_dollar_pushdef([yyla.value], [], [], [yyla.location])dnl
- b4_user_initial_action
-b4_dollar_popdef])[]dnl
-
- [ /* Initialize the stack. The initial state will be set in
- yynewstate, since the latter expects the semantical and the
- location values to have been already stored, initialize these
- stacks with a primary value. */
- yystack_.clear ();
- yypush_ (YY_NULLPTR, 0, YY_MOVE (yyla));
-
- /*-----------------------------------------------.
- | yynewstate -- push a new symbol on the stack. |
- `-----------------------------------------------*/
- yynewstate:
- YYCDEBUG << "Entering state " << int (yystack_[0].state) << '\n';
-
- // Accept?
- if (yystack_[0].state == yyfinal_)
- YYACCEPT;
-
- goto yybackup;
-
-
- /*-----------.
- | yybackup. |
- `-----------*/
- yybackup:
- // Try to take a decision without lookahead.
- yyn = yypact_[yystack_[0].state];
- if (yy_pact_value_is_default_ (yyn))
- goto yydefault;
-
- // Read a lookahead token.
- if (yyla.empty ())
- {
- YYCDEBUG << "Reading a token: ";
-#if YY_EXCEPTIONS
- try
-#endif // YY_EXCEPTIONS
- {]b4_token_ctor_if([[
- symbol_type yylookahead (]b4_lex[);
- yyla.move (yylookahead);]], [[
- yyla.type = yytranslate_ (]b4_lex[);]])[
- }
-#if YY_EXCEPTIONS
- catch (const syntax_error& yyexc)
+ symbol_kind_type yysym = YY_CAST (symbol_kind_type, yyx);
+ if (yysym != ]b4_symbol(error, kind)[
+ && yysym != ]b4_symbol(undef, kind)[
+ && yyparser_.yy_lac_check_ (yysym))
{
- YYCDEBUG << "Caught exception: " << yyexc.what() << '\n';
- error (yyexc);
- goto yyerrlab1;
+ if (!yyarg)
+ ++yycount;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = yysym;
}
-#endif // YY_EXCEPTIONS
- }
- YY_SYMBOL_PRINT ("Next token is", yyla);
-
- /* If the proper action on seeing token YYLA.TYPE is to reduce or
- to detect an error, take that action. */
- yyn += yyla.type_get ();
- if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yyla.type_get ())
- {]b4_lac_if([[
- if (!yy_lac_establish_ (yyla.type_get ()))
- goto yyerrlab;]])[
- goto yydefault;
- }
-
- // Reduce or error.
- yyn = yytable_[yyn];
- if (yyn <= 0)
+ }]], [[
+ const int yyn = yypact_[+yyparser_.yystack_[0].state];
+ if (!yy_pact_value_is_default_ (yyn))
{
- if (yy_table_value_is_error_ (yyn))
- goto yyerrlab;]b4_lac_if([[
- if (!yy_lac_establish_ (yyla.type_get ()))
- goto yyerrlab;
+ /* Start YYX at -YYN if negative to avoid negative indexes in
+ YYCHECK. In other words, skip the first -YYN actions for
+ this state because they are default actions. */
+ const int yyxbegin = yyn < 0 ? -yyn : 0;
+ // Stay within bounds of both yycheck and yytname.
+ const int yychecklim = yylast_ - yyn + 1;
+ const int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
+ for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
+ if (yycheck_[yyx + yyn] == yyx && yyx != ]b4_symbol(error, kind)[
+ && !yy_table_value_is_error_ (yytable_[yyx + yyn]))
+ {
+ if (!yyarg)
+ ++yycount;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = YY_CAST (symbol_kind_type, yyx);
+ }
+ }
]])[
- yyn = -yyn;
- goto yyreduce;
- }
-
- // Count tokens shifted since error; after three, turn off error status.
- if (yyerrstatus_)
- --yyerrstatus_;
-
- // Shift the lookahead token.
- yypush_ ("Shifting", static_cast<state_type> (yyn), YY_MOVE (yyla));]b4_lac_if([[
- yy_lac_discard_ ("shift");]])[
- goto yynewstate;
-
-
- /*-----------------------------------------------------------.
- | yydefault -- do the default action for the current state. |
- `-----------------------------------------------------------*/
- yydefault:
- yyn = yydefact_[yystack_[0].state];
- if (yyn == 0)
- goto yyerrlab;
- goto yyreduce;
-
-
- /*-----------------------------.
- | yyreduce -- do a reduction. |
- `-----------------------------*/
- yyreduce:
- yylen = yyr2_[yyn];
- {
- stack_symbol_type yylhs;
- yylhs.state = yy_lr_goto_state_ (yystack_[yylen].state, yyr1_[yyn]);]b4_variant_if([
- /* Variants are always initialized to an empty instance of the
- correct type. The default '$$ = $1' action is NOT applied
- when using variants. */
- b4_symbol_variant([[yyr1_@{yyn@}]], [yylhs.value], [emplace])], [
- /* If YYLEN is nonzero, implement the default value of the
- action: '$$ = $1'. Otherwise, use the top of the stack.
-
- Otherwise, the following line sets YYLHS.VALUE to garbage.
- This behavior is undocumented and Bison users should not rely
- upon it. */
- if (yylen)
- yylhs.value = yystack_@{yylen - 1@}.value;
- else
- yylhs.value = yystack_@{0@}.value;])[
-]b4_locations_if([dnl
-[
- // Default location.
- {
- stack_type::slice range (yystack_, yylen);
- YYLLOC_DEFAULT (yylhs.location, range, yylen);
- yyerror_range[1].location = yylhs.location;
- }]])[
-
- // Perform the reduction.
- YY_REDUCE_PRINT (yyn);
-#if YY_EXCEPTIONS
- try
-#endif // YY_EXCEPTIONS
- {
- switch (yyn)
- {
-]b4_user_actions[
- default:
- break;
- }
- }
-#if YY_EXCEPTIONS
- catch (const syntax_error& yyexc)
- {
- YYCDEBUG << "Caught exception: " << yyexc.what() << '\n';
- error (yyexc);
- YYERROR;
- }
-#endif // YY_EXCEPTIONS
- YY_SYMBOL_PRINT ("-> $$ =", yylhs);
- yypop_ (yylen);
- yylen = 0;
- YY_STACK_PRINT ();
-
- // Shift the result of the reduction.
- yypush_ (YY_NULLPTR, YY_MOVE (yylhs));
- }
- goto yynewstate;
-
-
- /*--------------------------------------.
- | yyerrlab -- here on detecting error. |
- `--------------------------------------*/
- yyerrlab:
- // If not already recovering from an error, report this error.
- if (!yyerrstatus_)
- {
- ++yynerrs_;
- error (]b4_join(b4_locations_if([yyla.location]),
- [[yysyntax_error_ (yystack_[0].state, yyla)]])[);
- }
-
-]b4_locations_if([[
- yyerror_range[1].location = yyla.location;]])[
- if (yyerrstatus_ == 3)
- {
- /* If just tried and failed to reuse lookahead token after an
- error, discard it. */
-
- // Return failure if at end of input.
- if (yyla.type_get () == yyeof_)
- YYABORT;
- else if (!yyla.empty ())
- {
- yy_destroy_ ("Error: discarding", yyla);
- yyla.clear ();
- }
- }
-
- // Else will try to reuse lookahead token after shifting the error token.
- goto yyerrlab1;
-
-
- /*---------------------------------------------------.
- | yyerrorlab -- error raised explicitly by YYERROR. |
- `---------------------------------------------------*/
- yyerrorlab:
- /* Pacify compilers when the user code never invokes YYERROR and
- the label yyerrorlab therefore never appears in user code. */
- if (false)
- YYERROR;
-
- /* Do not reclaim the symbols of the rule whose action triggered
- this YYERROR. */
- yypop_ (yylen);
- yylen = 0;
- goto yyerrlab1;
-
-
- /*-------------------------------------------------------------.
- | yyerrlab1 -- common code for both syntax error and YYERROR. |
- `-------------------------------------------------------------*/
- yyerrlab1:
- yyerrstatus_ = 3; // Each real token shifted decrements this.
- {
- stack_symbol_type error_token;
- for (;;)
- {
- yyn = yypact_[yystack_[0].state];
- if (!yy_pact_value_is_default_ (yyn))
- {
- yyn += yy_error_token_;
- if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == yy_error_token_)
- {
- yyn = yytable_[yyn];
- if (0 < yyn)
- break;
- }
- }
-
- // Pop the current state because it cannot handle the error token.
- if (yystack_.size () == 1)
- YYABORT;
-]b4_locations_if([[
- yyerror_range[1].location = yystack_[0].location;]])[
- yy_destroy_ ("Error: popping", yystack_[0]);
- yypop_ ();
- YY_STACK_PRINT ();
- }
-]b4_locations_if([[
- yyerror_range[2].location = yyla.location;
- YYLLOC_DEFAULT (error_token.location, yyerror_range, 2);]])[
-
- // Shift the error token.]b4_lac_if([[
- yy_lac_discard_ ("error recovery");]])[
- error_token.state = static_cast<state_type> (yyn);
- yypush_ ("Shifting", YY_MOVE (error_token));
- }
- goto yynewstate;
-
-
- /*-------------------------------------.
- | yyacceptlab -- YYACCEPT comes here. |
- `-------------------------------------*/
- yyacceptlab:
- yyresult = 0;
- goto yyreturn;
-
-
- /*-----------------------------------.
- | yyabortlab -- YYABORT comes here. |
- `-----------------------------------*/
- yyabortlab:
- yyresult = 1;
- goto yyreturn;
-
-
- /*-----------------------------------------------------.
- | yyreturn -- parsing is finished, return the result. |
- `-----------------------------------------------------*/
- yyreturn:
- if (!yyla.empty ())
- yy_destroy_ ("Cleanup: discarding lookahead", yyla);
-
- /* Do not reclaim the symbols of the rule whose action triggered
- this YYABORT or YYACCEPT. */
- yypop_ (yylen);
- while (1 < yystack_.size ())
- {
- yy_destroy_ ("Cleanup: popping", yystack_[0]);
- yypop_ ();
- }
-
- return yyresult;
- }
-#if YY_EXCEPTIONS
- catch (...)
- {
- YYCDEBUG << "Exception caught: cleaning lookahead and stack\n";
- // Do not try to display the values of the reclaimed symbols,
- // as their printers might throw an exception.
- if (!yyla.empty ())
- yy_destroy_ (YY_NULLPTR, yyla);
-
- while (1 < yystack_.size ())
- {
- yy_destroy_ (YY_NULLPTR, yystack_[0]);
- yypop_ ();
- }
- throw;
- }
-#endif // YY_EXCEPTIONS
+ if (yyarg && yycount == 0 && 0 < yyargn)
+ yyarg[0] = ]b4_symbol(empty, kind)[;
+ return yycount;
}
- void
- ]b4_parser_class[::error (const syntax_error& yyexc)
- {
- error (]b4_join(b4_locations_if([yyexc.location]),
- [[yyexc.what ()]])[);
- }]b4_lac_if([[
+]])[
+]b4_lac_if([[
bool
- ]b4_parser_class[::yy_lac_check_ (int yytoken) const
+ ]b4_parser_class[::yy_lac_check_ (symbol_kind_type yytoken) const
{
// Logically, the yylac_stack's lifetime is confined to this function.
// Clear it, to get rid of potential left-overs from previous call.
yylac_stack_.clear ();
// Reduce until we encounter a shift and thereby accept the token.
#if ]b4_api_PREFIX[DEBUG
- YYCDEBUG << "LAC: checking lookahead " << yytname_[yytoken] << ':';
+ YYCDEBUG << "LAC: checking lookahead " << symbol_name (yytoken) << ':';
#endif
std::ptrdiff_t lac_top = 0;
while (true)
@@ -1166,13 +1364,13 @@
state_type top_state = (yylac_stack_.empty ()
? yystack_[lac_top].state
: yylac_stack_.back ());
- int yyrule = yypact_[top_state];
+ int yyrule = yypact_[+top_state];
if (yy_pact_value_is_default_ (yyrule)
|| (yyrule += yytoken) < 0 || yylast_ < yyrule
|| yycheck_[yyrule] != yytoken)
{
// Use the default action.
- yyrule = yydefact_[top_state];
+ yyrule = yydefact_[+top_state];
if (yyrule == 0)
{
YYCDEBUG << " Err\n";
@@ -1222,14 +1420,14 @@
: yylac_stack_.back ());
// Push the resulting state of the reduction.
state_type state = yy_lr_goto_state_ (top_state, yyr1_[yyrule]);
- YYCDEBUG << " G" << state;
+ YYCDEBUG << " G" << int (state);
yylac_stack_.push_back (state);
}
}
// Establish the initial context if no initial context currently exists.
bool
- ]b4_parser_class[::yy_lac_establish_ (int yytoken)
+ ]b4_parser_class[::yy_lac_establish_ (symbol_kind_type yytoken)
{
/* Establish the initial context for the current lookahead if no initial
context is currently established.
@@ -1254,21 +1452,22 @@
follows. If no initial context is currently established for the
current lookahead, then check if that lookahead can eventually be
shifted if syntactic actions continue from the current context. */
- if (!yy_lac_established_)
+ if (yy_lac_established_)
+ return true;
+ else
{
#if ]b4_api_PREFIX[DEBUG
YYCDEBUG << "LAC: initial context established for "
- << yytname_[yytoken] << '\n';
+ << symbol_name (yytoken) << '\n';
#endif
yy_lac_established_ = true;
return yy_lac_check_ (yytoken);
}
- return true;
}
// Discard any previous initial lookahead context.
void
- ]b4_parser_class[::yy_lac_discard_ (const char* evt)
+ ]b4_parser_class[::yy_lac_discard_ (const char* event)
{
/* Discard any previous initial lookahead context because of Event,
which may be a lookahead change or an invalidation of the currently
@@ -1284,25 +1483,16 @@
if (yy_lac_established_)
{
YYCDEBUG << "LAC: initial context discarded due to "
- << evt << '\n';
+ << event << '\n';
yy_lac_established_ = false;
}
}]])[
- // Generate an error message.
- std::string
- ]b4_parser_class[::yysyntax_error_ (]dnl
-b4_error_verbose_if([state_type yystate, const symbol_type& yyla],
- [state_type, const symbol_type&])[) const
- {]b4_error_verbose_if([[
- // Number of reported tokens (one for the "unexpected", one per
- // "expected").
- std::ptrdiff_t yycount = 0;
- // Its maximum.
- enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 };
- // Arguments of yyformat.
- char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
-
+]b4_parse_error_bmatch([detailed\|verbose], [[
+ int
+ ]b4_parser_class[::yy_syntax_error_arguments_ (const context& yyctx,
+ symbol_kind_type yyarg[], int yyargn) const
+ {
/* There are many possibilities here to consider:
- If this state is a consistent state with a default action, then
the only way this function was invoked is if the default action
@@ -1333,46 +1523,26 @@
one exception: it will still contain any token that will not be
accepted due to an error action in a later state.]])[
*/
- if (!yyla.empty ())
+
+ if (!yyctx.lookahead ().empty ())
{
- symbol_number_type yytoken = yyla.type_get ();
- yyarg[yycount++] = yytname_[yytoken];]b4_lac_if([[
-
-#if ]b4_api_PREFIX[DEBUG
- // Execute LAC once. We don't care if it is succesful, we
- // only do it for the sake of debugging output.
- if (!yy_lac_established_)
- yy_lac_check_ (yytoken);
-#endif]])[
-
- int yyn = yypact_[yystate];
- if (!yy_pact_value_is_default_ (yyn))
- {]b4_lac_if([[
- for (int yyx = 0; yyx < yyntokens_; ++yyx)
- if (yyx != yy_error_token_ && yyx != yy_undef_token_
- && yy_lac_check_ (yyx))
- {]], [[
- /* Start YYX at -YYN if negative to avoid negative indexes in
- YYCHECK. In other words, skip the first -YYN actions for
- this state because they are default actions. */
- int yyxbegin = yyn < 0 ? -yyn : 0;
- // Stay within bounds of both yycheck and yytname.
- int yychecklim = yylast_ - yyn + 1;
- int yyxend = yychecklim < yyntokens_ ? yychecklim : yyntokens_;
- for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
- if (yycheck_[yyx + yyn] == yyx && yyx != yy_error_token_
- && !yy_table_value_is_error_ (yytable_[yyx + yyn]))
- {]])[
- if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
- {
- yycount = 1;
- break;
- }
- else
- yyarg[yycount++] = yytname_[yyx];
- }
- }
+ if (yyarg)
+ yyarg[0] = yyctx.token ();
+ int yyn = yyctx.expected_tokens (yyarg ? yyarg + 1 : yyarg, yyargn - 1);
+ return yyn + 1;
}
+ return 0;
+ }
+
+ // Generate an error message.
+ std::string
+ ]b4_parser_class[::yysyntax_error_ (const context& yyctx) const
+ {
+ // Its maximum.
+ enum { YYARGS_MAX = 5 };
+ // Arguments of yyformat.
+ symbol_kind_type yyarg[YYARGS_MAX];
+ int yycount = yy_syntax_error_arguments_ (yyctx, yyarg, YYARGS_MAX);
char const* yyformat = YY_NULLPTR;
switch (yycount)
@@ -1397,14 +1567,13 @@
for (char const* yyp = yyformat; *yyp; ++yyp)
if (yyp[0] == '%' && yyp[1] == 's' && yyi < yycount)
{
- yyres += yytnamerr_ (yyarg[yyi++]);
+ yyres += symbol_name (yyarg[yyi++]);
++yyp;
}
else
yyres += *yyp;
- return yyres;]], [[
- return YY_("syntax error");]])[
- }
+ return yyres;
+ }]])[
const ]b4_int_type(b4_pact_ninf, b4_pact_ninf) b4_parser_class::yypact_ninf_ = b4_pact_ninf[;
@@ -1413,21 +1582,23 @@
]b4_parser_tables_define[
-]b4_token_table_if([], [[#if ]b4_api_PREFIX[DEBUG]])[
+]b4_parse_error_bmatch([simple\|verbose],
+[[#if ]b4_api_PREFIX[DEBUG]b4_tname_if([[ || 1]])[
// YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
- // First, the terminals, then, starting at \a yyntokens_, nonterminals.
+ // First, the terminals, then, starting at \a YYNTOKENS, nonterminals.
const char*
const ]b4_parser_class[::yytname_[] =
{
]b4_tname[
};
+#endif
+]])[
-]b4_token_table_if([[#if ]b4_api_PREFIX[DEBUG]])[
+#if ]b4_api_PREFIX[DEBUG][
]b4_integral_parser_table_define([rline], [b4_rline])[
- // Print the state stack on the debug stream.
void
- ]b4_parser_class[::yystack_print_ ()
+ ]b4_parser_class[::yy_stack_print_ () const
{
*yycdebug_ << "Stack now";
for (stack_type::const_iterator
@@ -1438,9 +1609,8 @@
*yycdebug_ << '\n';
}
- // Report on the debug stream that the rule \a yyrule is going to be reduced.
void
- ]b4_parser_class[::yy_reduce_print_ (int yyrule)
+ ]b4_parser_class[::yy_reduce_print_ (int yyrule) const
{
int yylno = yyrline_[yyrule];
int yynrhs = yyr2_[yyrule];
diff --git a/common/bison/skeletons/lalr1.d b/common/bison/skeletons/lalr1.d
index 5aa2fb6..9730377 100644
--- a/common/bison/skeletons/lalr1.d
+++ b/common/bison/skeletons/lalr1.d
@@ -1,6 +1,6 @@
# D skeleton for Bison -*- autoconf -*-
-# Copyright (C) 2007-2012, 2019 Free Software Foundation, Inc.
+# Copyright (C) 2007-2012, 2019-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,15 +13,56 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
m4_include(b4_skeletonsdir/[d.m4])
+b4_header_if([b4_complain([%header/%defines does not make sense in D])])
+# parse.lac
+b4_percent_define_default([[parse.lac]], [[none]])
+b4_percent_define_check_values([[[[parse.lac]], [[full]], [[none]]]])
+b4_define_flag_if([lac])
+m4_define([b4_lac_flag],
+ [m4_if(b4_percent_define_get([[parse.lac]]),
+ [none], [[0]], [[1]])])
+
+
+## --------------- ##
+## api.push-pull. ##
+## --------------- ##
+
+b4_percent_define_default([[api.push-pull]], [[pull]])
+b4_percent_define_check_values([[[[api.push-pull]],
+ [[pull]], [[push]], [[both]]]])
+
+# Define m4 conditional macros that encode the value
+# of the api.push-pull flag.
+b4_define_flag_if([pull]) m4_define([b4_pull_flag], [[1]])
+b4_define_flag_if([push]) m4_define([b4_push_flag], [[1]])
+m4_case(b4_percent_define_get([[api.push-pull]]),
+ [pull], [m4_define([b4_push_flag], [[0]])],
+ [push], [m4_define([b4_pull_flag], [[0]])])
+
+# Define a macro to be true when api.push-pull has the value "both".
+m4_define([b4_both_if],[b4_push_if([b4_pull_if([$1],[$2])],[$2])])
+
+# Handle BISON_USE_PUSH_FOR_PULL for the test suite. So that push parsing
+# tests function as written, do not let BISON_USE_PUSH_FOR_PULL modify the
+# behavior of Bison at all when push parsing is already requested.
+b4_define_flag_if([use_push_for_pull])
+b4_use_push_for_pull_if([
+ b4_push_if([m4_define([b4_use_push_for_pull_flag], [[0]])],
+ [m4_define([b4_push_flag], [[1]])])])
+
+
+# Define a macro to encapsulate the parse state variables. This
+# allows them to be defined either in parse() when doing pull parsing,
+# or as class instance variable when doing push parsing.
b4_output_begin([b4_parser_file_name])
b4_copyright([Skeleton implementation for Bison LALR(1) parsers in D],
- [2007-2012, 2019])[
-
+ [2007-2012, 2019-2021])[
+]b4_disclaimer[
]b4_percent_define_ifdef([package], [module b4_percent_define_get([package]);
])[
version(D_Version2) {
@@ -33,6 +74,29 @@
]b4_user_post_prologue[
]b4_percent_code_get([[imports]])[
import std.format;
+import std.conv;
+
+/**
+ * Handle error message internationalisation.
+ */
+static if (!is(typeof(YY_))) {
+ version(YYENABLE_NLS)
+ {
+ version(ENABLE_NLS)
+ {
+ extern(C) char* dgettext(const char*, const char*);
+ string YY_(const char* s)
+ {
+ return to!string(dgettext("bison-runtime", s));
+ }
+ }
+ }
+ static if (!is(typeof(YY_)))
+ {
+ pragma(inline, true)
+ string YY_(string msg) { return msg; }
+ }
+}
/**
* A Bison parser, automatically generated from <tt>]m4_bpatsubst(b4_file_name, [^"\(.*\)"$], [\1])[</tt>.
@@ -46,29 +110,13 @@
* parser <tt>]b4_parser_class[</tt>.
*/
public interface Lexer
-{]b4_locations_if([[
- /**
- * Method to retrieve the beginning position of the last scanned token.
- * @@return the position at which the last scanned token starts. */
- @@property ]b4_position_type[ startPos ();
-
- /**
- * Method to retrieve the ending position of the last scanned token.
- * @@return the first position beyond the last scanned token. */
- @@property ]b4_position_type[ endPos ();
-
-]])[
- /**
- * Method to retrieve the semantic value of the last scanned token.
- * @@return the semantic value of the last scanned token. */
- @@property ]b4_yystype[ semanticVal ();
-
+{
/**
* Entry point for the scanner. Returns the token identifier corresponding
* to the next token and prepares to return the semantic value
* ]b4_locations_if([and beginning/ending positions ])[of the token.
* @@return the token identifier corresponding to the next token. */
- int yylex ();
+ Symbol yylex ();
/**
* Entry point for error reporting. Emits an error
@@ -77,16 +125,26 @@
* @@param loc The location of the element to which the
* error message is related]])[
* @@param s The string for the error message. */
- void yyerror (]b4_locations_if([b4_location_type[ loc, ]])[string s);
+ void yyerror (]b4_locations_if([[const Location loc, ]])[string s);
+]b4_parse_error_bmatch([custom], [[
+ /**
+ * Build and emit a "syntax error" message in a user-defined way.
+ *
+ * @@param ctx The context of the error.
+ */
+ void reportSyntaxError(]b4_parser_class[.Context ctx);
+]])[
}
+]b4_public_types_declare[
+
]b4_locations_if([b4_position_type_if([[
static assert(__traits(compiles,
- (new ]b4_position_type[[1])[0]=(new ]b4_position_type[[1])[0]),
- "struct/class ]b4_position_type[ must be default-constructible "
+ (new Position[1])[0]=(new Position[1])[0]),
+ "struct/class Position must be default-constructible "
"and assignable");
-static assert(__traits(compiles, (new string[1])[0]=(new ]b4_position_type[).toString()),
- "error: struct/class ]b4_position_type[ must have toString method");
+static assert(__traits(compiles, (new string[1])[0]=(new Position).toString()),
+ "error: struct/class Position must have toString method");
]], [[
/**
* A struct denoting a point in the input.*/
@@ -109,59 +167,65 @@
}
}
]])b4_location_type_if([[
-static assert(__traits(compiles, (new ]b4_location_type[((new ]b4_position_type[[1])[0]))) &&
- __traits(compiles, (new ]b4_location_type[((new ]b4_position_type[[1])[0], (new ]b4_position_type[[1])[0]))),
- "error: struct/class ]b4_location_type[ must have "
- "default constructor and constructors this(]b4_position_type[) and this(]b4_position_type[, ]b4_position_type[).");
-static assert(__traits(compiles, (new ]b4_location_type[[1])[0].begin=(new ]b4_location_type[[1])[0].begin) &&
- __traits(compiles, (new ]b4_location_type[[1])[0].begin=(new ]b4_location_type[[1])[0].end) &&
- __traits(compiles, (new ]b4_location_type[[1])[0].end=(new ]b4_location_type[[1])[0].begin) &&
- __traits(compiles, (new ]b4_location_type[[1])[0].end=(new ]b4_location_type[[1])[0].end),
- "error: struct/class ]b4_location_type[ must have assignment-compatible "
+static assert(__traits(compiles, (new Location((new Position[1])[0]))) &&
+ __traits(compiles, (new Location((new Position[1])[0], (new Position[1])[0]))),
+ "error: struct/class Location must have "
+ "default constructor and constructors this(Position) and this(Position, Position).");
+static assert(__traits(compiles, (new Location[1])[0].begin=(new Location[1])[0].begin) &&
+ __traits(compiles, (new Location[1])[0].begin=(new Location[1])[0].end) &&
+ __traits(compiles, (new Location[1])[0].end=(new Location[1])[0].begin) &&
+ __traits(compiles, (new Location[1])[0].end=(new Location[1])[0].end),
+ "error: struct/class Location must have assignment-compatible "
"members/properties 'begin' and 'end'.");
-static assert(__traits(compiles, (new string[1])[0]=(new ]b4_location_type[[1])[0].toString()),
- "error: struct/class ]b4_location_type[ must have toString method.");
+static assert(__traits(compiles, (new string[1])[0]=(new Location[1])[0].toString()),
+ "error: struct/class Location must have toString method.");
-private immutable bool yy_location_is_class = !__traits(compiles, *(new ]b4_location_type[((new ]b4_position_type[[1])[0])));]], [[
+private immutable bool yy_location_is_class = !__traits(compiles, *(new Location((new Position[1])[0])));]], [[
/**
- * A class defining a pair of positions. Positions, defined by the
- * <code>]b4_position_type[</code> class, denote a point in the input.
+ * A struct defining a pair of positions. Positions, defined by the
+ * <code>Position</code> struct, denote a point in the input.
* Locations represent a part of the input through the beginning
* and ending positions. */
-public class ]b4_location_type[
+public struct ]b4_location_type[
{
/** The first, inclusive, position in the range. */
- public ]b4_position_type[ begin;
+ public Position begin;
/** The first position beyond the range. */
- public ]b4_position_type[ end;
+ public Position end;
/**
- * Create a <code>]b4_location_type[</code> denoting an empty range located at
+ * Create a <code>Location</code> denoting an empty range located at
* a given point.
* @@param loc The position at which the range is anchored. */
- public this (]b4_position_type[ loc) {
+ public this(Position loc)
+ {
this.begin = this.end = loc;
}
- public this () {
- }
-
/**
- * Create a <code>]b4_location_type[</code> from the endpoints of the range.
+ * Create a <code>Location</code> from the endpoints of the range.
* @@param begin The first position included in the range.
* @@param end The first position beyond the range. */
- public this (]b4_position_type[ begin, ]b4_position_type[ end)
+ public this(Position begin, Position end)
{
this.begin = begin;
this.end = end;
}
/**
- * A representation of the location. For this to be correct,
- * <code>]b4_position_type[</code> should override the <code>toString</code>
- * method. */
- public override string toString () const {
+ * Reset initial location to final location.
+ */
+ public void step()
+ {
+ this.begin = this.end;
+ }
+
+ /**
+ * A representation of the location.
+ */
+ public string toString() const
+ {
auto end_col = 0 < end.column ? end.column - 1 : 0;
auto res = begin.toString ();
if (end.filename && begin.filename != end.filename)
@@ -174,32 +238,34 @@
}
}
-private immutable bool yy_location_is_class = true;
+private immutable bool yy_location_is_class = false;
-]])])m4_ifdef([b4_user_union_members], [private union YYSemanticType
+]])])[]b4_value_type_setup[]m4_ifdef([b4_user_union_members], [private union YYSemanticType
{
b4_user_union_members
};],
[m4_if(b4_tag_seen_flag, 0,
[[private alias int YYSemanticType;]])])[
-]b4_token_enums(b4_tokens)[
+]b4_token_enums[
]b4_parser_class_declaration[
{
]b4_identification[
+]b4_declare_symbol_enum[
+
]b4_locations_if([[
- private final ]b4_location_type[ yylloc_from_stack (ref YYStack rhs, int n)
+ private final Location yylloc_from_stack (ref YYStack rhs, int n)
{
static if (yy_location_is_class) {
if (n > 0)
- return new ]b4_location_type[ (rhs.locationAt (n-1).begin, rhs.locationAt (0).end);
+ return new Location (rhs.locationAt (n-1).begin, rhs.locationAt (0).end);
else
- return new ]b4_location_type[ (rhs.locationAt (0).end);
+ return new Location (rhs.locationAt (0).end);
} else {
if (n > 0)
- return ]b4_location_type[ (rhs.locationAt (n-1).begin, rhs.locationAt (0).end);
+ return Location (rhs.locationAt (n-1).begin, rhs.locationAt (0).end);
else
- return ]b4_location_type[ (rhs.locationAt (0).end);
+ return Location (rhs.locationAt (0).end);
}
}]])[
@@ -217,6 +283,9 @@
* Instantiate the Bison-generated parser.
*/
public this] (b4_parse_param_decl([b4_lex_param_decl])[) {
+]b4_percent_code_get([[init]])[]b4_lac_if([[
+ this.yylacStack = new int[];
+ this.yylacEstablished = false;]])[
this (new YYLexer(]b4_lex_param_call[));
}
]])[
@@ -231,6 +300,7 @@
]b4_parse_param_cons[
}
]b4_parse_trace_if([[
+ import std.stdio;
private File yyDebugStream;
/**
@@ -262,18 +332,29 @@
protected final void yycdebug (string s) {
if (0 < yydebug)
+ yyDebugStream.write (s);
+ }
+
+ protected final void yycdebugln (string s) {
+ if (0 < yydebug)
yyDebugStream.writeln (s);
}
]])[
- private final int yylex () {
+ private final ]b4_parser_class[.Symbol yylex () {
return yylexer.yylex ();
}
- protected final void yyerror (]b4_locations_if(ref [b4_location_type[ loc, ]])[string s) {
+ protected final void yyerror (]b4_locations_if([[const Location loc, ]])[string s) {
yylexer.yyerror (]b4_locations_if([loc, ])[s);
}
/**
+ * The number of syntax errors so far.
+ */
+ public int numberOfErrors() const { return yynerrs_; }
+ private int yynerrs_ = 0;
+
+ /**
* Returned by a Bison action in order to stop the parsing process and
* return success (<tt>true</tt>). */
public static immutable int YYACCEPT = 0;
@@ -282,6 +363,11 @@
* Returned by a Bison action in order to stop the parsing process and
* return failure (<tt>false</tt>). */
public static immutable int YYABORT = 1;
+]b4_push_if([
+ /**
+ * Returned by a Bison action in order to request a new token.
+ */
+ public static immutable int YYPUSH_MORE = 4;])[
/**
* Returned by a Bison action in order to start error recovery without
@@ -296,10 +382,43 @@
private static immutable int YYREDUCE = 6;
private static immutable int YYERRLAB1 = 7;
private static immutable int YYRETURN = 8;
+]b4_push_if([[ private static immutable int YYGETTOKEN = 9; /* Signify that a new token is expected when doing push-parsing. */]])[
+
]b4_locations_if([
private static immutable YYSemanticType yy_semantic_null;])[
private int yyerrstatus_ = 0;
+ private void yyerrok()
+ {
+ yyerrstatus_ = 0;
+ }
+
+ // Lookahead symbol kind.
+ SymbolKind yytoken = ]b4_symbol(empty, kind)[;
+
+ /* State. */
+ int yyn = 0;
+ int yylen = 0;
+ int yystate = 0;
+
+ YYStack yystack;
+
+ int label = YYNEWSTATE;
+
+ /* Error handling. */
+]b4_locations_if([[
+ /// The location where the error started.
+ Location yyerrloc;
+
+ /// Location of the lookahead.
+ Location yylloc;
+
+ /// @@$.
+ Location yyloc;]])[
+
+ /// Semantic value of the lookahead.
+ Value yylval;
+
/**
* Whether error recovery is being done. In this state, the parser
* reads token until it reaches a known state, and then restarts normal
@@ -309,10 +428,22 @@
return yyerrstatus_ == 0;
}
+ /** Compute post-reduction state.
+ * @@param yystate the current state
+ * @@param yysym the nonterminal to push on the stack
+ */
+ private int yyLRGotoState(int yystate, int yysym) {
+ int yyr = yypgoto_[yysym - yyntokens_] + yystate;
+ if (0 <= yyr && yyr <= yylast_ && yycheck_[yyr] == yystate)
+ return yytable_[yyr];
+ else
+ return yydefgoto_[yysym - yyntokens_];
+ }
+
private int yyaction (int yyn, ref YYStack yystack, int yylen)
{
- ]b4_yystype[ yyval;]b4_locations_if([[
- ]b4_location_type[ yyloc = yylloc_from_stack (yystack, yylen);]])[
+ Value yyval;]b4_locations_if([[
+ Location yyloc = yylloc_from_stack (yystack, yylen);]])[
/* If YYLEN is nonzero, implement the default value of the action:
`$$ = $1'. Otherwise, use the top of the stack.
@@ -335,82 +466,47 @@
}
]b4_parse_trace_if([[
- yy_symbol_print ("-> $$ =", yyr1_[yyn], yyval]b4_locations_if([, yyloc])[);]])[
+ yy_symbol_print ("-> $$ =", to!SymbolKind (yyr1_[yyn]), yyval]b4_locations_if([, yyloc])[);]])[
yystack.pop (yylen);
yylen = 0;
/* Shift the result of the reduction. */
- yyn = yyr1_[yyn];
- int yystate = yypgoto_[yyn - yyntokens_] + yystack.stateAt (0);
- if (0 <= yystate && yystate <= yylast_
- && yycheck_[yystate] == yystack.stateAt (0))
- yystate = yytable_[yystate];
- else
- yystate = yydefgoto_[yyn - yyntokens_];
-
+ int yystate = yyLRGotoState(yystack.stateAt(0), yyr1_[yyn]);
yystack.push (yystate, yyval]b4_locations_if([, yyloc])[);
return YYNEWSTATE;
}
- /* Return YYSTR after stripping away unnecessary quotes and
- backslashes, so that it's suitable for yyerror. The heuristic is
- that double-quoting is unnecessary unless the string contains an
- apostrophe, a comma, or backslash (other than backslash-backslash).
- YYSTR is taken from yytname. */
- private final string yytnamerr_ (string yystr)
- {
- if (yystr[0] == '"')
- {
- string yyr;
- strip_quotes:
- for (int i = 1; i < yystr.length; i++)
- switch (yystr[i])
- {
- case '\'':
- case ',':
- break strip_quotes;
-
- case '\\':
- if (yystr[++i] != '\\')
- break strip_quotes;
- goto default;
- default:
- yyr ~= yystr[i];
- break;
-
- case '"':
- return yyr;
- }
- }
- else if (yystr == "$end")
- return "end of input";
-
- return yystr;
- }
]b4_parse_trace_if([[
/*--------------------------------.
| Print this symbol on YYOUTPUT. |
`--------------------------------*/
- private final void yy_symbol_print (string s, int yytype,
- ref ]b4_yystype[ yyvaluep]dnl
-b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
+ private final void yy_symbol_print (string s, SymbolKind yykind,
+ ref Value yyval]b4_locations_if([, ref Location yyloc])[)
{
if (0 < yydebug)
{
- string message = s ~ (yytype < yyntokens_ ? " token " : " nterm ")
- ~ yytname_[yytype] ~ " ("]b4_locations_if([
- ~ yylocationp.toString() ~ ": "])[;
- static if (__traits(compiles, message ~= yyvaluep.toString ()))
- message ~= yyvaluep.toString ();
- else
- message ~= format ("%s", &yyvaluep);
- message ~= ")";
- yycdebug (message);
+ File yyo = yyDebugStream;
+ yyo.write(s);
+ yyo.write(yykind < yyntokens_ ? " token " : " nterm ");
+ yyo.write(format("%s", yykind));
+ yyo.write(" ("]b4_locations_if([ ~ yyloc.toString() ~ ": "])[);
+ ]b4_symbol_actions([printer])[
+ yyo.write(")\n");
}
}
]])[
+]b4_symbol_type_define[
+]b4_push_if([[
+ /**
+ * Push Parse input from external lexer
+ *
+ * @@param yyla current Symbol
+ *
+ * @@return <tt>YYACCEPT, YYABORT, YYPUSH_MORE</tt>
+ */
+ public int pushParse(Symbol yyla)]], [[
/**
* Parse input from the scanner that was specified at object construction
* time. Return whether the end of the input was reached successfully.
@@ -418,36 +514,23 @@
* @@return <tt>true</tt> if the parsing succeeds. Note that this does not
* imply that there were no syntax errors.
*/
- public bool parse ()
- {
- /// Lookahead and lookahead in internal form.
- int yychar = yyempty_;
- int yytoken = 0;
+ public bool parse()]])[
+ {]b4_push_if([[
+ if (!this.pushParseInitialized)
+ {
+ pushParseInitialize();
+ yyerrstatus_ = 0;
+ }
+ else
+ label = YYGETTOKEN;
- /* State. */
- int yyn = 0;
- int yylen = 0;
- int yystate = 0;
+ bool push_token_consumed = true;
+]], [[ bool yyresult;]b4_lac_if([[
+ // Discard the LAC context in case there still is one left from a
+ // previous invocation.
+ yylacDiscard("init");]])[]b4_parse_trace_if([[
- YYStack yystack;
-
- /* Error handling. */
- int yynerrs_ = 0;]b4_locations_if([[
- /// The location where the error started.
- ]b4_location_type[ yyerrloc = null;
-
- /// ]b4_location_type[ of the lookahead.
- ]b4_location_type[ yylloc;
-
- /// @@$.
- ]b4_location_type[ yyloc;]])[
-
- /// Semantic value of the lookahead.
- ]b4_yystype[ yylval;
-
- int yyresult;]b4_parse_trace_if([[
-
- yycdebug ("Starting parse\n");]])[
+ yycdebugln ("Starting parse");]])[
yyerrstatus_ = 0;
]m4_ifdef([b4_initial_action], [
@@ -461,81 +544,107 @@
[ /* Initialize the stack. */
yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
- int label = YYNEWSTATE;
+ label = YYNEWSTATE;]])[
for (;;)
final switch (label)
{
/* New state. Unlike in the C/C++ skeletons, the state is already
pushed when we come here. */
case YYNEWSTATE:]b4_parse_trace_if([[
- yycdebug (format("Entering state %d\n", yystate));
+ yycdebugln (format("Entering state %d", yystate));
if (0 < yydebug)
yystack.print (yyDebugStream);]])[
/* Accept? */
- if (yystate == yyfinal_)
- return true;
+ if (yystate == yyfinal_)]b4_push_if([[
+ {
+ label = YYACCEPT;
+ break;
+ }]], [[
+ return true;]])[
/* Take a decision. First try without lookahead. */
yyn = yypact_[yystate];
- if (yy_pact_value_is_default_ (yyn))
+ if (yyPactValueIsDefault(yyn))
{
label = YYDEFAULT;
break;
- }
+ }]b4_push_if([[
+ goto case;
+
+ case YYGETTOKEN:]])[
/* Read a lookahead token. */
- if (yychar == yyempty_)
- {]b4_parse_trace_if([[
- yycdebug ("Reading a token: ");]])[
- yychar = yylex ();]b4_locations_if([[
- static if (yy_location_is_class) {
- yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos);
- } else {
- yylloc = ]b4_location_type[(yylexer.startPos, yylexer.endPos);
- }]])
- yylval = yylexer.semanticVal;[
+ if (yytoken == ]b4_symbol(empty, kind)[)
+ {]b4_push_if([[
+ if (!push_token_consumed)
+ return YYPUSH_MORE;]])[]b4_parse_trace_if([[
+ yycdebugln ("Reading a token");]])[]b4_push_if([[
+ yytoken = yyla.token;
+ yylval = yyla.value;]b4_locations_if([[
+ yylloc = yyla.location;]])[
+ push_token_consumed = false;]], [[
+ Symbol yysymbol = yylex();
+ yytoken = yysymbol.token();
+ yylval = yysymbol.value();]b4_locations_if([[
+ yylloc = yysymbol.location();]])[]])[
}
- /* Convert token to internal form. */
- yytoken = yytranslate_ (yychar);]b4_parse_trace_if([[
- yy_symbol_print ("Next token is",
- yytoken, yylval]b4_locations_if([, yylloc])[);]])[
+ /* Token already converted to internal form. */]b4_parse_trace_if([[
+ yy_symbol_print ("Next token is", yytoken, yylval]b4_locations_if([, yylloc])[);]])[
- /* If the proper action on seeing token YYTOKEN is to reduce or to
- detect an error, take that action. */
- yyn += yytoken;
- if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken)
- label = YYDEFAULT;
-
- /* <= 0 means reduce or error. */
- else if ((yyn = yytable_[yyn]) <= 0)
+ if (yytoken == ]b4_symbol(error, kind)[)
{
- if (yy_table_value_is_error_ (yyn))
- label = YYERRLAB;
- else
- {
- yyn = -yyn;
- label = YYREDUCE;
- }
+ // The scanner already issued an error message, process directly
+ // to error recovery. But do not keep the error token as
+ // lookahead, it is too special and may lead us to an endless
+ // loop in error recovery. */
+ yytoken = ]b4_symbol(undef, kind)[;]b4_locations_if([[
+ yyerrloc = yylloc;]])[
+ label = YYERRLAB1;
}
else
{
- /* Shift the lookahead token. */]b4_parse_trace_if([[
- yy_symbol_print ("Shifting", yytoken,
- yylval]b4_locations_if([, yylloc])[);]])[
+ /* If the proper action on seeing token YYTOKEN is to reduce or to
+ detect an error, take that action. */
+ yyn += yytoken;
+ if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken) {]b4_lac_if([[
+ if (!yylacEstablish(yystack, yytoken))
+ label = YYERRLAB;
+ else]])[
+ label = YYDEFAULT;
+ }
+ /* <= 0 means reduce or error. */
+ else if ((yyn = yytable_[yyn]) <= 0)
+ {
+ if (yyTableValueIsError(yyn))
+ label = YYERRLAB;]b4_lac_if([[
+ else if (!yylacEstablish(yystack, yytoken))
+ label = YYERRLAB;]])[
+ else
+ {
+ yyn = -yyn;
+ label = YYREDUCE;
+ }
+ }
+ else
+ {
+ /* Shift the lookahead token. */]b4_parse_trace_if([[
+ yy_symbol_print ("Shifting", yytoken, yylval]b4_locations_if([, yylloc])[);]])[
- /* Discard the token being shifted. */
- yychar = yyempty_;
+ /* Discard the token being shifted. */
+ yytoken = ]b4_symbol(empty, kind)[;
- /* Count tokens shifted since error; after three, turn off error
- * status. */
- if (yyerrstatus_ > 0)
- --yyerrstatus_;
+ /* Count tokens shifted since error; after three, turn off error
+ * status. */
+ if (yyerrstatus_ > 0)
+ --yyerrstatus_;
- yystate = yyn;
- yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
- label = YYNEWSTATE;
+ yystate = yyn;
+ yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);]b4_lac_if([[
+ yylacDiscard("shift");]])[
+ label = YYNEWSTATE;
+ }
}
break;
@@ -559,33 +668,32 @@
yystate = yystack.stateAt (0);
break;
- /*------------------------------------.
- | yyerrlab -- here on detecting error |
- `------------------------------------*/
+ /*--------------------------------------.
+ | yyerrlab -- here on detecting error. |
+ `--------------------------------------*/
case YYERRLAB:
/* If not already recovering from an error, report this error. */
if (yyerrstatus_ == 0)
{
++yynerrs_;
- if (yychar == yyempty_)
- yytoken = yyempty_;
- yyerror (]b4_locations_if([yylloc, ])[yysyntax_error (yystate, yytoken));
+ yyreportSyntaxError(new Context(]b4_lac_if([[this, ]])[yystack, yytoken]b4_locations_if([[, yylloc]])[));
}
-
-]b4_locations_if([ yyerrloc = yylloc;])[
+]b4_locations_if([
+ yyerrloc = yylloc;])[
if (yyerrstatus_ == 3)
{
/* If just tried and failed to reuse lookahead token after an
* error, discard it. */
- if (yychar <= YYTokenType.EOF)
+ /* Return failure if at end of input. */
+ if (yytoken == ]b4_symbol(eof, [kind])[)]b4_push_if([[
{
- /* Return failure if at end of input. */
- if (yychar == YYTokenType.EOF)
- return false;
- }
+ label = YYABORT;
+ break;
+ }]], [[
+ return false;]])[
else
- yychar = yyempty_;
+ yytoken = ]b4_symbol(empty, kind)[;
}
/* Else will try to reuse lookahead token after shifting the error
@@ -612,13 +720,14 @@
case YYERRLAB1:
yyerrstatus_ = 3; /* Each real token shifted decrements this. */
+ // Pop stack until we find a state that shifts the error token.
for (;;)
{
yyn = yypact_[yystate];
- if (!yy_pact_value_is_default_ (yyn))
+ if (!yyPactValueIsDefault(yyn))
{
- yyn += yy_error_token_;
- if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == yy_error_token_)
+ yyn += ]b4_symbol(error, kind)[;
+ if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == ]b4_symbol(error, kind)[)
{
yyn = yytable_[yyn];
if (0 < yyn)
@@ -627,16 +736,23 @@
}
/* Pop the current state because it cannot handle the error token. */
- if (yystack.height == 1)
- return false;
+ if (yystack.height == 1)]b4_push_if([[
+ {
+ label = YYABORT;
+ break;
+ }]],[[
+ return false;]])[
]b4_locations_if([ yyerrloc = yystack.locationAt (0);])[
yystack.pop ();
yystate = yystack.stateAt (0);]b4_parse_trace_if([[
if (0 < yydebug)
yystack.print (yyDebugStream);]])[
- }
-
+ }]b4_push_if([[
+ if (label == YYABORT)
+ /* Leave the switch. */
+ break;
+]])[
]b4_locations_if([
/* Muck with the stack to setup for yylloc. */
yystack.push (0, yy_semantic_null, yylloc);
@@ -644,34 +760,145 @@
yyloc = yylloc_from_stack (yystack, 2);
yystack.pop (2);])[
- /* Shift the error token. */]b4_parse_trace_if([[
- yy_symbol_print ("Shifting", yystos_[yyn],
- yylval]b4_locations_if([, yyloc])[);]])[
-
+ /* Shift the error token. */]b4_lac_if([[
+ yylacDiscard("error recovery");]])[]b4_parse_trace_if([[
+ yy_symbol_print ("Shifting", to!SymbolKind (yystos_[yyn]), yylval]b4_locations_if([, yyloc])[);]])[
yystate = yyn;
yystack.push (yyn, yylval]b4_locations_if([, yyloc])[);
label = YYNEWSTATE;
break;
/* Accept. */
- case YYACCEPT:
- return true;
+ case YYACCEPT:]b4_push_if([[
+ this.pushParseInitialized = false;]b4_parse_trace_if([[
+ if (0 < yydebug)
+ yystack.print (yyDebugStream);]])[
+ return YYACCEPT;]], [[
+ yyresult = true;
+ label = YYRETURN;
+ break;]])[
/* Abort. */
- case YYABORT:
- return false;
+ case YYABORT:]b4_push_if([[
+ this.pushParseInitialized = false;]b4_parse_trace_if([[
+ if (0 < yydebug)
+ yystack.print (yyDebugStream);]])[
+ return YYABORT;]], [[
+ yyresult = false;
+ label = YYRETURN;
+ break;]])[
+]b4_push_if([[]], [[ ][case YYRETURN:]b4_parse_trace_if([[
+ if (0 < yydebug)
+ yystack.print (yyDebugStream);]])[
+ return yyresult;]])[
}
+ assert(0);
}
+]b4_push_if([[
+ bool pushParseInitialized = false;
+
+ /**
+ * (Re-)Initialize the state of the push parser.
+ */
+ public void pushParseInitialize()
+ {
+
+ /* Lookahead and lookahead in internal form. */
+ this.yytoken = ]b4_symbol(empty, kind)[;
+
+ /* State. */
+ this.yyn = 0;
+ this.yylen = 0;
+ this.yystate = 0;
+ destroy(this.yystack);
+ this.label = YYNEWSTATE;
+]b4_lac_if([[
+ destroy(this.yylacStack);
+ this.yylacEstablished = false;]])[
+
+ /* Error handling. */
+ this.yynerrs_ = 0;
+]b4_locations_if([
+ /* The location where the error started. */
+ this.yyerrloc = Location(Position(), Position());
+ this.yylloc = Location(Position(), Position());])[
+
+ /* Semantic value of the lookahead. */
+ //destroy(this.yylval);
+
+ /* Initialize the stack. */
+ yystack.push(this.yystate, this.yylval]b4_locations_if([, this.yylloc])[);
+
+ this.pushParseInitialized = true;
+ }]])[]b4_both_if([[
+ /**
+ * Parse input from the scanner that was specified at object construction
+ * time. Return whether the end of the input was reached successfully.
+ * This version of parse() is defined only when api.push-push=both.
+ *
+ * @@return <tt>true</tt> if the parsing succeeds. Note that this does not
+ * imply that there were no syntax errors.
+ */
+ bool parse()
+ {
+ int status = 0;
+ do {
+ status = this.pushParse(yylex());
+ } while (status == YYPUSH_MORE);
+ return status == YYACCEPT;
+ }]])[
+
// Generate an error message.
- private final string yysyntax_error (int yystate, int tok)
- {]b4_error_verbose_if([[
+ private final void yyreportSyntaxError(Context yyctx)
+ {]b4_parse_error_bmatch(
+[custom], [[
+ yylexer.reportSyntaxError(yyctx);]],
+[detailed], [[
+ if (yyctx.getToken() != ]b4_symbol(empty, kind)[)
+ {
+ // FIXME: This method of building the message is not compatible
+ // with internationalization.
+ immutable int argmax = 5;
+ SymbolKind[] yyarg = new SymbolKind[argmax];
+ int yycount = yysyntaxErrorArguments(yyctx, yyarg, argmax);
+ string res, yyformat;
+ switch (yycount)
+ {
+ case 1:
+ yyformat = YY_("syntax error, unexpected %s");
+ res = format(yyformat, yyarg[0]);
+ break;
+ case 2:
+ yyformat = YY_("syntax error, unexpected %s, expecting %s");
+ res = format(yyformat, yyarg[0], yyarg[1]);
+ break;
+ case 3:
+ yyformat = YY_("syntax error, unexpected %s, expecting %s or %s");
+ res = format(yyformat, yyarg[0], yyarg[1], yyarg[2]);
+ break;
+ case 4:
+ yyformat = YY_("syntax error, unexpected %s, expecting %s or %s or %s");
+ res = format(yyformat, yyarg[0], yyarg[1], yyarg[2], yyarg[3]);
+ break;
+ case 5:
+ yyformat = YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s");
+ res = format(yyformat, yyarg[0], yyarg[1], yyarg[2], yyarg[3], yyarg[4]);
+ break;
+ default:
+ res = YY_("syntax error");
+ break;
+ }
+ yyerror(]b4_locations_if([yyctx.getLocation(), ])[res);
+ }]],
+[[simple]], [[
+ yyerror(]b4_locations_if([yyctx.getLocation(), ])[YY_("syntax error"));]])[
+ }
+
+]b4_parse_error_bmatch(
+[detailed], [[
+ private int yysyntaxErrorArguments(Context yyctx, SymbolKind[] yyarg, int yyargn) {
/* There are many possibilities here to consider:
- - Assume YYFAIL is not used. It's too flawed to consider.
- See
- <http://lists.gnu.org/archive/html/bison-patches/2009-12/msg00024.html>
- for details. YYERROR is fine as it does not invoke this
- function.
- If this state is a consistent state with a default action,
then the only way this function was invoked is if the
default action is an error action. In that case, don't
@@ -696,15 +923,85 @@
list is correct for canonical LR with one exception: it
will still contain any token that will not be accepted due
to an error action in a later state.
- */
- if (tok != yyempty_)
+ */
+ int yycount = 0;
+ if (yyctx.getToken() != ]b4_symbol(empty, kind)[)
+ {
+ if (yyarg !is null)
+ yyarg[yycount] = yyctx.getToken();
+ yycount += 1;
+ yycount += yyctx.getExpectedTokens(yyarg, 1, yyargn);
+ }
+ return yycount;
+ }
+]])[
+
+
+ /**
+ * Information needed to get the list of expected tokens and to forge
+ * a syntax error diagnostic.
+ */
+ public static final class Context
+ {]b4_lac_if([[
+ private ]b4_parser_class[ yyparser;]])[
+ private const(YYStack) yystack;
+ private SymbolKind yytoken;]b4_locations_if([[
+ private const(Location) yylocation;]])[
+
+ this(]b4_lac_if([[]b4_parser_class[ parser, ]])[YYStack stack, SymbolKind kind]b4_locations_if([[, Location loc]])[)
+ {]b4_lac_if([[
+ yyparser = parser;]])[
+ yystack = stack;
+ yytoken = kind;]b4_locations_if([[
+ yylocation = loc;]])[
+ }
+
+ final SymbolKind getToken() const
{
- // FIXME: This method of building the message is not compatible
- // with internationalization.
- string res = "syntax error, unexpected ";
- res ~= yytnamerr_ (yytname_[tok]);
- int yyn = yypact_[yystate];
- if (!yy_pact_value_is_default_ (yyn))
+ return yytoken;
+ }]b4_locations_if([[
+
+ final const(Location) getLocation() const
+ {
+ return yylocation;
+ }]])[
+ /**
+ * Put in YYARG at most YYARGN of the expected tokens given the
+ * current YYCTX, and return the number of tokens stored in YYARG. If
+ * YYARG is null, return the number of expected tokens (guaranteed to
+ * be less than YYNTOKENS).
+ */
+ int getExpectedTokens(SymbolKind[] yyarg, int yyargn)]b4_lac_if([[]], [[ const]])[
+ {
+ return getExpectedTokens(yyarg, 0, yyargn);
+ }
+
+ int getExpectedTokens(SymbolKind[] yyarg, int yyoffset, int yyargn)]b4_lac_if([[]], [[ const]])[
+ {
+ int yycount = yyoffset;]b4_lac_if([b4_parse_trace_if([[
+ // Execute LAC once. We don't care if it is successful, we
+ // only do it for the sake of debugging output.
+
+ if (!yyparser.yylacEstablished)
+ yyparser.yylacCheck(yystack, yytoken);
+]])[
+ for (int yyx = 0; yyx < yyntokens_; ++yyx)
+ {
+ SymbolKind yysym = SymbolKind(yyx);
+ if (yysym != ]b4_symbol(error, kind)[
+ && yysym != ]b4_symbol(undef, kind)[
+ && yyparser.yylacCheck(yystack, yysym))
+ {
+ if (yyarg == null)
+ yycount += 1;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = yysym;
+ }
+ }]], [[
+ int yyn = yypact_[this.yystack.stateAt(0)];
+ if (!yyPactValueIsDefault(yyn))
{
/* Start YYX at -YYN if negative to avoid negative
indexes in YYCHECK. In other words, skip the first
@@ -714,33 +1011,181 @@
/* Stay within bounds of both yycheck and yytname. */
int yychecklim = yylast_ - yyn + 1;
int yyxend = yychecklim < yyntokens_ ? yychecklim : yyntokens_;
- int count = 0;
- for (int x = yyxbegin; x < yyxend; ++x)
- if (yycheck_[x + yyn] == x && x != yy_error_token_
- && !yy_table_value_is_error_ (yytable_[x + yyn]))
- ++count;
- if (count < 5)
+ for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
+ if (yycheck_[yyx + yyn] == yyx && yyx != ]b4_symbol(error, kind)[
+ && !yyTableValueIsError(yytable_[yyx + yyn]))
{
- count = 0;
- for (int x = yyxbegin; x < yyxend; ++x)
- if (yycheck_[x + yyn] == x && x != yy_error_token_
- && !yy_table_value_is_error_ (yytable_[x + yyn]))
- {
- res ~= count++ == 0 ? ", expecting " : " or ";
- res ~= yytnamerr_ (yytname_[x]);
- }
+ if (yyarg is null)
+ ++yycount;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = SymbolKind(yyx);
}
- }
- return res;
- }]])[
- return "syntax error";
+ }]])[
+ if (yyarg !is null && yycount == yyoffset && yyoffset < yyargn)
+ yyarg[yyoffset] = ]b4_symbol(empty, kind)[;
+ return yycount - yyoffset;
+ }
}
+]b4_lac_if([[
+ /** Check the lookahead yytoken.
+ * \returns true iff the token will be eventually shifted.
+ */
+ bool yylacCheck(const YYStack yystack, SymbolKind yytoken)
+ {
+ // Logically, the yylacStack's lifetime is confined to this function.
+ // Clear it, to get rid of potential left-overs from previous call.
+ destroy(yylacStack);
+ // Reduce until we encounter a shift and thereby accept the token.
+]b4_parse_trace_if([[
+ yycdebug("LAC: checking lookahead " ~ format("%s", yytoken) ~ ":");]])[
+ int lacTop = 0;
+ while (true)
+ {
+ int topState = (yylacStack.length == 0
+ ? yystack.stateAt(lacTop)
+ : yylacStack[$ - 1]);
+ int yyrule = yypact_[topState];
+ if (yyPactValueIsDefault(yyrule)
+ || (yyrule += yytoken) < 0 || yylast_ < yyrule
+ || yycheck_[yyrule] != yytoken)
+ {
+ // Use the default action.
+ yyrule = yydefact_[+topState];
+ if (yyrule == 0)
+ {]b4_parse_trace_if([[
+ yycdebugln(" Err");]])[
+ return false;
+ }
+ }
+ else
+ {
+ // Use the action from yytable.
+ yyrule = yytable_[yyrule];
+ if (yyTableValueIsError(yyrule))
+ {]b4_parse_trace_if([[
+ yycdebugln(" Err");]])[
+ return false;
+ }
+ if (0 < yyrule)
+ {]b4_parse_trace_if([[
+ yycdebugln(" S" ~ to!string(yyrule));]])[
+ return true;
+ }
+ yyrule = -yyrule;
+ }
+ // By now we know we have to simulate a reduce.
+]b4_parse_trace_if([[
+ yycdebug(" R" ~ to!string(yyrule - 1));]])[
+ // Pop the corresponding number of values from the stack.
+ {
+ int yylen = yyr2_[yyrule];
+ // First pop from the LAC stack as many tokens as possible.
+ int lacSize = cast (int) yylacStack.length;
+ if (yylen < lacSize)
+ {
+ yylacStack.length -= yylen;
+ yylen = 0;
+ }
+ else if (lacSize != 0)
+ {
+ destroy(yylacStack);
+ yylen -= lacSize;
+ }
+ // Only afterwards look at the main stack.
+ // We simulate popping elements by incrementing lacTop.
+ lacTop += yylen;
+ }
+ // Keep topState in sync with the updated stack.
+ topState = (yylacStack.length == 0
+ ? yystack.stateAt(lacTop)
+ : yylacStack[$ - 1]);
+ // Push the resulting state of the reduction.
+ int state = yyLRGotoState(topState, yyr1_[yyrule]);]b4_parse_trace_if([[
+ yycdebug(" G" ~ to!string(state));]])[
+ yylacStack.length++;
+ yylacStack[$ - 1] = state;
+ }
+ }
+
+ /** Establish the initial context if no initial context currently exists.
+ * \returns true iff the token will be eventually shifted.
+ */
+ bool yylacEstablish(YYStack yystack, SymbolKind yytoken)
+ {
+ /* Establish the initial context for the current lookahead if no initial
+ context is currently established.
+
+ We define a context as a snapshot of the parser stacks. We define
+ the initial context for a lookahead as the context in which the
+ parser initially examines that lookahead in order to select a
+ syntactic action. Thus, if the lookahead eventually proves
+ syntactically unacceptable (possibly in a later context reached via a
+ series of reductions), the initial context can be used to determine
+ the exact set of tokens that would be syntactically acceptable in the
+ lookahead's place. Moreover, it is the context after which any
+ further semantic actions would be erroneous because they would be
+ determined by a syntactically unacceptable token.
+
+ yylacEstablish should be invoked when a reduction is about to be
+ performed in an inconsistent state (which, for the purposes of LAC,
+ includes consistent states that don't know they're consistent because
+ their default reductions have been disabled).
+
+ For parse.lac=full, the implementation of yylacEstablish is as
+ follows. If no initial context is currently established for the
+ current lookahead, then check if that lookahead can eventually be
+ shifted if syntactic actions continue from the current context. */
+ if (yylacEstablished)
+ return true;
+ else
+ {]b4_parse_trace_if([[
+ yycdebugln("LAC: initial context established for " ~ format("%s", yytoken));]])[
+ yylacEstablished = true;
+ return yylacCheck(yystack, yytoken);
+ }
+ }
+
+ /** Discard any previous initial lookahead context because of event.
+ * \param event the event which caused the lookahead to be discarded.
+ * Only used for debbuging output. */
+ void yylacDiscard(string event)
+ {
+ /* Discard any previous initial lookahead context because of Event,
+ which may be a lookahead change or an invalidation of the currently
+ established initial context for the current lookahead.
+
+ The most common example of a lookahead change is a shift. An example
+ of both cases is syntax error recovery. That is, a syntax error
+ occurs when the lookahead is syntactically erroneous for the
+ currently established initial context, so error recovery manipulates
+ the parser stacks to try to find a new initial context in which the
+ current lookahead is syntactically acceptable. If it fails to find
+ such a context, it discards the lookahead. */
+ if (yylacEstablished)
+ {]b4_parse_trace_if([[
+ yycdebugln("LAC: initial context discarded due to " ~ event);]])[
+ yylacEstablished = false;
+ }
+ }
+
+ /** The stack for LAC.
+ * Logically, the yylacStack's lifetime is confined to the function
+ * yylacCheck. We just store it as a member of this class to hold
+ * on to the memory and to avoid frequent reallocations.
+ */
+ int[] yylacStack;
+ /** Whether an initial LAC context was established. */
+ bool yylacEstablished;
+]])[
+
/**
* Whether the given <code>yypact_</code> value indicates a defaulted state.
* @@param yyvalue the value to check
*/
- private static bool yy_pact_value_is_default_ (int yyvalue)
+ private static bool yyPactValueIsDefault(int yyvalue)
{
return yyvalue == yypact_ninf_;
}
@@ -749,7 +1194,7 @@
* Whether the given <code>yytable_</code> value indicates a syntax error.
* @@param yyvalue the value to check
*/
- private static bool yy_table_value_is_error_ (int yyvalue)
+ private static bool yyTableValueIsError(int yyvalue)
{
return yyvalue == yytable_ninf_;
}
@@ -765,13 +1210,6 @@
]b4_parser_tables_define[
- /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
- First, the terminals, then, starting at \a yyntokens_, nonterminals. */
- private static immutable string[] yytname_ =
- @{
- ]b4_tname[
- @};
-
]b4_parse_trace_if([[
/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */
private static immutable ]b4_int_type_for([b4_rline])[[] yyrline_ =
@@ -788,66 +1226,60 @@
int yylno = yyrline_[yyrule];
int yynrhs = yyr2_[yyrule];
/* Print the symbols being reduced, and their result. */
- yycdebug (format("Reducing stack by rule %d (line %d), ",
- yyrule - 1, yylno));
+ yycdebugln (format("Reducing stack by rule %d (line %d):",
+ yyrule - 1, yylno));
/* The symbols being reduced. */
for (int yyi = 0; yyi < yynrhs; yyi++)
yy_symbol_print (format(" $%d =", yyi + 1),
- yystos_[yystack.stateAt(yynrhs - (yyi + 1))],
+ to!SymbolKind (yystos_[yystack.stateAt(yynrhs - (yyi + 1))]),
]b4_rhs_value(yynrhs, yyi + 1)b4_locations_if([,
b4_rhs_location(yynrhs, yyi + 1)])[);
}
]])[
- private static token_number_type yytranslate_ (int t)
+ private static auto yytranslate_ (int t)
{
]b4_api_token_raw_if(
-[[ import std.conv : to;
- return to!byte (t);]],
+[[ return SymbolKind(t);]],
[[ /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */
- immutable token_number_type[] translate_table =
+ immutable ]b4_int_type_for([b4_translate])[[] translate_table =
@{
]b4_translate[
@};
- immutable int user_token_number_max_ = ]b4_user_token_number_max[;
- immutable token_number_type undef_token_ = ]b4_undef_token_number[;
+ // Last valid token kind.
+ immutable int code_max = ]b4_code_max[;
if (t <= 0)
- return YYTokenType.EOF;
- else if (t <= user_token_number_max_)
- return translate_table[t];
+ return ]b4_symbol(eof, kind)[;
+ else if (t <= code_max)
+ return SymbolKind(translate_table[t]);
else
- return undef_token_;]])[
+ return ]b4_symbol(undef, kind)[;]])[
}
- alias ]b4_int_type_for([b4_translate])[ token_number_type;
-
- private static immutable token_number_type yy_error_token_ = 1;
-
private static immutable int yylast_ = ]b4_last[;
private static immutable int yynnts_ = ]b4_nterms_number[;
- private static immutable int yyempty_ = -2;
private static immutable int yyfinal_ = ]b4_final_state_number[;
private static immutable int yyntokens_ = ]b4_tokens_number[;
private final struct YYStackElement {
int state;
- ]b4_yystype[ value;]b4_locations_if(
+ Value value;]b4_locations_if(
b4_location_type[[] location;])[
}
private final struct YYStack {
private YYStackElement[] stack = [];
- public final @@property ulong height()
+ public final ulong height()
{
return stack.length;
}
- public final void push (int state, ]b4_yystype[ value]dnl
- b4_locations_if([, ref ]b4_location_type[ loc])[)
+ public final void push (int state, Value value]dnl
+ b4_locations_if([, ref Location loc])[)
{
stack ~= YYStackElement(state, value]b4_locations_if([, loc])[);
}
@@ -862,18 +1294,18 @@
stack.length -= num;
}
- public final int stateAt (int i)
+ public final int stateAt (int i) const
{
return stack[$-i-1].state;
}
]b4_locations_if([[
- public final ref ]b4_location_type[ locationAt (int i)
+ public final ref Location locationAt (int i)
{
return stack[$-i-1].location;
}]])[
- public final ref ]b4_yystype[ valueAt (int i)
+ public final ref Value valueAt (int i)
{
return stack[$-i-1].value;
}
@@ -887,9 +1319,8 @@
stream.writeln ();
}]])[
}
-
- /* User implementation code. */
]b4_percent_code_get[
}
-]b4_epilogue[]dnl
+]b4_percent_code_get([[epilogue]])[]dnl
+b4_epilogue[]dnl
b4_output_end
diff --git a/common/bison/skeletons/lalr1.java b/common/bison/skeletons/lalr1.java
index 9ec0806..1bbecca 100644
--- a/common/bison/skeletons/lalr1.java
+++ b/common/bison/skeletons/lalr1.java
@@ -1,6 +1,6 @@
-# Java skeleton for Bison -*- autoconf -*-
+# Java skeleton for Bison -*- java -*-
-# Copyright (C) 2007-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2007-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,11 +13,11 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
m4_include(b4_skeletonsdir/[java.m4])
-b4_defines_if([b4_complain([%defines does not make sense in Java])])
+b4_header_if([b4_complain([%header/%defines does not make sense in Java])])
m4_define([b4_symbol_no_destructor_assert],
[b4_symbol_if([$1], [has_destructor],
@@ -25,7 +25,10 @@
[%destructor does not make sense in Java])])])
b4_symbol_foreach([b4_symbol_no_destructor_assert])
-# Setup some macros for api.push-pull.
+## --------------- ##
+## api.push-pull. ##
+## --------------- ##
+
b4_percent_define_default([[api.push-pull]], [[pull]])
b4_percent_define_check_values([[[[api.push-pull]],
[[pull]], [[push]], [[both]]]])
@@ -49,13 +52,15 @@
b4_push_if([m4_define([b4_use_push_for_pull_flag], [[0]])],
[m4_define([b4_push_flag], [[1]])])])
-# Define a macro to encapsulate the parse state variables.
-# This allows them to be defined either in parse() when doing
-# pull parsing, or as class instance variable when doing push parsing.
-m4_define([b4_define_state],[[
- /* Lookahead and lookahead in internal form. */
- int yychar = yyempty_;
- int yytoken = 0;
+# Define a macro to encapsulate the parse state variables. This
+# allows them to be defined either in parse() when doing pull parsing,
+# or as class instance variable when doing push parsing.
+m4_define([b4_define_state],
+[[
+ /* Lookahead token kind. */
+ int yychar = YYEMPTY_;
+ /* Lookahead symbol kind. */
+ SymbolKind yytoken = null;
/* State. */
int yyn = 0;
@@ -64,9 +69,8 @@
YYStack yystack = new YYStack ();
int label = YYNEWSTATE;
- /* Error handling. */
- int yynerrs_ = 0;
- ]b4_locations_if([[/* The location where the error started. */
+]b4_locations_if([[
+ /* The location where the error started. */
]b4_location_type[ yyerrloc = null;
/* Location. */
@@ -74,15 +78,31 @@
/* Semantic value of the lookahead. */
]b4_yystype[ yylval = null;
-]])[
+]])
-]b4_output_begin([b4_parser_file_name])[
+# parse.lac
+b4_percent_define_default([[parse.lac]], [[none]])
+b4_percent_define_check_values([[[[parse.lac]], [[full]], [[none]]]])
+b4_define_flag_if([lac])
+m4_define([b4_lac_flag],
+ [m4_if(b4_percent_define_get([[parse.lac]]),
+ [none], [[0]], [[1]])])
+
+
+## ------------- ##
+## Parser File. ##
+## ------------- ##
+
+b4_output_begin([b4_parser_file_name])[
]b4_copyright([Skeleton implementation for Bison LALR(1) parsers in Java],
- [2007-2015, 2018-2019])[
-]b4_percent_define_ifdef([package], [package b4_percent_define_get([package]);[
+ [2007-2015, 2018-2021])[
+]b4_disclaimer[
+]b4_percent_define_ifdef([api.package], [package b4_percent_define_get([api.package]);[
]])[
]b4_user_pre_prologue[
]b4_user_post_prologue[
+import java.text.MessageFormat;
+import java.util.ArrayList;
]b4_percent_code_get([[imports]])[
/**
* A Bison parser, automatically generated from <tt>]m4_bpatsubst(b4_file_name, [^"\(.*\)"$], [\1])[</tt>.
@@ -91,8 +111,10 @@
*/
]b4_parser_class_declaration[
{
- ]b4_identification[
-]b4_error_verbose_if([[
+]b4_identification[
+][
+]b4_parse_error_bmatch(
+ [detailed\|verbose], [[
/**
* True if verbose error messages are enabled.
*/
@@ -109,16 +131,16 @@
*/
public final void setErrorVerbose(boolean verbose)
{ yyErrorVerbose = verbose; }
-]])
+]])[
-b4_locations_if([[
+]b4_locations_if([[
/**
* A class defining a pair of positions. Positions, defined by the
* <code>]b4_position_type[</code> class, denote a point in the input.
* Locations represent a part of the input through the beginning
* and ending positions.
*/
- public class ]b4_location_type[ {
+ public static class ]b4_location_type[ {
/**
* The first, inclusive, position in the range.
*/
@@ -153,52 +175,50 @@
* <code>]b4_position_type[</code> should override the <code>equals</code>
* method.
*/
- public String toString () {
+ public String toString() {
if (begin.equals (end))
- return begin.toString ();
+ return begin.toString();
else
- return begin.toString () + "-" + end.toString ();
+ return begin.toString() + "-" + end.toString();
}
}
-]])[
-
-]b4_locations_if([[
- private ]b4_location_type[ yylloc (YYStack rhs, int n)
+ private ]b4_location_type[ yylloc(YYStack rhs, int n)
{
if (0 < n)
- return new ]b4_location_type[ (rhs.locationAt (n-1).begin, rhs.locationAt (0).end);
+ return new ]b4_location_type[(rhs.locationAt(n-1).begin, rhs.locationAt(0).end);
else
- return new ]b4_location_type[ (rhs.locationAt (0).end);
+ return new ]b4_location_type[(rhs.locationAt(0).end);
}]])[
+]b4_declare_symbol_enum[
+
/**
* Communication interface between the scanner and the Bison-generated
* parser <tt>]b4_parser_class[</tt>.
*/
public interface Lexer {
- /** Token returned by the scanner to signal the end of its input. */
- public static final int EOF = 0;
-
]b4_token_enums[
-
- ]b4_locations_if([[/**
+ /** Deprecated, use ]b4_symbol(eof, id)[ instead. */
+ public static final int EOF = ]b4_symbol(eof, id)[;
+]b4_pull_if([b4_locations_if([[
+ /**
* Method to retrieve the beginning position of the last scanned token.
* @@return the position at which the last scanned token starts.
*/
- ]b4_position_type[ getStartPos ();
+ ]b4_position_type[ getStartPos();
/**
* Method to retrieve the ending position of the last scanned token.
* @@return the first position beyond the last scanned token.
*/
- ]b4_position_type[ getEndPos ();]])[
+ ]b4_position_type[ getEndPos();]])[
/**
* Method to retrieve the semantic value of the last scanned token.
* @@return the semantic value of the last scanned token.
*/
- ]b4_yystype[ getLVal ();
+ ]b4_yystype[ getLVal();
/**
* Entry point for the scanner. Returns the token identifier corresponding
@@ -206,17 +226,26 @@
* ]b4_locations_if([and beginning/ending positions ])[of the token.
* @@return the token identifier corresponding to the next token.
*/
- int yylex () ]b4_maybe_throws([b4_lex_throws])[;
-
+ int yylex()]b4_maybe_throws([b4_lex_throws])[;
+]])[
/**
- * Entry point for error reporting. Emits an error
- * ]b4_locations_if([referring to the given location ])[in a user-defined way.
+ * Emit an error]b4_locations_if([ referring to the given location])[in a user-defined way.
*
- * ]b4_locations_if([[@@param loc The location of the element to which the
- * error message is related]])[
+ *]b4_locations_if([[ @@param loc The location of the element to which the
+ * error message is related.]])[
* @@param msg The string for the error message.
*/
- void yyerror (]b4_locations_if([b4_location_type[ loc, ]])[String msg);
+ void yyerror(]b4_locations_if([b4_location_type[ loc, ]])[String msg);
+
+]b4_parse_error_bmatch(
+ [custom], [[
+ /**
+ * Build and emit a "syntax error" message in a user-defined way.
+ *
+ * @@param ctx The context of the error.
+ */
+ void reportSyntaxError(Context ctx);
+]])[
}
]b4_lexer_if([[
@@ -224,22 +253,25 @@
]b4_percent_code_get([[lexer]])[
}
- ]])[/**
+]])[
+ /**
* The object doing lexical analysis for us.
*/
private Lexer yylexer;
- ]b4_parse_param_vars[
+]b4_parse_param_vars[
]b4_lexer_if([[
/**
* Instantiates the Bison-generated parser.
*/
- public ]b4_parser_class (b4_parse_param_decl([b4_lex_param_decl])[) ]b4_maybe_throws([b4_init_throws])[
+ public ]b4_parser_class[(]b4_parse_param_decl([b4_lex_param_decl])[)]b4_maybe_throws([b4_init_throws])[
{
- ]b4_percent_code_get([[init]])[
+]b4_percent_code_get([[init]])[]b4_lac_if([[
+ this.yylacStack = new ArrayList<Integer>();
+ this.yylacEstablished = false;]])[
this.yylexer = new YYLexer(]b4_lex_param_call[);
- ]b4_parse_param_cons[
+]b4_parse_param_cons[
}
]])[
@@ -247,11 +279,13 @@
* Instantiates the Bison-generated parser.
* @@param yylexer The scanner that will supply tokens to the parser.
*/
- ]b4_lexer_if([[protected]], [[public]]) b4_parser_class[ (]b4_parse_param_decl([[Lexer yylexer]])[) ]b4_maybe_throws([b4_init_throws])[
+ ]b4_lexer_if([[protected]], [[public]]) b4_parser_class[(]b4_parse_param_decl([[Lexer yylexer]])[)]b4_maybe_throws([b4_init_throws])[
{
- ]b4_percent_code_get([[init]])[
+]b4_percent_code_get([[init]])[]b4_lac_if([[
+ this.yylacStack = new ArrayList<Integer>();
+ this.yylacEstablished = false;]])[
this.yylexer = yylexer;
- ]b4_parse_param_cons[
+]b4_parse_param_cons[
}
]b4_parse_trace_if([[
@@ -260,7 +294,7 @@
/**
* The <tt>PrintStream</tt> on which the debugging output is printed.
*/
- public final java.io.PrintStream getDebugStream () { return yyDebugStream; }
+ public final java.io.PrintStream getDebugStream() { return yyDebugStream; }
/**
* Set the <tt>PrintStream</tt> on which the debug output is printed.
@@ -284,14 +318,20 @@
public final void setDebugLevel(int level) { yydebug = level; }
]])[
+ private int yynerrs = 0;
+
+ /**
+ * The number of syntax errors so far.
+ */
+ public final int getNumberOfErrors() { return yynerrs; }
+
/**
* Print an error message via the lexer.
*]b4_locations_if([[ Use a <code>null</code> location.]])[
* @@param msg The error message.
*/
- public final void yyerror (String msg)
- {
- yylexer.yyerror (]b4_locations_if([[(]b4_location_type[)null, ]])[msg);
+ public final void yyerror(String msg) {
+ yylexer.yyerror(]b4_locations_if([[(]b4_location_type[)null, ]])[msg);
}
]b4_locations_if([[
/**
@@ -299,9 +339,8 @@
* @@param loc The location associated with the message.
* @@param msg The error message.
*/
- public final void yyerror (]b4_location_type[ loc, String msg)
- {
- yylexer.yyerror (loc, msg);
+ public final void yyerror(]b4_location_type[ loc, String msg) {
+ yylexer.yyerror(loc, msg);
}
/**
@@ -309,84 +348,85 @@
* @@param pos The position associated with the message.
* @@param msg The error message.
*/
- public final void yyerror (]b4_position_type[ pos, String msg)
- {
- yylexer.yyerror (new ]b4_location_type[ (pos), msg);
+ public final void yyerror(]b4_position_type[ pos, String msg) {
+ yylexer.yyerror(new ]b4_location_type[ (pos), msg);
}]])[
]b4_parse_trace_if([[
- protected final void yycdebug (String s) {
+ protected final void yycdebugNnl(String s) {
if (0 < yydebug)
- yyDebugStream.println (s);
+ yyDebugStream.print(s);
+ }
+
+ protected final void yycdebug(String s) {
+ if (0 < yydebug)
+ yyDebugStream.println(s);
}]])[
private final class YYStack {
- private int[] stateStack = new int[16];
- ]b4_locations_if([[private ]b4_location_type[[] locStack = new ]b4_location_type[[16];]])[
+ private int[] stateStack = new int[16];]b4_locations_if([[
+ private ]b4_location_type[[] locStack = new ]b4_location_type[[16];]])[
private ]b4_yystype[[] valueStack = new ]b4_yystype[[16];
public int size = 16;
public int height = -1;
- public final void push (int state, ]b4_yystype[ value]dnl
- b4_locations_if([, ]b4_location_type[ loc])[) {
+ public final void push(int state, ]b4_yystype[ value]b4_locations_if([, ]b4_location_type[ loc])[) {
height++;
- if (size == height)
- {
- int[] newStateStack = new int[size * 2];
- System.arraycopy (stateStack, 0, newStateStack, 0, height);
- stateStack = newStateStack;
- ]b4_locations_if([[
- ]b4_location_type[[] newLocStack = new ]b4_location_type[[size * 2];
- System.arraycopy (locStack, 0, newLocStack, 0, height);
- locStack = newLocStack;]])
+ if (size == height) {
+ int[] newStateStack = new int[size * 2];
+ System.arraycopy(stateStack, 0, newStateStack, 0, height);
+ stateStack = newStateStack;]b4_locations_if([[
+ ]b4_location_type[[] newLocStack = new ]b4_location_type[[size * 2];
+ System.arraycopy(locStack, 0, newLocStack, 0, height);
+ locStack = newLocStack;]])
- b4_yystype[[] newValueStack = new ]b4_yystype[[size * 2];
- System.arraycopy (valueStack, 0, newValueStack, 0, height);
- valueStack = newValueStack;
+ b4_yystype[[] newValueStack = new ]b4_yystype[[size * 2];
+ System.arraycopy(valueStack, 0, newValueStack, 0, height);
+ valueStack = newValueStack;
- size *= 2;
- }
+ size *= 2;
+ }
- stateStack[height] = state;
- ]b4_locations_if([[locStack[height] = loc;]])[
+ stateStack[height] = state;]b4_locations_if([[
+ locStack[height] = loc;]])[
valueStack[height] = value;
}
- public final void pop () {
- pop (1);
+ public final void pop() {
+ pop(1);
}
- public final void pop (int num) {
+ public final void pop(int num) {
// Avoid memory leaks... garbage collection is a white lie!
if (0 < num) {
- java.util.Arrays.fill (valueStack, height - num + 1, height + 1, null);
- ]b4_locations_if([[java.util.Arrays.fill (locStack, height - num + 1, height + 1, null);]])[
+ java.util.Arrays.fill(valueStack, height - num + 1, height + 1, null);]b4_locations_if([[
+ java.util.Arrays.fill(locStack, height - num + 1, height + 1, null);]])[
}
height -= num;
}
- public final int stateAt (int i) {
+ public final int stateAt(int i) {
return stateStack[height - i];
}
+]b4_locations_if([[
- ]b4_locations_if([[public final ]b4_location_type[ locationAt (int i) {
+ public final ]b4_location_type[ locationAt(int i) {
return locStack[height - i];
}
-
- ]])[public final ]b4_yystype[ valueAt (int i) {
+]])[
+ public final ]b4_yystype[ valueAt(int i) {
return valueStack[height - i];
}
// Print the state stack on the debug stream.
- public void print (java.io.PrintStream out) {
+ public void print(java.io.PrintStream out) {
out.print ("Stack now");
- for (int i = 0; i <= height; i++)
- {
- out.print (' ');
- out.print (stateStack[i]);
- }
- out.println ();
+ for (int i = 0; i <= height; i++) {
+ out.print(' ');
+ out.print(stateStack[i]);
+ }
+ out.println();
}
}
@@ -443,16 +483,15 @@
* @@param yystate the current state
* @@param yysym the nonterminal to push on the stack
*/
- private int yyLRGotoState (int yystate, int yysym)
- {
- int yyr = yypgoto_[yysym - yyntokens_] + yystate;
- if (0 <= yyr && yyr <= yylast_ && yycheck_[yyr] == yystate)
+ private int yyLRGotoState(int yystate, int yysym) {
+ int yyr = yypgoto_[yysym - YYNTOKENS_] + yystate;
+ if (0 <= yyr && yyr <= YYLAST_ && yycheck_[yyr] == yystate)
return yytable_[yyr];
else
- return yydefgoto_[yysym - yyntokens_];
+ return yydefgoto_[yysym - YYNTOKENS_];
}
- private int yyaction (int yyn, YYStack yystack, int yylen) ]b4_maybe_throws([b4_throws])[
+ private int yyaction(int yyn, YYStack yystack, int yylen)]b4_maybe_throws([b4_throws])[
{
/* If YYLEN is nonzero, implement the default value of the action:
'$$ = $1'. Otherwise, use the top of the stack.
@@ -460,10 +499,10 @@
Otherwise, the following line sets YYVAL to garbage.
This behavior is undocumented and Bison
users should not rely upon it. */
- ]b4_yystype[ yyval = (0 < yylen) ? yystack.valueAt (yylen - 1) : yystack.valueAt (0);
- ]b4_locations_if([b4_location_type[ yyloc = yylloc (yystack, yylen);]])[]b4_parse_trace_if([[
+ ]b4_yystype[ yyval = (0 < yylen) ? yystack.valueAt(yylen - 1) : yystack.valueAt(0);]b4_locations_if([[
+ ]b4_location_type[ yyloc = yylloc(yystack, yylen);]])[]b4_parse_trace_if([[
- yyReducePrint (yyn, yystack);]])[
+ yyReducePrint(yyn, yystack);]])[
switch (yyn)
{
@@ -471,66 +510,30 @@
default: break;
}]b4_parse_trace_if([[
- yySymbolPrint ("-> $$ =", yyr1_[yyn], yyval]b4_locations_if([, yyloc])[);]])[
+ yySymbolPrint("-> $$ =", SymbolKind.get(yyr1_[yyn]), yyval]b4_locations_if([, yyloc])[);]])[
- yystack.pop (yylen);
+ yystack.pop(yylen);
yylen = 0;
-
/* Shift the result of the reduction. */
- int yystate = yyLRGotoState (yystack.stateAt (0), yyr1_[yyn]);
- yystack.push (yystate, yyval]b4_locations_if([, yyloc])[);
+ int yystate = yyLRGotoState(yystack.stateAt(0), yyr1_[yyn]);
+ yystack.push(yystate, yyval]b4_locations_if([, yyloc])[);
return YYNEWSTATE;
}
-]b4_error_verbose_if([[
- /* Return YYSTR after stripping away unnecessary quotes and
- backslashes, so that it's suitable for yyerror. The heuristic is
- that double-quoting is unnecessary unless the string contains an
- apostrophe, a comma, or backslash (other than backslash-backslash).
- YYSTR is taken from yytname. */
- private final String yytnamerr_ (String yystr)
- {
- if (yystr.charAt (0) == '"')
- {
- StringBuffer yyr = new StringBuffer ();
- strip_quotes: for (int i = 1; i < yystr.length (); i++)
- switch (yystr.charAt (i))
- {
- case '\'':
- case ',':
- break strip_quotes;
-
- case '\\':
- if (yystr.charAt(++i) != '\\')
- break strip_quotes;
- /* Fall through. */
- default:
- yyr.append (yystr.charAt (i));
- break;
-
- case '"':
- return yyr.toString ();
- }
- }
- else if (yystr.equals ("$end"))
- return "end of input";
-
- return yystr;
- }
-]])[
]b4_parse_trace_if([[
/*--------------------------------.
| Print this symbol on YYOUTPUT. |
`--------------------------------*/
- private void yySymbolPrint (String s, int yytype,
- ]b4_yystype[ yyvaluep]dnl
- b4_locations_if([, Object yylocationp])[)
- {
- yycdebug (s + (yytype < yyntokens_ ? " token " : " nterm ")
- + yytname_[yytype] + " ("]b4_locations_if([
- + yylocationp + ": "])[
- + (yyvaluep == null ? "(null)" : yyvaluep.toString ()) + ")");
+ private void yySymbolPrint(String s, SymbolKind yykind,
+ ]b4_yystype[ yyvalue]b4_locations_if([, ]b4_location_type[ yylocation])[) {
+ if (0 < yydebug) {
+ yycdebug(s
+ + (yykind.getCode() < YYNTOKENS_ ? " token " : " nterm ")
+ + yykind.getName() + " ("]b4_locations_if([
+ + yylocation + ": "])[
+ + (yyvalue == null ? "(null)" : yyvalue.toString()) + ")");
+ }
}]])[
]b4_push_if([],[[
@@ -541,29 +544,34 @@
* @@return <tt>true</tt> if the parsing succeeds. Note that this does not
* imply that there were no syntax errors.
*/
- public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[
+ public boolean parse()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[
]b4_push_if([
/**
* Push Parse input from external lexer
*
* @@param yylextoken current token
- * @@param yylexval current lval
-]b4_locations_if([ * @@param yylexloc current position])[
+ * @@param yylexval current lval]b4_locations_if([[
+ * @@param yylexloc current position]])[
*
* @@return <tt>YYACCEPT, YYABORT, YYPUSH_MORE</tt>
*/
- public int push_parse (int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc]))
- b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[
- {
- ]b4_locations_if([/* @@$. */
- b4_location_type yyloc;])[
+ public int push_parse(int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc]))b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[
+ {]b4_locations_if([[
+ /* @@$. */
+ ]b4_location_type[ yyloc;]])[
]b4_push_if([],[[
-]b4_define_state[]b4_parse_trace_if([[
- yycdebug ("Starting parse\n");]])[
+]b4_define_state[
+]b4_lac_if([[
+ // Discard the LAC context in case there still is one left from a
+ // previous invocation.
+ yylacDiscard("init");]])[
+]b4_parse_trace_if([[
+ yycdebug ("Starting parse");]])[
yyerrstatus_ = 0;
+ yynerrs = 0;
/* Initialize the stack. */
- yystack.push (yystate, yylval ]b4_locations_if([, yylloc])[);
+ yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
]m4_ifdef([b4_initial_action], [
b4_dollar_pushdef([yylval], [], [], [yylloc])dnl
b4_user_initial_action
@@ -579,7 +587,7 @@
b4_user_initial_action
b4_dollar_popdef[]dnl
])[]b4_parse_trace_if([[
- yycdebug ("Starting parse\n");]])[
+ yycdebug ("Starting parse");]])[
yyerrstatus_ = 0;
} else
label = YYGETTOKEN;
@@ -592,12 +600,12 @@
/* New state. Unlike in the C/C++ skeletons, the state is already
pushed when we come here. */
case YYNEWSTATE:]b4_parse_trace_if([[
- yycdebug ("Entering state " + yystate + "\n");
+ yycdebug ("Entering state " + yystate);
if (0 < yydebug)
yystack.print (yyDebugStream);]])[
/* Accept? */
- if (yystate == yyfinal_)
+ if (yystate == YYFINAL_)
]b4_push_if([{label = YYACCEPT; break;}],
[return true;])[
@@ -612,64 +620,84 @@
case YYGETTOKEN:])[
/* Read a lookahead token. */
- if (yychar == yyempty_)
+ if (yychar == YYEMPTY_)
{
]b4_push_if([[
if (!push_token_consumed)
return YYPUSH_MORE;]b4_parse_trace_if([[
- yycdebug ("Reading a token: ");]])[
+ yycdebug ("Reading a token");]])[
yychar = yylextoken;
yylval = yylexval;]b4_locations_if([
yylloc = yylexloc;])[
push_token_consumed = false;]], [b4_parse_trace_if([[
- yycdebug ("Reading a token: ");]])[
+ yycdebug ("Reading a token");]])[
yychar = yylexer.yylex ();
- yylval = yylexer.getLVal ();]b4_locations_if([
- yylloc = new b4_location_type (yylexer.getStartPos (),
- yylexer.getEndPos ());])[
+ yylval = yylexer.getLVal();]b4_locations_if([[
+ yylloc = new ]b4_location_type[(yylexer.getStartPos(),
+ yylexer.getEndPos());]])[
]])[
}
/* Convert token to internal form. */
yytoken = yytranslate_ (yychar);]b4_parse_trace_if([[
- yySymbolPrint ("Next token is", yytoken,
- yylval]b4_locations_if([, yylloc])[);]])[
+ yySymbolPrint("Next token is", yytoken,
+ yylval]b4_locations_if([, yylloc])[);]])[
- /* If the proper action on seeing token YYTOKEN is to reduce or to
- detect an error, take that action. */
- yyn += yytoken;
- if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken)
- label = YYDEFAULT;
-
- /* <= 0 means reduce or error. */
- else if ((yyn = yytable_[yyn]) <= 0)
+ if (yytoken == ]b4_symbol(error, kind)[)
{
- if (yyTableValueIsError (yyn))
- label = YYERRLAB;
- else
- {
- yyn = -yyn;
- label = YYREDUCE;
- }
+ // The scanner already issued an error message, process directly
+ // to error recovery. But do not keep the error token as
+ // lookahead, it is too special and may lead us to an endless
+ // loop in error recovery. */
+ yychar = Lexer.]b4_symbol(undef, id)[;
+ yytoken = ]b4_symbol(undef, kind)[;]b4_locations_if([[
+ yyerrloc = yylloc;]])[
+ label = YYERRLAB1;
}
-
else
{
- /* Shift the lookahead token. */]b4_parse_trace_if([[
- yySymbolPrint ("Shifting", yytoken,
- yylval]b4_locations_if([, yylloc])[);
+ /* If the proper action on seeing token YYTOKEN is to reduce or to
+ detect an error, take that action. */
+ yyn += yytoken.getCode();
+ if (yyn < 0 || YYLAST_ < yyn || yycheck_[yyn] != yytoken.getCode()) {]b4_lac_if([[
+ if (!yylacEstablish(yystack, yytoken)) {
+ label = YYERRLAB;
+ } else]])[
+ label = YYDEFAULT;
+ }
+
+ /* <= 0 means reduce or error. */
+ else if ((yyn = yytable_[yyn]) <= 0)
+ {
+ if (yyTableValueIsError(yyn)) {
+ label = YYERRLAB;
+ }]b4_lac_if([[ else if (!yylacEstablish(yystack, yytoken)) {
+ label = YYERRLAB;
+ }]])[ else {
+ yyn = -yyn;
+ label = YYREDUCE;
+ }
+ }
+
+ else
+ {
+ /* Shift the lookahead token. */]b4_parse_trace_if([[
+ yySymbolPrint("Shifting", yytoken,
+ yylval]b4_locations_if([, yylloc])[);
]])[
- /* Discard the token being shifted. */
- yychar = yyempty_;
+ /* Discard the token being shifted. */
+ yychar = YYEMPTY_;
- /* Count tokens shifted since error; after three, turn off error
- status. */
- if (yyerrstatus_ > 0)
- --yyerrstatus_;
+ /* Count tokens shifted since error; after three, turn off error
+ status. */
+ if (yyerrstatus_ > 0)
+ --yyerrstatus_;
- yystate = yyn;
- yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
- label = YYNEWSTATE;
+ yystate = yyn;
+ yystack.push(yystate, yylval]b4_locations_if([, yylloc])[);]b4_lac_if([[
+ yylacDiscard("shift");]])[
+ label = YYNEWSTATE;
+ }
}
break;
@@ -689,8 +717,8 @@
`-----------------------------*/
case YYREDUCE:
yylen = yyr2_[yyn];
- label = yyaction (yyn, yystack, yylen);
- yystate = yystack.stateAt (0);
+ label = yyaction(yyn, yystack, yylen);
+ yystate = yystack.stateAt(0);
break;
/*------------------------------------.
@@ -700,26 +728,26 @@
/* If not already recovering from an error, report this error. */
if (yyerrstatus_ == 0)
{
- ++yynerrs_;
- if (yychar == yyempty_)
- yytoken = yyempty_;
- yyerror (]b4_locations_if([yylloc, ])[yysyntax_error (yystate, yytoken));
+ ++yynerrs;
+ if (yychar == YYEMPTY_)
+ yytoken = null;
+ yyreportSyntaxError(new Context(this, yystack, yytoken]b4_locations_if([[, yylloc]])[));
}
-
- ]b4_locations_if([yyerrloc = yylloc;])[
+]b4_locations_if([[
+ yyerrloc = yylloc;]])[
if (yyerrstatus_ == 3)
{
/* If just tried and failed to reuse lookahead token after an
error, discard it. */
- if (yychar <= Lexer.EOF)
+ if (yychar <= Lexer.]b4_symbol(eof, id)[)
{
/* Return failure if at end of input. */
- if (yychar == Lexer.EOF)
+ if (yychar == Lexer.]b4_symbol(eof, id)[)
]b4_push_if([{label = YYABORT; break;}], [return false;])[
}
else
- yychar = yyempty_;
+ yychar = YYEMPTY_;
}
/* Else will try to reuse lookahead token after shifting the error
@@ -730,13 +758,13 @@
/*-------------------------------------------------.
| errorlab -- error raised explicitly by YYERROR. |
`-------------------------------------------------*/
- case YYERROR:
- ]b4_locations_if([yyerrloc = yystack.locationAt (yylen - 1);])[
+ case YYERROR:]b4_locations_if([[
+ yyerrloc = yystack.locationAt (yylen - 1);]])[
/* Do not reclaim the symbols of the rule which action triggered
this YYERROR. */
yystack.pop (yylen);
yylen = 0;
- yystate = yystack.stateAt (0);
+ yystate = yystack.stateAt(0);
label = YYERRLAB1;
break;
@@ -746,13 +774,15 @@
case YYERRLAB1:
yyerrstatus_ = 3; /* Each real token shifted decrements this. */
+ // Pop stack until we find a state that shifts the error token.
for (;;)
{
yyn = yypact_[yystate];
if (!yyPactValueIsDefault (yyn))
{
- yyn += yy_error_token_;
- if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == yy_error_token_)
+ yyn += ]b4_symbol(error, kind)[.getCode();
+ if (0 <= yyn && yyn <= YYLAST_
+ && yycheck_[yyn] == ]b4_symbol(error, kind)[.getCode())
{
yyn = yytable_[yyn];
if (0 < yyn)
@@ -765,27 +795,29 @@
if (yystack.height == 0)
]b4_push_if([{label = YYABORT; break;}],[return false;])[
- ]b4_locations_if([yyerrloc = yystack.locationAt (0);])[
+]b4_locations_if([[
+ yyerrloc = yystack.locationAt (0);]])[
yystack.pop ();
- yystate = yystack.stateAt (0);]b4_parse_trace_if([[
+ yystate = yystack.stateAt(0);]b4_parse_trace_if([[
if (0 < yydebug)
yystack.print (yyDebugStream);]])[
}
if (label == YYABORT)
- /* Leave the switch. */
- break;
+ /* Leave the switch. */
+ break;
-]b4_locations_if([
+]b4_locations_if([[
/* Muck with the stack to setup for yylloc. */
yystack.push (0, null, yylloc);
yystack.push (0, null, yyerrloc);
yyloc = yylloc (yystack, 2);
- yystack.pop (2);])[
+ yystack.pop (2);]])[
- /* Shift the error token. */]b4_parse_trace_if([[
- yySymbolPrint ("Shifting", yystos_[yyn],
- yylval]b4_locations_if([, yyloc])[);]])[
+ /* Shift the error token. */]b4_lac_if([[
+ yylacDiscard("error recovery");]])[]b4_parse_trace_if([[
+ yySymbolPrint("Shifting", SymbolKind.get(yystos_[yyn]),
+ yylval]b4_locations_if([, yyloc])[);]])[
yystate = yyn;
yystack.push (yyn, yylval]b4_locations_if([, yyloc])[);
@@ -809,24 +841,26 @@
/**
* (Re-)Initialize the state of the push parser.
*/
- public void push_parse_initialize()
+ public void push_parse_initialize ()
{
/* Lookahead and lookahead in internal form. */
- this.yychar = yyempty_;
- this.yytoken = 0;
+ this.yychar = YYEMPTY_;
+ this.yytoken = null;
/* State. */
this.yyn = 0;
this.yylen = 0;
this.yystate = 0;
- this.yystack = new YYStack ();
+ this.yystack = new YYStack();]b4_lac_if([[
+ this.yylacStack = new ArrayList<Integer>();
+ this.yylacEstablished = false;]])[
this.label = YYNEWSTATE;
/* Error handling. */
- this.yynerrs_ = 0;
- ]b4_locations_if([/* The location where the error started. */
+ this.yynerrs = 0;]b4_locations_if([[
+ /* The location where the error started. */
this.yyerrloc = null;
- this.yylloc = new b4_location_type (null, null);])[
+ this.yylloc = new ]b4_location_type[ (null, null);]])[
/* Semantic value of the lookahead. */
this.yylval = null;
@@ -836,7 +870,7 @@
this.push_parse_initialized = true;
}
-]b4_locations_if([
+]b4_locations_if([[
/**
* Push parse given input from an external lexer.
*
@@ -846,119 +880,354 @@
*
* @@return <tt>YYACCEPT, YYABORT, YYPUSH_MORE</tt>
*/
- public int push_parse (int yylextoken, b4_yystype yylexval, b4_position_type yylexpos)
- b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])
- {
- return push_parse (yylextoken, yylexval, new b4_location_type (yylexpos));
+ public int push_parse(int yylextoken, ]b4_yystype[ yylexval, ]b4_position_type[ yylexpos)]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ {
+ return push_parse(yylextoken, yylexval, new ]b4_location_type[(yylexpos));
}
-])[]])[
+]])])[
]b4_both_if([[
/**
* Parse input from the scanner that was specified at object construction
* time. Return whether the end of the input was reached successfully.
- * This version of parse () is defined only when api.push-push=both.
+ * This version of parse() is defined only when api.push-push=both.
*
* @@return <tt>true</tt> if the parsing succeeds. Note that this does not
* imply that there were no syntax errors.
*/
- public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[
- {
- if (yylexer == null)
- throw new NullPointerException("Null Lexer");
- int status;
- do {
- int token = yylexer.yylex();
- ]b4_yystype[ lval = yylexer.getLVal();
-]b4_locations_if([dnl
- b4_location_type yyloc = new b4_location_type (yylexer.getStartPos (),
- yylexer.getEndPos ());])[
- ]b4_locations_if([status = push_parse(token,lval,yyloc);],[
- status = push_parse(token,lval);])[
- } while (status == YYPUSH_MORE);
- return (status == YYACCEPT);
+ public boolean parse()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ {
+ if (yylexer == null)
+ throw new NullPointerException("Null Lexer");
+ int status;
+ do {
+ int token = yylexer.yylex();
+ ]b4_yystype[ lval = yylexer.getLVal();]b4_locations_if([[
+ ]b4_location_type[ yyloc = new ]b4_location_type[(yylexer.getStartPos(), yylexer.getEndPos());
+ status = push_parse(token, lval, yyloc);]], [[
+ status = push_parse(token, lval);]])[
+ } while (status == YYPUSH_MORE);
+ return status == YYACCEPT;
}
]])[
- // Generate an error message.
- private String yysyntax_error (int yystate, int tok)
- {]b4_error_verbose_if([[
- if (yyErrorVerbose)
- {
- /* There are many possibilities here to consider:
- - If this state is a consistent state with a default action,
- then the only way this function was invoked is if the
- default action is an error action. In that case, don't
- check for expected tokens because there are none.
- - The only way there can be no lookahead present (in tok) is
- if this state is a consistent state with a default action.
- Thus, detecting the absence of a lookahead is sufficient to
- determine that there is no unexpected or expected token to
- report. In that case, just report a simple "syntax error".
- - Don't assume there isn't a lookahead just because this
- state is a consistent state with a default action. There
- might have been a previous inconsistent state, consistent
- state with a non-default action, or user semantic action
- that manipulated yychar. (However, yychar is currently out
- of scope during semantic actions.)
- - Of course, the expected token list depends on states to
- have correct lookahead information, and it depends on the
- parser not to perform extra reductions after fetching a
- lookahead from the scanner and before detecting a syntax
- error. Thus, state merging (from LALR or IELR) and default
- reductions corrupt the expected token list. However, the
- list is correct for canonical LR with one exception: it
- will still contain any token that will not be accepted due
- to an error action in a later state.
- */
- if (tok != yyempty_)
- {
- /* FIXME: This method of building the message is not compatible
- with internationalization. */
- StringBuffer res =
- new StringBuffer ("syntax error, unexpected ");
- res.append (yytnamerr_ (yytname_[tok]));
- int yyn = yypact_[yystate];
- if (!yyPactValueIsDefault (yyn))
- {
- /* Start YYX at -YYN if negative to avoid negative
- indexes in YYCHECK. In other words, skip the first
- -YYN actions for this state because they are default
- actions. */
- int yyxbegin = yyn < 0 ? -yyn : 0;
- /* Stay within bounds of both yycheck and yytname. */
- int yychecklim = yylast_ - yyn + 1;
- int yyxend = yychecklim < yyntokens_ ? yychecklim : yyntokens_;
- int count = 0;
- for (int x = yyxbegin; x < yyxend; ++x)
- if (yycheck_[x + yyn] == x && x != yy_error_token_
- && !yyTableValueIsError (yytable_[x + yyn]))
- ++count;
- if (count < 5)
- {
- count = 0;
- for (int x = yyxbegin; x < yyxend; ++x)
- if (yycheck_[x + yyn] == x && x != yy_error_token_
- && !yyTableValueIsError (yytable_[x + yyn]))
- {
- res.append (count++ == 0 ? ", expecting " : " or ");
- res.append (yytnamerr_ (yytname_[x]));
- }
- }
- }
- return res.toString ();
- }
- }
+ /**
+ * Information needed to get the list of expected tokens and to forge
+ * a syntax error diagnostic.
+ */
+ public static final class Context {
+ Context(]b4_parser_class[ parser, YYStack stack, SymbolKind token]b4_locations_if([[, ]b4_location_type[ loc]])[) {
+ yyparser = parser;
+ yystack = stack;
+ yytoken = token;]b4_locations_if([[
+ yylocation = loc;]])[
+ }
+
+ private ]b4_parser_class[ yyparser;
+ private YYStack yystack;
+
+
+ /**
+ * The symbol kind of the lookahead token.
+ */
+ public final SymbolKind getToken() {
+ return yytoken;
+ }
+
+ private SymbolKind yytoken;]b4_locations_if([[
+
+ /**
+ * The location of the lookahead.
+ */
+ public final ]b4_location_type[ getLocation() {
+ return yylocation;
+ }
+
+ private ]b4_location_type[ yylocation;]])[
+ static final int NTOKENS = ]b4_parser_class[.YYNTOKENS_;
+
+ /**
+ * Put in YYARG at most YYARGN of the expected tokens given the
+ * current YYCTX, and return the number of tokens stored in YYARG. If
+ * YYARG is null, return the number of expected tokens (guaranteed to
+ * be less than YYNTOKENS).
+ */
+ int getExpectedTokens(SymbolKind yyarg[], int yyargn) {
+ return getExpectedTokens (yyarg, 0, yyargn);
+ }
+
+ int getExpectedTokens(SymbolKind yyarg[], int yyoffset, int yyargn) {
+ int yycount = yyoffset;]b4_lac_if([b4_parse_trace_if([[
+ // Execute LAC once. We don't care if it is successful, we
+ // only do it for the sake of debugging output.
+ if (!yyparser.yylacEstablished)
+ yyparser.yylacCheck(yystack, yytoken);
]])[
- return "syntax error";
+ for (int yyx = 0; yyx < YYNTOKENS_; ++yyx)
+ {
+ SymbolKind yysym = SymbolKind.get(yyx);
+ if (yysym != ]b4_symbol(error, kind)[
+ && yysym != ]b4_symbol(undef, kind)[
+ && yyparser.yylacCheck(yystack, yysym))
+ {
+ if (yyarg == null)
+ yycount += 1;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = yysym;
+ }
+ }]], [[
+ int yyn = yypact_[this.yystack.stateAt(0)];
+ if (!yyPactValueIsDefault(yyn))
+ {
+ /* Start YYX at -YYN if negative to avoid negative
+ indexes in YYCHECK. In other words, skip the first
+ -YYN actions for this state because they are default
+ actions. */
+ int yyxbegin = yyn < 0 ? -yyn : 0;
+ /* Stay within bounds of both yycheck and yytname. */
+ int yychecklim = YYLAST_ - yyn + 1;
+ int yyxend = yychecklim < NTOKENS ? yychecklim : NTOKENS;
+ for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
+ if (yycheck_[yyx + yyn] == yyx && yyx != ]b4_symbol(error, kind)[.getCode()
+ && !yyTableValueIsError(yytable_[yyx + yyn]))
+ {
+ if (yyarg == null)
+ yycount += 1;
+ else if (yycount == yyargn)
+ return 0; // FIXME: this is incorrect.
+ else
+ yyarg[yycount++] = SymbolKind.get(yyx);
+ }
+ }]])[
+ if (yyarg != null && yycount == yyoffset && yyoffset < yyargn)
+ yyarg[yycount] = null;
+ return yycount - yyoffset;
+ }
+ }
+
+]b4_lac_if([[
+ /** Check the lookahead yytoken.
+ * \returns true iff the token will be eventually shifted.
+ */
+ boolean yylacCheck(YYStack yystack, SymbolKind yytoken)
+ {
+ // Logically, the yylacStack's lifetime is confined to this function.
+ // Clear it, to get rid of potential left-overs from previous call.
+ yylacStack.clear();
+ // Reduce until we encounter a shift and thereby accept the token.
+ yycdebugNnl("LAC: checking lookahead " + yytoken.getName() + ":");
+ int lacTop = 0;
+ while (true)
+ {
+ int topState = (yylacStack.isEmpty()
+ ? yystack.stateAt(lacTop)
+ : yylacStack.get(yylacStack.size() - 1));
+ int yyrule = yypact_[topState];
+ if (yyPactValueIsDefault(yyrule)
+ || (yyrule += yytoken.getCode()) < 0 || YYLAST_ < yyrule
+ || yycheck_[yyrule] != yytoken.getCode())
+ {
+ // Use the default action.
+ yyrule = yydefact_[+topState];
+ if (yyrule == 0) {
+ yycdebug(" Err");
+ return false;
+ }
+ }
+ else
+ {
+ // Use the action from yytable.
+ yyrule = yytable_[yyrule];
+ if (yyTableValueIsError(yyrule)) {
+ yycdebug(" Err");
+ return false;
+ }
+ if (0 < yyrule) {
+ yycdebug(" S" + yyrule);
+ return true;
+ }
+ yyrule = -yyrule;
+ }
+ // By now we know we have to simulate a reduce.
+ yycdebugNnl(" R" + (yyrule - 1));
+ // Pop the corresponding number of values from the stack.
+ {
+ int yylen = yyr2_[yyrule];
+ // First pop from the LAC stack as many tokens as possible.
+ int lacSize = yylacStack.size();
+ if (yylen < lacSize) {
+ // yylacStack.setSize(lacSize - yylen);
+ for (/* Nothing */; 0 < yylen; yylen -= 1) {
+ yylacStack.remove(yylacStack.size() - 1);
+ }
+ yylen = 0;
+ } else if (lacSize != 0) {
+ yylacStack.clear();
+ yylen -= lacSize;
+ }
+ // Only afterwards look at the main stack.
+ // We simulate popping elements by incrementing lacTop.
+ lacTop += yylen;
+ }
+ // Keep topState in sync with the updated stack.
+ topState = (yylacStack.isEmpty()
+ ? yystack.stateAt(lacTop)
+ : yylacStack.get(yylacStack.size() - 1));
+ // Push the resulting state of the reduction.
+ int state = yyLRGotoState(topState, yyr1_[yyrule]);
+ yycdebugNnl(" G" + state);
+ yylacStack.add(state);
+ }
+ }
+
+ /** Establish the initial context if no initial context currently exists.
+ * \returns true iff the token will be eventually shifted.
+ */
+ boolean yylacEstablish(YYStack yystack, SymbolKind yytoken) {
+ /* Establish the initial context for the current lookahead if no initial
+ context is currently established.
+
+ We define a context as a snapshot of the parser stacks. We define
+ the initial context for a lookahead as the context in which the
+ parser initially examines that lookahead in order to select a
+ syntactic action. Thus, if the lookahead eventually proves
+ syntactically unacceptable (possibly in a later context reached via a
+ series of reductions), the initial context can be used to determine
+ the exact set of tokens that would be syntactically acceptable in the
+ lookahead's place. Moreover, it is the context after which any
+ further semantic actions would be erroneous because they would be
+ determined by a syntactically unacceptable token.
+
+ yylacEstablish should be invoked when a reduction is about to be
+ performed in an inconsistent state (which, for the purposes of LAC,
+ includes consistent states that don't know they're consistent because
+ their default reductions have been disabled).
+
+ For parse.lac=full, the implementation of yylacEstablish is as
+ follows. If no initial context is currently established for the
+ current lookahead, then check if that lookahead can eventually be
+ shifted if syntactic actions continue from the current context. */
+ if (yylacEstablished) {
+ return true;
+ } else {
+ yycdebug("LAC: initial context established for " + yytoken.getName());
+ yylacEstablished = true;
+ return yylacCheck(yystack, yytoken);
+ }
+ }
+
+ /** Discard any previous initial lookahead context because of event.
+ * \param event the event which caused the lookahead to be discarded.
+ * Only used for debbuging output. */
+ void yylacDiscard(String event) {
+ /* Discard any previous initial lookahead context because of Event,
+ which may be a lookahead change or an invalidation of the currently
+ established initial context for the current lookahead.
+
+ The most common example of a lookahead change is a shift. An example
+ of both cases is syntax error recovery. That is, a syntax error
+ occurs when the lookahead is syntactically erroneous for the
+ currently established initial context, so error recovery manipulates
+ the parser stacks to try to find a new initial context in which the
+ current lookahead is syntactically acceptable. If it fails to find
+ such a context, it discards the lookahead. */
+ if (yylacEstablished) {
+ yycdebug("LAC: initial context discarded due to " + event);
+ yylacEstablished = false;
+ }
+ }
+
+ /** The stack for LAC.
+ * Logically, the yylacStack's lifetime is confined to the function
+ * yylacCheck. We just store it as a member of this class to hold
+ * on to the memory and to avoid frequent reallocations.
+ */
+ ArrayList<Integer> yylacStack;
+ /** Whether an initial LAC context was established. */
+ boolean yylacEstablished;
+]])[
+
+]b4_parse_error_bmatch(
+[detailed\|verbose], [[
+ private int yysyntaxErrorArguments(Context yyctx, SymbolKind[] yyarg, int yyargn) {
+ /* There are many possibilities here to consider:
+ - If this state is a consistent state with a default action,
+ then the only way this function was invoked is if the
+ default action is an error action. In that case, don't
+ check for expected tokens because there are none.
+ - The only way there can be no lookahead present (in tok) is
+ if this state is a consistent state with a default action.
+ Thus, detecting the absence of a lookahead is sufficient to
+ determine that there is no unexpected or expected token to
+ report. In that case, just report a simple "syntax error".
+ - Don't assume there isn't a lookahead just because this
+ state is a consistent state with a default action. There
+ might have been a previous inconsistent state, consistent
+ state with a non-default action, or user semantic action
+ that manipulated yychar. (However, yychar is currently out
+ of scope during semantic actions.)
+ - Of course, the expected token list depends on states to
+ have correct lookahead information, and it depends on the
+ parser not to perform extra reductions after fetching a
+ lookahead from the scanner and before detecting a syntax
+ error. Thus, state merging (from LALR or IELR) and default
+ reductions corrupt the expected token list. However, the
+ list is correct for canonical LR with one exception: it
+ will still contain any token that will not be accepted due
+ to an error action in a later state.
+ */
+ int yycount = 0;
+ if (yyctx.getToken() != null)
+ {
+ if (yyarg != null)
+ yyarg[yycount] = yyctx.getToken();
+ yycount += 1;
+ yycount += yyctx.getExpectedTokens(yyarg, 1, yyargn);
+ }
+ return yycount;
+ }
+]])[
+
+ /**
+ * Build and emit a "syntax error" message in a user-defined way.
+ *
+ * @@param ctx The context of the error.
+ */
+ private void yyreportSyntaxError(Context yyctx) {]b4_parse_error_bmatch(
+[custom], [[
+ yylexer.reportSyntaxError(yyctx);]],
+[detailed\|verbose], [[
+ if (yyErrorVerbose) {
+ final int argmax = 5;
+ SymbolKind[] yyarg = new SymbolKind[argmax];
+ int yycount = yysyntaxErrorArguments(yyctx, yyarg, argmax);
+ String[] yystr = new String[yycount];
+ for (int yyi = 0; yyi < yycount; ++yyi) {
+ yystr[yyi] = yyarg[yyi].getName();
+ }
+ String yyformat;
+ switch (yycount) {
+ default:
+ case 0: yyformat = ]b4_trans(["syntax error"])[; break;
+ case 1: yyformat = ]b4_trans(["syntax error, unexpected {0}"])[; break;
+ case 2: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1}"])[; break;
+ case 3: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1} or {2}"])[; break;
+ case 4: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1} or {2} or {3}"])[; break;
+ case 5: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1} or {2} or {3} or {4}"])[; break;
+ }
+ yyerror(]b4_locations_if([[yyctx.yylocation, ]])[new MessageFormat(yyformat).format(yystr));
+ } else {
+ yyerror(]b4_locations_if([[yyctx.yylocation, ]])[]b4_trans(["syntax error"])[);
+ }]],
+[simple], [[
+ yyerror(]b4_locations_if([[yyctx.yylocation, ]])[]b4_trans(["syntax error"])[);]])[
}
/**
* Whether the given <code>yypact_</code> value indicates a defaulted state.
* @@param yyvalue the value to check
*/
- private static boolean yyPactValueIsDefault (int yyvalue)
- {
+ private static boolean yyPactValueIsDefault(int yyvalue) {
return yyvalue == yypact_ninf_;
}
@@ -967,19 +1236,14 @@
* value indicates a syntax error.
* @@param yyvalue the value to check
*/
- private static boolean yyTableValueIsError (int yyvalue)
- {
+ private static boolean yyTableValueIsError(int yyvalue) {
return yyvalue == yytable_ninf_;
}
private static final ]b4_int_type_for([b4_pact])[ yypact_ninf_ = ]b4_pact_ninf[;
private static final ]b4_int_type_for([b4_table])[ yytable_ninf_ = ]b4_table_ninf[;
- ]b4_parser_tables_define[
-
- /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
- First, the terminals, then, starting at \a yyntokens_, nonterminals. */
- ]b4_typed_parser_table_define([String], [tname], [b4_tname])[
+]b4_parser_tables_define[
]b4_parse_trace_if([[
]b4_integral_parser_table_define([rline], [b4_rline],
@@ -996,50 +1260,44 @@
int yynrhs = yyr2_[yyrule];
/* Print the symbols being reduced, and their result. */
yycdebug ("Reducing stack by rule " + (yyrule - 1)
- + " (line " + yylno + "), ");
+ + " (line " + yylno + "):");
/* The symbols being reduced. */
for (int yyi = 0; yyi < yynrhs; yyi++)
- yySymbolPrint (" $" + (yyi + 1) + " =",
- yystos_[yystack.stateAt(yynrhs - (yyi + 1))],
- ]b4_rhs_data(yynrhs, yyi + 1)b4_locations_if([,
- b4_rhs_location(yynrhs, yyi + 1)])[);
+ yySymbolPrint(" $" + (yyi + 1) + " =",
+ SymbolKind.get(yystos_[yystack.stateAt(yynrhs - (yyi + 1))]),
+ ]b4_rhs_data(yynrhs, yyi + 1)b4_locations_if([,
+ b4_rhs_location(yynrhs, yyi + 1)])[);
}]])[
/* YYTRANSLATE_(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
as returned by yylex, with out-of-bounds checking. */
- private static final ]b4_int_type_for([b4_translate])[ yytranslate_ (int t)
+ private static final SymbolKind yytranslate_(int t)
]b4_api_token_raw_if(dnl
[[ {
- return t;
+ return SymbolKind.get(t);
}
]],
[[ {
- int user_token_number_max_ = ]b4_user_token_number_max[;
- ]b4_int_type_for([b4_translate])[ undef_token_ = ]b4_undef_token_number[;
-
+ // Last valid token kind.
+ int code_max = ]b4_code_max[;
if (t <= 0)
- return Lexer.EOF;
- else if (t <= user_token_number_max_)
- return yytranslate_table_[t];
+ return ]b4_symbol(eof, kind)[;
+ else if (t <= code_max)
+ return SymbolKind.get(yytranslate_table_[t]);
else
- return undef_token_;
+ return ]b4_symbol(undef, kind)[;
}
]b4_integral_parser_table_define([translate_table], [b4_translate])[
]])[
- private static final ]b4_int_type_for([b4_translate])[ yy_error_token_ = 1;
+ private static final int YYLAST_ = ]b4_last[;
+ private static final int YYEMPTY_ = -2;
+ private static final int YYFINAL_ = ]b4_final_state_number[;
+ private static final int YYNTOKENS_ = ]b4_tokens_number[;
- private static final int yylast_ = ]b4_last[;
- private static final int yynnts_ = ]b4_nterms_number[;
- private static final int yyempty_ = -2;
- private static final int yyfinal_ = ]b4_final_state_number[;
- private static final int yyntokens_ = ]b4_tokens_number[;
-
-/* User implementation code. */
-]b4_percent_code_get[]dnl
-
+]b4_percent_code_get[
}
-
+]b4_percent_code_get([[epilogue]])[]dnl
b4_epilogue[]dnl
b4_output_end
diff --git a/common/bison/skeletons/location.cc b/common/bison/skeletons/location.cc
index 0258379..3870b2b 100644
--- a/common/bison/skeletons/location.cc
+++ b/common/bison/skeletons/location.cc
@@ -1,6 +1,6 @@
# C++ skeleton for Bison
-# Copyright (C) 2002-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,17 +13,10 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
m4_pushdef([b4_copyright_years],
- [2002-2015, 2018-2019])
-
-
-# b4_position_file
-# ----------------
-# Name of the file containing the position class, if we want this file.
-b4_defines_if([b4_required_version_if([302], [],
- [m4_define([b4_position_file], [position.hh])])])])
+ [2002-2015, 2018-2021])
# b4_location_file
@@ -32,7 +25,7 @@
# if we want this file.
b4_percent_define_check_file([b4_location_file],
[[api.location.file]],
- b4_defines_if([[location.hh]]))
+ b4_header_if([[location.hh]]))
# b4_location_include
# -------------------
@@ -48,12 +41,22 @@
["b4_location_file"])])
m4_define([b4_location_path],
b4_percent_define_get([[api.location.include]],
- ["b4_dir_prefix[]b4_location_file"]))
+ ["b4_mapped_dir_prefix[]b4_location_file"]))
m4_define([b4_location_path],
m4_substr(m4_defn([b4_location_path]), 1, m4_eval(m4_len(m4_defn([b4_location_path])) - 2)))
])
+# b4_position_file
+# ----------------
+# Name of the file containing the position class, if we want this file.
+b4_header_if(
+ [b4_required_version_if(
+ [30200], [],
+ [m4_ifdef([b4_location_file],
+ [m4_define([b4_position_file], [position.hh])])])])
+
+
# b4_location_define
# ------------------
@@ -63,11 +66,13 @@
class position
{
public:
+ /// Type for file name.
+ typedef ]b4_percent_define_get([[api.filename.type]])[ filename_type;
/// Type for line and column numbers.
typedef int counter_type;
]m4_ifdef([b4_location_constructors], [[
/// Construct a position.
- explicit position (]b4_percent_define_get([[filename_type]])[* f = YY_NULLPTR,
+ explicit position (filename_type* f = YY_NULLPTR,
counter_type l = ]b4_location_initial_line[,
counter_type c = ]b4_location_initial_column[)
: filename (f)
@@ -77,7 +82,7 @@
]])[
/// Initialization.
- void initialize (]b4_percent_define_get([[filename_type]])[* fn = YY_NULLPTR,
+ void initialize (filename_type* fn = YY_NULLPTR,
counter_type l = ]b4_location_initial_line[,
counter_type c = ]b4_location_initial_column[)
{
@@ -106,7 +111,7 @@
/** \} */
/// File name to which this position refers.
- ]b4_percent_define_get([[filename_type]])[* filename;
+ filename_type* filename;
/// Current line number.
counter_type line;
/// Current column number.
@@ -184,6 +189,8 @@
class location
{
public:
+ /// Type for file name.
+ typedef position::filename_type filename_type;
/// Type for line and column numbers.
typedef position::counter_type counter_type;
]m4_ifdef([b4_location_constructors], [
@@ -200,7 +207,7 @@
{}
/// Construct a 0-width location in \a f, \a l, \a c.
- explicit location (]b4_percent_define_get([[filename_type]])[* f,
+ explicit location (filename_type* f,
counter_type l = ]b4_location_initial_line[,
counter_type c = ]b4_location_initial_column[)
: begin (f, l, c)
@@ -209,7 +216,7 @@
])[
/// Initialization.
- void initialize (]b4_percent_define_get([[filename_type]])[* f = YY_NULLPTR,
+ void initialize (filename_type* f = YY_NULLPTR,
counter_type l = ]b4_location_initial_line[,
counter_type c = ]b4_location_initial_column[)
{
diff --git a/common/bison/skeletons/stack.hh b/common/bison/skeletons/stack.hh
index cb52bcc..9891325 100644
--- a/common/bison/skeletons/stack.hh
+++ b/common/bison/skeletons/stack.hh
@@ -1,6 +1,6 @@
# C++ skeleton for Bison
-# Copyright (C) 2002-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,14 +13,14 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
# b4_stack_file
# -------------
# Name of the file containing the stack class, if we want this file.
-b4_defines_if([b4_required_version_if([302], [],
- [m4_define([b4_stack_file], [stack.hh])])])
+b4_header_if([b4_required_version_if([30200], [],
+ [m4_define([b4_stack_file], [stack.hh])])])
# b4_stack_define
@@ -32,15 +32,22 @@
{
public:
// Hide our reversed order.
- typedef typename S::reverse_iterator iterator;
- typedef typename S::const_reverse_iterator const_iterator;
+ typedef typename S::iterator iterator;
+ typedef typename S::const_iterator const_iterator;
typedef typename S::size_type size_type;
typedef typename std::ptrdiff_t index_type;
- stack (size_type n = 200)
+ stack (size_type n = 200) YY_NOEXCEPT
: seq_ (n)
{}
+#if 201103L <= YY_CPLUSPLUS
+ /// Non copyable.
+ stack (const stack&) = delete;
+ /// Non copyable.
+ stack& operator= (const stack&) = delete;
+#endif
+
/// Random access.
///
/// Index 0 returns the topmost element.
@@ -91,31 +98,25 @@
return index_type (seq_.size ());
}
- std::ptrdiff_t
- ssize () const YY_NOEXCEPT
- {
- return std::ptrdiff_t (size ());
- }
-
/// Iterator on top of the stack (going downwards).
const_iterator
begin () const YY_NOEXCEPT
{
- return seq_.rbegin ();
+ return seq_.begin ();
}
/// Bottom of the stack.
const_iterator
end () const YY_NOEXCEPT
{
- return seq_.rend ();
+ return seq_.end ();
}
/// Present a slice of the top of a stack.
class slice
{
public:
- slice (const stack& stack, index_type range)
+ slice (const stack& stack, index_type range) YY_NOEXCEPT
: stack_ (stack)
, range_ (range)
{}
@@ -132,8 +133,12 @@
};
private:
+#if YY_CPLUSPLUS < 201103L
+ /// Non copyable.
stack (const stack&);
+ /// Non copyable.
stack& operator= (const stack&);
+#endif
/// The wrapped container.
S seq_;
};
diff --git a/common/bison/skeletons/traceon.m4 b/common/bison/skeletons/traceon.m4
new file mode 100644
index 0000000..344d7d1
--- /dev/null
+++ b/common/bison/skeletons/traceon.m4
@@ -0,0 +1,2 @@
+dnl GNU M4 treats -dV in a position-independent manner.
+m4_debugmode(V)m4_traceon()dnl
diff --git a/common/bison/skeletons/variant.hh b/common/bison/skeletons/variant.hh
index be4a3ce..2a490e8 100644
--- a/common/bison/skeletons/variant.hh
+++ b/common/bison/skeletons/variant.hh
@@ -1,6 +1,6 @@
# C++ skeleton for Bison
-# Copyright (C) 2002-2015, 2018-2019 Free Software Foundation, Inc.
+# Copyright (C) 2002-2015, 2018-2021 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,13 +13,20 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
## --------- ##
## variant. ##
## --------- ##
+# b4_assert
+# ---------
+# The name of YY_ASSERT.
+m4_define([b4_assert],
+ [b4_api_PREFIX[]_ASSERT])
+
+
# b4_symbol_variant(YYTYPE, YYVAL, ACTION, [ARGS])
# ------------------------------------------------
# Run some ACTION ("build", or "destroy") on YYVAL of symbol type
@@ -71,12 +78,12 @@
# -------------------
# The needed includes for variants support.
m4_define([b4_variant_includes],
-[b4_parse_assert_if([[#include <typeinfo>]])[
-#ifndef YY_ASSERT
+[b4_parse_assert_if([[#include <typeinfo>
+#ifndef ]b4_assert[
# include <cassert>
-# define YY_ASSERT assert
+# define ]b4_assert[ assert
#endif
-]])
+]])])
@@ -87,38 +94,45 @@
# b4_value_type_declare
# ---------------------
-# Define semantic_type.
+# Define value_type.
m4_define([b4_value_type_declare],
[[ /// A buffer to store and retrieve objects.
///
/// Sort of a variant, but does not keep track of the nature
/// of the stored data, since that knowledge is available
/// via the current parser state.
- class semantic_type
+ class value_type
{
public:
/// Type of *this.
- typedef semantic_type self_type;
+ typedef value_type self_type;
/// Empty construction.
- semantic_type () YY_NOEXCEPT
- : yybuffer_ ()]b4_parse_assert_if([
+ value_type () YY_NOEXCEPT
+ : yyraw_ ()]b4_parse_assert_if([
, yytypeid_ (YY_NULLPTR)])[
{}
/// Construct and fill.
template <typename T>
- semantic_type (YY_RVREF (T) t)]b4_parse_assert_if([
+ value_type (YY_RVREF (T) t)]b4_parse_assert_if([
: yytypeid_ (&typeid (T))])[
- {
- YY_ASSERT (sizeof (T) <= size);
+ {]b4_parse_assert_if([[
+ ]b4_assert[ (sizeof (T) <= size);]])[
new (yyas_<T> ()) T (YY_MOVE (t));
}
+#if 201103L <= YY_CPLUSPLUS
+ /// Non copyable.
+ value_type (const self_type&) = delete;
+ /// Non copyable.
+ self_type& operator= (const self_type&) = delete;
+#endif
+
/// Destruction, allowed only if empty.
- ~semantic_type () YY_NOEXCEPT
+ ~value_type () YY_NOEXCEPT
{]b4_parse_assert_if([
- YY_ASSERT (!yytypeid_);
+ ]b4_assert[ (!yytypeid_);
])[}
# if 201103L <= YY_CPLUSPLUS
@@ -126,10 +140,10 @@
template <typename T, typename... U>
T&
emplace (U&&... u)
- {]b4_parse_assert_if([
- YY_ASSERT (!yytypeid_);
- YY_ASSERT (sizeof (T) <= size);
- yytypeid_ = & typeid (T);])[
+ {]b4_parse_assert_if([[
+ ]b4_assert[ (!yytypeid_);
+ ]b4_assert[ (sizeof (T) <= size);
+ yytypeid_ = & typeid (T);]])[
return *new (yyas_<T> ()) T (std::forward <U>(u)...);
}
# else
@@ -137,10 +151,10 @@
template <typename T>
T&
emplace ()
- {]b4_parse_assert_if([
- YY_ASSERT (!yytypeid_);
- YY_ASSERT (sizeof (T) <= size);
- yytypeid_ = & typeid (T);])[
+ {]b4_parse_assert_if([[
+ ]b4_assert[ (!yytypeid_);
+ ]b4_assert[ (sizeof (T) <= size);
+ yytypeid_ = & typeid (T);]])[
return *new (yyas_<T> ()) T ();
}
@@ -148,10 +162,10 @@
template <typename T>
T&
emplace (const T& t)
- {]b4_parse_assert_if([
- YY_ASSERT (!yytypeid_);
- YY_ASSERT (sizeof (T) <= size);
- yytypeid_ = & typeid (T);])[
+ {]b4_parse_assert_if([[
+ ]b4_assert[ (!yytypeid_);
+ ]b4_assert[ (sizeof (T) <= size);
+ yytypeid_ = & typeid (T);]])[
return *new (yyas_<T> ()) T (t);
}
# endif
@@ -178,10 +192,10 @@
template <typename T>
T&
as () YY_NOEXCEPT
- {]b4_parse_assert_if([
- YY_ASSERT (yytypeid_);
- YY_ASSERT (*yytypeid_ == typeid (T));
- YY_ASSERT (sizeof (T) <= size);])[
+ {]b4_parse_assert_if([[
+ ]b4_assert[ (yytypeid_);
+ ]b4_assert[ (*yytypeid_ == typeid (T));
+ ]b4_assert[ (sizeof (T) <= size);]])[
return *yyas_<T> ();
}
@@ -189,10 +203,10 @@
template <typename T>
const T&
as () const YY_NOEXCEPT
- {]b4_parse_assert_if([
- YY_ASSERT (yytypeid_);
- YY_ASSERT (*yytypeid_ == typeid (T));
- YY_ASSERT (sizeof (T) <= size);])[
+ {]b4_parse_assert_if([[
+ ]b4_assert[ (yytypeid_);
+ ]b4_assert[ (*yytypeid_ == typeid (T));
+ ]b4_assert[ (sizeof (T) <= size);]])[
return *yyas_<T> ();
}
@@ -207,9 +221,9 @@
template <typename T>
void
swap (self_type& that) YY_NOEXCEPT
- {]b4_parse_assert_if([
- YY_ASSERT (yytypeid_);
- YY_ASSERT (*yytypeid_ == *that.yytypeid_);])[
+ {]b4_parse_assert_if([[
+ ]b4_assert[ (yytypeid_);
+ ]b4_assert[ (*yytypeid_ == *that.yytypeid_);]])[
std::swap (as<T> (), that.as<T> ());
}
@@ -258,16 +272,19 @@
}
private:
- /// Prohibit blind copies.
+#if YY_CPLUSPLUS < 201103L
+ /// Non copyable.
+ value_type (const self_type&);
+ /// Non copyable.
self_type& operator= (const self_type&);
- semantic_type (const self_type&);
+#endif
/// Accessor to raw memory as \a T.
template <typename T>
T*
yyas_ () YY_NOEXCEPT
{
- void *yyp = yybuffer_.yyraw;
+ void *yyp = yyraw_;
return static_cast<T*> (yyp);
}
@@ -276,7 +293,7 @@
const T*
yyas_ () const YY_NOEXCEPT
{
- const void *yyp = yybuffer_.yyraw;
+ const void *yyp = yyraw_;
return static_cast<const T*> (yyp);
}
@@ -291,10 +308,10 @@
union
{
/// Strongest alignment constraints.
- long double yyalign_me;
+ long double yyalign_me_;
/// A buffer large enough to store any of the semantic values.
- char yyraw[size];
- } yybuffer_;]b4_parse_assert_if([
+ char yyraw_[size];
+ };]b4_parse_assert_if([
/// Whether the content is built: if defined, the name of the stored type.
const std::type_info *yytypeid_;])[
@@ -378,18 +395,74 @@
])])
-m4_define([_b4_type_clause],
-[b4_symbol_if([$1], [is_token],
- [b4_symbol_if([$1], [has_id],
- [tok == token::b4_symbol([$1], [id])],
- [tok == b4_symbol([$1], [user_number])])])])
+# b4_token_kind(SYMBOL-NUM)
+# -------------------------
+# Some tokens don't have an ID.
+m4_define([b4_token_kind],
+[b4_symbol_if([$1], [has_id],
+ [token::b4_symbol([$1], [id])],
+ [b4_symbol([$1], [code])])])
-# _b4_token_constructor_define(SYMBOL-NUM...)
-# -------------------------------------------
-# Define a unique make_symbol for all the SYMBOL-NUM (they
+# _b4_tok_in(SYMBOL-NUM, ...)
+# ---------------------------
+# See b4_tok_in below. The SYMBOL-NUMs... are tokens only.
+#
+# We iterate over the tokens to group them by "range" of token numbers (not
+# symbols numbers!).
+#
+# b4_fst is the start of that range.
+# b4_prev is the previous value.
+# b4_val is the current value.
+# If b4_val is the successor of b4_prev in token numbers, update the latter,
+# otherwise emit the code for range b4_fst .. b4_prev.
+# $1 is also used as a terminator in the foreach, but it will not be printed.
+#
+m4_define([_b4_tok_in],
+[m4_pushdef([b4_prev], [$1])dnl
+m4_pushdef([b4_fst], [$1])dnl
+m4_pushdef([b4_sep], [])dnl
+m4_foreach([b4_val], m4_dquote(m4_shift($@, $1)),
+ [m4_if(b4_symbol(b4_val, [code]), m4_eval(b4_symbol(b4_prev, [code]) + 1), [],
+ [b4_sep[]m4_if(b4_fst, b4_prev,
+ [tok == b4_token_kind(b4_fst)],
+ [(b4_token_kind(b4_fst) <= tok && tok <= b4_token_kind(b4_prev))])[]dnl
+m4_define([b4_fst], b4_val)dnl
+m4_define([b4_sep], [
+ || ])])dnl
+m4_define([b4_prev], b4_val)])dnl
+m4_popdef([b4_sep])dnl
+m4_popdef([b4_fst])dnl
+m4_popdef([b4_prev])dnl
+])
+
+
+# _b4_filter_tokens(SYMBOL-NUM, ...)
+# ----------------------------------
+# Expand as the list of tokens amongst SYMBOL-NUM.
+m4_define([_b4_filter_tokens],
+[m4_pushdef([b4_sep])dnl
+m4_foreach([b4_val], [$@],
+ [b4_symbol_if(b4_val, [is_token], [b4_sep[]b4_val[]m4_define([b4_sep], [,])])])dnl
+m4_popdef([b4_sep])dnl
+])
+
+
+# b4_tok_in(SYMBOL-NUM, ...)
+# ---------------------------
+# A C++ conditional that checks that `tok` is a member of this list of symbol
+# numbers.
+m4_define([b4_tok_in],
+ [_$0(_b4_filter_tokens($@))])
+
+
+
+
+# _b4_symbol_constructor_define(SYMBOL-NUM...)
+# --------------------------------------------
+# Define a symbol_type constructor common to all the SYMBOL-NUM (they
# have the same type). Use at class-level.
-m4_define([_b4_token_constructor_define],
+m4_define([_b4_symbol_constructor_define],
[m4_ifval(_b4_includes_tokens($@),
[[#if 201103L <= YY_CPLUSPLUS
symbol_type (]b4_join(
@@ -397,25 +470,24 @@
b4_symbol_if([$1], [has_type],
[b4_symbol([$1], [type]) v]),
b4_locations_if([location_type l]))[)
- : super_type(]b4_join([token_type (tok)],
- b4_symbol_if([$1], [has_type], [std::move (v)]),
- b4_locations_if([std::move (l)]))[)
- {
- YY_ASSERT (]m4_join([ || ], m4_map_sep([_b4_type_clause], [, ], [$@]))[);
- }
+ : super_type (]b4_join([token_kind_type (tok)],
+ b4_symbol_if([$1], [has_type], [std::move (v)]),
+ b4_locations_if([std::move (l)]))[)
#else
symbol_type (]b4_join(
[int tok],
b4_symbol_if([$1], [has_type],
[const b4_symbol([$1], [type])& v]),
b4_locations_if([const location_type& l]))[)
- : super_type(]b4_join([token_type (tok)],
- b4_symbol_if([$1], [has_type], [v]),
- b4_locations_if([l]))[)
- {
- YY_ASSERT (]m4_join([ || ], m4_map_sep([_b4_type_clause], [, ], [$@]))[);
- }
+ : super_type (]b4_join([token_kind_type (tok)],
+ b4_symbol_if([$1], [has_type], [v]),
+ b4_locations_if([l]))[)
#endif
+ {]b4_parse_assert_if([[
+#if !defined _MSC_VER || defined __clang__
+ ]b4_assert[ (]b4_tok_in($@)[);
+#endif
+ ]])[}
]])])
@@ -447,7 +519,7 @@
# b4_token_constructor_define
# ---------------------------
-# Define the overloaded versions of make_symbol for all the value types.
+# Define the overloaded versions of make_FOO for all the token kinds.
m4_define([b4_token_constructor_define],
-[ // Implementation of make_symbol for each symbol type.
+[ // Implementation of make_symbol for each token kind.
b4_symbol_foreach([_b4_token_maker_define])])
diff --git a/common/bison/skeletons/yacc.c b/common/bison/skeletons/yacc.c
index 2830c1d..64b9ac6 100644
--- a/common/bison/skeletons/yacc.c
+++ b/common/bison/skeletons/yacc.c
@@ -1,11 +1,11 @@
# -*- C -*-
# Yacc compatible skeleton for Bison
-# Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2019 Free Software
+# Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software
# Foundation, Inc.
m4_pushdef([b4_copyright_years],
- [1984, 1989-1990, 2000-2015, 2018-2019])
+ [1984, 1989-1990, 2000-2015, 2018-2021])
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -18,10 +18,11 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
m4_include(b4_skeletonsdir/[c.m4])
+
## ---------- ##
## api.pure. ##
## ---------- ##
@@ -48,6 +49,10 @@
## api.push-pull. ##
## --------------- ##
+# b4_pull_if, b4_push_if
+# ----------------------
+# Whether the pull/push APIs are needed. Both can be enabled.
+
b4_percent_define_default([[api.push-pull]], [[pull]])
b4_percent_define_check_values([[[[api.push-pull]],
[[pull]], [[push]], [[both]]]])
@@ -97,6 +102,16 @@
[1], [m4_ifset([b4_parse_param], [$1])],
[2], [$1])])])
+# b4_yyerror_formals
+# ------------------
+m4_define([b4_yyerror_formals],
+[b4_pure_if([b4_locations_if([, [[const ]b4_api_PREFIX[LTYPE *yyllocp], [&yylloc]]])[]dnl
+m4_ifdef([b4_parse_param], [, b4_parse_param])[]dnl
+,])dnl
+[[const char *msg], [msg]]])
+
+
+
# b4_yyerror_args
# ---------------
# Arguments passed to yyerror: user args plus yylloc.
@@ -111,6 +126,16 @@
## ----------------- ##
+# b4_accept([SYMBOL-NUM])
+# -----------------------
+# Used in actions of the rules of accept, the initial symbol, to call
+# YYACCEPT. If SYMBOL-NUM is specified, run "yyvalue->SLOT = $2;"
+# before, using the slot of SYMBOL-NUM.
+m4_define([b4_accept],
+[m4_ifval([$1],
+ [b4_symbol_value(yyimpl->yyvalue, [$1]) = b4_rhs_value(2, 1, [$1]); ]) YYACCEPT])
+
+
# b4_lhs_value(SYMBOL-NUM, [TYPE])
# --------------------------------
# See README.
@@ -132,6 +157,7 @@
# b4_lhs_location()
# -----------------
# Expansion of @$.
+# Overparenthetized to avoid obscure problems with "foo$$bar = foo$1bar".
m4_define([b4_lhs_location],
[(yyloc)])
@@ -140,6 +166,7 @@
# ---------------------------------
# Expansion of @POS, where the current rule has RULE-LENGTH symbols
# on RHS.
+# Overparenthetized to avoid obscure problems with "foo$$bar = foo$1bar".
m4_define([b4_rhs_location],
[(yylsp@{b4_subtract([$2], [$1])@})])
@@ -148,14 +175,61 @@
## Declarations. ##
## -------------- ##
+# _b4_declare_sub_yyparse(START-SYMBOL-NUM, SWITCHING-TOKEN-SYMBOL-NUM)
+# ---------------------------------------------------------------------
+# Define the return type of the parsing function for SYMBOL-NUM, and
+# declare its parsing function.
+m4_define([_b4_declare_sub_yyparse],
+[[
+// Return type when parsing one ]_b4_symbol($1, tag)[.
+typedef struct
+{]b4_symbol_if([$1], [has_type], [[
+ ]_b4_symbol($1, type)[ yyvalue;]])[
+ int yystatus;
+ int yynerrs;
+} ]b4_prefix[parse_]_b4_symbol($1, id)[_t;
+
+// Parse one ]_b4_symbol($1, tag)[.
+]b4_prefix[parse_]_b4_symbol($1, id)[_t ]b4_prefix[parse_]_b4_symbol($1, id)[ (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[);
+]])
+
+
+# _b4_first_switching_token
+# -------------------------
+m4_define([b4_first], [$1])
+m4_define([b4_second], [$2])
+m4_define([_b4_first_switching_token],
+[b4_second(b4_first(b4_start_symbols))])
+
+
+# _b4_define_sub_yyparse(START-SYMBOL-NUM, SWITCHING-TOKEN-SYMBOL-NUM)
+# --------------------------------------------------------------------
+# Define the parsing function for START-SYMBOL-NUM.
+m4_define([_b4_define_sub_yyparse],
+[[
+]b4_prefix[parse_]_b4_symbol($1, id)[_t
+]b4_prefix[parse_]_b4_symbol($1, id)[ (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)
+{
+ ]b4_prefix[parse_]_b4_symbol($1, id)[_t yyres;
+ yy_parse_impl_t yyimpl;
+ yyres.yystatus = yy_parse_impl (]b4_symbol($2, id)[, &yyimpl]m4_ifset([b4_parse_param],
+ [[, ]b4_args(b4_parse_param)])[);]b4_symbol_if([$1], [has_type], [[
+ yyres.yyvalue = yyimpl.yyvalue.]b4_symbol($1, slot)[;]])[
+ yyres.yynerrs = yyimpl.yynerrs;
+ return yyres;
+}
+]])
+
+
# b4_declare_scanner_communication_variables
# ------------------------------------------
# Declare the variables that are global, or local to YYPARSE if
# pure-parser.
m4_define([b4_declare_scanner_communication_variables], [[
-/* The lookahead symbol. */
+]m4_ifdef([b4_start_symbols], [],
+[[/* Lookahead token kind. */
int yychar;
-
+]])[
]b4_pure_if([[
/* The semantic value of the lookahead symbol. */
/* Default value used for initialization, for pacifying older GCCs
@@ -174,49 +248,79 @@
int yynerrs;]])])
-# b4_declare_parser_state_variables
-# ---------------------------------
+# b4_declare_parser_state_variables([INIT])
+# -----------------------------------------
# Declare all the variables that are needed to maintain the parser state
# between calls to yypush_parse.
-m4_define([b4_declare_parser_state_variables], [b4_pure_if([[
+# If INIT is non-null, initialize these variables.
+m4_define([b4_declare_parser_state_variables],
+[b4_pure_if([[
/* Number of syntax errors so far. */
- int yynerrs;
+ int yynerrs]m4_ifval([$1], [ = 0])[;
]])[
- yy_state_fast_t yystate;
+ yy_state_fast_t yystate]m4_ifval([$1], [ = 0])[;
/* Number of tokens to shift before error messages enabled. */
- int yyerrstatus;
+ int yyerrstatus]m4_ifval([$1], [ = 0])[;
- /* The stacks and their tools:
- 'yyss': related to states.
- 'yyvs': related to semantic values.]b4_locations_if([[
- 'yyls': related to locations.]])[
-
- Refer to the stacks through separate pointers, to allow yyoverflow
+ /* Refer to the stacks through separate pointers, to allow yyoverflow
to reallocate them elsewhere. */
- /* The state stack. */
+ /* Their size. */
+ YYPTRDIFF_T yystacksize]m4_ifval([$1], [ = YYINITDEPTH])[;
+
+ /* The state stack: array, bottom, top. */
yy_state_t yyssa[YYINITDEPTH];
- yy_state_t *yyss;
- yy_state_t *yyssp;
+ yy_state_t *yyss]m4_ifval([$1], [ = yyssa])[;
+ yy_state_t *yyssp]m4_ifval([$1], [ = yyss])[;
- /* The semantic value stack. */
+ /* The semantic value stack: array, bottom, top. */
YYSTYPE yyvsa[YYINITDEPTH];
- YYSTYPE *yyvs;
- YYSTYPE *yyvsp;]b4_locations_if([[
+ YYSTYPE *yyvs]m4_ifval([$1], [ = yyvsa])[;
+ YYSTYPE *yyvsp]m4_ifval([$1], [ = yyvs])[;]b4_locations_if([[
- /* The location stack. */
+ /* The location stack: array, bottom, top. */
YYLTYPE yylsa[YYINITDEPTH];
- YYLTYPE *yyls;
- YYLTYPE *yylsp;
-
- /* The locations where the error started and ended. */
- YYLTYPE yyerror_range[3];]])[
-
- YYPTRDIFF_T yystacksize;]b4_lac_if([[
+ YYLTYPE *yyls]m4_ifval([$1], [ = yylsa])[;
+ YYLTYPE *yylsp]m4_ifval([$1], [ = yyls])[;]])[]b4_lac_if([[
yy_state_t yyesa@{]b4_percent_define_get([[parse.lac.es-capacity-initial]])[@};
- yy_state_t *yyes;
- YYPTRDIFF_T yyes_capacity;]])])
+ yy_state_t *yyes]m4_ifval([$1], [ = yyesa])[;
+ YYPTRDIFF_T yyes_capacity][]m4_ifval([$1],
+ [m4_do([ = b4_percent_define_get([[parse.lac.es-capacity-initial]]) < YYMAXDEPTH],
+ [ ? b4_percent_define_get([[parse.lac.es-capacity-initial]])],
+ [ : YYMAXDEPTH])])[;]])])
+
+
+m4_define([b4_macro_define],
+[[#]define $1 $2])
+
+m4_define([b4_macro_undef],
+[[#]undef $1])
+
+m4_define([b4_pstate_macro_define],
+[b4_macro_define([$1], [yyps->$1])])
+
+# b4_parse_state_variable_macros(b4_macro_define|b4_macro_undef)
+# --------------------------------------------------------------
+m4_define([b4_parse_state_variable_macros],
+[b4_pure_if([$1([b4_prefix[]nerrs])])
+$1([yystate])
+$1([yyerrstatus])
+$1([yyssa])
+$1([yyss])
+$1([yyssp])
+$1([yyvsa])
+$1([yyvs])
+$1([yyvsp])[]b4_locations_if([
+$1([yylsa])
+$1([yyls])
+$1([yylsp])])
+$1([yystacksize])[]b4_lac_if([
+$1([yyesa])
+$1([yyes])
+$1([yyes_capacity])])])
+
+
# _b4_declare_yyparse_push
@@ -231,27 +335,23 @@
typedef struct ]b4_prefix[pstate ]b4_prefix[pstate;
-]b4_pull_if([b4_function_declare([b4_prefix[parse]], [[int]], b4_parse_param)
-])b4_function_declare([b4_prefix[push_parse]], [[int]],
- [[b4_prefix[pstate *ps]], [[ps]]]b4_pure_if([,
- [[[int pushed_char]], [[pushed_char]]],
- [[b4_api_PREFIX[STYPE const *pushed_val]], [[pushed_val]]]b4_locations_if([,
- [[b4_api_PREFIX[LTYPE *pushed_loc]], [[pushed_loc]]]])])m4_ifset([b4_parse_param], [,
- b4_parse_param]))
-b4_pull_if([b4_function_declare([b4_prefix[pull_parse]], [[int]],
- [[b4_prefix[pstate *ps]], [[ps]]]m4_ifset([b4_parse_param], [,
- b4_parse_param]))])
-b4_function_declare([b4_prefix[pstate_new]], [b4_prefix[pstate *]],
- [[[void]], []])
-b4_function_declare([b4_prefix[pstate_delete]], [[void]],
- [[b4_prefix[pstate *ps]], [[ps]]])dnl
-])
+]b4_pull_if([[
+int ]b4_prefix[parse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[);]])[
+int ]b4_prefix[push_parse (]b4_prefix[pstate *ps]b4_pure_if([[,
+ int pushed_char, ]b4_api_PREFIX[STYPE const *pushed_val]b4_locations_if([[, ]b4_api_PREFIX[LTYPE *pushed_loc]])])b4_user_formals[);
+]b4_pull_if([[int ]b4_prefix[pull_parse (]b4_prefix[pstate *ps]b4_user_formals[);]])[
+]b4_prefix[pstate *]b4_prefix[pstate_new (void);
+void ]b4_prefix[pstate_delete (]b4_prefix[pstate *ps);
+]])
+
# _b4_declare_yyparse
# -------------------
# When not the push parser.
m4_define([_b4_declare_yyparse],
-[b4_function_declare(b4_prefix[parse], [int], b4_parse_param)])
+[[int ]b4_prefix[parse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[);
+]m4_ifdef([b4_start_symbols],
+ [m4_map([_b4_declare_sub_yyparse], m4_defn([b4_start_symbols]))])])
# b4_declare_yyparse
@@ -262,19 +362,35 @@
])
+# b4_declare_yyerror_and_yylex
+# ----------------------------
+# Comply with POSIX Yacc.
+# <https://austingroupbugs.net/view.php?id=1388#c5220>
+m4_define([b4_declare_yyerror_and_yylex],
+[b4_posix_if([[#if !defined ]b4_prefix[error && !defined ]b4_api_PREFIX[ERROR_IS_DECLARED
+]b4_function_declare([b4_prefix[error]], void, b4_yyerror_formals)[
+#endif
+#if !defined ]b4_prefix[lex && !defined ]b4_api_PREFIX[LEX_IS_DECLARED
+]b4_function_declare([b4_prefix[lex]], int, b4_yylex_formals)[
+#endif
+]])dnl
+])
+
+
# b4_shared_declarations
# ----------------------
-# Declaration that might either go into the header (if --defines)
-# or open coded in the parser body.
+# Declarations that might either go into the header (if --header)
+# or into the implementation file.
m4_define([b4_shared_declarations],
-[b4_cpp_guard_open([b4_spec_header_file])[
+[b4_cpp_guard_open([b4_spec_mapped_header_file])[
]b4_declare_yydebug[
]b4_percent_code_get([[requires]])[
]b4_token_enums_defines[
]b4_declare_yylstype[
+]b4_declare_yyerror_and_yylex[
]b4_declare_yyparse[
]b4_percent_code_get([[provides]])[
-]b4_cpp_guard_close([b4_spec_header_file])[]dnl
+]b4_cpp_guard_close([b4_spec_mapped_header_file])[]dnl
])
@@ -289,7 +405,7 @@
[$2])],
[$2])])
-m4_if(b4_spec_header_file, [[y.tab.h]],
+m4_if(b4_spec_header_file, [y.tab.h], [],
[b4_percent_define_default([[api.header.include]],
[["@basename(]b4_spec_header_file[@)"]])])
@@ -301,19 +417,20 @@
## -------------- ##
-b4_defines_if([[
+b4_header_if([[
]b4_output_begin([b4_spec_header_file])[
]b4_copyright([Bison interface for Yacc-like parsers in C])[
]b4_disclaimer[
]b4_shared_declarations[
]b4_output_end[
-]])# b4_defines_if
+]])# b4_header_if
b4_output_begin([b4_parser_file_name])[
]b4_copyright([Bison implementation for Yacc-like parsers in C])[
/* C LALR(1) parser skeleton written by Richard Stallman, by
simplifying the original so-called "semantic" parser. */
+]b4_disclaimer[
/* All symbols defined below should begin with yy or YY, to avoid
infringing on user name space. This should be done even for local
variables, as they might otherwise be expanded by user macros.
@@ -321,7 +438,6 @@
define necessary library symbols; they are noted "INFRINGES ON
USER NAME SPACE" below. */
-]b4_disclaimer[
]b4_identification[
]b4_percent_code_get([[top]])[]dnl
m4_if(b4_api_prefix, [yy], [],
@@ -334,6 +450,7 @@
#define yypush_parse ]b4_prefix[push_parse]b4_pull_if([[
#define yypull_parse ]b4_prefix[pull_parse]])[
#define yypstate_new ]b4_prefix[pstate_new
+#define yypstate_clear ]b4_prefix[pstate_clear
#define yypstate_delete ]b4_prefix[pstate_delete
#define yypstate ]b4_prefix[pstate]])[
#define yylex ]b4_prefix[lex
@@ -348,65 +465,18 @@
]b4_cast_define[
]b4_null_define[
-/* Enabling verbose error messages. */
-#ifdef YYERROR_VERBOSE
-# undef YYERROR_VERBOSE
-# define YYERROR_VERBOSE 1
-#else
-# define YYERROR_VERBOSE ]b4_error_verbose_if([1], [0])[
-#endif
-
]b4_header_include_if([[#include ]b4_percent_define_get([[api.header.include]])],
[m4_ifval(m4_quote(b4_spec_header_file),
[/* Use api.header.include to #include this header
instead of duplicating it here. */
])b4_shared_declarations])[
+]b4_declare_symbol_enum[
]b4_user_post_prologue[
-]b4_percent_code_get[]dnl
-
-[#ifdef short
-# undef short
-#endif
-
+]b4_percent_code_get[
]b4_c99_int_type_define[
-#ifndef YYPTRDIFF_T
-# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__
-# define YYPTRDIFF_T __PTRDIFF_TYPE__
-# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__
-# elif defined PTRDIFF_MAX
-# ifndef ptrdiff_t
-# include <stddef.h> /* INFRINGES ON USER NAME SPACE */
-# endif
-# define YYPTRDIFF_T ptrdiff_t
-# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX
-# else
-# define YYPTRDIFF_T long
-# define YYPTRDIFF_MAXIMUM LONG_MAX
-# endif
-#endif
-
-#ifndef YYSIZE_T
-# ifdef __SIZE_TYPE__
-# define YYSIZE_T __SIZE_TYPE__
-# elif defined size_t
-# define YYSIZE_T size_t
-# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__
-# include <stddef.h> /* INFRINGES ON USER NAME SPACE */
-# define YYSIZE_T size_t
-# else
-# define YYSIZE_T unsigned
-# endif
-#endif
-
-#define YYSIZE_MAXIMUM \
- YY_CAST (YYPTRDIFF_T, \
- (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \
- ? YYPTRDIFF_MAXIMUM \
- : YY_CAST (YYSIZE_T, -1)))
-
-#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X))
+]b4_sizes_types_define[
/* Stored state numbers (used for stacks). */
typedef ]b4_int_type(0, m4_eval(b4_states_number - 1))[ yy_state_t;
@@ -425,6 +495,11 @@
# define YY_(Msgid) Msgid
# endif
#endif
+]b4_has_translations_if([
+#ifndef N_
+# define N_(Msgid) Msgid
+#endif
+])[
]b4_attribute_define[
@@ -437,7 +512,7 @@
]],
[[#define YY_ASSERT(E) ((void) (0 && (E)))]])[
-#if ]b4_lac_if([[1]], [[! defined yyoverflow || YYERROR_VERBOSE]])[
+#if ]b4_lac_if([[1]], [b4_parse_error_case([simple], [[!defined yyoverflow]], [[1]])])[
/* The parser invokes alloca or malloc; define the necessary symbols. */]dnl
b4_push_if([], [b4_lac_if([], [[
@@ -504,8 +579,7 @@
# endif
# endif]b4_lac_if([[
# define YYCOPY_NEEDED 1]])[
-#endif]b4_lac_if([], [[ /* ! defined yyoverflow || YYERROR_VERBOSE */]])[
-
+#endif /* ]b4_lac_if([[1]], [b4_parse_error_case([simple], [[!defined yyoverflow]], [[1]])])[ */
#if (! defined yyoverflow \
&& (! defined __cplusplus \
@@ -588,16 +662,18 @@
/* YYNSTATES -- Number of states. */
#define YYNSTATES ]b4_states_number[
-#define YYUNDEFTOK ]b4_undef_token_number[
-#define YYMAXUTOK ]b4_user_token_number_max[
+/* YYMAXUTOK -- Last valid token kind. */
+#define YYMAXUTOK ]b4_code_max[
/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
as returned by yylex, with out-of-bounds checking. */
]b4_api_token_raw_if(dnl
-[[#define YYTRANSLATE(YYX) (YYX)]],
-[[#define YYTRANSLATE(YYX) \
- (0 <= (YYX) && (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
+[[#define YYTRANSLATE(YYX) YY_CAST (yysymbol_kind_t, YYX)]],
+[[#define YYTRANSLATE(YYX) \
+ (0 <= (YYX) && (YYX) <= YYMAXUTOK \
+ ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \
+ : ]b4_symbol_prefix[YYUNDEF)
/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM
as returned by yylex. */
@@ -611,23 +687,46 @@
[[YYRLINE[YYN] -- Source line where rule number YYN was defined.]])[
#endif
-#if ]b4_api_PREFIX[DEBUG || YYERROR_VERBOSE || ]b4_token_table_flag[
-/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+/** Accessing symbol of state STATE. */
+#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State])
+
+#if ]b4_parse_error_case([simple], [b4_api_PREFIX[DEBUG || ]b4_token_table_flag], [[1]])[
+/* The user-facing name of the symbol whose (internal) number is
+ YYSYMBOL. No bounds checking. */
+static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED;
+
+]b4_parse_error_bmatch([simple\|verbose],
+[[/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
First, the terminals, then, starting at YYNTOKENS, nonterminals. */
static const char *const yytname[] =
{
]b4_tname[
};
-#endif
-# ifdef YYPRINT
-/* YYTOKNUM[NUM] -- (External) token number corresponding to the
- (internal) symbol number NUM (which must be that of a token). */
-static const ]b4_int_type_for([b4_toknum])[ yytoknum[] =
+static const char *
+yysymbol_name (yysymbol_kind_t yysymbol)
{
- ]b4_toknum[
-};
-# endif
+ return yytname[yysymbol];
+}]],
+[[static const char *
+yysymbol_name (yysymbol_kind_t yysymbol)
+{
+ static const char *const yy_sname[] =
+ {
+ ]b4_symbol_names[
+ };]b4_has_translations_if([[
+ /* YYTRANSLATABLE[SYMBOL-NUM] -- Whether YY_SNAME[SYMBOL-NUM] is
+ internationalizable. */
+ static ]b4_int_type_for([b4_translatable])[ yytranslatable[] =
+ {
+ ]b4_translatable[
+ };
+ return (yysymbol < YYNTOKENS && yytranslatable[yysymbol]
+ ? _(yy_sname[yysymbol])
+ : yy_sname[yysymbol]);]], [[
+ return yy_sname[yysymbol];]])[
+}]])[
+#endif
#define YYPACT_NINF (]b4_pact_ninf[)
@@ -641,21 +740,22 @@
]b4_parser_tables_define[
+enum { YYENOMEM = -2 };
+
#define yyerrok (yyerrstatus = 0)
-#define yyclearin (yychar = YYEMPTY)
-#define YYEMPTY (-2)
-#define YYEOF 0
+#define yyclearin (yychar = ]b4_symbol(empty, id)[)
#define YYACCEPT goto yyacceptlab
#define YYABORT goto yyabortlab
#define YYERROR goto yyerrorlab
+#define YYNOMEM goto yyexhaustedlab
#define YYRECOVERING() (!!yyerrstatus)
#define YYBACKUP(Token, Value) \
do \
- if (yychar == YYEMPTY) \
+ if (yychar == ]b4_symbol(empty, id)[) \
{ \
yychar = (Token); \
yylval = (Value); \
@@ -671,10 +771,9 @@
} \
while (0)
-/* Error token number */
-#define YYTERROR 1
-#define YYERRCODE 256
-
+/* Backward compatibility with an undocumented macro.
+ Use ]b4_symbol(error, id)[ or ]b4_symbol(undef, id)[. */
+#define YYERRCODE ]b4_symbol(undef, id)[
]b4_locations_if([[
]b4_yylloc_default_define[
#define YYRHSLOC(Rhs, K) ((Rhs)[K])
@@ -694,15 +793,15 @@
YYFPRINTF Args; \
} while (0)
-]b4_yy_location_print_define[
+]b4_yylocation_print_define[
-# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \
+# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) \
do { \
if (yydebug) \
{ \
YYFPRINTF (stderr, "%s ", Title); \
yy_symbol_print (stderr, \
- Type, Value]b4_locations_if([, Location])[]b4_user_args[); \
+ Kind, Value]b4_locations_if([, Location])[]b4_user_args[); \
YYFPRINTF (stderr, "\n"); \
} \
} while (0)
@@ -714,9 +813,8 @@
| TOP (included). |
`------------------------------------------------------------------*/
-]b4_function_define([yy_stack_print], [static void],
- [[yy_state_t *yybottom], [yybottom]],
- [[yy_state_t *yytop], [yytop]])[
+static void
+yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop)
{
YYFPRINTF (stderr, "Stack now");
for (; yybottom <= yytop; yybottom++)
@@ -738,12 +836,9 @@
| Report that the YYRULE is going to be reduced. |
`------------------------------------------------*/
-]b4_function_define([yy_reduce_print], [static void],
- [[yy_state_t *yyssp], [yyssp]],
- [[YYSTYPE *yyvsp], [yyvsp]],
- b4_locations_if([[[YYLTYPE *yylsp], [yylsp]],
- ])[[int yyrule], [yyrule]]m4_ifset([b4_parse_param], [,
- b4_parse_param]))[
+static void
+yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp,]b4_locations_if([[ YYLTYPE *yylsp,]])[
+ int yyrule]b4_user_formals[)
{
int yylno = yyrline[yyrule];
int yynrhs = yyr2[yyrule];
@@ -755,10 +850,9 @@
{
YYFPRINTF (stderr, " $%d = ", yyi + 1);
yy_symbol_print (stderr,
- yystos[yyssp[yyi + 1 - yynrhs]],
- &]b4_rhs_value(yynrhs, yyi + 1)[
- ]b4_locations_if([, &]b4_rhs_location(yynrhs, yyi + 1))[]dnl
- b4_user_args[);
+ YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]),
+ &]b4_rhs_value(yynrhs, yyi + 1)[]b4_locations_if([,
+ &]b4_rhs_location(yynrhs, yyi + 1))[]b4_user_args[);
YYFPRINTF (stderr, "\n");
}
}
@@ -773,8 +867,8 @@
multiple parsers can coexist. */
int yydebug;
#else /* !]b4_api_PREFIX[DEBUG */
-# define YYDPRINTF(Args)
-# define YY_SYMBOL_PRINT(Title, Type, Value, Location)
+# define YYDPRINTF(Args) ((void) 0)
+# define YY_SYMBOL_PRINT(Title, Kind, Value, Location)
# define YY_STACK_PRINT(Bottom, Top)
# define YY_REDUCE_PRINT(Rule)
#endif /* !]b4_api_PREFIX[DEBUG */
@@ -794,7 +888,18 @@
#ifndef YYMAXDEPTH
# define YYMAXDEPTH ]b4_stack_depth_max[
-#endif]b4_lac_if([[
+#endif]b4_push_if([[
+/* Parser data structure. */
+struct yypstate
+ {]b4_declare_parser_state_variables[
+ /* Whether this instance has not started parsing yet.
+ * If 2, it corresponds to a finished parsing. */
+ int yynew;
+ };]b4_pure_if([], [[
+
+/* Whether the only allowed instance of yypstate is allocated. */
+static char yypstate_allocated = 0;]])])[
+]b4_lac_if([[
/* Given a state stack such that *YYBOTTOM is its bottom, such that
*YYTOP is either its top or is YYTOP_EMPTY to indicate an empty
@@ -805,7 +910,7 @@
*YYTOP, and *YYCAPACITY to reflect the new capacity and memory
location. If *YYBOTTOM != YYBOTTOM_NO_FREE, then free the old stack
using YYSTACK_FREE. Return 0 if successful or if no reallocation is
- required. Return 1 if memory is exhausted. */
+ required. Return YYENOMEM if memory is exhausted. */
static int
yy_lac_stack_realloc (YYPTRDIFF_T *yycapacity, YYPTRDIFF_T yyadd,
#if ]b4_api_PREFIX[DEBUG
@@ -830,7 +935,7 @@
{
YYDPRINTF ((stderr, "%smax size exceeded%s", yydebug_prefix,
yydebug_suffix));
- return 1;
+ return YYENOMEM;
}
if (YYMAXDEPTH < yyalloc)
yyalloc = YYMAXDEPTH;
@@ -842,7 +947,7 @@
{
YYDPRINTF ((stderr, "%srealloc failed%s", yydebug_prefix,
yydebug_suffix));
- return 1;
+ return YYENOMEM;
}
if (*yytop != yytop_empty)
{
@@ -888,23 +993,22 @@
current lookahead, then check if that lookahead can eventually be
shifted if syntactic actions continue from the current context.
Report a syntax error if it cannot. */
-#define YY_LAC_ESTABLISH \
-do { \
- if (!yy_lac_established) \
- { \
- YYDPRINTF ((stderr, \
- "LAC: initial context established for %s\n", \
- yytname[yytoken])); \
- yy_lac_established = 1; \
- { \
- int yy_lac_status = \
- yy_lac (yyesa, &yyes, &yyes_capacity, yyssp, yytoken); \
- if (yy_lac_status == 2) \
- goto yyexhaustedlab; \
- if (yy_lac_status == 1) \
- goto yyerrlab; \
- } \
- } \
+#define YY_LAC_ESTABLISH \
+do { \
+ if (!yy_lac_established) \
+ { \
+ YYDPRINTF ((stderr, \
+ "LAC: initial context established for %s\n", \
+ yysymbol_name (yytoken))); \
+ yy_lac_established = 1; \
+ switch (yy_lac (yyesa, &yyes, &yyes_capacity, yyssp, yytoken)) \
+ { \
+ case YYENOMEM: \
+ YYNOMEM; \
+ case 1: \
+ goto yyerrlab; \
+ } \
+ } \
} while (0)
/* Discard any previous initial lookahead context because of Event,
@@ -923,9 +1027,8 @@
do { \
if (yy_lac_established) \
{ \
- if (yydebug) \
- YYFPRINTF (stderr, "LAC: initial context discarded due to " \
- Event "\n"); \
+ YYDPRINTF ((stderr, "LAC: initial context discarded due to " \
+ Event "\n")); \
yy_lac_established = 0; \
} \
} while (0)
@@ -935,7 +1038,7 @@
/* Given the stack whose top is *YYSSP, return 0 iff YYTOKEN can
eventually (after perhaps some reductions) be shifted, return 1 if
- not, or return 2 if memory is exhausted. As preconditions and
+ not, or return YYENOMEM if memory is exhausted. As preconditions and
postconditions: *YYES_CAPACITY is the allocated size of the array to
which *YYES points, and either *YYES = YYESA or *YYES points to an
array allocated with YYSTACK_ALLOC. yy_lac may overwrite the
@@ -943,24 +1046,26 @@
any old *YYES other than YYESA. */
static int
yy_lac (yy_state_t *yyesa, yy_state_t **yyes,
- YYPTRDIFF_T *yyes_capacity, yy_state_t *yyssp, int yytoken)
+ YYPTRDIFF_T *yyes_capacity, yy_state_t *yyssp, yysymbol_kind_t yytoken)
{
yy_state_t *yyes_prev = yyssp;
yy_state_t *yyesp = yyes_prev;
- YYDPRINTF ((stderr, "LAC: checking lookahead %s:", yytname[yytoken]));
- if (yytoken == YYUNDEFTOK)
+ /* Reduce until we encounter a shift and thereby accept the token. */
+ YYDPRINTF ((stderr, "LAC: checking lookahead %s:", yysymbol_name (yytoken)));
+ if (yytoken == ]b4_symbol_prefix[YYUNDEF)
{
YYDPRINTF ((stderr, " Always Err\n"));
return 1;
}
while (1)
{
- int yyrule = yypact[*yyesp];
+ int yyrule = yypact[+*yyesp];
if (yypact_value_is_default (yyrule)
|| (yyrule += yytoken) < 0 || YYLAST < yyrule
|| yycheck[yyrule] != yytoken)
{
- yyrule = yydefact[*yyesp];
+ /* Use the default action. */
+ yyrule = yydefact[+*yyesp];
if (yyrule == 0)
{
YYDPRINTF ((stderr, " Err\n"));
@@ -969,6 +1074,7 @@
}
else
{
+ /* Use the action from yytable. */
yyrule = yytable[yyrule];
if (yytable_value_is_error (yyrule))
{
@@ -982,9 +1088,12 @@
}
yyrule = -yyrule;
}
+ /* By now we know we have to simulate a reduce. */
+ YYDPRINTF ((stderr, " R%d", yyrule - 1));
{
+ /* Pop the corresponding number of values from the stack. */
YYPTRDIFF_T yylen = yyr2[yyrule];
- YYDPRINTF ((stderr, " R%d", yyrule - 1));
+ /* First pop from the LAC stack as many tokens as possible. */
if (yyesp != yyes_prev)
{
YYPTRDIFF_T yysize = yyesp - *yyes + 1;
@@ -995,13 +1104,15 @@
}
else
{
- yylen -= yysize;
yyesp = yyes_prev;
+ yylen -= yysize;
}
}
+ /* Only afterwards look at the main stack. */
if (yylen)
yyesp = yyes_prev -= yylen;
}
+ /* Push the resulting state of the reduction. */
{
yy_state_fast_t yystate;
{
@@ -1027,7 +1138,7 @@
yyes, yyesa, &yyesp, yyes_prev))
{
YYDPRINTF ((stderr, "\n"));
- return 2;
+ return YYENOMEM;
}
YY_IGNORE_USELESS_CAST_BEGIN
*++yyesp = YY_CAST (yy_state_t, yystate);
@@ -1038,33 +1149,144 @@
}
}]])[
+]b4_parse_error_case([simple], [],
+[[/* Context of a parse error. */
+typedef struct
+{]b4_push_if([[
+ yypstate* yyps;]], [[
+ yy_state_t *yyssp;]b4_lac_if([[
+ yy_state_t *yyesa;
+ yy_state_t **yyes;
+ YYPTRDIFF_T *yyes_capacity;]])])[
+ yysymbol_kind_t yytoken;]b4_locations_if([[
+ YYLTYPE *yylloc;]])[
+} yypcontext_t;
-#if YYERROR_VERBOSE
+/* Put in YYARG at most YYARGN of the expected tokens given the
+ current YYCTX, and return the number of tokens stored in YYARG. If
+ YYARG is null, return the number of expected tokens (guaranteed to
+ be less than YYNTOKENS). Return YYENOMEM on memory exhaustion.
+ Return 0 if there are more than YYARGN expected tokens, yet fill
+ YYARG up to YYARGN. */]b4_push_if([[
+static int
+yypstate_expected_tokens (yypstate *yyps,
+ yysymbol_kind_t yyarg[], int yyargn)]], [[
+static int
+yypcontext_expected_tokens (const yypcontext_t *yyctx,
+ yysymbol_kind_t yyarg[], int yyargn)]])[
+{
+ /* Actual size of YYARG. */
+ int yycount = 0;
+]b4_lac_if([[
+ int yyx;
+ for (yyx = 0; yyx < YYNTOKENS; ++yyx)
+ {
+ yysymbol_kind_t yysym = YY_CAST (yysymbol_kind_t, yyx);
+ if (yysym != ]b4_symbol(error, kind)[ && yysym != ]b4_symbol_prefix[YYUNDEF)
+ switch (yy_lac (]b4_push_if([[yyps->yyesa, &yyps->yyes, &yyps->yyes_capacity, yyps->yyssp, yysym]],
+ [[yyctx->yyesa, yyctx->yyes, yyctx->yyes_capacity, yyctx->yyssp, yysym]])[))
+ {
+ case YYENOMEM:
+ return YYENOMEM;
+ case 1:
+ continue;
+ default:
+ if (!yyarg)
+ ++yycount;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = yysym;
+ }
+ }]],
+[[ int yyn = yypact@{+*]b4_push_if([yyps], [yyctx])[->yyssp@};
+ if (!yypact_value_is_default (yyn))
+ {
+ /* Start YYX at -YYN if negative to avoid negative indexes in
+ YYCHECK. In other words, skip the first -YYN actions for
+ this state because they are default actions. */
+ int yyxbegin = yyn < 0 ? -yyn : 0;
+ /* Stay within bounds of both yycheck and yytname. */
+ int yychecklim = YYLAST - yyn + 1;
+ int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
+ int yyx;
+ for (yyx = yyxbegin; yyx < yyxend; ++yyx)
+ if (yycheck[yyx + yyn] == yyx && yyx != ]b4_symbol(error, kind)[
+ && !yytable_value_is_error (yytable[yyx + yyn]))
+ {
+ if (!yyarg)
+ ++yycount;
+ else if (yycount == yyargn)
+ return 0;
+ else
+ yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx);
+ }
+ }]])[
+ if (yyarg && yycount == 0 && 0 < yyargn)
+ yyarg[0] = ]b4_symbol(empty, kind)[;
+ return yycount;
+}
-# ifndef yystrlen
-# if defined __GLIBC__ && defined _STRING_H
-# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S)))
-# else
+]b4_push_if([[
+/* Similar to the previous function. */
+static int
+yypcontext_expected_tokens (const yypcontext_t *yyctx,
+ yysymbol_kind_t yyarg[], int yyargn)
+{
+ return yypstate_expected_tokens (yyctx->yyps, yyarg, yyargn);
+}]])[
+]])[
+
+]b4_parse_error_bmatch(
+ [custom],
+[[/* The kind of the lookahead of this context. */
+static yysymbol_kind_t
+yypcontext_token (const yypcontext_t *yyctx) YY_ATTRIBUTE_UNUSED;
+
+static yysymbol_kind_t
+yypcontext_token (const yypcontext_t *yyctx)
+{
+ return yyctx->yytoken;
+}
+
+]b4_locations_if([[/* The location of the lookahead of this context. */
+static YYLTYPE *
+yypcontext_location (const yypcontext_t *yyctx) YY_ATTRIBUTE_UNUSED;
+
+static YYLTYPE *
+yypcontext_location (const yypcontext_t *yyctx)
+{
+ return yyctx->yylloc;
+}]])[
+
+/* User defined function to report a syntax error. */
+static int
+yyreport_syntax_error (const yypcontext_t *yyctx]b4_user_formals[);]],
+ [detailed\|verbose],
+[[#ifndef yystrlen
+# if defined __GLIBC__ && defined _STRING_H
+# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S)))
+# else
/* Return the length of YYSTR. */
-]b4_function_define([yystrlen], [static YYPTRDIFF_T],
- [[const char *yystr], [yystr]])[
+static YYPTRDIFF_T
+yystrlen (const char *yystr)
{
YYPTRDIFF_T yylen;
for (yylen = 0; yystr[yylen]; yylen++)
continue;
return yylen;
}
-# endif
# endif
+#endif
-# ifndef yystpcpy
-# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE
-# define yystpcpy stpcpy
-# else
+#ifndef yystpcpy
+# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE
+# define yystpcpy stpcpy
+# else
/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in
YYDEST. */
-]b4_function_define([yystpcpy], [static char *],
- [[char *yydest], [yydest]], [[const char *yysrc], [yysrc]])[
+static char *
+yystpcpy (char *yydest, const char *yysrc)
{
char *yyd = yydest;
const char *yys = yysrc;
@@ -1074,10 +1296,12 @@
return yyd - 1;
}
-# endif
# endif
+#endif
-# ifndef yytnamerr
+]b4_parse_error_case(
+ [verbose],
+[[#ifndef yytnamerr
/* Copy to YYRES the contents of YYSTR after stripping away unnecessary
quotes and backslashes, so that it's suitable for yyerror. The
heuristic is that double-quoting is unnecessary unless the string
@@ -1092,7 +1316,6 @@
{
YYPTRDIFF_T yyn = 0;
char const *yyp = yystr;
-
for (;;)
switch (*++yyp)
{
@@ -1126,34 +1349,15 @@
else
return yystrlen (yystr);
}
-# endif
+#endif
+]])[
-/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message
- about the unexpected token YYTOKEN for the state stack whose top is
- YYSSP.]b4_lac_if([[ In order to see if a particular token T is a
- valid looakhead, invoke yy_lac (YYESA, YYES, YYES_CAPACITY, YYSSP, T).]])[
-
- Return 0 if *YYMSG was successfully written. Return 1 if *YYMSG is
- not large enough to hold the message. In that case, also set
- *YYMSG_ALLOC to the required number of bytes. Return 2 if the
- required number of bytes is too large to store]b4_lac_if([[ or if
- yy_lac returned 2]])[. */
static int
-yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
- ]b4_lac_if([[yy_state_t *yyesa, yy_state_t **yyes,
- YYPTRDIFF_T *yyes_capacity, ]])[yy_state_t *yyssp, int yytoken)
+yy_syntax_error_arguments (const yypcontext_t *yyctx,
+ yysymbol_kind_t yyarg[], int yyargn)
{
- enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 };
- /* Internationalized format string. */
- const char *yyformat = YY_NULLPTR;
- /* Arguments of yyformat: reported tokens (one for the "unexpected",
- one per "expected"). */
- char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
/* Actual size of YYARG. */
int yycount = 0;
- /* Cumulated lengths of YYARG. */
- YYPTRDIFF_T yysize = 0;
-
/* There are many possibilities here to consider:
- If this state is a consistent state with a default action, then
the only way this function was invoked is if the default action
@@ -1182,70 +1386,59 @@
one exception: it will still contain any token that will not be
accepted due to an error action in a later state.]])[
*/
- if (yytoken != YYEMPTY)
+ if (yyctx->yytoken != ]b4_symbol(empty, kind)[)
{
- int yyn = yypact[*yyssp];
- YYPTRDIFF_T yysize0 = yytnamerr (YY_NULLPTR, yytname[yytoken]);
- yysize = yysize0;]b4_lac_if([[
+ int yyn;]b4_lac_if([[
YYDPRINTF ((stderr, "Constructing syntax error message\n"));]])[
- yyarg[yycount++] = yytname[yytoken];
- if (!yypact_value_is_default (yyn))
- {]b4_lac_if([[
- int yyx;
-
- for (yyx = 0; yyx < YYNTOKENS; ++yyx)
- if (yyx != YYTERROR && yyx != YYUNDEFTOK)
- {
- {
- int yy_lac_status = yy_lac (yyesa, yyes, yyes_capacity,
- yyssp, yyx);
- if (yy_lac_status == 2)
- return 2;
- if (yy_lac_status == 1)
- continue;
- }]], [[
- /* Start YYX at -YYN if negative to avoid negative indexes in
- YYCHECK. In other words, skip the first -YYN actions for
- this state because they are default actions. */
- int yyxbegin = yyn < 0 ? -yyn : 0;
- /* Stay within bounds of both yycheck and yytname. */
- int yychecklim = YYLAST - yyn + 1;
- int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
- int yyx;
-
- for (yyx = yyxbegin; yyx < yyxend; ++yyx)
- if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR
- && !yytable_value_is_error (yytable[yyx + yyn]))
- {]])[
- if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
- {
- yycount = 1;
- yysize = yysize0;
- break;
- }
- yyarg[yycount++] = yytname[yyx];
- {
- YYPTRDIFF_T yysize1
- = yysize + yytnamerr (YY_NULLPTR, yytname[yyx]);
- if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)
- yysize = yysize1;
- else
- return 2;
- }
- }
- }]b4_lac_if([[
-# if ]b4_api_PREFIX[DEBUG
- else if (yydebug)
- YYFPRINTF (stderr, "No expected tokens.\n");
-# endif]])[
+ if (yyarg)
+ yyarg[yycount] = yyctx->yytoken;
+ ++yycount;
+ yyn = yypcontext_expected_tokens (yyctx,
+ yyarg ? yyarg + 1 : yyarg, yyargn - 1);
+ if (yyn == YYENOMEM)
+ return YYENOMEM;]b4_lac_if([[
+ else if (yyn == 0)
+ YYDPRINTF ((stderr, "No expected tokens.\n"));]])[
+ else
+ yycount += yyn;
}
+ return yycount;
+}
+
+/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message
+ about the unexpected token YYTOKEN for the state stack whose top is
+ YYSSP.]b4_lac_if([[ In order to see if a particular token T is a
+ valid looakhead, invoke yy_lac (YYESA, YYES, YYES_CAPACITY, YYSSP, T).]])[
+
+ Return 0 if *YYMSG was successfully written. Return -1 if *YYMSG is
+ not large enough to hold the message. In that case, also set
+ *YYMSG_ALLOC to the required number of bytes. Return YYENOMEM if the
+ required number of bytes is too large to store]b4_lac_if([[ or if
+ yy_lac returned YYENOMEM]])[. */
+static int
+yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
+ const yypcontext_t *yyctx)
+{
+ enum { YYARGS_MAX = 5 };
+ /* Internationalized format string. */
+ const char *yyformat = YY_NULLPTR;
+ /* Arguments of yyformat: reported tokens (one for the "unexpected",
+ one per "expected"). */
+ yysymbol_kind_t yyarg[YYARGS_MAX];
+ /* Cumulated lengths of YYARG. */
+ YYPTRDIFF_T yysize = 0;
+
+ /* Actual size of YYARG. */
+ int yycount = yy_syntax_error_arguments (yyctx, yyarg, YYARGS_MAX);
+ if (yycount == YYENOMEM)
+ return YYENOMEM;
switch (yycount)
{
-# define YYCASE_(N, S) \
+#define YYCASE_(N, S) \
case N: \
yyformat = S; \
- break
+ break
default: /* Avoid compiler warnings. */
YYCASE_(0, YY_("syntax error"));
YYCASE_(1, YY_("syntax error, unexpected %s"));
@@ -1253,17 +1446,25 @@
YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s"));
YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s"));
YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"));
-# undef YYCASE_
+#undef YYCASE_
}
+ /* Compute error message size. Don't count the "%s"s, but reserve
+ room for the terminator. */
+ yysize = yystrlen (yyformat) - 2 * yycount + 1;
{
- /* Don't count the "%s"s in the final size, but reserve room for
- the terminator. */
- YYPTRDIFF_T yysize1 = yysize + (yystrlen (yyformat) - 2 * yycount) + 1;
- if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)
- yysize = yysize1;
- else
- return 2;
+ int yyi;
+ for (yyi = 0; yyi < yycount; ++yyi)
+ {
+ YYPTRDIFF_T yysize1
+ = yysize + ]b4_parse_error_case(
+ [verbose], [[yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]])]],
+ [[yystrlen (yysymbol_name (yyarg[yyi]))]]);[
+ if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)
+ yysize = yysize1;
+ else
+ return YYENOMEM;
+ }
}
if (*yymsg_alloc < yysize)
@@ -1272,7 +1473,7 @@
if (! (yysize <= *yymsg_alloc
&& *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM))
*yymsg_alloc = YYSTACK_ALLOC_MAXIMUM;
- return 1;
+ return -1;
}
/* Avoid sprintf, as that infringes on the user's name space.
@@ -1283,8 +1484,9 @@
int yyi = 0;
while ((*yyp = *yyformat) != '\0')
if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount)
- {
- yyp += yytnamerr (yyp, yyarg[yyi++]);
+ {]b4_parse_error_case([verbose], [[
+ yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]);]], [[
+ yyp = yystpcpy (yyp, yysymbol_name (yyarg[yyi++]));]])[
yyformat += 2;
}
else
@@ -1295,153 +1497,175 @@
}
return 0;
}
-#endif /* YYERROR_VERBOSE */
+]])[
]b4_yydestruct_define[
-]b4_pure_if([], [
+]b4_pure_if([], [b4_declare_scanner_communication_variables])[
-b4_declare_scanner_communication_variables])[]b4_push_if([[
+]b4_push_if([b4_pull_if([[
-struct yypstate
- {]b4_declare_parser_state_variables[
- /* Used to determine if this is the first time this instance has
- been used. */
- int yynew;
- };]b4_pure_if([], [[
-
-static char yypstate_allocated = 0;]])b4_pull_if([
-
-b4_function_define([[yyparse]], [[int]], b4_parse_param)[
+int
+yyparse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)
{
- return yypull_parse (YY_NULLPTR]m4_ifset([b4_parse_param],
- [[, ]b4_args(b4_parse_param)])[);
+ yypstate *yyps = yypstate_new ();
+ if (!yyps)
+ {]b4_pure_if([b4_locations_if([[
+ static YYLTYPE yyloc_default][]b4_yyloc_default[;
+ YYLTYPE yylloc = yyloc_default;]])[
+ yyerror (]b4_yyerror_args[YY_("memory exhausted"));]], [[
+ if (!yypstate_allocated)
+ yyerror (]b4_yyerror_args[YY_("memory exhausted"));]])[
+ return 2;
+ }
+ int yystatus = yypull_parse (yyps]b4_user_args[);
+ yypstate_delete (yyps);
+ return yystatus;
}
-]b4_function_define([[yypull_parse]], [[int]],
- [[[yypstate *yyps]], [[yyps]]]m4_ifset([b4_parse_param], [,
- b4_parse_param]))[
-{]b4_pure_if([b4_locations_if([[
+int
+yypull_parse (yypstate *yyps]b4_user_formals[)
+{
+ YY_ASSERT (yyps);]b4_pure_if([b4_locations_if([[
static YYLTYPE yyloc_default][]b4_yyloc_default[;
YYLTYPE yylloc = yyloc_default;]])])[
- yypstate *yyps_local;
- if (yyps)
- yyps_local = yyps;
- else
- {
- yyps_local = yypstate_new ();
- if (!yyps_local)
- {]b4_pure_if([[
- yyerror (]b4_yyerror_args[YY_("memory exhausted"));]], [[
- if (!yypstate_allocated)
- yyerror (]b4_yyerror_args[YY_("memory exhausted"));]])[
- return 2;
- }
- }
int yystatus;
- do {]b4_pure_if([[
- YYSTYPE yylval;
- int ]])[yychar = ]b4_lex[;
- yystatus =
- yypush_parse (yyps_local]b4_pure_if([[, yychar, &yylval]b4_locations_if([[, &yylloc]])])m4_ifset([b4_parse_param], [, b4_args(b4_parse_param)])[);
+ do {
+]b4_pure_if([[ YYSTYPE yylval;
+ int ]])[yychar = ]b4_yylex[;
+ yystatus = yypush_parse (yyps]b4_pure_if([[, yychar, &yylval]b4_locations_if([[, &yylloc]])])m4_ifset([b4_parse_param], [, b4_args(b4_parse_param)])[);
} while (yystatus == YYPUSH_MORE);
- if (!yyps)
- yypstate_delete (yyps_local);
return yystatus;
}]])[
+]b4_parse_state_variable_macros([b4_pstate_macro_define])[
+
/* Initialize the parser data structure. */
-]b4_function_define([[yypstate_new]], [[yypstate *]])[
+static void
+yypstate_clear (yypstate *yyps)
+{
+ yynerrs = 0;
+ yystate = 0;
+ yyerrstatus = 0;
+
+ yyssp = yyss;
+ yyvsp = yyvs;]b4_locations_if([[
+ yylsp = yyls;]])[
+
+ /* Initialize the state stack, in case yypcontext_expected_tokens is
+ called before the first call to yyparse. */
+ *yyssp = 0;
+ yyps->yynew = 1;
+}
+
+/* Initialize the parser data structure. */
+yypstate *
+yypstate_new (void)
{
yypstate *yyps;]b4_pure_if([], [[
if (yypstate_allocated)
return YY_NULLPTR;]])[
- yyps = YY_CAST (yypstate *, malloc (sizeof *yyps));
+ yyps = YY_CAST (yypstate *, YYMALLOC (sizeof *yyps));
if (!yyps)
- return YY_NULLPTR;
- yyps->yynew = 1;]b4_pure_if([], [[
+ return YY_NULLPTR;]b4_pure_if([], [[
yypstate_allocated = 1;]])[
+ yystacksize = YYINITDEPTH;
+ yyss = yyssa;
+ yyvs = yyvsa;]b4_locations_if([[
+ yyls = yylsa;]])[]b4_lac_if([[
+ yyes = yyesa;
+ yyes_capacity = ]b4_percent_define_get([[parse.lac.es-capacity-initial]])[;
+ if (YYMAXDEPTH < yyes_capacity)
+ yyes_capacity = YYMAXDEPTH;]])[
+ yypstate_clear (yyps);
return yyps;
}
-]b4_function_define([[yypstate_delete]], [[void]],
- [[[yypstate *yyps]], [[yyps]]])[
+void
+yypstate_delete (yypstate *yyps)
{
if (yyps)
{
#ifndef yyoverflow
/* If the stack was reallocated but the parse did not complete, then the
stack still needs to be freed. */
- if (!yyps->yynew && yyps->yyss != yyps->yyssa)
- YYSTACK_FREE (yyps->yyss);
+ if (yyss != yyssa)
+ YYSTACK_FREE (yyss);
#endif]b4_lac_if([[
- if (!yyps->yynew && yyps->yyes != yyps->yyesa)
- YYSTACK_FREE (yyps->yyes);]])[
- free (yyps);]b4_pure_if([], [[
+ if (yyes != yyesa)
+ YYSTACK_FREE (yyes);]])[
+ YYFREE (yyps);]b4_pure_if([], [[
yypstate_allocated = 0;]])[
}
}
-]b4_pure_if([[
-#define ]b4_prefix[nerrs yyps->]b4_prefix[nerrs]])[
-#define yystate yyps->yystate
-#define yyerrstatus yyps->yyerrstatus
-#define yyssa yyps->yyssa
-#define yyss yyps->yyss
-#define yyssp yyps->yyssp
-#define yyvsa yyps->yyvsa
-#define yyvs yyps->yyvs
-#define yyvsp yyps->yyvsp]b4_locations_if([[
-#define yylsa yyps->yylsa
-#define yyls yyps->yyls
-#define yylsp yyps->yylsp
-#define yyerror_range yyps->yyerror_range]])[
-#define yystacksize yyps->yystacksize]b4_lac_if([[
-#define yyesa yyps->yyesa
-#define yyes yyps->yyes
-#define yyes_capacity yyps->yyes_capacity]])[
+]])[
-
+]b4_push_if([[
/*---------------.
| yypush_parse. |
`---------------*/
-]b4_function_define([[yypush_parse]], [[int]],
- [[[yypstate *yyps]], [[yyps]]]b4_pure_if([,
- [[[int yypushed_char]], [[yypushed_char]]],
- [[[YYSTYPE const *yypushed_val]], [[yypushed_val]]]b4_locations_if([,
- [[[YYLTYPE *yypushed_loc]], [[yypushed_loc]]]])])m4_ifset([b4_parse_param], [,
- b4_parse_param]))], [[
-
-
+int
+yypush_parse (yypstate *yyps]b4_pure_if([[,
+ int yypushed_char, YYSTYPE const *yypushed_val]b4_locations_if([[, YYLTYPE *yypushed_loc]])])b4_user_formals[)]],
+[[
/*----------.
| yyparse. |
`----------*/
-]b4_function_define([yyparse], [int], b4_parse_param)])[
+]m4_ifdef([b4_start_symbols],
+[[// Extract data from the parser.
+typedef struct
+{
+ YYSTYPE yyvalue;
+ int yynerrs;
+} yy_parse_impl_t;
+
+// Run a full parse, using YYCHAR as switching token.
+static int
+yy_parse_impl (int yychar, yy_parse_impl_t *yyimpl]m4_ifset([b4_parse_param], [, b4_formals(b4_parse_param)])[);
+
+]m4_map([_b4_define_sub_yyparse], m4_defn([b4_start_symbols]))[
+
+int
+yyparse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)
+{
+ return yy_parse_impl (]b4_symbol(_b4_first_switching_token, id)[, YY_NULLPTR]m4_ifset([b4_parse_param],
+ [[, ]b4_args(b4_parse_param)])[);
+}
+
+static int
+yy_parse_impl (int yychar, yy_parse_impl_t *yyimpl]m4_ifset([b4_parse_param], [, b4_formals(b4_parse_param)])[)]],
+[[int
+yyparse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)]])])[
{]b4_pure_if([b4_declare_scanner_communication_variables
])b4_push_if([b4_pure_if([], [[
int yypushed_char = yychar;
YYSTYPE yypushed_val = yylval;]b4_locations_if([[
YYLTYPE yypushed_loc = yylloc;]])
])],
- [b4_declare_parser_state_variables
+ [b4_declare_parser_state_variables([init])
])b4_lac_if([[
+ /* Whether LAC context is established. A Boolean. */
int yy_lac_established = 0;]])[
int yyn;
+ /* The return value of yyparse. */
int yyresult;
- /* Lookahead token as an internal (translated) token number. */
- int yytoken = 0;
+ /* Lookahead symbol kind. */
+ yysymbol_kind_t yytoken = ]b4_symbol(empty, kind)[;
/* The variables used to return semantic value and location from the
action routines. */
YYSTYPE yyval;]b4_locations_if([[
- YYLTYPE yyloc;]])[
+ YYLTYPE yyloc;
-#if YYERROR_VERBOSE
- /* Buffer for error messages, and its allocated size. */
+ /* The locations where the error started and ended. */
+ YYLTYPE yyerror_range[3];]])[
+
+]b4_parse_error_bmatch([detailed\|verbose],
+[[ /* Buffer for error messages, and its allocated size. */
char yymsgbuf[128];
char *yymsg = yymsgbuf;
- YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf;
-#endif
+ YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf;]])[
#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)]b4_locations_if([, yylsp -= (N)])[)
@@ -1449,28 +1673,25 @@
Keep to zero when no symbol should be popped. */
int yylen = 0;]b4_push_if([[
- if (!yyps->yynew)
+ switch (yyps->yynew)
{
+ case 0:
yyn = yypact[yystate];
goto yyread_pushed_token;
+
+ case 2:
+ yypstate_clear (yyps);
+ break;
+
+ default:
+ break;
}]])[
- yyssp = yyss = yyssa;
- yyvsp = yyvs = yyvsa;]b4_locations_if([[
- yylsp = yyls = yylsa;]])[
- yystacksize = YYINITDEPTH;]b4_lac_if([[
-
- yyes = yyesa;
- yyes_capacity = ]b4_percent_define_get([[parse.lac.es-capacity-initial]])[;
- if (YYMAXDEPTH < yyes_capacity)
- yyes_capacity = YYMAXDEPTH;]])[
-
YYDPRINTF ((stderr, "Starting parse\n"));
- yystate = 0;
- yyerrstatus = 0;
- yynerrs = 0;
- yychar = YYEMPTY; /* Cause a token to be read. */
+]m4_ifdef([b4_start_symbols], [],
+[[ yychar = ]b4_symbol(empty, id)[; /* Cause a token to be read. */
+]])[
]m4_ifdef([b4_initial_action], [
b4_dollar_pushdef([m4_define([b4_dollar_dollar_used])yylval], [], [],
[b4_push_if([b4_pure_if([*])yypushed_loc], [yylloc])])dnl
@@ -1501,10 +1722,11 @@
YY_IGNORE_USELESS_CAST_BEGIN
*yyssp = YY_CAST (yy_state_t, yystate);
YY_IGNORE_USELESS_CAST_END
+ YY_STACK_PRINT (yyss, yyssp);
if (yyss + yystacksize - 1 <= yyssp)
#if !defined yyoverflow && !defined YYSTACK_RELOCATE
- goto yyexhaustedlab;
+ YYNOMEM;
#else
{
/* Get the current used size of the three stacks, in elements. */
@@ -1535,7 +1757,7 @@
# else /* defined YYSTACK_RELOCATE */
/* Extend the stack our own way. */
if (YYMAXDEPTH <= yystacksize)
- goto yyexhaustedlab;
+ YYNOMEM;
yystacksize *= 2;
if (YYMAXDEPTH < yystacksize)
yystacksize = YYMAXDEPTH;
@@ -1546,11 +1768,11 @@
YY_CAST (union yyalloc *,
YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize))));
if (! yyptr)
- goto yyexhaustedlab;
+ YYNOMEM;
YYSTACK_RELOCATE (yyss_alloc, yyss);
YYSTACK_RELOCATE (yyvs_alloc, yyvs);]b4_locations_if([
YYSTACK_RELOCATE (yyls_alloc, yyls);])[
-# undef YYSTACK_RELOCATE
+# undef YYSTACK_RELOCATE
if (yyss1 != yyssa)
YYSTACK_FREE (yyss1);
}
@@ -1570,8 +1792,9 @@
}
#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */
+]m4_ifdef([b4_start_symbols], [], [[
if (yystate == YYFINAL)
- YYACCEPT;
+ YYACCEPT;]])[
goto yybackup;
@@ -1590,8 +1813,8 @@
/* Not known => get a lookahead token if don't already have one. */
- /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */
- if (yychar == YYEMPTY)
+ /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */
+ if (yychar == ]b4_symbol(empty, id)[)
{]b4_push_if([[
if (!yyps->yynew)
{]b4_use_push_for_pull_if([], [[
@@ -1607,20 +1830,32 @@
yylval = yypushed_val;]b4_locations_if([[
yylloc = yypushed_loc;]])])[
yyread_pushed_token:]])[
- YYDPRINTF ((stderr, "Reading a token: "));]b4_push_if([b4_pure_if([[
+ YYDPRINTF ((stderr, "Reading a token\n"));]b4_push_if([b4_pure_if([[
yychar = yypushed_char;
if (yypushed_val)
yylval = *yypushed_val;]b4_locations_if([[
if (yypushed_loc)
yylloc = *yypushed_loc;]])])], [[
- yychar = ]b4_lex[;]])[
+ yychar = ]b4_yylex[;]])[
}
- if (yychar <= YYEOF)
+ if (yychar <= ]b4_symbol(eof, [id])[)
{
- yychar = yytoken = YYEOF;
+ yychar = ]b4_symbol(eof, [id])[;
+ yytoken = ]b4_symbol(eof, [kind])[;
YYDPRINTF ((stderr, "Now at end of input.\n"));
}
+ else if (yychar == ]b4_symbol(error, [id])[)
+ {
+ /* The scanner already issued an error message, process directly
+ to error recovery. But do not keep the error token as
+ lookahead, it is too special and may lead us to an endless
+ loop in error recovery. */
+ yychar = ]b4_symbol(undef, [id])[;
+ yytoken = ]b4_symbol(error, [kind])[;]b4_locations_if([[
+ yyerror_range[1] = yylloc;]])[
+ goto yyerrlab1;
+ }
else
{
yytoken = YYTRANSLATE (yychar);
@@ -1640,9 +1875,9 @@
if (yyn <= 0)
{
if (yytable_value_is_error (yyn))
- goto yyerrlab;]b4_lac_if([[
+ goto yyerrlab;
+ yyn = -yyn;]b4_lac_if([[
YY_LAC_ESTABLISH;]])[
- yyn = -yyn;
goto yyreduce;
}
@@ -1660,7 +1895,7 @@
*++yylsp = yylloc;])[
/* Discard the shifted token. */
- yychar = YYEMPTY;]b4_lac_if([[
+ yychar = ]b4_symbol(empty, id)[;]b4_lac_if([[
YY_LAC_DISCARD ("shift");]])[
goto yynewstate;
@@ -1723,11 +1958,10 @@
case of YYERROR or YYBACKUP, subsequent parser actions might lead
to an incorrect destructor call or verbose syntax error message
before the lookahead is translated. */
- YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc);
+ YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc);
YYPOPSTACK (yylen);
yylen = 0;
- YY_STACK_PRINT (yyss, yyssp);
*++yyvsp = yyval;]b4_locations_if([
*++yylsp = yyloc;])[
@@ -1752,69 +1986,76 @@
yyerrlab:
/* Make sure we have latest lookahead translation. See comments at
user semantic actions for why this is necessary. */
- yytoken = yychar == YYEMPTY ? YYEMPTY : YYTRANSLATE (yychar);
-
+ yytoken = yychar == ]b4_symbol(empty, id)[ ? ]b4_symbol(empty, kind)[ : YYTRANSLATE (yychar);
/* If not already recovering from an error, report this error. */
if (!yyerrstatus)
{
++yynerrs;
-#if ! YYERROR_VERBOSE
- yyerror (]b4_yyerror_args[YY_("syntax error"));
-#else
-# define YYSYNTAX_ERROR yysyntax_error (&yymsg_alloc, &yymsg, \]b4_lac_if([[
- yyesa, &yyes, &yyes_capacity, \]])[
- yyssp, yytoken)
- {
+]b4_parse_error_case(
+ [custom],
+[[ {
+ yypcontext_t yyctx
+ = {]b4_push_if([[yyps]], [[yyssp]b4_lac_if([[, yyesa, &yyes, &yyes_capacity]])])[, yytoken]b4_locations_if([[, &yylloc]])[};]b4_lac_if([[
+ if (yychar != ]b4_symbol(empty, id)[)
+ YY_LAC_ESTABLISH;]])[
+ if (yyreport_syntax_error (&yyctx]m4_ifset([b4_parse_param],
+ [[, ]b4_args(b4_parse_param)])[) == 2)
+ YYNOMEM;
+ }]],
+ [simple],
+[[ yyerror (]b4_yyerror_args[YY_("syntax error"));]],
+[[ {
+ yypcontext_t yyctx
+ = {]b4_push_if([[yyps]], [[yyssp]b4_lac_if([[, yyesa, &yyes, &yyes_capacity]])])[, yytoken]b4_locations_if([[, &yylloc]])[};
char const *yymsgp = YY_("syntax error");
int yysyntax_error_status;]b4_lac_if([[
- if (yychar != YYEMPTY)
+ if (yychar != ]b4_symbol(empty, id)[)
YY_LAC_ESTABLISH;]])[
- yysyntax_error_status = YYSYNTAX_ERROR;
+ yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx);
if (yysyntax_error_status == 0)
yymsgp = yymsg;
- else if (yysyntax_error_status == 1)
+ else if (yysyntax_error_status == -1)
{
if (yymsg != yymsgbuf)
YYSTACK_FREE (yymsg);
- yymsg = YY_CAST (char *, YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc)));
- if (!yymsg)
+ yymsg = YY_CAST (char *,
+ YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc)));
+ if (yymsg)
{
- yymsg = yymsgbuf;
- yymsg_alloc = sizeof yymsgbuf;
- yysyntax_error_status = 2;
+ yysyntax_error_status
+ = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx);
+ yymsgp = yymsg;
}
else
{
- yysyntax_error_status = YYSYNTAX_ERROR;
- yymsgp = yymsg;
+ yymsg = yymsgbuf;
+ yymsg_alloc = sizeof yymsgbuf;
+ yysyntax_error_status = YYENOMEM;
}
}
yyerror (]b4_yyerror_args[yymsgp);
- if (yysyntax_error_status == 2)
- goto yyexhaustedlab;
- }
-# undef YYSYNTAX_ERROR
-#endif
+ if (yysyntax_error_status == YYENOMEM)
+ YYNOMEM;
+ }]])[
}
-
-]b4_locations_if([[ yyerror_range[1] = yylloc;]])[
-
+]b4_locations_if([[
+ yyerror_range[1] = yylloc;]])[
if (yyerrstatus == 3)
{
/* If just tried and failed to reuse lookahead token after an
error, discard it. */
- if (yychar <= YYEOF)
+ if (yychar <= ]b4_symbol(eof, [id])[)
{
/* Return failure if at end of input. */
- if (yychar == YYEOF)
+ if (yychar == ]b4_symbol(eof, [id])[)
YYABORT;
}
else
{
yydestruct ("Error: discarding",
yytoken, &yylval]b4_locations_if([, &yylloc])[]b4_user_args[);
- yychar = YYEMPTY;
+ yychar = ]b4_symbol(empty, id)[;
}
}
@@ -1831,6 +2072,7 @@
label yyerrorlab therefore never appears in user code. */
if (0)
YYERROR;
+ ++yynerrs;
/* Do not reclaim the symbols of the rule whose action triggered
this YYERROR. */
@@ -1847,13 +2089,14 @@
yyerrlab1:
yyerrstatus = 3; /* Each real token shifted decrements this. */
+ /* Pop stack until we find a state that shifts the error token. */
for (;;)
{
yyn = yypact[yystate];
if (!yypact_value_is_default (yyn))
{
- yyn += YYTERROR;
- if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR)
+ yyn += ]b4_symbol(error, kind)[;
+ if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == ]b4_symbol(error, kind)[)
{
yyn = yytable[yyn];
if (0 < yyn)
@@ -1867,7 +2110,7 @@
]b4_locations_if([[ yyerror_range[1] = *yylsp;]])[
yydestruct ("Error: popping",
- yystos[yystate], yyvsp]b4_locations_if([, yylsp])[]b4_user_args[);
+ YY_ACCESSING_SYMBOL (yystate), yyvsp]b4_locations_if([, yylsp])[]b4_user_args[);
YYPOPSTACK (1);
yystate = *yyssp;
YY_STACK_PRINT (yyss, yyssp);
@@ -1882,13 +2125,11 @@
YY_IGNORE_MAYBE_UNINITIALIZED_END
]b4_locations_if([[
yyerror_range[2] = yylloc;
- /* Using YYLLOC is tempting, but would change the location of
- the lookahead. YYLOC is available though. */
- YYLLOC_DEFAULT (yyloc, yyerror_range, 2);
- *++yylsp = yyloc;]])[
+ ++yylsp;
+ YYLLOC_DEFAULT (*yylsp, yyerror_range, 2);]])[
/* Shift the error token. */
- YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp);
+ YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp);
yystate = yyn;
goto yynewstate;
@@ -1899,7 +2140,7 @@
`-------------------------------------*/
yyacceptlab:
yyresult = 0;
- goto yyreturn;
+ goto yyreturnlab;
/*-----------------------------------.
@@ -1907,25 +2148,23 @@
`-----------------------------------*/
yyabortlab:
yyresult = 1;
- goto yyreturn;
+ goto yyreturnlab;
-#if ]b4_lac_if([[1]], [[!defined yyoverflow || YYERROR_VERBOSE]])[
-/*-------------------------------------------------.
-| yyexhaustedlab -- memory exhaustion comes here. |
-`-------------------------------------------------*/
+/*-----------------------------------------------------------.
+| yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here. |
+`-----------------------------------------------------------*/
yyexhaustedlab:
yyerror (]b4_yyerror_args[YY_("memory exhausted"));
yyresult = 2;
- /* Fall through. */
-#endif
+ goto yyreturnlab;
-/*-----------------------------------------------------.
-| yyreturn -- parsing is finished, return the result. |
-`-----------------------------------------------------*/
-yyreturn:
- if (yychar != YYEMPTY)
+/*----------------------------------------------------------.
+| yyreturnlab -- parsing is finished, clean up and return. |
+`----------------------------------------------------------*/
+yyreturnlab:
+ if (yychar != ]b4_symbol(empty, id)[)
{
/* Make sure we have latest lookahead translation. See comments at
user semantic actions for why this is necessary. */
@@ -1940,27 +2179,31 @@
while (yyssp != yyss)
{
yydestruct ("Cleanup: popping",
- yystos[*yyssp], yyvsp]b4_locations_if([, yylsp])[]b4_user_args[);
+ YY_ACCESSING_SYMBOL (+*yyssp), yyvsp]b4_locations_if([, yylsp])[]b4_user_args[);
YYPOPSTACK (1);
- }
+ }]b4_push_if([[
+ yyps->yynew = 2;
+ goto yypushreturn;
+
+
+/*-------------------------.
+| yypushreturn -- return. |
+`-------------------------*/
+yypushreturn:]], [[
#ifndef yyoverflow
if (yyss != yyssa)
YYSTACK_FREE (yyss);
#endif]b4_lac_if([[
if (yyes != yyesa)
- YYSTACK_FREE (yyes);]])b4_push_if([[
- yyps->yynew = 1;
-
-
-/*-----------------------------------------.
-| yypushreturn -- ask for the next token. |
-`-----------------------------------------*/
-yypushreturn:]])[
-#if YYERROR_VERBOSE
- if (yymsg != yymsgbuf)
- YYSTACK_FREE (yymsg);
-#endif
+ YYSTACK_FREE (yyes);]])])[
+]b4_parse_error_bmatch([detailed\|verbose],
+[[ if (yymsg != yymsgbuf)
+ YYSTACK_FREE (yymsg);]])[]m4_ifdef([b4_start_symbols], [[
+ if (yyimpl)
+ yyimpl->yynerrs = yynerrs;]])[
return yyresult;
}
-]b4_epilogue[]dnl
+]b4_push_if([b4_parse_state_variable_macros([b4_macro_undef])])[
+]b4_percent_code_get([[epilogue]])[]dnl
+b4_epilogue[]dnl
b4_output_end
diff --git a/common/bison/xslt/bison.xsl b/common/bison/xslt/bison.xsl
index 5c1a358..989a343 100644
--- a/common/bison/xslt/bison.xsl
+++ b/common/bison/xslt/bison.xsl
@@ -3,7 +3,7 @@
<!--
bison.xsl - common templates for Bison XSLT.
- Copyright (C) 2007-2015, 2018-2019 Free Software Foundation, Inc.
+ Copyright (C) 2007-2015, 2018-2021 Free Software Foundation, Inc.
This file is part of Bison, the GNU Compiler Compiler.
@@ -18,12 +18,12 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- xmlns:bison="http://www.gnu.org/software/bison/">
+ xmlns:bison="https://www.gnu.org/software/bison/">
<xsl:key
name="bison:symbolByName"
diff --git a/common/bison/xslt/xml2dot.xsl b/common/bison/xslt/xml2dot.xsl
index 0a7419f..7715d1a 100644
--- a/common/bison/xslt/xml2dot.xsl
+++ b/common/bison/xslt/xml2dot.xsl
@@ -3,7 +3,7 @@
<!--
xml2dot.xsl - transform Bison XML Report into DOT.
- Copyright (C) 2007-2015, 2018-2019 Free Software Foundation, Inc.
+ Copyright (C) 2007-2015, 2018-2021 Free Software Foundation, Inc.
This file is part of Bison, the GNU Compiler Compiler.
@@ -18,14 +18,14 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
Written by Wojciech Polak <polak@gnu.org>.
-->
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- xmlns:bison="http://www.gnu.org/software/bison/">
+ xmlns:bison="https://www.gnu.org/software/bison/">
<xsl:import href="bison.xsl"/>
<xsl:output method="text" encoding="UTF-8" indent="no"/>
@@ -159,7 +159,7 @@
<xsl:param name="prev-rule-number"
select="preceding-sibling::item[1]/@rule-number"/>
<xsl:apply-templates select="key('bison:ruleByNumber', @rule-number)">
- <xsl:with-param name="point" select="@point"/>
+ <xsl:with-param name="dot" select="@dot"/>
<xsl:with-param name="num" select="@rule-number"/>
<xsl:with-param name="prev-lhs"
select="key('bison:ruleByNumber', $prev-rule-number)/lhs[text()]"
@@ -169,7 +169,7 @@
</xsl:template>
<xsl:template match="rule">
- <xsl:param name="point"/>
+ <xsl:param name="dot"/>
<xsl:param name="num"/>
<xsl:param name="prev-lhs"/>
<xsl:text> </xsl:text>
@@ -198,14 +198,14 @@
<xsl:text>:</xsl:text>
</xsl:otherwise>
</xsl:choose>
- <xsl:if test="$point = 0">
+ <xsl:if test="$dot = 0">
<xsl:text> .</xsl:text>
</xsl:if>
<!-- RHS -->
<xsl:for-each select="rhs/symbol|rhs/empty">
<xsl:apply-templates select="."/>
- <xsl:if test="$point = position()">
+ <xsl:if test="$dot = position()">
<xsl:text> .</xsl:text>
</xsl:if>
</xsl:for-each>
diff --git a/common/bison/xslt/xml2text.xsl b/common/bison/xslt/xml2text.xsl
index 72b56e7..1fc5731 100644
--- a/common/bison/xslt/xml2text.xsl
+++ b/common/bison/xslt/xml2text.xsl
@@ -3,7 +3,7 @@
<!--
xml2text.xsl - transform Bison XML Report into plain text.
- Copyright (C) 2007-2015, 2018-2019 Free Software Foundation, Inc.
+ Copyright (C) 2007-2015, 2018-2021 Free Software Foundation, Inc.
This file is part of Bison, the GNU Compiler Compiler.
@@ -18,14 +18,14 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
Written by Wojciech Polak <polak@gnu.org>.
-->
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- xmlns:bison="http://www.gnu.org/software/bison/">
+ xmlns:bison="https://www.gnu.org/software/bison/">
<xsl:import href="bison.xsl"/>
<xsl:output method="text" encoding="UTF-8" indent="no"/>
@@ -52,7 +52,7 @@
<xsl:if test="nonterminal[@usefulness='useless-in-grammar']">
<xsl:text>Nonterminals useless in grammar </xsl:text>
<xsl:for-each select="nonterminal[@usefulness='useless-in-grammar']">
- <xsl:text> </xsl:text>
+ <xsl:text> </xsl:text>
<xsl:value-of select="@name"/>
<xsl:text> </xsl:text>
</xsl:for-each>
@@ -65,7 +65,7 @@
<xsl:text>Terminals unused in grammar </xsl:text>
<xsl:for-each select="terminal[@usefulness='unused-in-grammar']">
<xsl:sort select="@symbol-number" data-type="number"/>
- <xsl:text> </xsl:text>
+ <xsl:text> </xsl:text>
<xsl:value-of select="@name"/>
<xsl:text> </xsl:text>
</xsl:for-each>
@@ -136,6 +136,7 @@
</xsl:template>
<xsl:template match="terminal">
+ <xsl:text> </xsl:text>
<xsl:value-of select="@name"/>
<xsl:call-template name="line-wrap">
<xsl:with-param name="first-line-length">
@@ -148,6 +149,9 @@
</xsl:with-param>
<xsl:with-param name="line-length" select="66" />
<xsl:with-param name="text">
+ <xsl:if test="string-length(@type) != 0">
+ <xsl:value-of select="concat(' <', @type, '>')"/>
+ </xsl:if>
<xsl:value-of select="concat(' (', @token-number, ')')"/>
<xsl:for-each select="key('bison:ruleByRhs', @name)">
<xsl:value-of select="concat(' ', @number)"/>
@@ -157,14 +161,18 @@
</xsl:template>
<xsl:template match="nonterminal">
+ <xsl:text> </xsl:text>
<xsl:value-of select="@name"/>
+ <xsl:if test="string-length(@type) != 0">
+ <xsl:value-of select="concat(' <', @type, '>')"/>
+ </xsl:if>
<xsl:value-of select="concat(' (', @symbol-number, ')')"/>
<xsl:text> </xsl:text>
<xsl:variable name="output">
<xsl:call-template name="line-wrap">
<xsl:with-param name="line-length" select="66" />
<xsl:with-param name="text">
- <xsl:text> </xsl:text>
+ <xsl:text> </xsl:text>
<xsl:if test="key('bison:ruleByLhs', @name)">
<xsl:text>on@left:</xsl:text>
<xsl:for-each select="key('bison:ruleByLhs', @name)">
@@ -173,7 +181,7 @@
</xsl:if>
<xsl:if test="key('bison:ruleByRhs', @name)">
<xsl:if test="key('bison:ruleByLhs', @name)">
- <xsl:text>, </xsl:text>
+ <xsl:text> </xsl:text>
</xsl:if>
<xsl:text>on@right:</xsl:text>
<xsl:for-each select="key('bison:ruleByRhs', @name)">
@@ -300,7 +308,7 @@
name="prev-lhs"
select="key('bison:ruleByNumber', $prev-rule-number)/lhs[text()]"
/>
- <xsl:with-param name="point" select="@point"/>
+ <xsl:with-param name="dot" select="@dot"/>
<xsl:with-param name="lookaheads">
<xsl:apply-templates select="lookaheads"/>
</xsl:with-param>
@@ -311,7 +319,7 @@
<xsl:param name="itemset"/>
<xsl:param name="pad"/>
<xsl:param name="prev-lhs"/>
- <xsl:param name="point"/>
+ <xsl:param name="dot"/>
<xsl:param name="lookaheads"/>
<xsl:if test="$itemset != 'true' and not($prev-lhs = lhs[text()])">
@@ -347,12 +355,12 @@
<!-- RHS -->
<xsl:for-each select="rhs/*">
- <xsl:if test="position() = $point + 1">
- <xsl:text> .</xsl:text>
+ <xsl:if test="position() = $dot + 1">
+ <xsl:text> •</xsl:text>
</xsl:if>
<xsl:apply-templates select="."/>
- <xsl:if test="position() = last() and position() = $point">
- <xsl:text> .</xsl:text>
+ <xsl:if test="position() = last() and position() = $dot">
+ <xsl:text> •</xsl:text>
</xsl:if>
</xsl:for-each>
<xsl:if test="$lookaheads">
diff --git a/common/bison/xslt/xml2xhtml.xsl b/common/bison/xslt/xml2xhtml.xsl
index 087b035..aaa5dba 100644
--- a/common/bison/xslt/xml2xhtml.xsl
+++ b/common/bison/xslt/xml2xhtml.xsl
@@ -3,7 +3,7 @@
<!--
xml2html.xsl - transform Bison XML Report into XHTML.
- Copyright (C) 2007-2015, 2018-2019 Free Software Foundation, Inc.
+ Copyright (C) 2007-2015, 2018-2021 Free Software Foundation, Inc.
This file is part of Bison, the GNU Compiler Compiler.
@@ -18,7 +18,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
Written by Wojciech Polak <polak@gnu.org>.
-->
@@ -26,7 +26,7 @@
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns="http://www.w3.org/1999/xhtml"
- xmlns:bison="http://www.gnu.org/software/bison/">
+ xmlns:bison="https://www.gnu.org/software/bison/">
<xsl:import href="bison.xsl"/>
@@ -75,7 +75,7 @@
ol.lower-alpha {
list-style-type: lower-alpha;
}
- .point {
+ .dot {
color: #cc0000;
}
#footer {
@@ -88,7 +88,7 @@
<xsl:apply-templates select="bison-xml-report"/>
<xsl:text> </xsl:text>
<div id="footer"><hr />This document was generated using
- <a href="http://www.gnu.org/software/bison/" title="GNU Bison">
+ <a href="https://www.gnu.org/software/bison/" title="GNU Bison">
GNU Bison <xsl:value-of select="/bison-xml-report/@version"/></a>
XML Automaton Report.<br />
<!-- default copying notice -->
@@ -227,6 +227,7 @@
<xsl:text> </xsl:text>
<p class="pre">
<xsl:call-template name="style-rule-set">
+ <xsl:with-param name="anchor" select="'true'" />
<xsl:with-param
name="rule-set" select="rules/rule[@usefulness!='useless-in-grammar']"
/>
@@ -238,9 +239,11 @@
</xsl:template>
<xsl:template name="style-rule-set">
+ <xsl:param name="anchor"/>
<xsl:param name="rule-set"/>
<xsl:for-each select="$rule-set">
<xsl:apply-templates select=".">
+ <xsl:with-param name="anchor" select="$anchor"/>
<xsl:with-param name="pad" select="'3'"/>
<xsl:with-param name="prev-lhs">
<xsl:if test="position()>1">
@@ -306,9 +309,10 @@
<xsl:text> Terminals, with rules where they appear</xsl:text>
</h3>
<xsl:text> </xsl:text>
- <p class="pre">
+ <ul>
+ <xsl:text> </xsl:text>
<xsl:apply-templates select="terminal"/>
- </p>
+ </ul>
<xsl:text> </xsl:text>
</xsl:template>
@@ -318,41 +322,64 @@
<xsl:text> Nonterminals, with rules where they appear</xsl:text>
</h3>
<xsl:text> </xsl:text>
- <p class="pre">
+ <ul>
+ <xsl:text> </xsl:text>
<xsl:apply-templates
select="nonterminal[@usefulness!='useless-in-grammar']"
/>
- </p>
+ </ul>
</xsl:template>
<xsl:template match="terminal">
- <b><xsl:value-of select="@name"/></b>
- <xsl:value-of select="concat(' (', @token-number, ')')"/>
- <xsl:for-each select="key('bison:ruleByRhs', @name)">
- <xsl:apply-templates select="." mode="number-link"/>
- </xsl:for-each>
+ <xsl:text> </xsl:text>
+ <li>
+ <b><xsl:value-of select="@name"/></b>
+ <xsl:if test="string-length(@type) != 0">
+ <xsl:value-of select="concat(' <', @type, '>')"/>
+ </xsl:if>
+ <xsl:value-of select="concat(' (', @token-number, ')')"/>
+ <xsl:for-each select="key('bison:ruleByRhs', @name)">
+ <xsl:apply-templates select="." mode="number-link"/>
+ </xsl:for-each>
+ </li>
<xsl:text> </xsl:text>
</xsl:template>
<xsl:template match="nonterminal">
- <b><xsl:value-of select="@name"/></b>
- <xsl:value-of select="concat(' (', @symbol-number, ')')"/>
- <xsl:text> </xsl:text>
- <xsl:if test="key('bison:ruleByLhs', @name)">
- <xsl:text>on left:</xsl:text>
- <xsl:for-each select="key('bison:ruleByLhs', @name)">
- <xsl:apply-templates select="." mode="number-link"/>
- </xsl:for-each>
- </xsl:if>
- <xsl:if test="key('bison:ruleByRhs', @name)">
- <xsl:if test="key('bison:ruleByLhs', @name)">
- <xsl:text> </xsl:text>
+ <xsl:text> </xsl:text>
+ <li>
+ <b><xsl:value-of select="@name"/></b>
+ <xsl:if test="string-length(@type) != 0">
+ <xsl:value-of select="concat(' <', @type, '>')"/>
</xsl:if>
- <xsl:text>on right:</xsl:text>
- <xsl:for-each select="key('bison:ruleByRhs', @name)">
- <xsl:apply-templates select="." mode="number-link"/>
- </xsl:for-each>
- </xsl:if>
+ <xsl:value-of select="concat(' (', @symbol-number, ')')"/>
+ <xsl:text> </xsl:text>
+ <ul>
+ <xsl:text> </xsl:text>
+ <xsl:if test="key('bison:ruleByLhs', @name)">
+ <xsl:text> </xsl:text>
+ <li>
+ <xsl:text>on left:</xsl:text>
+ <xsl:for-each select="key('bison:ruleByLhs', @name)">
+ <xsl:apply-templates select="." mode="number-link"/>
+ </xsl:for-each>
+ </li>
+ <xsl:text> </xsl:text>
+ </xsl:if>
+ <xsl:if test="key('bison:ruleByRhs', @name)">
+ <xsl:text> </xsl:text>
+ <li>
+ <xsl:text>on right:</xsl:text>
+ <xsl:for-each select="key('bison:ruleByRhs', @name)">
+ <xsl:apply-templates select="." mode="number-link"/>
+ </xsl:for-each>
+ </li>
+ <xsl:text> </xsl:text>
+ </xsl:if>
+ <xsl:text> </xsl:text>
+ </ul>
+ <xsl:text> </xsl:text>
+ </li>
<xsl:text> </xsl:text>
</xsl:template>
@@ -385,7 +412,7 @@
<xsl:value-of select="concat('state_', @number)"/>
</xsl:attribute>
</a>
- <xsl:text>state </xsl:text>
+ <xsl:text>State </xsl:text>
<xsl:value-of select="@number"/>
</h3>
<xsl:text> </xsl:text>
@@ -457,35 +484,44 @@
<xsl:with-param name="prev-lhs"
select="key('bison:ruleByNumber', $prev-rule-number)/lhs[text()]"
/>
- <xsl:with-param name="point" select="@point"/>
+ <xsl:with-param name="dot" select="@dot"/>
<xsl:with-param name="lookaheads">
<xsl:apply-templates select="lookaheads"/>
</xsl:with-param>
</xsl:apply-templates>
</xsl:template>
+<!--
+anchor = 'true': define as an <a> anchor.
+itemset = 'true': show the items.
+ -->
<xsl:template match="rule">
+ <xsl:param name="anchor"/>
<xsl:param name="itemset"/>
<xsl:param name="pad"/>
<xsl:param name="prev-lhs"/>
- <xsl:param name="point"/>
+ <xsl:param name="dot"/>
<xsl:param name="lookaheads"/>
<xsl:if test="$itemset != 'true' and not($prev-lhs = lhs[text()])">
<xsl:text> </xsl:text>
</xsl:if>
- <xsl:if test="$itemset != 'true'">
- <a>
- <xsl:attribute name="name">
- <xsl:value-of select="concat('rule_', @number)"/>
- </xsl:attribute>
- </a>
- </xsl:if>
<xsl:text> </xsl:text>
<xsl:choose>
- <xsl:when test="$itemset = 'true'">
+ <xsl:when test="$anchor = 'true'">
+ <a>
+ <xsl:attribute name="name">
+ <xsl:value-of select="concat('rule_', @number)"/>
+ </xsl:attribute>
+ <xsl:call-template name="lpad">
+ <xsl:with-param name="str" select="string(@number)"/>
+ <xsl:with-param name="pad" select="number($pad)"/>
+ </xsl:call-template>
+ </a>
+ </xsl:when>
+ <xsl:otherwise>
<a>
<xsl:attribute name="href">
<xsl:value-of select="concat('#rule_', @number)"/>
@@ -495,25 +531,13 @@
<xsl:with-param name="pad" select="number($pad)"/>
</xsl:call-template>
</a>
- </xsl:when>
- <xsl:otherwise>
- <xsl:call-template name="lpad">
- <xsl:with-param name="str" select="string(@number)"/>
- <xsl:with-param name="pad" select="number($pad)"/>
- </xsl:call-template>
</xsl:otherwise>
</xsl:choose>
<xsl:text> </xsl:text>
<!-- LHS -->
<xsl:choose>
- <xsl:when test="$itemset != 'true' and $prev-lhs = lhs[text()]">
- <xsl:call-template name="lpad">
- <xsl:with-param name="str" select="'|'"/>
- <xsl:with-param name="pad" select="number(string-length(lhs[text()])) + 2"/>
- </xsl:call-template>
- </xsl:when>
- <xsl:when test="$itemset = 'true' and $prev-lhs = lhs[text()]">
+ <xsl:when test="$prev-lhs = lhs[text()]">
<xsl:call-template name="lpad">
<xsl:with-param name="str" select="'|'"/>
<xsl:with-param name="pad" select="number(string-length(lhs[text()])) + 2"/>
@@ -529,14 +553,14 @@
<!-- RHS -->
<xsl:for-each select="rhs/*">
- <xsl:if test="position() = $point + 1">
+ <xsl:if test="position() = $dot + 1">
<xsl:text> </xsl:text>
- <span class="point">.</span>
+ <span class="dot">•</span>
</xsl:if>
<xsl:apply-templates select="."/>
- <xsl:if test="position() = last() and position() = $point">
+ <xsl:if test="position() = last() and position() = $dot">
<xsl:text> </xsl:text>
- <span class="point">.</span>
+ <span class="dot">•</span>
</xsl:if>
</xsl:for-each>
<xsl:if test="$lookaheads">
diff --git a/common/py3-stdlib/LICENSE b/common/py3-stdlib/LICENSE
index f42f8ad..02a5145 100644
--- a/common/py3-stdlib/LICENSE
+++ b/common/py3-stdlib/LICENSE
@@ -84,7 +84,7 @@
distribute, and otherwise use Python alone or in any derivative version,
provided, however, that PSF's License Agreement and PSF's notice of copyright,
i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Python Software Foundation;
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation;
All Rights Reserved" are retained in Python alone or in any derivative version
prepared by Licensee.
@@ -191,9 +191,9 @@
Agreement, Licensee may substitute the following text (omitting the
quotes): "Python 1.6.1 is made available subject to the terms and
conditions in CNRI's License Agreement. This Agreement together with
-Python 1.6.1 may be located on the Internet using the following
+Python 1.6.1 may be located on the internet using the following
unique, persistent identifier (known as a handle): 1895.22/1013. This
-Agreement may also be obtained from a proxy server on the Internet
+Agreement may also be obtained from a proxy server on the internet
using the following URL: http://hdl.handle.net/1895.22/1013".
3. In the event Licensee prepares a derivative work that is based on
diff --git a/common/py3-stdlib/__future__.py b/common/py3-stdlib/__future__.py
index 0e7b555..97dc90c 100644
--- a/common/py3-stdlib/__future__.py
+++ b/common/py3-stdlib/__future__.py
@@ -42,7 +42,7 @@
argument to the builtin function compile() to enable the feature in
dynamically compiled code. This flag is stored in the .compiler_flag
attribute on _Future instances. These values must match the appropriate
-#defines of CO_xxx flags in Include/compile.h.
+#defines of CO_xxx flags in Include/cpython/compile.h.
No feature line is ever to be deleted from this file.
"""
@@ -143,5 +143,5 @@
CO_FUTURE_GENERATOR_STOP)
annotations = _Feature((3, 7, 0, "beta", 1),
- (3, 10, 0, "alpha", 0),
+ (3, 11, 0, "alpha", 0),
CO_FUTURE_ANNOTATIONS)
diff --git a/common/py3-stdlib/_bootlocale.py b/common/py3-stdlib/_bootlocale.py
deleted file mode 100644
index 3273a3b..0000000
--- a/common/py3-stdlib/_bootlocale.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""A minimal subset of the locale module used at interpreter startup
-(imported by the _io module), in order to reduce startup time.
-
-Don't import directly from third-party code; use the `locale` module instead!
-"""
-
-import sys
-import _locale
-
-if sys.platform.startswith("win"):
- def getpreferredencoding(do_setlocale=True):
- if sys.flags.utf8_mode:
- return 'UTF-8'
- return _locale._getdefaultlocale()[1]
-else:
- try:
- _locale.CODESET
- except AttributeError:
- if hasattr(sys, 'getandroidapilevel'):
- # On Android langinfo.h and CODESET are missing, and UTF-8 is
- # always used in mbstowcs() and wcstombs().
- def getpreferredencoding(do_setlocale=True):
- return 'UTF-8'
- else:
- def getpreferredencoding(do_setlocale=True):
- if sys.flags.utf8_mode:
- return 'UTF-8'
- # This path for legacy systems needs the more complex
- # getdefaultlocale() function, import the full locale module.
- import locale
- return locale.getpreferredencoding(do_setlocale)
- else:
- def getpreferredencoding(do_setlocale=True):
- assert not do_setlocale
- if sys.flags.utf8_mode:
- return 'UTF-8'
- result = _locale.nl_langinfo(_locale.CODESET)
- if not result and sys.platform == 'darwin':
- # nl_langinfo can return an empty string
- # when the setting has an invalid value.
- # Default to UTF-8 in that case because
- # UTF-8 is the default charset on OSX and
- # returning nothing will crash the
- # interpreter.
- result = 'UTF-8'
- return result
diff --git a/common/py3-stdlib/_collections_abc.py b/common/py3-stdlib/_collections_abc.py
index 36cd993..40417dc 100644
--- a/common/py3-stdlib/_collections_abc.py
+++ b/common/py3-stdlib/_collections_abc.py
@@ -10,6 +10,10 @@
import sys
GenericAlias = type(list[int])
+EllipsisType = type(...)
+def _f(): pass
+FunctionType = type(_f)
+del _f
__all__ = ["Awaitable", "Coroutine",
"AsyncIterable", "AsyncIterator", "AsyncGenerator",
@@ -409,6 +413,143 @@
return NotImplemented
+class _CallableGenericAlias(GenericAlias):
+ """ Represent `Callable[argtypes, resulttype]`.
+
+ This sets ``__args__`` to a tuple containing the flattened ``argtypes``
+ followed by ``resulttype``.
+
+ Example: ``Callable[[int, str], float]`` sets ``__args__`` to
+ ``(int, str, float)``.
+ """
+
+ __slots__ = ()
+
+ def __new__(cls, origin, args):
+ if not (isinstance(args, tuple) and len(args) == 2):
+ raise TypeError(
+ "Callable must be used as Callable[[arg, ...], result].")
+ t_args, t_result = args
+ if isinstance(t_args, list):
+ args = (*t_args, t_result)
+ elif not _is_param_expr(t_args):
+ raise TypeError(f"Expected a list of types, an ellipsis, "
+ f"ParamSpec, or Concatenate. Got {t_args}")
+ return super().__new__(cls, origin, args)
+
+ @property
+ def __parameters__(self):
+ params = []
+ for arg in self.__args__:
+ # Looks like a genericalias
+ if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple):
+ params.extend(arg.__parameters__)
+ else:
+ if _is_typevarlike(arg):
+ params.append(arg)
+ return tuple(dict.fromkeys(params))
+
+ def __repr__(self):
+ if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
+ return super().__repr__()
+ return (f'collections.abc.Callable'
+ f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
+ f'{_type_repr(self.__args__[-1])}]')
+
+ def __reduce__(self):
+ args = self.__args__
+ if not (len(args) == 2 and _is_param_expr(args[0])):
+ args = list(args[:-1]), args[-1]
+ return _CallableGenericAlias, (Callable, args)
+
+ def __getitem__(self, item):
+ # Called during TypeVar substitution, returns the custom subclass
+ # rather than the default types.GenericAlias object. Most of the
+ # code is copied from typing's _GenericAlias and the builtin
+ # types.GenericAlias.
+
+ # A special case in PEP 612 where if X = Callable[P, int],
+ # then X[int, str] == X[[int, str]].
+ param_len = len(self.__parameters__)
+ if param_len == 0:
+ raise TypeError(f'{self} is not a generic class')
+ if not isinstance(item, tuple):
+ item = (item,)
+ if (param_len == 1 and _is_param_expr(self.__parameters__[0])
+ and item and not _is_param_expr(item[0])):
+ item = (list(item),)
+ item_len = len(item)
+ if item_len != param_len:
+ raise TypeError(f'Too {"many" if item_len > param_len else "few"}'
+ f' arguments for {self};'
+ f' actual {item_len}, expected {param_len}')
+ subst = dict(zip(self.__parameters__, item))
+ new_args = []
+ for arg in self.__args__:
+ if _is_typevarlike(arg):
+ if _is_param_expr(arg):
+ arg = subst[arg]
+ if not _is_param_expr(arg):
+ raise TypeError(f"Expected a list of types, an ellipsis, "
+ f"ParamSpec, or Concatenate. Got {arg}")
+ else:
+ arg = subst[arg]
+ # Looks like a GenericAlias
+ elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple):
+ subparams = arg.__parameters__
+ if subparams:
+ subargs = tuple(subst[x] for x in subparams)
+ arg = arg[subargs]
+ if isinstance(arg, tuple):
+ new_args.extend(arg)
+ else:
+ new_args.append(arg)
+
+ # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
+ if not isinstance(new_args[0], list):
+ t_result = new_args[-1]
+ t_args = new_args[:-1]
+ new_args = (t_args, t_result)
+ return _CallableGenericAlias(Callable, tuple(new_args))
+
+
+def _is_typevarlike(arg):
+ obj = type(arg)
+ # looks like a TypeVar/ParamSpec
+ return (obj.__module__ == 'typing'
+ and obj.__name__ in {'ParamSpec', 'TypeVar'})
+
+def _is_param_expr(obj):
+ """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
+ ``_ConcatenateGenericAlias`` from typing.py
+ """
+ if obj is Ellipsis:
+ return True
+ if isinstance(obj, list):
+ return True
+ obj = type(obj)
+ names = ('ParamSpec', '_ConcatenateGenericAlias')
+ return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names)
+
+def _type_repr(obj):
+ """Return the repr() of an object, special-casing types (internal helper).
+
+ Copied from :mod:`typing` since collections.abc
+ shouldn't depend on that module.
+ """
+ if isinstance(obj, GenericAlias):
+ return repr(obj)
+ if isinstance(obj, type):
+ if obj.__module__ == 'builtins':
+ return obj.__qualname__
+ return f'{obj.__module__}.{obj.__qualname__}'
+ if obj is Ellipsis:
+ return '...'
+ if isinstance(obj, FunctionType):
+ return obj.__name__
+ return repr(obj)
+
+
class Callable(metaclass=ABCMeta):
__slots__ = ()
@@ -423,14 +564,13 @@
return _check_methods(C, "__call__")
return NotImplemented
- __class_getitem__ = classmethod(GenericAlias)
+ __class_getitem__ = classmethod(_CallableGenericAlias)
### SETS ###
class Set(Collection):
-
"""A set is a finite, iterable container.
This class provides concrete generic implementations of all
@@ -558,6 +698,7 @@
hx = hash(x)
h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167
h &= MASK
+ h ^= (h >> 11) ^ (h >> 25)
h = h * 69069 + 907133923
h &= MASK
if h > MAX:
@@ -655,19 +796,19 @@
### MAPPINGS ###
-
class Mapping(Collection):
-
- __slots__ = ()
-
"""A Mapping is a generic container for associating key/value
pairs.
This class provides concrete generic implementations of all
methods except for __getitem__, __iter__, and __len__.
-
"""
+ __slots__ = ()
+
+ # Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set.
+ __abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING
+
@abstractmethod
def __getitem__(self, key):
raise KeyError
@@ -706,7 +847,6 @@
__reversed__ = None
-
Mapping.register(mappingproxy)
@@ -731,7 +871,7 @@
__slots__ = ()
@classmethod
- def _from_iterable(self, it):
+ def _from_iterable(cls, it):
return set(it)
def __contains__(self, key):
@@ -749,7 +889,7 @@
__slots__ = ()
@classmethod
- def _from_iterable(self, it):
+ def _from_iterable(cls, it):
return set(it)
def __contains__(self, item):
@@ -789,18 +929,16 @@
class MutableMapping(Mapping):
-
- __slots__ = ()
-
"""A MutableMapping is a generic container for associating
key/value pairs.
This class provides concrete generic implementations of all
methods except for __getitem__, __setitem__, __delitem__,
__iter__, and __len__.
-
"""
+ __slots__ = ()
+
@abstractmethod
def __setitem__(self, key, value):
raise KeyError
@@ -877,9 +1015,7 @@
### SEQUENCES ###
-
class Sequence(Reversible, Collection):
-
"""All the operations on a read-only sequence.
Concrete subclasses must override __new__ or __init__,
@@ -888,6 +1024,9 @@
__slots__ = ()
+ # Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set.
+ __abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE
+
@abstractmethod
def __getitem__(self, index):
raise IndexError
@@ -939,7 +1078,6 @@
'S.count(value) -> integer -- return number of occurrences of value'
return sum(1 for v in self if v is value or v == value)
-
Sequence.register(tuple)
Sequence.register(str)
Sequence.register(range)
@@ -947,7 +1085,6 @@
class ByteString(Sequence):
-
"""This unifies bytes and bytearray.
XXX Should add all their methods.
@@ -960,16 +1097,14 @@
class MutableSequence(Sequence):
-
- __slots__ = ()
-
"""All the operations on a read-write sequence.
Concrete subclasses must provide __new__ or __init__,
__getitem__, __setitem__, __delitem__, __len__, and insert().
-
"""
+ __slots__ = ()
+
@abstractmethod
def __setitem__(self, index, value):
raise IndexError
diff --git a/common/py3-stdlib/_compression.py b/common/py3-stdlib/_compression.py
index b00f31b..e8b70aa 100644
--- a/common/py3-stdlib/_compression.py
+++ b/common/py3-stdlib/_compression.py
@@ -1,7 +1,7 @@
"""Internal classes used by the gzip, lzma and bz2 modules"""
import io
-
+import sys
BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size
@@ -110,6 +110,16 @@
self._pos += len(data)
return data
+ def readall(self):
+ chunks = []
+ # sys.maxsize means the max length of output buffer is unlimited,
+ # so that the whole input buffer can be decompressed within one
+ # .decompress() call.
+ while data := self.read(sys.maxsize):
+ chunks.append(data)
+
+ return b"".join(chunks)
+
# Rewind the file to the beginning of the data stream.
def _rewind(self):
self._fp.seek(0)
diff --git a/common/py3-stdlib/_markupbase.py b/common/py3-stdlib/_markupbase.py
index 2af5f1c..3ad7e27 100644
--- a/common/py3-stdlib/_markupbase.py
+++ b/common/py3-stdlib/_markupbase.py
@@ -29,10 +29,6 @@
raise RuntimeError(
"_markupbase.ParserBase must be subclassed")
- def error(self, message):
- raise NotImplementedError(
- "subclasses of ParserBase must override error()")
-
def reset(self):
self.lineno = 1
self.offset = 0
@@ -131,12 +127,11 @@
# also in data attribute specifications of attlist declaration
# also link type declaration subsets in linktype declarations
# also link attribute specification lists in link declarations
- self.error("unsupported '[' char in %s declaration" % decltype)
+ raise AssertionError("unsupported '[' char in %s declaration" % decltype)
else:
- self.error("unexpected '[' char in declaration")
+ raise AssertionError("unexpected '[' char in declaration")
else:
- self.error(
- "unexpected %r char in declaration" % rawdata[j])
+ raise AssertionError("unexpected %r char in declaration" % rawdata[j])
if j < 0:
return j
return -1 # incomplete
@@ -156,7 +151,9 @@
# look for MS Office ]> ending
match= _msmarkedsectionclose.search(rawdata, i+3)
else:
- self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
+ raise AssertionError(
+ 'unknown status keyword %r in marked section' % rawdata[i+3:j]
+ )
if not match:
return -1
if report:
@@ -168,7 +165,7 @@
def parse_comment(self, i, report=1):
rawdata = self.rawdata
if rawdata[i:i+4] != '<!--':
- self.error('unexpected call to parse_comment()')
+ raise AssertionError('unexpected call to parse_comment()')
match = _commentclose.search(rawdata, i+4)
if not match:
return -1
@@ -192,7 +189,9 @@
return -1
if s != "<!":
self.updatepos(declstartpos, j + 1)
- self.error("unexpected char in internal subset (in %r)" % s)
+ raise AssertionError(
+ "unexpected char in internal subset (in %r)" % s
+ )
if (j + 2) == n:
# end of buffer; incomplete
return -1
@@ -209,8 +208,9 @@
return -1
if name not in {"attlist", "element", "entity", "notation"}:
self.updatepos(declstartpos, j + 2)
- self.error(
- "unknown declaration %r in internal subset" % name)
+ raise AssertionError(
+ "unknown declaration %r in internal subset" % name
+ )
# handle the individual names
meth = getattr(self, "_parse_doctype_" + name)
j = meth(j, declstartpos)
@@ -234,14 +234,14 @@
if rawdata[j] == ">":
return j
self.updatepos(declstartpos, j)
- self.error("unexpected char after internal subset")
+ raise AssertionError("unexpected char after internal subset")
else:
return -1
elif c.isspace():
j = j + 1
else:
self.updatepos(declstartpos, j)
- self.error("unexpected char %r in internal subset" % c)
+ raise AssertionError("unexpected char %r in internal subset" % c)
# end of buffer reached
return -1
@@ -387,8 +387,9 @@
return name.lower(), m.end()
else:
self.updatepos(declstartpos, i)
- self.error("expected name token at %r"
- % rawdata[declstartpos:declstartpos+20])
+ raise AssertionError(
+ "expected name token at %r" % rawdata[declstartpos:declstartpos+20]
+ )
# To be overridden -- handlers for unknown objects
def unknown_decl(self, data):
diff --git a/common/py3-stdlib/_osx_support.py b/common/py3-stdlib/_osx_support.py
index 37975fe..aa66c8b 100644
--- a/common/py3-stdlib/_osx_support.py
+++ b/common/py3-stdlib/_osx_support.py
@@ -96,7 +96,7 @@
if _SYSTEM_VERSION is None:
_SYSTEM_VERSION = ''
try:
- f = open('/System/Library/CoreServices/SystemVersion.plist')
+ f = open('/System/Library/CoreServices/SystemVersion.plist', encoding="utf-8")
except OSError:
# We're on a plain darwin box, fall back to the default
# behaviour.
@@ -156,9 +156,9 @@
if _cache_default_sysroot is not None:
return _cache_default_sysroot
-
+
contents = _read_output('%s -c -E -v - </dev/null' % (cc,), True)
- in_incdirs = False
+ in_incdirs = False
for line in contents.splitlines():
if line.startswith("#include <...>"):
in_incdirs = True
@@ -428,10 +428,9 @@
break
if sysroot and not os.path.isdir(sysroot):
- from distutils import log
- log.warn("Compiling with an SDK that doesn't seem to exist: %s",
- sysroot)
- log.warn("Please check your Xcode installation")
+ sys.stderr.write(f"Compiling with an SDK that doesn't seem to exist: {sysroot}\n")
+ sys.stderr.write("Please check your Xcode installation\n")
+ sys.stderr.flush()
return compiler_so
@@ -482,7 +481,7 @@
This customization is performed when the first
extension module build is requested
- in distutils.sysconfig.customize_compiler).
+ in distutils.sysconfig.customize_compiler.
"""
# Find a compiler to use for extension module builds
@@ -525,10 +524,10 @@
try:
macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
except ValueError:
- macrelease = (10, 0)
+ macrelease = (10, 3)
else:
# assume no universal support
- macrelease = (10, 0)
+ macrelease = (10, 3)
if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
# The universal build will build fat binaries, but not on
diff --git a/common/py3-stdlib/_pydecimal.py b/common/py3-stdlib/_pydecimal.py
index ab989e5..3d6cece 100644
--- a/common/py3-stdlib/_pydecimal.py
+++ b/common/py3-stdlib/_pydecimal.py
@@ -951,7 +951,7 @@
if self.is_snan():
raise TypeError('Cannot hash a signaling NaN value.')
elif self.is_nan():
- return _PyHASH_NAN
+ return object.__hash__(self)
else:
if self._sign:
return -_PyHASH_INF
diff --git a/common/py3-stdlib/_pyio.py b/common/py3-stdlib/_pyio.py
index 4804ed2..fb867fb 100644
--- a/common/py3-stdlib/_pyio.py
+++ b/common/py3-stdlib/_pyio.py
@@ -40,6 +40,36 @@
_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE
+def text_encoding(encoding, stacklevel=2):
+ """
+ A helper function to choose the text encoding.
+
+ When encoding is not None, just return it.
+ Otherwise, return the default text encoding (i.e. "locale").
+
+ This function emits an EncodingWarning if *encoding* is None and
+ sys.flags.warn_default_encoding is true.
+
+ This can be used in APIs with an encoding=None parameter
+ that pass it to TextIOWrapper or open.
+ However, please consider using encoding="utf-8" for new APIs.
+ """
+ if encoding is None:
+ encoding = "locale"
+ if sys.flags.warn_default_encoding:
+ import warnings
+ warnings.warn("'encoding' argument not specified.",
+ EncodingWarning, stacklevel + 1)
+ return encoding
+
+
+# Wrapper for builtins.open
+#
+# Trick so that open() won't become a bound method when stored
+# as a class variable (as dbm.dumb does).
+#
+# See init_set_builtins_open() in Python/pylifecycle.c.
+@staticmethod
def open(file, mode="r", buffering=-1, encoding=None, errors=None,
newline=None, closefd=True, opener=None):
@@ -248,6 +278,7 @@
result = buffer
if binary:
return result
+ encoding = text_encoding(encoding)
text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
result = text
text.mode = mode
@@ -280,27 +311,20 @@
open_code = _open_code_with_warning
-class DocDescriptor:
- """Helper for builtins.open.__doc__
- """
- def __get__(self, obj, typ=None):
- return (
- "open(file, mode='r', buffering=-1, encoding=None, "
- "errors=None, newline=None, closefd=True)\n\n" +
- open.__doc__)
-
-class OpenWrapper:
- """Wrapper for builtins.open
-
- Trick so that open won't become a bound method when stored
- as a class variable (as dbm.dumb does).
-
- See initstdio() in Python/pylifecycle.c.
- """
- __doc__ = DocDescriptor()
-
- def __new__(cls, *args, **kwargs):
- return open(*args, **kwargs)
+def __getattr__(name):
+ if name == "OpenWrapper":
+ # bpo-43680: Until Python 3.9, _pyio.open was not a static method and
+ # builtins.open was set to OpenWrapper to not become a bound method
+ # when set to a class variable. _io.open is a built-in function whereas
+ # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now
+ # a static method, and builtins.open() is now io.open().
+ import warnings
+ warnings.warn('OpenWrapper is deprecated, use open instead',
+ DeprecationWarning, stacklevel=2)
+ global OpenWrapper
+ OpenWrapper = open
+ return OpenWrapper
+ raise AttributeError(name)
# In normal operation, both `UnsupportedOperation`s should be bound to the
@@ -314,8 +338,7 @@
class IOBase(metaclass=abc.ABCMeta):
- """The abstract base class for all I/O classes, acting on streams of
- bytes. There is no public constructor.
+ """The abstract base class for all I/O classes.
This class provides dummy implementations for many methods that
derived classes can override selectively; the default implementations
@@ -1821,7 +1844,7 @@
"""Base class for text I/O.
This class provides a character and line based interface to stream
- I/O. There is no public constructor.
+ I/O.
"""
def read(self, size=-1):
@@ -2004,19 +2027,22 @@
def __init__(self, buffer, encoding=None, errors=None, newline=None,
line_buffering=False, write_through=False):
self._check_newline(newline)
- if encoding is None:
+ encoding = text_encoding(encoding)
+
+ if encoding == "locale":
try:
- encoding = os.device_encoding(buffer.fileno())
+ encoding = os.device_encoding(buffer.fileno()) or "locale"
except (AttributeError, UnsupportedOperation):
pass
- if encoding is None:
- try:
- import locale
- except ImportError:
- # Importing locale may fail if Python is being built
- encoding = "ascii"
- else:
- encoding = locale.getpreferredencoding(False)
+
+ if encoding == "locale":
+ try:
+ import locale
+ except ImportError:
+ # Importing locale may fail if Python is being built
+ encoding = "utf-8"
+ else:
+ encoding = locale.getpreferredencoding(False)
if not isinstance(encoding, str):
raise ValueError("invalid encoding: %r" % encoding)
diff --git a/common/py3-stdlib/_sitebuiltins.py b/common/py3-stdlib/_sitebuiltins.py
index c29cf4b..c66269a 100644
--- a/common/py3-stdlib/_sitebuiltins.py
+++ b/common/py3-stdlib/_sitebuiltins.py
@@ -47,7 +47,7 @@
data = None
for filename in self.__filenames:
try:
- with open(filename, "r") as fp:
+ with open(filename, encoding='utf-8') as fp:
data = fp.read()
break
except OSError:
diff --git a/common/py3-stdlib/_strptime.py b/common/py3-stdlib/_strptime.py
index 5df37f5..b97dfcc 100644
--- a/common/py3-stdlib/_strptime.py
+++ b/common/py3-stdlib/_strptime.py
@@ -201,7 +201,7 @@
#XXX: Does 'Y' need to worry about having less or more than
# 4 digits?
'Y': r"(?P<Y>\d\d\d\d)",
- 'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|Z)",
+ 'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))",
'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
diff --git a/common/py3-stdlib/_weakrefset.py b/common/py3-stdlib/_weakrefset.py
index b267780..2a27684 100644
--- a/common/py3-stdlib/_weakrefset.py
+++ b/common/py3-stdlib/_weakrefset.py
@@ -51,10 +51,14 @@
self.update(data)
def _commit_removals(self):
- l = self._pending_removals
+ pop = self._pending_removals.pop
discard = self.data.discard
- while l:
- discard(l.pop())
+ while True:
+ try:
+ item = pop()
+ except IndexError:
+ return
+ discard(item)
def __iter__(self):
with _IterationGuard(self):
diff --git a/common/py3-stdlib/abc.py b/common/py3-stdlib/abc.py
index 431b640..3c552ce 100644
--- a/common/py3-stdlib/abc.py
+++ b/common/py3-stdlib/abc.py
@@ -28,7 +28,14 @@
class abstractclassmethod(classmethod):
"""A decorator indicating abstract classmethods.
- Deprecated, use 'classmethod' with 'abstractmethod' instead.
+ Deprecated, use 'classmethod' with 'abstractmethod' instead:
+
+ class C(ABC):
+ @classmethod
+ @abstractmethod
+ def my_abstract_classmethod(cls, ...):
+ ...
+
"""
__isabstractmethod__ = True
@@ -41,7 +48,14 @@
class abstractstaticmethod(staticmethod):
"""A decorator indicating abstract staticmethods.
- Deprecated, use 'staticmethod' with 'abstractmethod' instead.
+ Deprecated, use 'staticmethod' with 'abstractmethod' instead:
+
+ class C(ABC):
+ @staticmethod
+ @abstractmethod
+ def my_abstract_staticmethod(...):
+ ...
+
"""
__isabstractmethod__ = True
@@ -54,7 +68,14 @@
class abstractproperty(property):
"""A decorator indicating abstract properties.
- Deprecated, use 'property' with 'abstractmethod' instead.
+ Deprecated, use 'property' with 'abstractmethod' instead:
+
+ class C(ABC):
+ @property
+ @abstractmethod
+ def my_abstract_property(self):
+ ...
+
"""
__isabstractmethod__ = True
@@ -122,6 +143,44 @@
_reset_caches(cls)
+def update_abstractmethods(cls):
+ """Recalculate the set of abstract methods of an abstract class.
+
+ If a class has had one of its abstract methods implemented after the
+ class was created, the method will not be considered implemented until
+ this function is called. Alternatively, if a new abstract method has been
+ added to the class, it will only be considered an abstract method of the
+ class after this function is called.
+
+ This function should be called before any use is made of the class,
+ usually in class decorators that add methods to the subject class.
+
+ Returns cls, to allow usage as a class decorator.
+
+ If cls is not an instance of ABCMeta, does nothing.
+ """
+ if not hasattr(cls, '__abstractmethods__'):
+ # We check for __abstractmethods__ here because cls might by a C
+ # implementation or a python implementation (especially during
+ # testing), and we want to handle both cases.
+ return cls
+
+ abstracts = set()
+ # Check the existing abstract methods of the parents, keep only the ones
+ # that are not implemented.
+ for scls in cls.__bases__:
+ for name in getattr(scls, '__abstractmethods__', ()):
+ value = getattr(cls, name, None)
+ if getattr(value, "__isabstractmethod__", False):
+ abstracts.add(name)
+ # Also add any other newly added abstract methods.
+ for name, value in cls.__dict__.items():
+ if getattr(value, "__isabstractmethod__", False):
+ abstracts.add(name)
+ cls.__abstractmethods__ = frozenset(abstracts)
+ return cls
+
+
class ABC(metaclass=ABCMeta):
"""Helper class that provides a standard way to create an ABC using
inheritance.
diff --git a/common/py3-stdlib/argparse.py b/common/py3-stdlib/argparse.py
index 2fb1da5..2c0dd85 100644
--- a/common/py3-stdlib/argparse.py
+++ b/common/py3-stdlib/argparse.py
@@ -392,6 +392,9 @@
group_actions = set()
inserts = {}
for group in groups:
+ if not group._group_actions:
+ raise ValueError(f'empty group {group}')
+
try:
start = actions.index(group._group_actions[0])
except ValueError:
@@ -526,12 +529,13 @@
parts = [action_header]
# if there was help for the action, add lines of help text
- if action.help:
+ if action.help and action.help.strip():
help_text = self._expand_help(action)
- help_lines = self._split_lines(help_text, help_width)
- parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
- for line in help_lines[1:]:
- parts.append('%*s%s\n' % (help_position, '', line))
+ if help_text:
+ help_lines = self._split_lines(help_text, help_width)
+ parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+ for line in help_lines[1:]:
+ parts.append('%*s%s\n' % (help_position, '', line))
# or add a newline if the description doesn't end with one
elif not action_header.endswith('\n'):
@@ -722,11 +726,13 @@
if argument is None:
return None
elif argument.option_strings:
- return '/'.join(argument.option_strings)
+ return '/'.join(argument.option_strings)
elif argument.metavar not in (None, SUPPRESS):
return argument.metavar
elif argument.dest not in (None, SUPPRESS):
return argument.dest
+ elif argument.choices:
+ return '{' + ','.join(argument.choices) + '}'
else:
return None
@@ -872,8 +878,8 @@
option_string = '--no-' + option_string[2:]
_option_strings.append(option_string)
- if help is not None and default is not None:
- help += f" (default: {default})"
+ if help is not None and default is not None and default is not SUPPRESS:
+ help += " (default: %(default)s)"
super().__init__(
option_strings=_option_strings,
@@ -1250,9 +1256,9 @@
# the special argument "-" means sys.std{in,out}
if string == '-':
if 'r' in self._mode:
- return _sys.stdin
- elif 'w' in self._mode:
- return _sys.stdout
+ return _sys.stdin.buffer if 'b' in self._mode else _sys.stdin
+ elif any(c in self._mode for c in 'wax'):
+ return _sys.stdout.buffer if 'b' in self._mode else _sys.stdout
else:
msg = _('argument "-" with mode %r') % self._mode
raise ValueError(msg)
@@ -1666,7 +1672,8 @@
"""Object for parsing command line strings into Python objects.
Keyword Arguments:
- - prog -- The name of the program (default: sys.argv[0])
+ - prog -- The name of the program (default:
+ ``os.path.basename(sys.argv[0])``)
- usage -- A usage message (default: auto-generated from arguments)
- description -- A description of what the program does
- epilog -- Text following the argument descriptions
@@ -1719,7 +1726,7 @@
add_group = self.add_argument_group
self._positionals = add_group(_('positional arguments'))
- self._optionals = add_group(_('optional arguments'))
+ self._optionals = add_group(_('options'))
self._subparsers = None
# register types
diff --git a/common/py3-stdlib/ast.py b/common/py3-stdlib/ast.py
index ecd4895..f4d2f6e 100644
--- a/common/py3-stdlib/ast.py
+++ b/common/py3-stdlib/ast.py
@@ -59,11 +59,14 @@
sets, booleans, and None.
"""
if isinstance(node_or_string, str):
- node_or_string = parse(node_or_string, mode='eval')
+ node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval')
if isinstance(node_or_string, Expression):
node_or_string = node_or_string.body
def _raise_malformed_node(node):
- raise ValueError(f'malformed node or string: {node!r}')
+ msg = "malformed node or string"
+ if lno := getattr(node, 'lineno', None):
+ msg += f' on line {lno}'
+ raise ValueError(msg + f': {node!r}')
def _convert_num(node):
if not isinstance(node, Constant) or type(node.value) not in (int, float, complex):
_raise_malformed_node(node)
@@ -794,6 +797,9 @@
else:
super().visit(node)
+ # Note: as visit() resets the output text, do NOT rely on
+ # NodeVisitor.generic_visit to handle any nodes (as it calls back in to
+ # the subclass visit() method, which resets self._source to an empty list)
def visit(self, node):
"""Outputs a source code string that, if converted back to an ast
(using ast.parse) will generate an AST equivalent to *node*"""
@@ -1196,8 +1202,13 @@
def _write_constant(self, value):
if isinstance(value, (float, complex)):
- # Substitute overflowing decimal literal for AST infinities.
- self.write(repr(value).replace("inf", _INFSTR))
+ # Substitute overflowing decimal literal for AST infinities,
+ # and inf - inf for NaNs.
+ self.write(
+ repr(value)
+ .replace("inf", _INFSTR)
+ .replace("nan", f"({_INFSTR}-{_INFSTR})")
+ )
elif self._avoid_backslashes and isinstance(value, str):
self._write_str_avoiding_backslashes(value)
else:
@@ -1270,10 +1281,13 @@
self.traverse(node.orelse)
def visit_Set(self, node):
- if not node.elts:
- raise ValueError("Set node should have at least one item")
- with self.delimit("{", "}"):
- self.interleave(lambda: self.write(", "), self.traverse, node.elts)
+ if node.elts:
+ with self.delimit("{", "}"):
+ self.interleave(lambda: self.write(", "), self.traverse, node.elts)
+ else:
+ # `{}` would be interpreted as a dictionary literal, and
+ # `set` might be shadowed. Thus:
+ self.write('{*()}')
def visit_Dict(self, node):
def write_key_value_pair(k, v):
@@ -1440,9 +1454,9 @@
def visit_Subscript(self, node):
def is_simple_tuple(slice_value):
- # when unparsing a non-empty tuple, the parantheses can be safely
+ # when unparsing a non-empty tuple, the parentheses can be safely
# omitted if there aren't any elements that explicitly requires
- # parantheses (such as starred expressions).
+ # parentheses (such as starred expressions).
return (
isinstance(slice_value, Tuple)
and slice_value.elts
@@ -1475,6 +1489,13 @@
self.write(":")
self.traverse(node.step)
+ def visit_Match(self, node):
+ self.fill("match ")
+ self.traverse(node.subject)
+ with self.block():
+ for case in node.cases:
+ self.traverse(case)
+
def visit_arg(self, node):
self.write(node.arg)
if node.annotation:
@@ -1559,6 +1580,94 @@
self.write(" as ")
self.traverse(node.optional_vars)
+ def visit_match_case(self, node):
+ self.fill("case ")
+ self.traverse(node.pattern)
+ if node.guard:
+ self.write(" if ")
+ self.traverse(node.guard)
+ with self.block():
+ self.traverse(node.body)
+
+ def visit_MatchValue(self, node):
+ self.traverse(node.value)
+
+ def visit_MatchSingleton(self, node):
+ self._write_constant(node.value)
+
+ def visit_MatchSequence(self, node):
+ with self.delimit("[", "]"):
+ self.interleave(
+ lambda: self.write(", "), self.traverse, node.patterns
+ )
+
+ def visit_MatchStar(self, node):
+ name = node.name
+ if name is None:
+ name = "_"
+ self.write(f"*{name}")
+
+ def visit_MatchMapping(self, node):
+ def write_key_pattern_pair(pair):
+ k, p = pair
+ self.traverse(k)
+ self.write(": ")
+ self.traverse(p)
+
+ with self.delimit("{", "}"):
+ keys = node.keys
+ self.interleave(
+ lambda: self.write(", "),
+ write_key_pattern_pair,
+ zip(keys, node.patterns, strict=True),
+ )
+ rest = node.rest
+ if rest is not None:
+ if keys:
+ self.write(", ")
+ self.write(f"**{rest}")
+
+ def visit_MatchClass(self, node):
+ self.set_precedence(_Precedence.ATOM, node.cls)
+ self.traverse(node.cls)
+ with self.delimit("(", ")"):
+ patterns = node.patterns
+ self.interleave(
+ lambda: self.write(", "), self.traverse, patterns
+ )
+ attrs = node.kwd_attrs
+ if attrs:
+ def write_attr_pattern(pair):
+ attr, pattern = pair
+ self.write(f"{attr}=")
+ self.traverse(pattern)
+
+ if patterns:
+ self.write(", ")
+ self.interleave(
+ lambda: self.write(", "),
+ write_attr_pattern,
+ zip(attrs, node.kwd_patterns, strict=True),
+ )
+
+ def visit_MatchAs(self, node):
+ name = node.name
+ pattern = node.pattern
+ if name is None:
+ self.write("_")
+ elif pattern is None:
+ self.write(node.name)
+ else:
+ with self.require_parens(_Precedence.TEST, node):
+ self.set_precedence(_Precedence.BOR, node.pattern)
+ self.traverse(node.pattern)
+ self.write(f" as {node.name}")
+
+ def visit_MatchOr(self, node):
+ with self.require_parens(_Precedence.BOR, node):
+ self.set_precedence(_Precedence.BOR.next(), *node.patterns)
+ self.interleave(lambda: self.write(" | "), self.traverse, node.patterns)
+
def unparse(ast_obj):
unparser = _Unparser()
return unparser.visit(ast_obj)
diff --git a/common/py3-stdlib/asynchat.py b/common/py3-stdlib/asynchat.py
index f4ba361..de26ffa 100644
--- a/common/py3-stdlib/asynchat.py
+++ b/common/py3-stdlib/asynchat.py
@@ -48,6 +48,14 @@
import asyncore
from collections import deque
+from warnings import warn
+warn(
+ 'The asynchat module is deprecated. '
+ 'The recommended replacement is asyncio',
+ DeprecationWarning,
+ stacklevel=2)
+
+
class async_chat(asyncore.dispatcher):
"""This is an abstract class. You must derive from this class, and add
diff --git a/common/py3-stdlib/asyncio/__init__.py b/common/py3-stdlib/asyncio/__init__.py
index eb84bfb..200b14c 100644
--- a/common/py3-stdlib/asyncio/__init__.py
+++ b/common/py3-stdlib/asyncio/__init__.py
@@ -20,10 +20,6 @@
from .threads import *
from .transports import *
-# Exposed for _asynciomodule.c to implement now deprecated
-# Task.all_tasks() method. This function will be removed in 3.9.
-from .tasks import _all_tasks_compat # NoQA
-
__all__ = (base_events.__all__ +
coroutines.__all__ +
events.__all__ +
diff --git a/common/py3-stdlib/asyncio/base_events.py b/common/py3-stdlib/asyncio/base_events.py
index b2d446a..952da11 100644
--- a/common/py3-stdlib/asyncio/base_events.py
+++ b/common/py3-stdlib/asyncio/base_events.py
@@ -49,7 +49,7 @@
from .log import logger
-__all__ = 'BaseEventLoop',
+__all__ = 'BaseEventLoop','Server',
# Minimum number of _scheduled timer handles before cleanup of
@@ -202,6 +202,11 @@
pass
+def _check_ssl_socket(sock):
+ if ssl is not None and isinstance(sock, ssl.SSLSocket):
+ raise TypeError("Socket cannot be of type SSLSocket")
+
+
class _SendfileFallbackProtocol(protocols.Protocol):
def __init__(self, transp):
if not isinstance(transp, transports._FlowControlMixin):
@@ -350,7 +355,7 @@
self._start_serving()
# Skip one loop iteration so that all 'loop.add_reader'
# go through.
- await tasks.sleep(0, loop=self._loop)
+ await tasks.sleep(0)
async def serve_forever(self):
if self._serving_forever_fut is not None:
@@ -541,8 +546,7 @@
results = await tasks.gather(
*[ag.aclose() for ag in closing_agens],
- return_exceptions=True,
- loop=self)
+ return_exceptions=True)
for result, agen in zip(results, closing_agens):
if isinstance(result, Exception):
@@ -864,6 +868,7 @@
*, fallback=True):
if self._debug and sock.gettimeout() != 0:
raise ValueError("the socket must be non-blocking")
+ _check_ssl_socket(sock)
self._check_sendfile_params(sock, file, offset, count)
try:
return await self._sock_sendfile_native(sock, file,
@@ -973,7 +978,7 @@
happy_eyeballs_delay=None, interleave=None):
"""Connect to a TCP server.
- Create a streaming transport connection to a given Internet host and
+ Create a streaming transport connection to a given internet host and
port: socket family AF_INET or socket.AF_INET6 depending on host (or
family if specified), socket type SOCK_STREAM. protocol_factory must be
a callable returning a protocol instance.
@@ -1005,6 +1010,9 @@
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if sock is not None:
+ _check_ssl_socket(sock)
+
if happy_eyeballs_delay is not None and interleave is None:
# If using happy eyeballs, default to interleave addresses by family
interleave = 1
@@ -1438,6 +1446,9 @@
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if sock is not None:
+ _check_ssl_socket(sock)
+
if host is not None or port is not None:
if sock is not None:
raise ValueError(
@@ -1457,7 +1468,7 @@
fs = [self._create_server_getaddrinfo(host, port, family=family,
flags=flags)
for host in hosts]
- infos = await tasks.gather(*fs, loop=self)
+ infos = await tasks.gather(*fs)
infos = set(itertools.chain.from_iterable(infos))
completed = False
@@ -1515,7 +1526,7 @@
server._start_serving()
# Skip one loop iteration so that all 'loop.add_reader'
# go through.
- await tasks.sleep(0, loop=self)
+ await tasks.sleep(0)
if self._debug:
logger.info("%r is serving", server)
@@ -1525,14 +1536,6 @@
self, protocol_factory, sock,
*, ssl=None,
ssl_handshake_timeout=None):
- """Handle an accepted connection.
-
- This is used by servers that accept connections outside of
- asyncio but that use asyncio to handle connections.
-
- This method is a coroutine. When completed, the coroutine
- returns a (transport, protocol) pair.
- """
if sock.type != socket.SOCK_STREAM:
raise ValueError(f'A Stream Socket was expected, got {sock!r}')
@@ -1540,6 +1543,9 @@
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if sock is not None:
+ _check_ssl_socket(sock)
+
transport, protocol = await self._create_connection_transport(
sock, protocol_factory, ssl, '', server_side=True,
ssl_handshake_timeout=ssl_handshake_timeout)
diff --git a/common/py3-stdlib/asyncio/events.py b/common/py3-stdlib/asyncio/events.py
index 0dce87b..5ab1acc 100644
--- a/common/py3-stdlib/asyncio/events.py
+++ b/common/py3-stdlib/asyncio/events.py
@@ -258,13 +258,13 @@
"""Notification that a TimerHandle has been cancelled."""
raise NotImplementedError
- def call_soon(self, callback, *args):
- return self.call_later(0, callback, *args)
+ def call_soon(self, callback, *args, context=None):
+ return self.call_later(0, callback, *args, context=context)
- def call_later(self, delay, callback, *args):
+ def call_later(self, delay, callback, *args, context=None):
raise NotImplementedError
- def call_at(self, when, callback, *args):
+ def call_at(self, when, callback, *args, context=None):
raise NotImplementedError
def time(self):
@@ -280,7 +280,7 @@
# Methods for interacting with threads.
- def call_soon_threadsafe(self, callback, *args):
+ def call_soon_threadsafe(self, callback, *args, context=None):
raise NotImplementedError
def run_in_executor(self, executor, func, *args):
@@ -418,6 +418,20 @@
"""
raise NotImplementedError
+ async def connect_accepted_socket(
+ self, protocol_factory, sock,
+ *, ssl=None,
+ ssl_handshake_timeout=None):
+ """Handle an accepted connection.
+
+ This is used by servers that accept connections outside of
+ asyncio, but use asyncio to handle connections.
+
+ This method is a coroutine. When completed, the coroutine
+ returns a (transport, protocol) pair.
+ """
+ raise NotImplementedError
+
async def create_datagram_endpoint(self, protocol_factory,
local_addr=None, remote_addr=None, *,
family=0, proto=0, flags=0,
@@ -465,7 +479,7 @@
# The reason to accept file-like object instead of just file descriptor
# is: we need to own pipe and close it at transport finishing
# Can got complicated errors if pass f.fileno(),
- # close fd in pipe transport then close f and vise versa.
+ # close fd in pipe transport then close f and vice versa.
raise NotImplementedError
async def connect_write_pipe(self, protocol_factory, pipe):
@@ -478,7 +492,7 @@
# The reason to accept file-like object instead of just file descriptor
# is: we need to own pipe and close it at transport finishing
# Can got complicated errors if pass f.fileno(),
- # close fd in pipe transport then close f and vise versa.
+ # close fd in pipe transport then close f and vice versa.
raise NotImplementedError
async def subprocess_shell(self, protocol_factory, cmd, *,
@@ -745,9 +759,16 @@
the result of `get_event_loop_policy().get_event_loop()` call.
"""
# NOTE: this function is implemented in C (see _asynciomodule.c)
+ return _py__get_event_loop()
+
+
+def _get_event_loop(stacklevel=3):
current_loop = _get_running_loop()
if current_loop is not None:
return current_loop
+ import warnings
+ warnings.warn('There is no current event loop',
+ DeprecationWarning, stacklevel=stacklevel)
return get_event_loop_policy().get_event_loop()
@@ -777,6 +798,7 @@
_py__set_running_loop = _set_running_loop
_py_get_running_loop = get_running_loop
_py_get_event_loop = get_event_loop
+_py__get_event_loop = _get_event_loop
try:
@@ -784,7 +806,7 @@
# functions in asyncio. Pure Python implementation is
# about 4 times slower than C-accelerated.
from _asyncio import (_get_running_loop, _set_running_loop,
- get_running_loop, get_event_loop)
+ get_running_loop, get_event_loop, _get_event_loop)
except ImportError:
pass
else:
@@ -793,3 +815,4 @@
_c__set_running_loop = _set_running_loop
_c_get_running_loop = get_running_loop
_c_get_event_loop = get_event_loop
+ _c__get_event_loop = _get_event_loop
diff --git a/common/py3-stdlib/asyncio/futures.py b/common/py3-stdlib/asyncio/futures.py
index bed4da5..8e8cd87 100644
--- a/common/py3-stdlib/asyncio/futures.py
+++ b/common/py3-stdlib/asyncio/futures.py
@@ -8,6 +8,7 @@
import contextvars
import logging
import sys
+from types import GenericAlias
from . import base_futures
from . import events
@@ -76,7 +77,7 @@
the default event loop.
"""
if loop is None:
- self._loop = events.get_event_loop()
+ self._loop = events._get_event_loop()
else:
self._loop = loop
self._callbacks = []
@@ -106,8 +107,7 @@
context['source_traceback'] = self._source_traceback
self._loop.call_exception_handler(context)
- def __class_getitem__(cls, type):
- return cls
+ __class_getitem__ = classmethod(GenericAlias)
@property
def _log_traceback(self):
@@ -115,7 +115,7 @@
@_log_traceback.setter
def _log_traceback(self, val):
- if bool(val):
+ if val:
raise ValueError('_log_traceback can only be set to False')
self.__log_traceback = False
@@ -408,7 +408,7 @@
assert isinstance(future, concurrent.futures.Future), \
f'concurrent.futures.Future is expected, got {future!r}'
if loop is None:
- loop = events.get_event_loop()
+ loop = events._get_event_loop()
new_future = loop.create_future()
_chain_future(future, new_future)
return new_future
diff --git a/common/py3-stdlib/asyncio/locks.py b/common/py3-stdlib/asyncio/locks.py
index f1ce732..4fef64e 100644
--- a/common/py3-stdlib/asyncio/locks.py
+++ b/common/py3-stdlib/asyncio/locks.py
@@ -3,10 +3,9 @@
__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore')
import collections
-import warnings
-from . import events
from . import exceptions
+from . import mixins
class _ContextManagerMixin:
@@ -20,7 +19,7 @@
self.release()
-class Lock(_ContextManagerMixin):
+class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
"""Primitive lock objects.
A primitive lock is a synchronization primitive that is not owned
@@ -74,16 +73,10 @@
"""
- def __init__(self, *, loop=None):
+ def __init__(self, *, loop=mixins._marker):
+ super().__init__(loop=loop)
self._waiters = None
self._locked = False
- if loop is None:
- self._loop = events.get_event_loop()
- else:
- self._loop = loop
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
def __repr__(self):
res = super().__repr__()
@@ -109,7 +102,7 @@
if self._waiters is None:
self._waiters = collections.deque()
- fut = self._loop.create_future()
+ fut = self._get_loop().create_future()
self._waiters.append(fut)
# Finally block should be called before the CancelledError
@@ -161,7 +154,7 @@
fut.set_result(True)
-class Event:
+class Event(mixins._LoopBoundMixin):
"""Asynchronous equivalent to threading.Event.
Class implementing event objects. An event manages a flag that can be set
@@ -170,16 +163,10 @@
false.
"""
- def __init__(self, *, loop=None):
+ def __init__(self, *, loop=mixins._marker):
+ super().__init__(loop=loop)
self._waiters = collections.deque()
self._value = False
- if loop is None:
- self._loop = events.get_event_loop()
- else:
- self._loop = loop
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
def __repr__(self):
res = super().__repr__()
@@ -220,7 +207,7 @@
if self._value:
return True
- fut = self._loop.create_future()
+ fut = self._get_loop().create_future()
self._waiters.append(fut)
try:
await fut
@@ -229,7 +216,7 @@
self._waiters.remove(fut)
-class Condition(_ContextManagerMixin):
+class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
"""Asynchronous equivalent to threading.Condition.
This class implements condition variable objects. A condition variable
@@ -239,19 +226,10 @@
A new Lock object is created and used as the underlying lock.
"""
- def __init__(self, lock=None, *, loop=None):
- if loop is None:
- self._loop = events.get_event_loop()
- else:
- self._loop = loop
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
-
+ def __init__(self, lock=None, *, loop=mixins._marker):
+ super().__init__(loop=loop)
if lock is None:
- lock = Lock(loop=loop)
- elif lock._loop is not self._loop:
- raise ValueError("loop argument must agree with lock")
+ lock = Lock()
self._lock = lock
# Export the lock's locked(), acquire() and release() methods.
@@ -284,7 +262,7 @@
self.release()
try:
- fut = self._loop.create_future()
+ fut = self._get_loop().create_future()
self._waiters.append(fut)
try:
await fut
@@ -351,7 +329,7 @@
self.notify(len(self._waiters))
-class Semaphore(_ContextManagerMixin):
+class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
"""A Semaphore implementation.
A semaphore manages an internal counter which is decremented by each
@@ -366,18 +344,12 @@
ValueError is raised.
"""
- def __init__(self, value=1, *, loop=None):
+ def __init__(self, value=1, *, loop=mixins._marker):
+ super().__init__(loop=loop)
if value < 0:
raise ValueError("Semaphore initial value must be >= 0")
self._value = value
self._waiters = collections.deque()
- if loop is None:
- self._loop = events.get_event_loop()
- else:
- self._loop = loop
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
def __repr__(self):
res = super().__repr__()
@@ -407,7 +379,7 @@
True.
"""
while self._value <= 0:
- fut = self._loop.create_future()
+ fut = self._get_loop().create_future()
self._waiters.append(fut)
try:
await fut
@@ -436,12 +408,7 @@
above the initial value.
"""
- def __init__(self, value=1, *, loop=None):
- if loop:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
-
+ def __init__(self, value=1, *, loop=mixins._marker):
self._bound_value = value
super().__init__(value, loop=loop)
diff --git a/common/py3-stdlib/asyncio/mixins.py b/common/py3-stdlib/asyncio/mixins.py
new file mode 100644
index 0000000..650df05
--- /dev/null
+++ b/common/py3-stdlib/asyncio/mixins.py
@@ -0,0 +1,31 @@
+"""Event loop mixins."""
+
+import threading
+from . import events
+
+_global_lock = threading.Lock()
+
+# Used as a sentinel for loop parameter
+_marker = object()
+
+
+class _LoopBoundMixin:
+ _loop = None
+
+ def __init__(self, *, loop=_marker):
+ if loop is not _marker:
+ raise TypeError(
+ f'As of 3.10, the *loop* parameter was removed from '
+ f'{type(self).__name__}() since it is no longer necessary'
+ )
+
+ def _get_loop(self):
+ loop = events._get_running_loop()
+
+ if self._loop is None:
+ with _global_lock:
+ if self._loop is None:
+ self._loop = loop
+ if loop is not self._loop:
+ raise RuntimeError(f'{self!r} is bound to a different event loop')
+ return loop
diff --git a/common/py3-stdlib/asyncio/proactor_events.py b/common/py3-stdlib/asyncio/proactor_events.py
index b4cd414..411685b 100644
--- a/common/py3-stdlib/asyncio/proactor_events.py
+++ b/common/py3-stdlib/asyncio/proactor_events.py
@@ -158,7 +158,7 @@
# end then it may fail with ERROR_NETNAME_DELETED if we
# just close our end. First calling shutdown() seems to
# cure it, but maybe using DisconnectEx() would be better.
- if hasattr(self._sock, 'shutdown'):
+ if hasattr(self._sock, 'shutdown') and self._sock.fileno() != -1:
self._sock.shutdown(socket.SHUT_RDWR)
self._sock.close()
self._sock = None
@@ -179,11 +179,12 @@
"""Transport for read pipes."""
def __init__(self, loop, sock, protocol, waiter=None,
- extra=None, server=None):
- self._pending_data = None
+ extra=None, server=None, buffer_size=65536):
+ self._pending_data_length = -1
self._paused = True
super().__init__(loop, sock, protocol, waiter, extra, server)
+ self._data = bytearray(buffer_size)
self._loop.call_soon(self._loop_reading)
self._paused = False
@@ -217,12 +218,12 @@
if self._read_fut is None:
self._loop.call_soon(self._loop_reading, None)
- data = self._pending_data
- self._pending_data = None
- if data is not None:
+ length = self._pending_data_length
+ self._pending_data_length = -1
+ if length > -1:
# Call the protocol methode after calling _loop_reading(),
# since the protocol can decide to pause reading again.
- self._loop.call_soon(self._data_received, data)
+ self._loop.call_soon(self._data_received, self._data[:length], length)
if self._loop.get_debug():
logger.debug("%r resumes reading", self)
@@ -243,15 +244,15 @@
if not keep_open:
self.close()
- def _data_received(self, data):
+ def _data_received(self, data, length):
if self._paused:
# Don't call any protocol method while reading is paused.
# The protocol will be called on resume_reading().
- assert self._pending_data is None
- self._pending_data = data
+ assert self._pending_data_length == -1
+ self._pending_data_length = length
return
- if not data:
+ if length == 0:
self._eof_received()
return
@@ -269,6 +270,7 @@
self._protocol.data_received(data)
def _loop_reading(self, fut=None):
+ length = -1
data = None
try:
if fut is not None:
@@ -277,18 +279,18 @@
self._read_fut = None
if fut.done():
# deliver data later in "finally" clause
- data = fut.result()
+ length = fut.result()
+ if length == 0:
+ # we got end-of-file so no need to reschedule a new read
+ return
+
+ data = self._data[:length]
else:
# the future will be replaced by next proactor.recv call
fut.cancel()
if self._closing:
# since close() has been called we ignore any read data
- data = None
- return
-
- if data == b'':
- # we got end-of-file so no need to reschedule a new read
return
# bpo-33694: buffer_updated() has currently no fast path because of
@@ -296,7 +298,7 @@
if not self._paused:
# reschedule a new read
- self._read_fut = self._loop._proactor.recv(self._sock, 32768)
+ self._read_fut = self._loop._proactor.recv_into(self._sock, self._data)
except ConnectionAbortedError as exc:
if not self._closing:
self._fatal_error(exc, 'Fatal read error on pipe transport')
@@ -314,8 +316,8 @@
if not self._paused:
self._read_fut.add_done_callback(self._loop_reading)
finally:
- if data is not None:
- self._data_received(data)
+ if length > -1:
+ self._data_received(data, length)
class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
@@ -450,7 +452,8 @@
self.close()
-class _ProactorDatagramTransport(_ProactorBasePipeTransport):
+class _ProactorDatagramTransport(_ProactorBasePipeTransport,
+ transports.DatagramTransport):
max_size = 256 * 1024
def __init__(self, loop, sock, protocol, address=None,
waiter=None, extra=None):
diff --git a/common/py3-stdlib/asyncio/protocols.py b/common/py3-stdlib/asyncio/protocols.py
index 69fa43e..09987b1 100644
--- a/common/py3-stdlib/asyncio/protocols.py
+++ b/common/py3-stdlib/asyncio/protocols.py
@@ -109,10 +109,6 @@
class BufferedProtocol(BaseProtocol):
"""Interface for stream protocol with manual buffer control.
- Important: this has been added to asyncio in Python 3.7
- *on a provisional basis*! Consider it as an experimental API that
- might be changed or removed in Python 3.8.
-
Event methods, such as `create_server` and `create_connection`,
accept factories that return protocols that implement this interface.
diff --git a/common/py3-stdlib/asyncio/queues.py b/common/py3-stdlib/asyncio/queues.py
index cd3f7c6..10dd689 100644
--- a/common/py3-stdlib/asyncio/queues.py
+++ b/common/py3-stdlib/asyncio/queues.py
@@ -2,10 +2,10 @@
import collections
import heapq
-import warnings
+from types import GenericAlias
-from . import events
from . import locks
+from . import mixins
class QueueEmpty(Exception):
@@ -18,7 +18,7 @@
pass
-class Queue:
+class Queue(mixins._LoopBoundMixin):
"""A queue, useful for coordinating producer and consumer coroutines.
If maxsize is less than or equal to zero, the queue size is infinite. If it
@@ -30,14 +30,8 @@
interrupted between calling qsize() and doing an operation on the Queue.
"""
- def __init__(self, maxsize=0, *, loop=None):
- if loop is None:
- self._loop = events.get_event_loop()
- else:
- self._loop = loop
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ def __init__(self, maxsize=0, *, loop=mixins._marker):
+ super().__init__(loop=loop)
self._maxsize = maxsize
# Futures.
@@ -45,7 +39,7 @@
# Futures.
self._putters = collections.deque()
self._unfinished_tasks = 0
- self._finished = locks.Event(loop=loop)
+ self._finished = locks.Event()
self._finished.set()
self._init(maxsize)
@@ -76,8 +70,7 @@
def __str__(self):
return f'<{type(self).__name__} {self._format()}>'
- def __class_getitem__(cls, type):
- return cls
+ __class_getitem__ = classmethod(GenericAlias)
def _format(self):
result = f'maxsize={self._maxsize!r}'
@@ -122,7 +115,7 @@
slot is available before adding item.
"""
while self.full():
- putter = self._loop.create_future()
+ putter = self._get_loop().create_future()
self._putters.append(putter)
try:
await putter
@@ -160,7 +153,7 @@
If queue is empty, wait until an item is available.
"""
while self.empty():
- getter = self._loop.create_future()
+ getter = self._get_loop().create_future()
self._getters.append(getter)
try:
await getter
diff --git a/common/py3-stdlib/asyncio/runners.py b/common/py3-stdlib/asyncio/runners.py
index 268635d..9a5e9a4 100644
--- a/common/py3-stdlib/asyncio/runners.py
+++ b/common/py3-stdlib/asyncio/runners.py
@@ -60,8 +60,7 @@
for task in to_cancel:
task.cancel()
- loop.run_until_complete(
- tasks.gather(*to_cancel, loop=loop, return_exceptions=True))
+ loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True))
for task in to_cancel:
if task.cancelled():
diff --git a/common/py3-stdlib/asyncio/selector_events.py b/common/py3-stdlib/asyncio/selector_events.py
index 59cb6b1..71080b8 100644
--- a/common/py3-stdlib/asyncio/selector_events.py
+++ b/common/py3-stdlib/asyncio/selector_events.py
@@ -40,11 +40,6 @@
return bool(key.events & event)
-def _check_ssl_socket(sock):
- if ssl is not None and isinstance(sock, ssl.SSLSocket):
- raise TypeError("Socket cannot be of type SSLSocket")
-
-
class BaseSelectorEventLoop(base_events.BaseEventLoop):
"""Selector event loop.
@@ -357,7 +352,7 @@
The maximum amount of data to be received at once is specified by
nbytes.
"""
- _check_ssl_socket(sock)
+ base_events._check_ssl_socket(sock)
if self._debug and sock.gettimeout() != 0:
raise ValueError("the socket must be non-blocking")
try:
@@ -398,7 +393,7 @@
The received data is written into *buf* (a writable buffer).
The return value is the number of bytes written.
"""
- _check_ssl_socket(sock)
+ base_events._check_ssl_socket(sock)
if self._debug and sock.gettimeout() != 0:
raise ValueError("the socket must be non-blocking")
try:
@@ -439,7 +434,7 @@
raised, and there is no way to determine how much data, if any, was
successfully processed by the receiving end of the connection.
"""
- _check_ssl_socket(sock)
+ base_events._check_ssl_socket(sock)
if self._debug and sock.gettimeout() != 0:
raise ValueError("the socket must be non-blocking")
try:
@@ -488,13 +483,15 @@
This method is a coroutine.
"""
- _check_ssl_socket(sock)
+ base_events._check_ssl_socket(sock)
if self._debug and sock.gettimeout() != 0:
raise ValueError("the socket must be non-blocking")
if not hasattr(socket, 'AF_UNIX') or sock.family != socket.AF_UNIX:
resolved = await self._ensure_resolved(
- address, family=sock.family, proto=sock.proto, loop=self)
+ address, family=sock.family, type=sock.type, proto=sock.proto,
+ loop=self,
+ )
_, _, _, _, address = resolved[0]
fut = self.create_future()
@@ -553,7 +550,7 @@
object usable to send and receive data on the connection, and address
is the address bound to the socket on the other end of the connection.
"""
- _check_ssl_socket(sock)
+ base_events._check_ssl_socket(sock)
if self._debug and sock.gettimeout() != 0:
raise ValueError("the socket must be non-blocking")
fut = self.create_future()
diff --git a/common/py3-stdlib/asyncio/sslproto.py b/common/py3-stdlib/asyncio/sslproto.py
index cad25b2..00fc16c 100644
--- a/common/py3-stdlib/asyncio/sslproto.py
+++ b/common/py3-stdlib/asyncio/sslproto.py
@@ -367,6 +367,12 @@
"""Return the current size of the write buffer."""
return self._ssl_protocol._transport.get_write_buffer_size()
+ def get_write_buffer_limits(self):
+ """Get the high and low watermarks for write flow control.
+ Return a tuple (low, high) where low and high are
+ positive number of bytes."""
+ return self._ssl_protocol._transport.get_write_buffer_limits()
+
@property
def _protocol_paused(self):
# Required for sendfile fallback pause_writing/resume_writing logic
diff --git a/common/py3-stdlib/asyncio/streams.py b/common/py3-stdlib/asyncio/streams.py
index 3c80bb8..080d8a6 100644
--- a/common/py3-stdlib/asyncio/streams.py
+++ b/common/py3-stdlib/asyncio/streams.py
@@ -23,7 +23,7 @@
async def open_connection(host=None, port=None, *,
- loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ limit=_DEFAULT_LIMIT, **kwds):
"""A wrapper for create_connection() returning a (reader, writer) pair.
The reader returned is a StreamReader instance; the writer is a
@@ -41,12 +41,7 @@
StreamReaderProtocol classes, just copy the code -- there's
really nothing special here except some convenience.)
"""
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events.get_running_loop()
reader = StreamReader(limit=limit, loop=loop)
protocol = StreamReaderProtocol(reader, loop=loop)
transport, _ = await loop.create_connection(
@@ -56,7 +51,7 @@
async def start_server(client_connected_cb, host=None, port=None, *,
- loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ limit=_DEFAULT_LIMIT, **kwds):
"""Start a socket server, call back for each client connected.
The first parameter, `client_connected_cb`, takes two parameters:
@@ -78,12 +73,7 @@
The return value is the same as loop.create_server(), i.e. a
Server object which can be used to stop the service.
"""
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events.get_running_loop()
def factory():
reader = StreamReader(limit=limit, loop=loop)
@@ -98,14 +88,10 @@
# UNIX Domain Sockets are supported on this platform
async def open_unix_connection(path=None, *,
- loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ limit=_DEFAULT_LIMIT, **kwds):
"""Similar to `open_connection` but works with UNIX Domain Sockets."""
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events.get_running_loop()
+
reader = StreamReader(limit=limit, loop=loop)
protocol = StreamReaderProtocol(reader, loop=loop)
transport, _ = await loop.create_unix_connection(
@@ -114,14 +100,9 @@
return reader, writer
async def start_unix_server(client_connected_cb, path=None, *,
- loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ limit=_DEFAULT_LIMIT, **kwds):
"""Similar to `start_server` but works with UNIX Domain Sockets."""
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events.get_running_loop()
def factory():
reader = StreamReader(limit=limit, loop=loop)
@@ -144,7 +125,7 @@
def __init__(self, loop=None):
if loop is None:
- self._loop = events.get_event_loop()
+ self._loop = events._get_event_loop(stacklevel=4)
else:
self._loop = loop
self._paused = False
@@ -302,9 +283,13 @@
def __del__(self):
# Prevent reports about unhandled exceptions.
# Better than self._closed._log_traceback = False hack
- closed = self._closed
- if closed.done() and not closed.cancelled():
- closed.exception()
+ try:
+ closed = self._closed
+ except AttributeError:
+ pass # failed constructor
+ else:
+ if closed.done() and not closed.cancelled():
+ closed.exception()
class StreamWriter:
@@ -400,7 +385,7 @@
self._limit = limit
if loop is None:
- self._loop = events.get_event_loop()
+ self._loop = events._get_event_loop()
else:
self._loop = loop
self._buffer = bytearray()
diff --git a/common/py3-stdlib/asyncio/subprocess.py b/common/py3-stdlib/asyncio/subprocess.py
index c9506b1..cd10231 100644
--- a/common/py3-stdlib/asyncio/subprocess.py
+++ b/common/py3-stdlib/asyncio/subprocess.py
@@ -1,7 +1,6 @@
__all__ = 'create_subprocess_exec', 'create_subprocess_shell'
import subprocess
-import warnings
from . import events
from . import protocols
@@ -193,24 +192,14 @@
stderr = self._read_stream(2)
else:
stderr = self._noop()
- stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr,
- loop=self._loop)
+ stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)
await self.wait()
return (stdout, stderr)
async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
- loop=None, limit=streams._DEFAULT_LIMIT,
- **kwds):
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8 "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning,
- stacklevel=2
- )
-
+ limit=streams._DEFAULT_LIMIT, **kwds):
+ loop = events.get_running_loop()
protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
loop=loop)
transport, protocol = await loop.subprocess_shell(
@@ -221,16 +210,9 @@
async def create_subprocess_exec(program, *args, stdin=None, stdout=None,
- stderr=None, loop=None,
- limit=streams._DEFAULT_LIMIT, **kwds):
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8 "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning,
- stacklevel=2
- )
+ stderr=None, limit=streams._DEFAULT_LIMIT,
+ **kwds):
+ loop = events.get_running_loop()
protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
loop=loop)
transport, protocol = await loop.subprocess_exec(
diff --git a/common/py3-stdlib/asyncio/tasks.py b/common/py3-stdlib/asyncio/tasks.py
index f486b67..c4bedb5 100644
--- a/common/py3-stdlib/asyncio/tasks.py
+++ b/common/py3-stdlib/asyncio/tasks.py
@@ -17,6 +17,7 @@
import types
import warnings
import weakref
+from types import GenericAlias
from . import base_tasks
from . import coroutines
@@ -61,30 +62,6 @@
if futures._get_loop(t) is loop and not t.done()}
-def _all_tasks_compat(loop=None):
- # Different from "all_task()" by returning *all* Tasks, including
- # the completed ones. Used to implement deprecated "Tasks.all_task()"
- # method.
- if loop is None:
- loop = events.get_event_loop()
- # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another
- # thread while we do so. Therefore we cast it to list prior to filtering. The list
- # cast itself requires iteration, so we repeat it several times ignoring
- # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for
- # details.
- i = 0
- while True:
- try:
- tasks = list(_all_tasks)
- except RuntimeError:
- i += 1
- if i >= 1000:
- raise
- else:
- break
- return {t for t in tasks if futures._get_loop(t) is loop}
-
-
def _set_task_name(task, name):
if name is not None:
try:
@@ -147,8 +124,7 @@
self._loop.call_exception_handler(context)
super().__del__()
- def __class_getitem__(cls, type):
- return cls
+ __class_getitem__ = classmethod(GenericAlias)
def _repr_info(self):
return base_tasks._task_repr_info(self)
@@ -370,7 +346,7 @@
ALL_COMPLETED = concurrent.futures.ALL_COMPLETED
-async def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED):
+async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED):
"""Wait for the Futures and coroutines given by fs to complete.
The fs iterable must not be empty.
@@ -393,12 +369,7 @@
if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED):
raise ValueError(f'Invalid return_when value: {return_when}')
- if loop is None:
- loop = events.get_running_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events.get_running_loop()
fs = set(fs)
@@ -418,7 +389,7 @@
waiter.set_result(None)
-async def wait_for(fut, timeout, *, loop=None):
+async def wait_for(fut, timeout):
"""Wait for the single Future or coroutine to complete, with timeout.
Coroutine will be wrapped in Task.
@@ -431,12 +402,7 @@
This function is a coroutine.
"""
- if loop is None:
- loop = events.get_running_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events.get_running_loop()
if timeout is None:
return await fut
@@ -449,11 +415,9 @@
await _cancel_and_wait(fut, loop=loop)
try:
- fut.result()
+ return fut.result()
except exceptions.CancelledError as exc:
raise exceptions.TimeoutError() from exc
- else:
- raise exceptions.TimeoutError()
waiter = loop.create_future()
timeout_handle = loop.call_later(timeout, _release_waiter, waiter)
@@ -471,7 +435,10 @@
return fut.result()
else:
fut.remove_done_callback(cb)
- fut.cancel()
+ # We must ensure that the task is not running
+ # after wait_for() returns.
+ # See https://bugs.python.org/issue32751
+ await _cancel_and_wait(fut, loop=loop)
raise
if fut.done():
@@ -486,11 +453,9 @@
# exception, we should re-raise it
# See https://bugs.python.org/issue40607
try:
- fut.result()
+ return fut.result()
except exceptions.CancelledError as exc:
raise exceptions.TimeoutError() from exc
- else:
- raise exceptions.TimeoutError()
finally:
timeout_handle.cancel()
@@ -556,7 +521,7 @@
# This is *not* a @coroutine! It is just an iterator (yielding Futures).
-def as_completed(fs, *, loop=None, timeout=None):
+def as_completed(fs, *, timeout=None):
"""Return an iterator whose values are coroutines.
When waiting for the yielded coroutines you'll get the results (or
@@ -578,14 +543,9 @@
raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}")
from .queues import Queue # Import here to avoid circular import problem.
- done = Queue(loop=loop)
+ done = Queue()
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events._get_event_loop()
todo = {ensure_future(f, loop=loop) for f in set(fs)}
timeout_handle = None
@@ -630,19 +590,13 @@
yield
-async def sleep(delay, result=None, *, loop=None):
+async def sleep(delay, result=None):
"""Coroutine that completes after a given time (in seconds)."""
if delay <= 0:
await __sleep0()
return result
- if loop is None:
- loop = events.get_running_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
-
+ loop = events.get_running_loop()
future = loop.create_future()
h = loop.call_later(delay,
futures._set_result_unless_cancelled,
@@ -658,23 +612,32 @@
If the argument is a Future, it is returned directly.
"""
- if coroutines.iscoroutine(coro_or_future):
- if loop is None:
- loop = events.get_event_loop()
- task = loop.create_task(coro_or_future)
- if task._source_traceback:
- del task._source_traceback[-1]
- return task
- elif futures.isfuture(coro_or_future):
+ return _ensure_future(coro_or_future, loop=loop)
+
+
+def _ensure_future(coro_or_future, *, loop=None):
+ if futures.isfuture(coro_or_future):
if loop is not None and loop is not futures._get_loop(coro_or_future):
raise ValueError('The future belongs to a different loop than '
- 'the one specified as the loop argument')
+ 'the one specified as the loop argument')
return coro_or_future
- elif inspect.isawaitable(coro_or_future):
- return ensure_future(_wrap_awaitable(coro_or_future), loop=loop)
- else:
- raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
- 'required')
+ called_wrap_awaitable = False
+ if not coroutines.iscoroutine(coro_or_future):
+ if inspect.isawaitable(coro_or_future):
+ coro_or_future = _wrap_awaitable(coro_or_future)
+ called_wrap_awaitable = True
+ else:
+ raise TypeError('An asyncio.Future, a coroutine or an awaitable '
+ 'is required')
+
+ if loop is None:
+ loop = events._get_event_loop(stacklevel=4)
+ try:
+ return loop.create_task(coro_or_future)
+ except RuntimeError:
+ if not called_wrap_awaitable:
+ coro_or_future.close()
+ raise
@types.coroutine
@@ -697,7 +660,8 @@
cancelled.
"""
- def __init__(self, children, *, loop=None):
+ def __init__(self, children, *, loop):
+ assert loop is not None
super().__init__(loop=loop)
self._children = children
self._cancel_requested = False
@@ -717,7 +681,7 @@
return ret
-def gather(*coros_or_futures, loop=None, return_exceptions=False):
+def gather(*coros_or_futures, return_exceptions=False):
"""Return a future aggregating results from the given coroutines/futures.
Coroutines will be wrapped in a future and scheduled in the event
@@ -748,12 +712,7 @@
gather won't cancel any other awaitables.
"""
if not coros_or_futures:
- if loop is None:
- loop = events.get_event_loop()
- else:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
+ loop = events._get_event_loop()
outer = loop.create_future()
outer.set_result([])
return outer
@@ -762,7 +721,7 @@
nonlocal nfinished
nfinished += 1
- if outer.done():
+ if outer is None or outer.done():
if not fut.cancelled():
# Mark exception retrieved.
fut.exception()
@@ -817,9 +776,11 @@
children = []
nfuts = 0
nfinished = 0
+ loop = None
+ outer = None # bpo-46672
for arg in coros_or_futures:
if arg not in arg_to_fut:
- fut = ensure_future(arg, loop=loop)
+ fut = _ensure_future(arg, loop=loop)
if loop is None:
loop = futures._get_loop(fut)
if fut is not arg:
@@ -843,7 +804,7 @@
return outer
-def shield(arg, *, loop=None):
+def shield(arg):
"""Wait for a future, shielding it from cancellation.
The statement
@@ -869,11 +830,7 @@
except CancelledError:
res = None
"""
- if loop is not None:
- warnings.warn("The loop argument is deprecated since Python 3.8, "
- "and scheduled for removal in Python 3.10.",
- DeprecationWarning, stacklevel=2)
- inner = ensure_future(arg, loop=loop)
+ inner = _ensure_future(arg)
if inner.done():
# Shortcut.
return inner
diff --git a/common/py3-stdlib/asyncio/threads.py b/common/py3-stdlib/asyncio/threads.py
index 34b7513..db048a8 100644
--- a/common/py3-stdlib/asyncio/threads.py
+++ b/common/py3-stdlib/asyncio/threads.py
@@ -13,7 +13,7 @@
"""Asynchronously run function *func* in a separate thread.
Any *args and **kwargs supplied for this function are directly passed
- to *func*. Also, the current :class:`contextvars.Context` is propogated,
+ to *func*. Also, the current :class:`contextvars.Context` is propagated,
allowing context variables from the main thread to be accessed in the
separate thread.
diff --git a/common/py3-stdlib/asyncio/transports.py b/common/py3-stdlib/asyncio/transports.py
index 45e155c..73b1fa2 100644
--- a/common/py3-stdlib/asyncio/transports.py
+++ b/common/py3-stdlib/asyncio/transports.py
@@ -99,6 +99,12 @@
"""Return the current size of the write buffer."""
raise NotImplementedError
+ def get_write_buffer_limits(self):
+ """Get the high and low watermarks for write flow control.
+ Return a tuple (low, high) where low and high are
+ positive number of bytes."""
+ raise NotImplementedError
+
def write(self, data):
"""Write some data bytes to the transport.
diff --git a/common/py3-stdlib/asyncio/unix_events.py b/common/py3-stdlib/asyncio/unix_events.py
index f34a5b4..c88b818 100644
--- a/common/py3-stdlib/asyncio/unix_events.py
+++ b/common/py3-stdlib/asyncio/unix_events.py
@@ -44,6 +44,16 @@
pass
+def waitstatus_to_exitcode(status):
+ try:
+ return os.waitstatus_to_exitcode(status)
+ except ValueError:
+ # The child exited, but we don't understand its status.
+ # This shouldn't happen, but if it does, let's just
+ # return that status; perhaps that helps debug it.
+ return status
+
+
class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop):
"""Unix event loop.
@@ -323,7 +333,7 @@
server._start_serving()
# Skip one loop iteration so that all 'loop.add_reader'
# go through.
- await tasks.sleep(0, loop=self)
+ await tasks.sleep(0)
return server
@@ -941,7 +951,7 @@
" will report returncode 255",
pid)
else:
- returncode = _compute_returncode(status)
+ returncode = waitstatus_to_exitcode(status)
os.close(pidfd)
callback(pid, returncode, *args)
@@ -956,20 +966,6 @@
return True
-def _compute_returncode(status):
- if os.WIFSIGNALED(status):
- # The child process died because of a signal.
- return -os.WTERMSIG(status)
- elif os.WIFEXITED(status):
- # The child process exited (e.g sys.exit()).
- return os.WEXITSTATUS(status)
- else:
- # The child exited, but we don't understand its status.
- # This shouldn't happen, but if it does, let's just
- # return that status; perhaps that helps debug it.
- return status
-
-
class BaseChildWatcher(AbstractChildWatcher):
def __init__(self):
@@ -1080,7 +1076,7 @@
# The child process is still alive.
return
- returncode = _compute_returncode(status)
+ returncode = waitstatus_to_exitcode(status)
if self._loop.get_debug():
logger.debug('process %s exited with returncode %s',
expected_pid, returncode)
@@ -1173,7 +1169,7 @@
# A child process is still alive.
return
- returncode = _compute_returncode(status)
+ returncode = waitstatus_to_exitcode(status)
with self._lock:
try:
@@ -1230,13 +1226,15 @@
def close(self):
self._callbacks.clear()
- if self._saved_sighandler is not None:
- handler = signal.getsignal(signal.SIGCHLD)
- if handler != self._sig_chld:
- logger.warning("SIGCHLD handler was changed by outside code")
- else:
- signal.signal(signal.SIGCHLD, self._saved_sighandler)
- self._saved_sighandler = None
+ if self._saved_sighandler is None:
+ return
+
+ handler = signal.getsignal(signal.SIGCHLD)
+ if handler != self._sig_chld:
+ logger.warning("SIGCHLD handler was changed by outside code")
+ else:
+ signal.signal(signal.SIGCHLD, self._saved_sighandler)
+ self._saved_sighandler = None
def __enter__(self):
return self
@@ -1263,15 +1261,17 @@
# The reason to do it here is that attach_loop() is called from
# unix policy only for the main thread.
# Main thread is required for subscription on SIGCHLD signal
- if self._saved_sighandler is None:
- self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld)
- if self._saved_sighandler is None:
- logger.warning("Previous SIGCHLD handler was set by non-Python code, "
- "restore to default handler on watcher close.")
- self._saved_sighandler = signal.SIG_DFL
+ if self._saved_sighandler is not None:
+ return
- # Set SA_RESTART to limit EINTR occurrences.
- signal.siginterrupt(signal.SIGCHLD, False)
+ self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld)
+ if self._saved_sighandler is None:
+ logger.warning("Previous SIGCHLD handler was set by non-Python code, "
+ "restore to default handler on watcher close.")
+ self._saved_sighandler = signal.SIG_DFL
+
+ # Set SA_RESTART to limit EINTR occurrences.
+ signal.siginterrupt(signal.SIGCHLD, False)
def _do_waitpid_all(self):
for pid in list(self._callbacks):
@@ -1296,7 +1296,7 @@
# The child process is still alive.
return
- returncode = _compute_returncode(status)
+ returncode = waitstatus_to_exitcode(status)
debug_log = True
try:
loop, callback, args = self._callbacks.pop(pid)
@@ -1379,7 +1379,7 @@
def remove_child_handler(self, pid):
# asyncio never calls remove_child_handler() !!!
# The method is no-op but is implemented because
- # abstract base classe requires it
+ # abstract base classes require it.
return True
def attach_loop(self, loop):
@@ -1399,7 +1399,7 @@
"Unknown child process pid %d, will report returncode 255",
pid)
else:
- returncode = _compute_returncode(status)
+ returncode = waitstatus_to_exitcode(status)
if loop.get_debug():
logger.debug('process %s exited with returncode %s',
expected_pid, returncode)
diff --git a/common/py3-stdlib/asyncio/windows_events.py b/common/py3-stdlib/asyncio/windows_events.py
index 5e7cd79..da81ab4 100644
--- a/common/py3-stdlib/asyncio/windows_events.py
+++ b/common/py3-stdlib/asyncio/windows_events.py
@@ -1,5 +1,10 @@
"""Selector and proactor event loops for Windows."""
+import sys
+
+if sys.platform != 'win32': # pragma: no cover
+ raise ImportError('win32 only')
+
import _overlapped
import _winapi
import errno
diff --git a/common/py3-stdlib/asyncore.py b/common/py3-stdlib/asyncore.py
index ce16f11..b1eea4b 100644
--- a/common/py3-stdlib/asyncore.py
+++ b/common/py3-stdlib/asyncore.py
@@ -57,6 +57,13 @@
ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
errorcode
+warnings.warn(
+ 'The asyncore module is deprecated. '
+ 'The recommended replacement is asyncio',
+ DeprecationWarning,
+ stacklevel=2)
+
+
_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
EBADF})
@@ -113,7 +120,7 @@
if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
obj.handle_close()
except OSError as e:
- if e.args[0] not in _DISCONNECTED:
+ if e.errno not in _DISCONNECTED:
obj.handle_error()
else:
obj.handle_close()
@@ -236,7 +243,7 @@
try:
self.addr = sock.getpeername()
except OSError as err:
- if err.args[0] in (ENOTCONN, EINVAL):
+ if err.errno in (ENOTCONN, EINVAL):
# To handle the case where we got an unconnected
# socket.
self.connected = False
@@ -346,7 +353,7 @@
except TypeError:
return None
except OSError as why:
- if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
+ if why.errno in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
return None
else:
raise
@@ -358,9 +365,9 @@
result = self.socket.send(data)
return result
except OSError as why:
- if why.args[0] == EWOULDBLOCK:
+ if why.errno == EWOULDBLOCK:
return 0
- elif why.args[0] in _DISCONNECTED:
+ elif why.errno in _DISCONNECTED:
self.handle_close()
return 0
else:
@@ -378,7 +385,7 @@
return data
except OSError as why:
# winsock sometimes raises ENOTCONN
- if why.args[0] in _DISCONNECTED:
+ if why.errno in _DISCONNECTED:
self.handle_close()
return b''
else:
@@ -393,7 +400,7 @@
try:
self.socket.close()
except OSError as why:
- if why.args[0] not in (ENOTCONN, EBADF):
+ if why.errno not in (ENOTCONN, EBADF):
raise
# log and log_info may be overridden to provide more sophisticated
@@ -557,7 +564,7 @@
try:
x.close()
except OSError as x:
- if x.args[0] == EBADF:
+ if x.errno == EBADF:
pass
elif not ignore_all:
raise
diff --git a/common/py3-stdlib/base64.py b/common/py3-stdlib/base64.py
index a28109f..e1256ad 100755
--- a/common/py3-stdlib/base64.py
+++ b/common/py3-stdlib/base64.py
@@ -16,7 +16,7 @@
'encode', 'decode', 'encodebytes', 'decodebytes',
# Generalized interface for other encodings
'b64encode', 'b64decode', 'b32encode', 'b32decode',
- 'b16encode', 'b16decode',
+ 'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
# Base85 and Ascii85 encodings
'b85encode', 'b85decode', 'a85encode', 'a85decode',
# Standard Base64 encoding
@@ -135,19 +135,40 @@
# Base32 encoding/decoding must be done in Python
-_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
-_b32tab2 = None
-_b32rev = None
+_B32_ENCODE_DOCSTRING = '''
+Encode the bytes-like objects using {encoding} and return a bytes object.
+'''
+_B32_DECODE_DOCSTRING = '''
+Decode the {encoding} encoded bytes-like object or ASCII string s.
-def b32encode(s):
- """Encode the bytes-like object s using Base32 and return a bytes object.
- """
+Optional casefold is a flag specifying whether a lowercase alphabet is
+acceptable as input. For security purposes, the default is False.
+{extra_args}
+The result is returned as a bytes object. A binascii.Error is raised if
+the input is incorrectly padded or if there are non-alphabet
+characters present in the input.
+'''
+_B32_DECODE_MAP01_DOCSTRING = '''
+RFC 3548 allows for optional mapping of the digit 0 (zero) to the
+letter O (oh), and for optional mapping of the digit 1 (one) to
+either the letter I (eye) or letter L (el). The optional argument
+map01 when not None, specifies which letter the digit 1 should be
+mapped to (when map01 is not None, the digit 0 is always mapped to
+the letter O). For security purposes the default is None, so that
+0 and 1 are not allowed in the input.
+'''
+_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
+_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV'
+_b32tab2 = {}
+_b32rev = {}
+
+def _b32encode(alphabet, s):
global _b32tab2
# Delay the initialization of the table to not waste memory
# if the function is never called
- if _b32tab2 is None:
- b32tab = [bytes((i,)) for i in _b32alphabet]
- _b32tab2 = [a + b for a in b32tab for b in b32tab]
+ if alphabet not in _b32tab2:
+ b32tab = [bytes((i,)) for i in alphabet]
+ _b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab]
b32tab = None
if not isinstance(s, bytes_types):
@@ -158,7 +179,7 @@
s = s + b'\0' * (5 - leftover) # Don't use += !
encoded = bytearray()
from_bytes = int.from_bytes
- b32tab2 = _b32tab2
+ b32tab2 = _b32tab2[alphabet]
for i in range(0, len(s), 5):
c = from_bytes(s[i: i + 5], 'big')
encoded += (b32tab2[c >> 30] + # bits 1 - 10
@@ -177,29 +198,12 @@
encoded[-1:] = b'='
return bytes(encoded)
-def b32decode(s, casefold=False, map01=None):
- """Decode the Base32 encoded bytes-like object or ASCII string s.
-
- Optional casefold is a flag specifying whether a lowercase alphabet is
- acceptable as input. For security purposes, the default is False.
-
- RFC 3548 allows for optional mapping of the digit 0 (zero) to the
- letter O (oh), and for optional mapping of the digit 1 (one) to
- either the letter I (eye) or letter L (el). The optional argument
- map01 when not None, specifies which letter the digit 1 should be
- mapped to (when map01 is not None, the digit 0 is always mapped to
- the letter O). For security purposes the default is None, so that
- 0 and 1 are not allowed in the input.
-
- The result is returned as a bytes object. A binascii.Error is raised if
- the input is incorrectly padded or if there are non-alphabet
- characters present in the input.
- """
+def _b32decode(alphabet, s, casefold=False, map01=None):
global _b32rev
# Delay the initialization of the table to not waste memory
# if the function is never called
- if _b32rev is None:
- _b32rev = {v: k for k, v in enumerate(_b32alphabet)}
+ if alphabet not in _b32rev:
+ _b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)}
s = _bytes_from_decode_data(s)
if len(s) % 8:
raise binascii.Error('Incorrect padding')
@@ -220,7 +224,7 @@
padchars = l - len(s)
# Now decode the full quanta
decoded = bytearray()
- b32rev = _b32rev
+ b32rev = _b32rev[alphabet]
for i in range(0, len(s), 8):
quanta = s[i: i + 8]
acc = 0
@@ -241,6 +245,26 @@
return bytes(decoded)
+def b32encode(s):
+ return _b32encode(_b32alphabet, s)
+b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32')
+
+def b32decode(s, casefold=False, map01=None):
+ return _b32decode(_b32alphabet, s, casefold, map01)
+b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32',
+ extra_args=_B32_DECODE_MAP01_DOCSTRING)
+
+def b32hexencode(s):
+ return _b32encode(_b32hexalphabet, s)
+b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex')
+
+def b32hexdecode(s, casefold=False):
+ # base32hex does not have the 01 mapping
+ return _b32decode(_b32hexalphabet, s, casefold)
+b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
+ extra_args='')
+
+
# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
# lowercase. The RFC also recommends against accepting input case
# insensitively.
@@ -320,7 +344,7 @@
global _a85chars, _a85chars2
# Delay the initialization of tables to not waste memory
# if the function is never called
- if _a85chars is None:
+ if _a85chars2 is None:
_a85chars = [bytes((i,)) for i in range(33, 118)]
_a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
@@ -428,7 +452,7 @@
global _b85chars, _b85chars2
# Delay the initialization of tables to not waste memory
# if the function is never called
- if _b85chars is None:
+ if _b85chars2 is None:
_b85chars = [bytes((i,)) for i in _b85alphabet]
_b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
return _85encode(b, _b85chars, _b85chars2, pad)
diff --git a/common/py3-stdlib/bdb.py b/common/py3-stdlib/bdb.py
index b18a061..75d6113 100644
--- a/common/py3-stdlib/bdb.py
+++ b/common/py3-stdlib/bdb.py
@@ -34,6 +34,8 @@
self.fncache = {}
self.frame_returning = None
+ self._load_breaks()
+
def canonic(self, filename):
"""Return canonical form of filename.
@@ -117,7 +119,7 @@
"""Invoke user function and return trace function for call event.
If the debugger stops on this function call, invoke
- self.user_call(). Raise BbdQuit if self.quitting is set.
+ self.user_call(). Raise BdbQuit if self.quitting is set.
Return self.trace_dispatch to continue tracing in this scope.
"""
# XXX 'arg' is no longer used
@@ -365,6 +367,12 @@
# Call self.get_*break*() to see the breakpoints or better
# for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
+ def _add_to_breaks(self, filename, lineno):
+ """Add breakpoint to breaks, if not already there."""
+ bp_linenos = self.breaks.setdefault(filename, [])
+ if lineno not in bp_linenos:
+ bp_linenos.append(lineno)
+
def set_break(self, filename, lineno, temporary=False, cond=None,
funcname=None):
"""Set a new breakpoint for filename:lineno.
@@ -377,12 +385,21 @@
line = linecache.getline(filename, lineno)
if not line:
return 'Line %s:%d does not exist' % (filename, lineno)
- list = self.breaks.setdefault(filename, [])
- if lineno not in list:
- list.append(lineno)
+ self._add_to_breaks(filename, lineno)
bp = Breakpoint(filename, lineno, temporary, cond, funcname)
return None
+ def _load_breaks(self):
+ """Apply all breakpoints (set in other instances) to this one.
+
+ Populates this instance's breaks list from the Breakpoint class's
+ list, which can have breakpoints set by another Bdb instance. This
+ is necessary for interactive sessions to keep the breakpoints
+ active across multiple calls to run().
+ """
+ for (filename, lineno) in Breakpoint.bplist.keys():
+ self._add_to_breaks(filename, lineno)
+
def _prune_breaks(self, filename, lineno):
"""Prune breakpoints for filename:lineno.
@@ -681,6 +698,12 @@
else:
self.bplist[file, line] = [self]
+ @staticmethod
+ def clearBreakpoints():
+ Breakpoint.next = 1
+ Breakpoint.bplist = {}
+ Breakpoint.bpbynumber = [None]
+
def deleteMe(self):
"""Delete the breakpoint from the list associated to a file:line.
diff --git a/common/py3-stdlib/bisect.py b/common/py3-stdlib/bisect.py
index 8f3f6a3..d37da74 100644
--- a/common/py3-stdlib/bisect.py
+++ b/common/py3-stdlib/bisect.py
@@ -1,6 +1,7 @@
"""Bisection algorithms."""
-def insort_right(a, x, lo=0, hi=None):
+
+def insort_right(a, x, lo=0, hi=None, *, key=None):
"""Insert item x in list a, and keep it sorted assuming a is sorted.
If x is already in a, insert it to the right of the rightmost x.
@@ -8,15 +9,18 @@
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
"""
-
- lo = bisect_right(a, x, lo, hi)
+ if key is None:
+ lo = bisect_right(a, x, lo, hi)
+ else:
+ lo = bisect_right(a, key(x), lo, hi, key=key)
a.insert(lo, x)
-def bisect_right(a, x, lo=0, hi=None):
+
+def bisect_right(a, x, lo=0, hi=None, *, key=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e <= x, and all e in
- a[i:] have e > x. So if x already appears in the list, a.insert(x) will
+ a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will
insert just after the rightmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
@@ -27,14 +31,26 @@
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
- while lo < hi:
- mid = (lo+hi)//2
- # Use __lt__ to match the logic in list.sort() and in heapq
- if x < a[mid]: hi = mid
- else: lo = mid+1
+ # Note, the comparison uses "<" to match the
+ # __lt__() logic in list.sort() and in heapq.
+ if key is None:
+ while lo < hi:
+ mid = (lo + hi) // 2
+ if x < a[mid]:
+ hi = mid
+ else:
+ lo = mid + 1
+ else:
+ while lo < hi:
+ mid = (lo + hi) // 2
+ if x < key(a[mid]):
+ hi = mid
+ else:
+ lo = mid + 1
return lo
-def insort_left(a, x, lo=0, hi=None):
+
+def insort_left(a, x, lo=0, hi=None, *, key=None):
"""Insert item x in list a, and keep it sorted assuming a is sorted.
If x is already in a, insert it to the left of the leftmost x.
@@ -43,15 +59,17 @@
slice of a to be searched.
"""
- lo = bisect_left(a, x, lo, hi)
+ if key is None:
+ lo = bisect_left(a, x, lo, hi)
+ else:
+ lo = bisect_left(a, key(x), lo, hi, key=key)
a.insert(lo, x)
-
-def bisect_left(a, x, lo=0, hi=None):
+def bisect_left(a, x, lo=0, hi=None, *, key=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e < x, and all e in
- a[i:] have e >= x. So if x already appears in the list, a.insert(x) will
+ a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will
insert just before the leftmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
@@ -62,13 +80,25 @@
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
- while lo < hi:
- mid = (lo+hi)//2
- # Use __lt__ to match the logic in list.sort() and in heapq
- if a[mid] < x: lo = mid+1
- else: hi = mid
+ # Note, the comparison uses "<" to match the
+ # __lt__() logic in list.sort() and in heapq.
+ if key is None:
+ while lo < hi:
+ mid = (lo + hi) // 2
+ if a[mid] < x:
+ lo = mid + 1
+ else:
+ hi = mid
+ else:
+ while lo < hi:
+ mid = (lo + hi) // 2
+ if key(a[mid]) < x:
+ lo = mid + 1
+ else:
+ hi = mid
return lo
+
# Overwrite above definitions with a fast C implementation
try:
from _bisect import *
diff --git a/common/py3-stdlib/bz2.py b/common/py3-stdlib/bz2.py
index ce07ebe..fabe4f7 100644
--- a/common/py3-stdlib/bz2.py
+++ b/common/py3-stdlib/bz2.py
@@ -13,7 +13,6 @@
import io
import os
import _compression
-from threading import RLock
from _bz2 import BZ2Compressor, BZ2Decompressor
@@ -53,9 +52,6 @@
If mode is 'r', the input file may be the concatenation of
multiple compressed streams.
"""
- # This lock must be recursive, so that BufferedIOBase's
- # writelines() does not deadlock.
- self._lock = RLock()
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
@@ -104,24 +100,23 @@
May be called more than once without error. Once the file is
closed, any other operation on it will raise a ValueError.
"""
- with self._lock:
- if self._mode == _MODE_CLOSED:
- return
+ if self._mode == _MODE_CLOSED:
+ return
+ try:
+ if self._mode == _MODE_READ:
+ self._buffer.close()
+ elif self._mode == _MODE_WRITE:
+ self._fp.write(self._compressor.flush())
+ self._compressor = None
+ finally:
try:
- if self._mode == _MODE_READ:
- self._buffer.close()
- elif self._mode == _MODE_WRITE:
- self._fp.write(self._compressor.flush())
- self._compressor = None
+ if self._closefp:
+ self._fp.close()
finally:
- try:
- if self._closefp:
- self._fp.close()
- finally:
- self._fp = None
- self._closefp = False
- self._mode = _MODE_CLOSED
- self._buffer = None
+ self._fp = None
+ self._closefp = False
+ self._mode = _MODE_CLOSED
+ self._buffer = None
@property
def closed(self):
@@ -153,12 +148,11 @@
Always returns at least one byte of data, unless at EOF.
The exact number of bytes returned is unspecified.
"""
- with self._lock:
- self._check_can_read()
- # Relies on the undocumented fact that BufferedReader.peek()
- # always returns at least one byte (except at EOF), independent
- # of the value of n
- return self._buffer.peek(n)
+ self._check_can_read()
+ # Relies on the undocumented fact that BufferedReader.peek()
+ # always returns at least one byte (except at EOF), independent
+ # of the value of n
+ return self._buffer.peek(n)
def read(self, size=-1):
"""Read up to size uncompressed bytes from the file.
@@ -166,9 +160,8 @@
If size is negative or omitted, read until EOF is reached.
Returns b'' if the file is already at EOF.
"""
- with self._lock:
- self._check_can_read()
- return self._buffer.read(size)
+ self._check_can_read()
+ return self._buffer.read(size)
def read1(self, size=-1):
"""Read up to size uncompressed bytes, while trying to avoid
@@ -177,20 +170,18 @@
Returns b'' if the file is at EOF.
"""
- with self._lock:
- self._check_can_read()
- if size < 0:
- size = io.DEFAULT_BUFFER_SIZE
- return self._buffer.read1(size)
+ self._check_can_read()
+ if size < 0:
+ size = io.DEFAULT_BUFFER_SIZE
+ return self._buffer.read1(size)
def readinto(self, b):
"""Read bytes into b.
Returns the number of bytes read (0 for EOF).
"""
- with self._lock:
- self._check_can_read()
- return self._buffer.readinto(b)
+ self._check_can_read()
+ return self._buffer.readinto(b)
def readline(self, size=-1):
"""Read a line of uncompressed bytes from the file.
@@ -203,9 +194,8 @@
if not hasattr(size, "__index__"):
raise TypeError("Integer argument expected")
size = size.__index__()
- with self._lock:
- self._check_can_read()
- return self._buffer.readline(size)
+ self._check_can_read()
+ return self._buffer.readline(size)
def readlines(self, size=-1):
"""Read a list of lines of uncompressed bytes from the file.
@@ -218,23 +208,29 @@
if not hasattr(size, "__index__"):
raise TypeError("Integer argument expected")
size = size.__index__()
- with self._lock:
- self._check_can_read()
- return self._buffer.readlines(size)
+ self._check_can_read()
+ return self._buffer.readlines(size)
def write(self, data):
"""Write a byte string to the file.
Returns the number of uncompressed bytes written, which is
- always len(data). Note that due to buffering, the file on disk
- may not reflect the data written until close() is called.
+ always the length of data in bytes. Note that due to buffering,
+ the file on disk may not reflect the data written until close()
+ is called.
"""
- with self._lock:
- self._check_can_write()
- compressed = self._compressor.compress(data)
- self._fp.write(compressed)
- self._pos += len(data)
- return len(data)
+ self._check_can_write()
+ if isinstance(data, (bytes, bytearray)):
+ length = len(data)
+ else:
+ # accept any data that supports the buffer protocol
+ data = memoryview(data)
+ length = data.nbytes
+
+ compressed = self._compressor.compress(data)
+ self._fp.write(compressed)
+ self._pos += length
+ return length
def writelines(self, seq):
"""Write a sequence of byte strings to the file.
@@ -244,8 +240,7 @@
Line separators are not added between the written byte strings.
"""
- with self._lock:
- return _compression.BaseStream.writelines(self, seq)
+ return _compression.BaseStream.writelines(self, seq)
def seek(self, offset, whence=io.SEEK_SET):
"""Change the file position.
@@ -262,17 +257,15 @@
Note that seeking is emulated, so depending on the parameters,
this operation may be extremely slow.
"""
- with self._lock:
- self._check_can_seek()
- return self._buffer.seek(offset, whence)
+ self._check_can_seek()
+ return self._buffer.seek(offset, whence)
def tell(self):
"""Return the current file position."""
- with self._lock:
- self._check_not_closed()
- if self._mode == _MODE_READ:
- return self._buffer.tell()
- return self._pos
+ self._check_not_closed()
+ if self._mode == _MODE_READ:
+ return self._buffer.tell()
+ return self._pos
def open(filename, mode="rb", compresslevel=9,
@@ -311,6 +304,7 @@
binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
if "t" in mode:
+ encoding = io.text_encoding(encoding)
return io.TextIOWrapper(binary_file, encoding, errors, newline)
else:
return binary_file
diff --git a/common/py3-stdlib/cProfile.py b/common/py3-stdlib/cProfile.py
index 59b4699..22a7d0a 100755
--- a/common/py3-stdlib/cProfile.py
+++ b/common/py3-stdlib/cProfile.py
@@ -175,7 +175,12 @@
'__package__': None,
'__cached__': None,
}
- runctx(code, globs, None, options.outfile, options.sort)
+ try:
+ runctx(code, globs, None, options.outfile, options.sort)
+ except BrokenPipeError as exc:
+ # Prevent "Exception ignored" during interpreter shutdown.
+ sys.stdout = None
+ sys.exit(exc.errno)
else:
parser.print_usage()
return parser
diff --git a/common/py3-stdlib/calendar.py b/common/py3-stdlib/calendar.py
index 7550d52..cbea9ec 100644
--- a/common/py3-stdlib/calendar.py
+++ b/common/py3-stdlib/calendar.py
@@ -15,7 +15,9 @@
"monthcalendar", "prmonth", "month", "prcal", "calendar",
"timegm", "month_name", "month_abbr", "day_name", "day_abbr",
"Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar",
- "LocaleHTMLCalendar", "weekheader"]
+ "LocaleHTMLCalendar", "weekheader",
+ "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY",
+ "SATURDAY", "SUNDAY"]
# Exception raised for bad input (with string parameter for details)
error = ValueError
@@ -571,19 +573,11 @@
def formatweekday(self, day, width):
with different_locale(self.locale):
- if width >= 9:
- names = day_name
- else:
- names = day_abbr
- name = names[day]
- return name[:width].center(width)
+ return super().formatweekday(day, width)
def formatmonthname(self, theyear, themonth, width, withyear=True):
with different_locale(self.locale):
- s = month_name[themonth]
- if withyear:
- s = "%s %r" % (s, theyear)
- return s.center(width)
+ return super().formatmonthname(theyear, themonth, width, withyear)
class LocaleHTMLCalendar(HTMLCalendar):
@@ -601,16 +595,11 @@
def formatweekday(self, day):
with different_locale(self.locale):
- s = day_abbr[day]
- return '<th class="%s">%s</th>' % (self.cssclasses[day], s)
+ return super().formatweekday(day)
def formatmonthname(self, theyear, themonth, withyear=True):
with different_locale(self.locale):
- s = month_name[themonth]
- if withyear:
- s = '%s %s' % (s, theyear)
- return '<tr><th colspan="7" class="month">%s</th></tr>' % s
-
+ return super().formatmonthname(theyear, themonth, withyear)
# Support for old module level interface
c = TextCalendar()
diff --git a/common/py3-stdlib/cgi.py b/common/py3-stdlib/cgi.py
index 77ab703..6cb8cf2 100755
--- a/common/py3-stdlib/cgi.py
+++ b/common/py3-stdlib/cgi.py
@@ -41,6 +41,7 @@
import html
import locale
import tempfile
+import warnings
__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart",
"parse_header", "test", "print_exception", "print_environ",
@@ -77,9 +78,11 @@
"""
global log, logfile, logfp
+ warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead",
+ DeprecationWarning, stacklevel=2)
if logfile and not logfp:
try:
- logfp = open(logfile, "a")
+ logfp = open(logfile, "a", encoding="locale")
except OSError:
pass
if not logfp:
@@ -115,7 +118,8 @@
# 0 ==> unlimited input
maxlen = 0
-def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
+def parse(fp=None, environ=os.environ, keep_blank_values=0,
+ strict_parsing=0, separator='&'):
"""Parse a query in the environment or from a file (default stdin)
Arguments, all optional:
@@ -134,6 +138,9 @@
strict_parsing: flag indicating what to do with parsing errors.
If false (the default), errors are silently ignored.
If true, errors raise a ValueError exception.
+
+ separator: str. The symbol to use for separating the query arguments.
+ Defaults to &.
"""
if fp is None:
fp = sys.stdin
@@ -154,7 +161,7 @@
if environ['REQUEST_METHOD'] == 'POST':
ctype, pdict = parse_header(environ['CONTENT_TYPE'])
if ctype == 'multipart/form-data':
- return parse_multipart(fp, pdict)
+ return parse_multipart(fp, pdict, separator=separator)
elif ctype == 'application/x-www-form-urlencoded':
clength = int(environ['CONTENT_LENGTH'])
if maxlen and clength > maxlen:
@@ -178,10 +185,10 @@
qs = ""
environ['QUERY_STRING'] = qs # XXX Shouldn't, really
return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
- encoding=encoding)
+ encoding=encoding, separator=separator)
-def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"):
+def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'):
"""Parse multipart input.
Arguments:
@@ -194,7 +201,7 @@
value is a list of values for that field. For non-file fields, the value
is a list of strings.
"""
- # RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always
+ # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always
# represented as 7bit US-ASCII.
boundary = pdict['boundary'].decode('ascii')
ctype = "multipart/form-data; boundary={}".format(boundary)
@@ -205,7 +212,7 @@
except KeyError:
pass
fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors,
- environ={'REQUEST_METHOD': 'POST'})
+ environ={'REQUEST_METHOD': 'POST'}, separator=separator)
return {k: fs.getlist(k) for k in fs}
def _parseparam(s):
@@ -315,7 +322,7 @@
def __init__(self, fp=None, headers=None, outerboundary=b'',
environ=os.environ, keep_blank_values=0, strict_parsing=0,
limit=None, encoding='utf-8', errors='replace',
- max_num_fields=None):
+ max_num_fields=None, separator='&'):
"""Constructor. Read multipart/* until last part.
Arguments, all optional:
@@ -363,6 +370,7 @@
self.keep_blank_values = keep_blank_values
self.strict_parsing = strict_parsing
self.max_num_fields = max_num_fields
+ self.separator = separator
if 'REQUEST_METHOD' in environ:
method = environ['REQUEST_METHOD'].upper()
self.qs_on_post = None
@@ -589,7 +597,7 @@
query = urllib.parse.parse_qsl(
qs, self.keep_blank_values, self.strict_parsing,
encoding=self.encoding, errors=self.errors,
- max_num_fields=self.max_num_fields)
+ max_num_fields=self.max_num_fields, separator=self.separator)
self.list = [MiniFieldStorage(key, value) for key, value in query]
self.skip_lines()
@@ -605,7 +613,7 @@
query = urllib.parse.parse_qsl(
self.qs_on_post, self.keep_blank_values, self.strict_parsing,
encoding=self.encoding, errors=self.errors,
- max_num_fields=self.max_num_fields)
+ max_num_fields=self.max_num_fields, separator=self.separator)
self.list.extend(MiniFieldStorage(key, value) for key, value in query)
klass = self.FieldStorageClass or self.__class__
@@ -649,7 +657,7 @@
else self.limit - self.bytes_read
part = klass(self.fp, headers, ib, environ, keep_blank_values,
strict_parsing, limit,
- self.encoding, self.errors, max_num_fields)
+ self.encoding, self.errors, max_num_fields, self.separator)
if max_num_fields is not None:
max_num_fields -= 1
diff --git a/common/py3-stdlib/cgitb.py b/common/py3-stdlib/cgitb.py
index 4f81271..17ddda3 100644
--- a/common/py3-stdlib/cgitb.py
+++ b/common/py3-stdlib/cgitb.py
@@ -181,8 +181,8 @@
<!-- The above is a description of an error in a Python program, formatted
- for a Web browser because the 'cgitb' module was enabled. In case you
- are not reading this in a Web browser, here is the original traceback:
+ for a web browser because the 'cgitb' module was enabled. In case you
+ are not reading this in a web browser, here is the original traceback:
%s
-->
diff --git a/common/py3-stdlib/codecs.py b/common/py3-stdlib/codecs.py
index 7f23e97..e6ad6e3 100644
--- a/common/py3-stdlib/codecs.py
+++ b/common/py3-stdlib/codecs.py
@@ -83,7 +83,7 @@
class CodecInfo(tuple):
"""Codec details when looking up the codec registry"""
- # Private API to allow Python 3.4 to blacklist the known non-Unicode
+ # Private API to allow Python 3.4 to denylist the known non-Unicode
# codecs in the standard library. A more general mechanism to
# reliably distinguish test encodings from other codecs will hopefully
# be defined for Python 3.5
@@ -386,7 +386,7 @@
def reset(self):
- """ Flushes and resets the codec buffers used for keeping state.
+ """ Resets the codec buffers used for keeping internal state.
Calling this method should ensure that the data on the
output is put into a clean state, that allows appending
@@ -620,7 +620,7 @@
def reset(self):
- """ Resets the codec buffers used for keeping state.
+ """ Resets the codec buffers used for keeping internal state.
Note that no stream repositioning should take place.
This method is primarily intended to be able to recover
diff --git a/common/py3-stdlib/codeop.py b/common/py3-stdlib/codeop.py
index 4c10470..568e9bb 100644
--- a/common/py3-stdlib/codeop.py
+++ b/common/py3-stdlib/codeop.py
@@ -10,30 +10,6 @@
syntax error (OverflowError and ValueError can be produced by
malformed literals).
-Approach:
-
-First, check if the source consists entirely of blank lines and
-comments; if so, replace it with 'pass', because the built-in
-parser doesn't always do the right thing for these.
-
-Compile three times: as is, with \n, and with \n\n appended. If it
-compiles as is, it's complete. If it compiles with one \n appended,
-we expect more. If it doesn't compile either way, we compare the
-error we get when compiling with \n or \n\n appended. If the errors
-are the same, the code is broken. But if the errors are different, we
-expect more. Not intuitive; not even guaranteed to hold in future
-releases; but this matches the compiler's behavior from Python 1.4
-through 2.2, at least.
-
-Caveat:
-
-It is possible (but not likely) that the parser stops parsing with a
-successful outcome before reaching the end of the source; in this
-case, trailing symbols may be ignored instead of causing an error.
-For example, a backslash followed by two newlines may be followed by
-arbitrary garbage. This will be fixed once the API for the parser is
-better.
-
The two interfaces are:
compile_command(source, filename, symbol):
@@ -64,24 +40,25 @@
__all__ = ["compile_command", "Compile", "CommandCompiler"]
-PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h
+# The following flags match the values from Include/cpython/compile.h
+# Caveat emptor: These flags are undocumented on purpose and depending
+# on their effect outside the standard library is **unsupported**.
+PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
def _maybe_compile(compiler, source, filename, symbol):
- # Check for source consisting of only blank lines and comments
+ # Check for source consisting of only blank lines and comments.
for line in source.split("\n"):
line = line.strip()
if line and line[0] != '#':
- break # Leave it alone
+ break # Leave it alone.
else:
if symbol != "eval":
source = "pass" # Replace it with a 'pass' statement
- err = err1 = err2 = None
- code = code1 = code2 = None
-
try:
- code = compiler(source, filename, symbol)
- except SyntaxError:
+ return compiler(source, filename, symbol)
+ except SyntaxError: # Let other compile() errors propagate.
pass
# Catch syntax warnings after the first compile
@@ -90,25 +67,23 @@
warnings.simplefilter("error")
try:
- code1 = compiler(source + "\n", filename, symbol)
+ compiler(source + "\n", filename, symbol)
except SyntaxError as e:
- err1 = e
+ if "incomplete input" in str(e):
+ return None
+ raise
- try:
- code2 = compiler(source + "\n\n", filename, symbol)
- except SyntaxError as e:
- err2 = e
-
- try:
- if code:
- return code
- if not code1 and repr(err1) == repr(err2):
- raise err1
- finally:
- err1 = err2 = None
+def _is_syntax_error(err1, err2):
+ rep1 = repr(err1)
+ rep2 = repr(err2)
+ if "was never closed" in rep1 and "was never closed" in rep2:
+ return False
+ if rep1 == rep2:
+ return True
+ return False
def _compile(source, filename, symbol):
- return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
+ return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT)
def compile_command(source, filename="<input>", symbol="single"):
r"""Compile a command and determine whether it is incomplete.
@@ -137,7 +112,7 @@
statement, it "remembers" and compiles all subsequent program texts
with the statement in force."""
def __init__(self):
- self.flags = PyCF_DONT_IMPLY_DEDENT
+ self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
def __call__(self, source, filename, symbol):
codeob = compile(source, filename, symbol, self.flags, True)
diff --git a/common/py3-stdlib/collections/__init__.py b/common/py3-stdlib/collections/__init__.py
index bc69a67..818588f 100644
--- a/common/py3-stdlib/collections/__init__.py
+++ b/common/py3-stdlib/collections/__init__.py
@@ -27,7 +27,6 @@
]
import _collections_abc
-import heapq as _heapq
import sys as _sys
from itertools import chain as _chain
@@ -52,22 +51,6 @@
pass
-def __getattr__(name):
- # For backwards compatibility, continue to make the collections ABCs
- # through Python 3.6 available through the collections module.
- # Note, no new collections ABCs were added in Python 3.7
- if name in _collections_abc.__all__:
- obj = getattr(_collections_abc, name)
- import warnings
- warnings.warn("Using or importing the ABCs from 'collections' instead "
- "of from 'collections.abc' is deprecated since Python 3.3, "
- "and in 3.10 it will stop working",
- DeprecationWarning, stacklevel=2)
- globals()[name] = obj
- return obj
- raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
-
-
################################################################################
### OrderedDict
################################################################################
@@ -424,7 +407,7 @@
namespace = {
'_tuple_new': tuple_new,
- '__builtins__': None,
+ '__builtins__': {},
'__name__': f'namedtuple_{typename}',
}
code = f'lambda _cls, {arg_list}: _tuple_new(_cls, ({arg_list}))'
@@ -489,6 +472,7 @@
'__repr__': __repr__,
'_asdict': _asdict,
'__getnewargs__': __getnewargs__,
+ '__match_args__': field_names,
}
for index, name in enumerate(field_names):
doc = _sys.intern(f'Alias for field number {index}')
@@ -597,6 +581,10 @@
# Needed so that self[missing_item] does not raise KeyError
return 0
+ def total(self):
+ 'Sum of the counts'
+ return sum(self.values())
+
def most_common(self, n=None):
'''List the n most common elements and their counts from the most
common to the least. If n is None, then list all element counts.
@@ -608,7 +596,10 @@
# Emulate Bag.sortedByCount from Smalltalk
if n is None:
return sorted(self.items(), key=_itemgetter(1), reverse=True)
- return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
+
+ # Lazy import to speedup Python startup time
+ import heapq
+ return heapq.nlargest(n, self.items(), key=_itemgetter(1))
def elements(self):
'''Iterator over elements repeating each as many times as its count.
@@ -719,6 +710,42 @@
if elem in self:
super().__delitem__(elem)
+ def __eq__(self, other):
+ 'True if all counts agree. Missing counts are treated as zero.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return all(self[e] == other[e] for c in (self, other) for e in c)
+
+ def __ne__(self, other):
+ 'True if any counts disagree. Missing counts are treated as zero.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return not self == other
+
+ def __le__(self, other):
+ 'True if all counts in self are a subset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return all(self[e] <= other[e] for c in (self, other) for e in c)
+
+ def __lt__(self, other):
+ 'True if all counts in self are a proper subset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return self <= other and self != other
+
+ def __ge__(self, other):
+ 'True if all counts in self are a superset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return all(self[e] >= other[e] for c in (self, other) for e in c)
+
+ def __gt__(self, other):
+ 'True if all counts in self are a proper superset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return self >= other and self != other
+
def __repr__(self):
if not self:
return f'{self.__class__.__name__}()'
@@ -739,12 +766,30 @@
# To strip negative and zero counts, add-in an empty counter:
# c += Counter()
#
- # Rich comparison operators for multiset subset and superset tests
- # are deliberately omitted due to semantic conflicts with the
- # existing inherited dict equality method. Subset and superset
- # semantics ignore zero counts and require that p≤q ∧ p≥q → p=q;
- # however, that would not be the case for p=Counter(a=1, b=0)
- # and q=Counter(a=1) where the dictionaries are not equal.
+ # Results are ordered according to when an element is first
+ # encountered in the left operand and then by the order
+ # encountered in the right operand.
+ #
+ # When the multiplicities are all zero or one, multiset operations
+ # are guaranteed to be equivalent to the corresponding operations
+ # for regular sets.
+ # Given counter multisets such as:
+ # cp = Counter(a=1, b=0, c=1)
+ # cq = Counter(c=1, d=0, e=1)
+ # The corresponding regular sets would be:
+ # sp = {'a', 'c'}
+ # sq = {'c', 'e'}
+ # All of the following relations would hold:
+ # set(cp + cq) == sp | sq
+ # set(cp - cq) == sp - sq
+ # set(cp | cq) == sp | sq
+ # set(cp & cq) == sp & sq
+ # (cp == cq) == (sp == sq)
+ # (cp != cq) == (sp != sq)
+ # (cp <= cq) == (sp <= sq)
+ # (cp < cq) == (sp < sq)
+ # (cp >= cq) == (sp >= sq)
+ # (cp > cq) == (sp > sq)
def __add__(self, other):
'''Add counts from two counters.
@@ -973,12 +1018,15 @@
__copy__ = copy
- def new_child(self, m=None): # like Django's Context.push()
+ def new_child(self, m=None, **kwargs): # like Django's Context.push()
'''New ChainMap with a new map followed by all previous maps.
If no map is provided, an empty dict is used.
+ Keyword arguments update the map or new empty dict.
'''
if m is None:
- m = {}
+ m = kwargs
+ elif kwargs:
+ m.update(kwargs)
return self.__class__(m, *self.maps)
@property
diff --git a/common/py3-stdlib/collections/abc.py b/common/py3-stdlib/collections/abc.py
index 891600d..86ca8b8 100644
--- a/common/py3-stdlib/collections/abc.py
+++ b/common/py3-stdlib/collections/abc.py
@@ -1,2 +1,3 @@
from _collections_abc import *
from _collections_abc import __all__
+from _collections_abc import _CallableGenericAlias
diff --git a/common/py3-stdlib/colorsys.py b/common/py3-stdlib/colorsys.py
index b93e384..0f52512 100644
--- a/common/py3-stdlib/colorsys.py
+++ b/common/py3-stdlib/colorsys.py
@@ -75,17 +75,18 @@
def rgb_to_hls(r, g, b):
maxc = max(r, g, b)
minc = min(r, g, b)
- # XXX Can optimize (maxc+minc) and (maxc-minc)
- l = (minc+maxc)/2.0
+ sumc = (maxc+minc)
+ rangec = (maxc-minc)
+ l = sumc/2.0
if minc == maxc:
return 0.0, l, 0.0
if l <= 0.5:
- s = (maxc-minc) / (maxc+minc)
+ s = rangec / sumc
else:
- s = (maxc-minc) / (2.0-maxc-minc)
- rc = (maxc-r) / (maxc-minc)
- gc = (maxc-g) / (maxc-minc)
- bc = (maxc-b) / (maxc-minc)
+ s = rangec / (2.0-sumc)
+ rc = (maxc-r) / rangec
+ gc = (maxc-g) / rangec
+ bc = (maxc-b) / rangec
if r == maxc:
h = bc-gc
elif g == maxc:
diff --git a/common/py3-stdlib/compileall.py b/common/py3-stdlib/compileall.py
index fe7f450..3755e76 100644
--- a/common/py3-stdlib/compileall.py
+++ b/common/py3-stdlib/compileall.py
@@ -84,12 +84,14 @@
if workers < 0:
raise ValueError('workers must be greater or equal to 0')
if workers != 1:
+ # Check if this is a system where ProcessPoolExecutor can function.
+ from concurrent.futures.process import _check_system_limits
try:
- # Only import when needed, as low resource platforms may
- # fail to import it
- from concurrent.futures import ProcessPoolExecutor
- except ImportError:
+ _check_system_limits()
+ except NotImplementedError:
workers = 1
+ else:
+ from concurrent.futures import ProcessPoolExecutor
if maxlevels is None:
maxlevels = sys.getrecursionlimit()
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
@@ -219,8 +221,8 @@
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
- expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER,
- 0, mtime)
+ expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
+ 0, mtime & 0xFFFF_FFFF)
for cfile in opt_cfiles.values():
with open(cfile, 'rb') as chandle:
actual = chandle.read(12)
@@ -252,9 +254,8 @@
else:
print('*** ', end='')
# escape non-printable characters in msg
- msg = err.msg.encode(sys.stdout.encoding,
- errors='backslashreplace')
- msg = msg.decode(sys.stdout.encoding)
+ encoding = sys.stdout.encoding or sys.getdefaultencoding()
+ msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding)
print(msg)
except (SyntaxError, UnicodeError, OSError) as e:
success = False
@@ -366,9 +367,9 @@
'environment variable is set, and '
'"timestamp" otherwise.'))
parser.add_argument('-o', action='append', type=int, dest='opt_levels',
- help=('Optimization levels to run compilation with.'
- 'Default is -1 which uses optimization level of'
- 'Python interpreter itself (specified by -O).'))
+ help=('Optimization levels to run compilation with. '
+ 'Default is -1 which uses the optimization level '
+ 'of the Python interpreter itself (see -O).'))
parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
help='Ignore symlinks pointing outsite of the DIR')
parser.add_argument('--hardlink-dupes', action='store_true',
@@ -405,7 +406,8 @@
# if flist is provided then load it
if args.flist:
try:
- with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
+ with (sys.stdin if args.flist=='-' else
+ open(args.flist, encoding="utf-8")) as f:
for line in f:
compile_dests.append(line.strip())
except OSError:
diff --git a/common/py3-stdlib/concurrent/futures/_base.py b/common/py3-stdlib/concurrent/futures/_base.py
index 00eb548..5c00f2e 100644
--- a/common/py3-stdlib/concurrent/futures/_base.py
+++ b/common/py3-stdlib/concurrent/futures/_base.py
@@ -284,13 +284,14 @@
A named 2-tuple of sets. The first set, named 'done', contains the
futures that completed (is finished or cancelled) before the wait
completed. The second set, named 'not_done', contains uncompleted
- futures.
+ futures. Duplicate futures given to *fs* are removed and will be
+ returned only once.
"""
+ fs = set(fs)
with _AcquireFutures(fs):
- done = set(f for f in fs
- if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
- not_done = set(fs) - done
-
+ done = {f for f in fs
+ if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
+ not_done = fs - done
if (return_when == FIRST_COMPLETED) and done:
return DoneAndNotDoneFutures(done, not_done)
elif (return_when == FIRST_EXCEPTION) and done:
@@ -309,7 +310,7 @@
f._waiters.remove(waiter)
done.update(waiter.finished_futures)
- return DoneAndNotDoneFutures(done, set(fs) - done)
+ return DoneAndNotDoneFutures(done, fs - done)
class Future(object):
"""Represents the result of an asynchronous computation."""
@@ -386,7 +387,11 @@
def __get_result(self):
if self._exception:
- raise self._exception
+ try:
+ raise self._exception
+ finally:
+ # Break a reference cycle with the exception in self._exception
+ self = None
else:
return self._result
@@ -426,20 +431,24 @@
timeout.
Exception: If the call raised then that exception will be raised.
"""
- with self._condition:
- if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
- raise CancelledError()
- elif self._state == FINISHED:
- return self.__get_result()
+ try:
+ with self._condition:
+ if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+ raise CancelledError()
+ elif self._state == FINISHED:
+ return self.__get_result()
- self._condition.wait(timeout)
+ self._condition.wait(timeout)
- if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
- raise CancelledError()
- elif self._state == FINISHED:
- return self.__get_result()
- else:
- raise TimeoutError()
+ if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+ raise CancelledError()
+ elif self._state == FINISHED:
+ return self.__get_result()
+ else:
+ raise TimeoutError()
+ finally:
+ # Break a reference cycle with the exception in self._exception
+ self = None
def exception(self, timeout=None):
"""Return the exception raised by the call that the future represents.
diff --git a/common/py3-stdlib/concurrent/futures/process.py b/common/py3-stdlib/concurrent/futures/process.py
index 90bc98b..6ee2ce6 100644
--- a/common/py3-stdlib/concurrent/futures/process.py
+++ b/common/py3-stdlib/concurrent/futures/process.py
@@ -373,7 +373,7 @@
assert not self.thread_wakeup._closed
wakeup_reader = self.thread_wakeup._reader
readers = [result_reader, wakeup_reader]
- worker_sentinels = [p.sentinel for p in self.processes.values()]
+ worker_sentinels = [p.sentinel for p in list(self.processes.values())]
ready = mp.connection.wait(readers + worker_sentinels)
cause = None
@@ -533,6 +533,14 @@
raise NotImplementedError(_system_limited)
_system_limits_checked = True
try:
+ import multiprocessing.synchronize
+ except ImportError:
+ _system_limited = (
+ "This Python build lacks multiprocessing.synchronize, usually due "
+ "to named semaphores being unavailable on this platform."
+ )
+ raise NotImplementedError(_system_limited)
+ try:
nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")
except (AttributeError, ValueError):
# sysconf not available or setting not available
diff --git a/common/py3-stdlib/concurrent/futures/thread.py b/common/py3-stdlib/concurrent/futures/thread.py
index b7a2cac..51c942f 100644
--- a/common/py3-stdlib/concurrent/futures/thread.py
+++ b/common/py3-stdlib/concurrent/futures/thread.py
@@ -36,6 +36,12 @@
# See bpo-39812 for context.
threading._register_atexit(_python_exit)
+# At fork, reinitialize the `_global_shutdown_lock` lock in the child process
+if hasattr(os, 'register_at_fork'):
+ os.register_at_fork(before=_global_shutdown_lock.acquire,
+ after_in_child=_global_shutdown_lock._at_fork_reinit,
+ after_in_parent=_global_shutdown_lock.release)
+
class _WorkItem(object):
def __init__(self, future, fn, args, kwargs):
diff --git a/common/py3-stdlib/configparser.py b/common/py3-stdlib/configparser.py
index 924cc56..3470624 100644
--- a/common/py3-stdlib/configparser.py
+++ b/common/py3-stdlib/configparser.py
@@ -316,7 +316,7 @@
def filename(self):
"""Deprecated, use `source'."""
warnings.warn(
- "The 'filename' attribute will be removed in future versions. "
+ "The 'filename' attribute will be removed in Python 3.12. "
"Use 'source' instead.",
DeprecationWarning, stacklevel=2
)
@@ -326,7 +326,7 @@
def filename(self, value):
"""Deprecated, user `source'."""
warnings.warn(
- "The 'filename' attribute will be removed in future versions. "
+ "The 'filename' attribute will be removed in Python 3.12. "
"Use 'source' instead.",
DeprecationWarning, stacklevel=2
)
@@ -563,7 +563,7 @@
# Regular expressions for parsing section headers and options
_SECT_TMPL = r"""
\[ # [
- (?P<header>[^]]+) # very permissive!
+ (?P<header>.+) # very permissive!
\] # ]
"""
_OPT_TMPL = r"""
@@ -690,6 +690,7 @@
"""
if isinstance(filenames, (str, bytes, os.PathLike)):
filenames = [filenames]
+ encoding = io.text_encoding(encoding)
read_ok = []
for filename in filenames:
try:
@@ -756,7 +757,7 @@
def readfp(self, fp, filename=None):
"""Deprecated, use read_file instead."""
warnings.warn(
- "This method will be removed in future versions. "
+ "This method will be removed in Python 3.12. "
"Use 'parser.read_file()' instead.",
DeprecationWarning, stacklevel=2
)
@@ -907,6 +908,9 @@
If `space_around_delimiters' is True (the default), delimiters
between keys and values are surrounded by spaces.
+
+ Please note that comments in the original configuration file are not
+ preserved when writing the configuration back.
"""
if space_around_delimiters:
d = " {} ".format(self._delimiters[0])
@@ -1005,7 +1009,7 @@
Configuration files may include comments, prefixed by specific
characters (`#' and `;' by default). Comments may appear on their own
in an otherwise empty line or may be entered in lines holding values or
- section names.
+ section names. Please note that comments get stripped off when reading configuration files.
"""
elements_added = set()
cursect = None # None, or a dictionary
@@ -1228,7 +1232,7 @@
super().__init__(*args, **kwargs)
warnings.warn(
"The SafeConfigParser class has been renamed to ConfigParser "
- "in Python 3.2. This alias will be removed in future versions."
+ "in Python 3.2. This alias will be removed in Python 3.12."
" Use ConfigParser directly instead.",
DeprecationWarning, stacklevel=2
)
diff --git a/common/py3-stdlib/contextlib.py b/common/py3-stdlib/contextlib.py
index ff92d9f..c63a849 100644
--- a/common/py3-stdlib/contextlib.py
+++ b/common/py3-stdlib/contextlib.py
@@ -9,7 +9,7 @@
__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
"AbstractContextManager", "AbstractAsyncContextManager",
"AsyncExitStack", "ContextDecorator", "ExitStack",
- "redirect_stdout", "redirect_stderr", "suppress"]
+ "redirect_stdout", "redirect_stderr", "suppress", "aclosing"]
class AbstractContextManager(abc.ABC):
@@ -80,6 +80,22 @@
return inner
+class AsyncContextDecorator(object):
+ "A base class or mixin that enables async context managers to work as decorators."
+
+ def _recreate_cm(self):
+ """Return a recreated instance of self.
+ """
+ return self
+
+ def __call__(self, func):
+ @wraps(func)
+ async def inner(*args, **kwds):
+ async with self._recreate_cm():
+ return await func(*args, **kwds)
+ return inner
+
+
class _GeneratorContextManagerBase:
"""Shared functionality for @contextmanager and @asynccontextmanager."""
@@ -97,18 +113,20 @@
# for the class instead.
# See http://bugs.python.org/issue19404 for more details.
-
-class _GeneratorContextManager(_GeneratorContextManagerBase,
- AbstractContextManager,
- ContextDecorator):
- """Helper for @contextmanager decorator."""
-
def _recreate_cm(self):
- # _GCM instances are one-shot context managers, so the
+ # _GCMB instances are one-shot context managers, so the
# CM must be recreated each time a decorated function is
# called
return self.__class__(self.func, self.args, self.kwds)
+
+class _GeneratorContextManager(
+ _GeneratorContextManagerBase,
+ AbstractContextManager,
+ ContextDecorator,
+):
+ """Helper for @contextmanager decorator."""
+
def __enter__(self):
# do not keep args and kwds alive unnecessarily
# they are only needed for recreation, which is not possible anymore
@@ -118,8 +136,8 @@
except StopIteration:
raise RuntimeError("generator didn't yield") from None
- def __exit__(self, type, value, traceback):
- if type is None:
+ def __exit__(self, typ, value, traceback):
+ if typ is None:
try:
next(self.gen)
except StopIteration:
@@ -130,9 +148,9 @@
if value is None:
# Need to force instantiation so we can reliably
# tell if we get the same exception back
- value = type()
+ value = typ()
try:
- self.gen.throw(type, value, traceback)
+ self.gen.throw(typ, value, traceback)
except StopIteration as exc:
# Suppress StopIteration *unless* it's the same exception that
# was passed to throw(). This prevents a StopIteration
@@ -142,74 +160,93 @@
# Don't re-raise the passed in exception. (issue27122)
if exc is value:
return False
- # Likewise, avoid suppressing if a StopIteration exception
+ # Avoid suppressing if a StopIteration exception
# was passed to throw() and later wrapped into a RuntimeError
- # (see PEP 479).
- if type is StopIteration and exc.__cause__ is value:
+ # (see PEP 479 for sync generators; async generators also
+ # have this behavior). But do this only if the exception wrapped
+ # by the RuntimeError is actually Stop(Async)Iteration (see
+ # issue29692).
+ if (
+ isinstance(value, StopIteration)
+ and exc.__cause__ is value
+ ):
return False
raise
- except:
+ except BaseException as exc:
# only re-raise if it's *not* the exception that was
# passed to throw(), because __exit__() must not raise
# an exception unless __exit__() itself failed. But throw()
# has to raise the exception to signal propagation, so this
# fixes the impedance mismatch between the throw() protocol
# and the __exit__() protocol.
- #
- # This cannot use 'except BaseException as exc' (as in the
- # async implementation) to maintain compatibility with
- # Python 2, where old-style class exceptions are not caught
- # by 'except BaseException'.
- if sys.exc_info()[1] is value:
- return False
- raise
+ if exc is not value:
+ raise
+ return False
raise RuntimeError("generator didn't stop after throw()")
-
-class _AsyncGeneratorContextManager(_GeneratorContextManagerBase,
- AbstractAsyncContextManager):
- """Helper for @asynccontextmanager."""
+class _AsyncGeneratorContextManager(
+ _GeneratorContextManagerBase,
+ AbstractAsyncContextManager,
+ AsyncContextDecorator,
+):
+ """Helper for @asynccontextmanager decorator."""
async def __aenter__(self):
+ # do not keep args and kwds alive unnecessarily
+ # they are only needed for recreation, which is not possible anymore
+ del self.args, self.kwds, self.func
try:
- return await self.gen.__anext__()
+ return await anext(self.gen)
except StopAsyncIteration:
raise RuntimeError("generator didn't yield") from None
async def __aexit__(self, typ, value, traceback):
if typ is None:
try:
- await self.gen.__anext__()
+ await anext(self.gen)
except StopAsyncIteration:
- return
+ return False
else:
raise RuntimeError("generator didn't stop")
else:
if value is None:
+ # Need to force instantiation so we can reliably
+ # tell if we get the same exception back
value = typ()
- # See _GeneratorContextManager.__exit__ for comments on subtleties
- # in this implementation
try:
await self.gen.athrow(typ, value, traceback)
- raise RuntimeError("generator didn't stop after athrow()")
except StopAsyncIteration as exc:
+ # Suppress StopIteration *unless* it's the same exception that
+ # was passed to throw(). This prevents a StopIteration
+ # raised inside the "with" statement from being suppressed.
return exc is not value
except RuntimeError as exc:
+ # Don't re-raise the passed in exception. (issue27122)
if exc is value:
return False
- # Avoid suppressing if a StopIteration exception
- # was passed to throw() and later wrapped into a RuntimeError
+ # Avoid suppressing if a Stop(Async)Iteration exception
+ # was passed to athrow() and later wrapped into a RuntimeError
# (see PEP 479 for sync generators; async generators also
# have this behavior). But do this only if the exception wrapped
# by the RuntimeError is actully Stop(Async)Iteration (see
# issue29692).
- if isinstance(value, (StopIteration, StopAsyncIteration)):
- if exc.__cause__ is value:
- return False
+ if (
+ isinstance(value, (StopIteration, StopAsyncIteration))
+ and exc.__cause__ is value
+ ):
+ return False
raise
except BaseException as exc:
+ # only re-raise if it's *not* the exception that was
+ # passed to throw(), because __exit__() must not raise
+ # an exception unless __exit__() itself failed. But throw()
+ # has to raise the exception to signal propagation, so this
+ # fixes the impedance mismatch between the throw() protocol
+ # and the __exit__() protocol.
if exc is not value:
raise
+ return False
+ raise RuntimeError("generator didn't stop after athrow()")
def contextmanager(func):
@@ -303,6 +340,32 @@
self.thing.close()
+class aclosing(AbstractAsyncContextManager):
+ """Async context manager for safely finalizing an asynchronously cleaned-up
+ resource such as an async generator, calling its ``aclose()`` method.
+
+ Code like this:
+
+ async with aclosing(<module>.fetch(<arguments>)) as agen:
+ <block>
+
+ is equivalent to this:
+
+ agen = <module>.fetch(<arguments>)
+ try:
+ <block>
+ finally:
+ await agen.aclose()
+
+ """
+ def __init__(self, thing):
+ self.thing = thing
+ async def __aenter__(self):
+ return self.thing
+ async def __aexit__(self, *exc_info):
+ await self.thing.aclose()
+
+
class _RedirectStream(AbstractContextManager):
_stream = None
@@ -477,10 +540,10 @@
# Context may not be correct, so find the end of the chain
while 1:
exc_context = new_exc.__context__
- if exc_context is old_exc:
+ if exc_context is None or exc_context is old_exc:
# Context is already set correctly (see issue 20317)
return
- if exc_context is None or exc_context is frame_exc:
+ if exc_context is frame_exc:
break
new_exc = exc_context
# Change the end of the chain to point to the exception
@@ -611,10 +674,10 @@
# Context may not be correct, so find the end of the chain
while 1:
exc_context = new_exc.__context__
- if exc_context is old_exc:
+ if exc_context is None or exc_context is old_exc:
# Context is already set correctly (see issue 20317)
return
- if exc_context is None or exc_context is frame_exc:
+ if exc_context is frame_exc:
break
new_exc = exc_context
# Change the end of the chain to point to the exception
@@ -655,7 +718,7 @@
return received_exc and suppressed_exc
-class nullcontext(AbstractContextManager):
+class nullcontext(AbstractContextManager, AbstractAsyncContextManager):
"""Context manager that does no additional processing.
Used as a stand-in for a normal context manager, when a particular
@@ -674,3 +737,9 @@
def __exit__(self, *excinfo):
pass
+
+ async def __aenter__(self):
+ return self.enter_result
+
+ async def __aexit__(self, *excinfo):
+ pass
diff --git a/common/py3-stdlib/copy.py b/common/py3-stdlib/copy.py
index 41873f2..69bac98 100644
--- a/common/py3-stdlib/copy.py
+++ b/common/py3-stdlib/copy.py
@@ -39,8 +39,8 @@
set of components copied
This version does not copy types like module, class, function, method,
-nor stack trace, stack frame, nor file, socket, window, nor array, nor
-any similar types.
+nor stack trace, stack frame, nor file, socket, window, nor any
+similar types.
Classes can use the same interfaces to control copying that they use
to control pickling: they can define methods called __getinitargs__(),
@@ -192,6 +192,7 @@
d[str] = _deepcopy_atomic
d[types.CodeType] = _deepcopy_atomic
d[type] = _deepcopy_atomic
+d[range] = _deepcopy_atomic
d[types.BuiltinFunctionType] = _deepcopy_atomic
d[types.FunctionType] = _deepcopy_atomic
d[weakref.ref] = _deepcopy_atomic
diff --git a/common/py3-stdlib/copyreg.py b/common/py3-stdlib/copyreg.py
index 7ab8c12..356db6f 100644
--- a/common/py3-stdlib/copyreg.py
+++ b/common/py3-stdlib/copyreg.py
@@ -36,6 +36,12 @@
pickle(complex, pickle_complex, complex)
+def pickle_union(obj):
+ import functools, operator
+ return functools.reduce, (operator.or_, obj.__args__)
+
+pickle(type(int | str), pickle_union)
+
# Support for pickling new-style objects
def _reconstructor(cls, base, state):
diff --git a/common/py3-stdlib/csv.py b/common/py3-stdlib/csv.py
index dc85077..bb3ee26 100644
--- a/common/py3-stdlib/csv.py
+++ b/common/py3-stdlib/csv.py
@@ -409,14 +409,10 @@
continue # skip rows that have irregular number of columns
for col in list(columnTypes.keys()):
-
- for thisType in [int, float, complex]:
- try:
- thisType(row[col])
- break
- except (ValueError, OverflowError):
- pass
- else:
+ thisType = complex
+ try:
+ thisType(row[col])
+ except (ValueError, OverflowError):
# fallback to length of string
thisType = len(row[col])
diff --git a/common/py3-stdlib/ctypes/_aix.py b/common/py3-stdlib/ctypes/_aix.py
index 190cac6..fc3e95c 100644
--- a/common/py3-stdlib/ctypes/_aix.py
+++ b/common/py3-stdlib/ctypes/_aix.py
@@ -163,7 +163,7 @@
return member
else:
# 32-bit legacy names - both shr.o and shr4.o exist.
- # shr.o is the preffered name so we look for shr.o first
+ # shr.o is the preferred name so we look for shr.o first
# i.e., shr4.o is returned only when shr.o does not exist
for name in ['shr.o', 'shr4.o']:
member = get_one_match(re.escape(name), members)
@@ -282,7 +282,7 @@
if path.exists(archive):
members = get_shared(get_ld_headers(archive))
member = get_member(re.escape(name), members)
- if member != None:
+ if member is not None:
return (base, member)
else:
return (None, None)
@@ -307,7 +307,7 @@
libpaths = get_libpaths()
(base, member) = find_shared(libpaths, name)
- if base != None:
+ if base is not None:
return f"{base}({member})"
# To get here, a member in an archive has not been found
diff --git a/common/py3-stdlib/ctypes/test/__init__.py b/common/py3-stdlib/ctypes/test/__init__.py
index 26a70b7..6e496fa 100644
--- a/common/py3-stdlib/ctypes/test/__init__.py
+++ b/common/py3-stdlib/ctypes/test/__init__.py
@@ -1,9 +1,11 @@
import os
import unittest
from test import support
+from test.support import import_helper
+
# skip tests if _ctypes was not built
-ctypes = support.import_module('ctypes')
+ctypes = import_helper.import_module('ctypes')
ctypes_symbols = dir(ctypes)
def need_symbol(name):
diff --git a/common/py3-stdlib/ctypes/test/test_bitfields.py b/common/py3-stdlib/ctypes/test/test_bitfields.py
index 992b8c4..66acd62 100644
--- a/common/py3-stdlib/ctypes/test/test_bitfields.py
+++ b/common/py3-stdlib/ctypes/test/test_bitfields.py
@@ -1,5 +1,6 @@
from ctypes import *
from ctypes.test import need_symbol
+from test import support
import unittest
import os
@@ -39,6 +40,8 @@
setattr(b, name, i)
self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii')))
+ # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior
+ @support.skip_if_sanitizer(ub=True)
def test_shorts(self):
b = BITS()
name = "M"
diff --git a/common/py3-stdlib/ctypes/test/test_find.py b/common/py3-stdlib/ctypes/test/test_find.py
index 92ac184..1ff9d01 100644
--- a/common/py3-stdlib/ctypes/test/test_find.py
+++ b/common/py3-stdlib/ctypes/test/test_find.py
@@ -3,6 +3,7 @@
import os.path
import sys
import test.support
+from test.support import os_helper
from ctypes import *
from ctypes.util import find_library
@@ -66,8 +67,8 @@
self.gle.gleGetJoinStyle
def test_shell_injection(self):
- result = find_library('; echo Hello shell > ' + test.support.TESTFN)
- self.assertFalse(os.path.lexists(test.support.TESTFN))
+ result = find_library('; echo Hello shell > ' + os_helper.TESTFN)
+ self.assertFalse(os.path.lexists(os_helper.TESTFN))
self.assertIsNone(result)
@@ -89,7 +90,7 @@
srcname = os.path.join(d, 'dummy.c')
libname = 'py_ctypes_test_dummy'
dstname = os.path.join(d, 'lib%s.so' % libname)
- with open(srcname, 'w') as f:
+ with open(srcname, 'wb') as f:
pass
self.assertTrue(os.path.exists(srcname))
# compile the file to a shared library
@@ -101,7 +102,7 @@
# LD_LIBRARY_PATH)
self.assertIsNone(find_library(libname))
# now add the location to LD_LIBRARY_PATH
- with test.support.EnvironmentVarGuard() as env:
+ with os_helper.EnvironmentVarGuard() as env:
KEY = 'LD_LIBRARY_PATH'
if KEY not in env:
v = d
diff --git a/common/py3-stdlib/ctypes/test/test_functions.py b/common/py3-stdlib/ctypes/test/test_functions.py
index 7562892..bdb044e 100644
--- a/common/py3-stdlib/ctypes/test/test_functions.py
+++ b/common/py3-stdlib/ctypes/test/test_functions.py
@@ -35,34 +35,24 @@
# wasn't checked, and it even crashed Python.
# Found by Greg Chapman.
- try:
+ with self.assertRaises(TypeError):
class X(object, Array):
_length_ = 5
_type_ = "i"
- except TypeError:
- pass
-
from _ctypes import _Pointer
- try:
+ with self.assertRaises(TypeError):
class X(object, _Pointer):
pass
- except TypeError:
- pass
from _ctypes import _SimpleCData
- try:
+ with self.assertRaises(TypeError):
class X(object, _SimpleCData):
_type_ = "i"
- except TypeError:
- pass
- try:
+ with self.assertRaises(TypeError):
class X(object, Structure):
_fields_ = []
- except TypeError:
- pass
-
@need_symbol('c_wchar')
def test_wchar_parm(self):
@@ -389,7 +379,7 @@
(9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
def test_sf1651235(self):
- # see http://www.python.org/sf/1651235
+ # see https://www.python.org/sf/1651235
proto = CFUNCTYPE(c_int, RECT, POINT)
def callback(*args):
diff --git a/common/py3-stdlib/ctypes/test/test_loading.py b/common/py3-stdlib/ctypes/test/test_loading.py
index ba655bc..ea89227 100644
--- a/common/py3-stdlib/ctypes/test/test_loading.py
+++ b/common/py3-stdlib/ctypes/test/test_loading.py
@@ -5,6 +5,8 @@
import sys
import unittest
import test.support
+from test.support import import_helper
+from test.support import os_helper
from ctypes.util import find_library
libc_name = None
@@ -91,7 +93,7 @@
# NOT fit into a 32-bit integer. FreeLibrary must be able
# to accept this address.
- # These are tests for http://www.python.org/sf/1703286
+ # These are tests for https://www.python.org/sf/1703286
handle = LoadLibrary("advapi32")
FreeLibrary(handle)
@@ -117,14 +119,14 @@
@unittest.skipUnless(os.name == "nt",
'test specific to Windows')
def test_load_dll_with_flags(self):
- _sqlite3 = test.support.import_module("_sqlite3")
+ _sqlite3 = import_helper.import_module("_sqlite3")
src = _sqlite3.__file__
if src.lower().endswith("_d.pyd"):
ext = "_d.dll"
else:
ext = ".dll"
- with test.support.temp_dir() as tmp:
+ with os_helper.temp_dir() as tmp:
# We copy two files and load _sqlite3.dll (formerly .pyd),
# which has a dependency on sqlite3.dll. Then we test
# loading it in subprocesses to avoid it starting in memory
diff --git a/common/py3-stdlib/ctypes/test/test_numbers.py b/common/py3-stdlib/ctypes/test/test_numbers.py
index c6d843b..db500e8 100644
--- a/common/py3-stdlib/ctypes/test/test_numbers.py
+++ b/common/py3-stdlib/ctypes/test/test_numbers.py
@@ -134,8 +134,7 @@
for t in signed_types + unsigned_types:
self.assertRaises(TypeError, t, 3.14)
self.assertRaises(TypeError, t, f)
- with self.assertWarns(DeprecationWarning):
- self.assertEqual(t(d).value, 2)
+ self.assertRaises(TypeError, t, d)
self.assertEqual(t(i).value, 2)
def test_sizes(self):
diff --git a/common/py3-stdlib/ctypes/test/test_parameters.py b/common/py3-stdlib/ctypes/test/test_parameters.py
index e4c25fd..38af7ac 100644
--- a/common/py3-stdlib/ctypes/test/test_parameters.py
+++ b/common/py3-stdlib/ctypes/test/test_parameters.py
@@ -201,6 +201,49 @@
with self.assertRaises(ZeroDivisionError):
WorseStruct().__setstate__({}, b'foo')
+ def test_parameter_repr(self):
+ from ctypes import (
+ c_bool,
+ c_char,
+ c_wchar,
+ c_byte,
+ c_ubyte,
+ c_short,
+ c_ushort,
+ c_int,
+ c_uint,
+ c_long,
+ c_ulong,
+ c_longlong,
+ c_ulonglong,
+ c_float,
+ c_double,
+ c_longdouble,
+ c_char_p,
+ c_wchar_p,
+ c_void_p,
+ )
+ self.assertRegex(repr(c_bool.from_param(True)), r"^<cparam '\?' at 0x[A-Fa-f0-9]+>$")
+ self.assertEqual(repr(c_char.from_param(97)), "<cparam 'c' ('a')>")
+ self.assertRegex(repr(c_wchar.from_param('a')), r"^<cparam 'u' at 0x[A-Fa-f0-9]+>$")
+ self.assertEqual(repr(c_byte.from_param(98)), "<cparam 'b' (98)>")
+ self.assertEqual(repr(c_ubyte.from_param(98)), "<cparam 'B' (98)>")
+ self.assertEqual(repr(c_short.from_param(511)), "<cparam 'h' (511)>")
+ self.assertEqual(repr(c_ushort.from_param(511)), "<cparam 'H' (511)>")
+ self.assertRegex(repr(c_int.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
+ self.assertRegex(repr(c_uint.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
+ self.assertRegex(repr(c_long.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
+ self.assertRegex(repr(c_ulong.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
+ self.assertRegex(repr(c_longlong.from_param(20000)), r"^<cparam '[liq]' \(20000\)>$")
+ self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^<cparam '[LIQ]' \(20000\)>$")
+ self.assertEqual(repr(c_float.from_param(1.5)), "<cparam 'f' (1.5)>")
+ self.assertEqual(repr(c_double.from_param(1.5)), "<cparam 'd' (1.5)>")
+ self.assertEqual(repr(c_double.from_param(1e300)), "<cparam 'd' (1e+300)>")
+ self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^<cparam ('d' \(1.5\)|'g' at 0x[A-Fa-f0-9]+)>$")
+ self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^<cparam 'z' \(0x[A-Fa-f0-9]+\)>$")
+ self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$")
+ self.assertRegex(repr(c_void_p.from_param(0x12)), r"^<cparam 'P' \(0x0*12\)>$")
+
################################################################
if __name__ == '__main__':
diff --git a/common/py3-stdlib/ctypes/test/test_python_api.py b/common/py3-stdlib/ctypes/test/test_python_api.py
index 9c13746..49571f9 100644
--- a/common/py3-stdlib/ctypes/test/test_python_api.py
+++ b/common/py3-stdlib/ctypes/test/test_python_api.py
@@ -1,5 +1,5 @@
from ctypes import *
-import unittest, sys
+import unittest
from test import support
################################################################
@@ -10,10 +10,6 @@
################################################################
from sys import getrefcount as grc
-if sys.version_info > (2, 4):
- c_py_ssize_t = c_size_t
-else:
- c_py_ssize_t = c_int
class PythonAPITestCase(unittest.TestCase):
@@ -21,7 +17,7 @@
PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize
PyBytes_FromStringAndSize.restype = py_object
- PyBytes_FromStringAndSize.argtypes = c_char_p, c_py_ssize_t
+ PyBytes_FromStringAndSize.argtypes = c_char_p, c_size_t
self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc")
diff --git a/common/py3-stdlib/ctypes/test/test_struct_fields.py b/common/py3-stdlib/ctypes/test/test_struct_fields.py
index 8045cc8..ee8415f 100644
--- a/common/py3-stdlib/ctypes/test/test_struct_fields.py
+++ b/common/py3-stdlib/ctypes/test/test_struct_fields.py
@@ -46,6 +46,14 @@
Y._fields_ = []
self.assertRaises(AttributeError, setattr, X, "_fields_", [])
+ def test_5(self):
+ class X(Structure):
+ _fields_ = (("char", c_char * 5),)
+
+ x = X(b'#' * 5)
+ x.char = b'a\0b\0'
+ self.assertEqual(bytes(x), b'a\x00###')
+
# __set__ and __get__ should raise a TypeError in case their self
# argument is not a ctype instance.
def test___set__(self):
diff --git a/common/py3-stdlib/ctypes/test/test_structures.py b/common/py3-stdlib/ctypes/test/test_structures.py
index 245cd94..97ad2b8 100644
--- a/common/py3-stdlib/ctypes/test/test_structures.py
+++ b/common/py3-stdlib/ctypes/test/test_structures.py
@@ -443,7 +443,7 @@
s = Test(1, 2, 3)
# Test the StructUnionType_paramfunc() code path which copies the
- # structure: if the stucture is larger than sizeof(void*).
+ # structure: if the structure is larger than sizeof(void*).
self.assertGreater(sizeof(s), sizeof(c_void_p))
dll = CDLL(_ctypes_test.__file__)
@@ -451,7 +451,7 @@
func.argtypes = (Test,)
func.restype = None
func(s)
- # bpo-37140: Passing the structure by refrence must not call
+ # bpo-37140: Passing the structure by reference must not call
# its finalizer!
self.assertEqual(finalizer_calls, [])
self.assertEqual(s.first, 1)
diff --git a/common/py3-stdlib/ctypes/test/test_unicode.py b/common/py3-stdlib/ctypes/test/test_unicode.py
index c200af7..60c7542 100644
--- a/common/py3-stdlib/ctypes/test/test_unicode.py
+++ b/common/py3-stdlib/ctypes/test/test_unicode.py
@@ -26,6 +26,14 @@
self.assertEqual(buf[::2], 'a\xe4\xfc')
self.assertEqual(buf[6:5:-1], "")
+ def test_embedded_null(self):
+ class TestStruct(ctypes.Structure):
+ _fields_ = [("unicode", ctypes.c_wchar_p)]
+ t = TestStruct()
+ # This would raise a ValueError:
+ t.unicode = "foo\0bar\0\0"
+
+
func = ctypes.CDLL(_ctypes_test.__file__)._testfunc_p_p
class StringTestCase(UnicodeTestCase):
diff --git a/common/py3-stdlib/ctypes/test/test_values.py b/common/py3-stdlib/ctypes/test/test_values.py
index 87eb919..7514fe8 100644
--- a/common/py3-stdlib/ctypes/test/test_values.py
+++ b/common/py3-stdlib/ctypes/test/test_values.py
@@ -80,9 +80,9 @@
continue
items.append((entry.name.decode("ascii"), entry.size))
- expected = [("__hello__", 141),
- ("__phello__", -141),
- ("__phello__.spam", 141),
+ expected = [("__hello__", 137),
+ ("__phello__", -137),
+ ("__phello__.spam", 137),
]
self.assertEqual(items, expected, "PyImport_FrozenModules example "
"in Doc/library/ctypes.rst may be out of date")
diff --git a/common/py3-stdlib/dataclasses.py b/common/py3-stdlib/dataclasses.py
index 530d3e9..105a95b 100644
--- a/common/py3-stdlib/dataclasses.py
+++ b/common/py3-stdlib/dataclasses.py
@@ -6,8 +6,9 @@
import keyword
import builtins
import functools
+import abc
import _thread
-from types import GenericAlias
+from types import FunctionType, GenericAlias
__all__ = ['dataclass',
@@ -15,6 +16,7 @@
'Field',
'FrozenInstanceError',
'InitVar',
+ 'KW_ONLY',
'MISSING',
# Helper functions.
@@ -151,6 +153,20 @@
#
# See _hash_action (below) for a coded version of this table.
+# __match_args__
+#
+# +--- match_args= parameter
+# |
+# v | | |
+# | no | yes | <--- class has __match_args__ in __dict__?
+# +=======+=======+=======+
+# | False | | |
+# +-------+-------+-------+
+# | True | add | | <- the default
+# +=======+=======+=======+
+# __match_args__ is always added unless the class already defines it. It is a
+# tuple of __init__ parameter names; non-init fields must be matched by keyword.
+
# Raised when an attempt is made to modify a frozen class.
class FrozenInstanceError(AttributeError): pass
@@ -169,6 +185,12 @@
pass
MISSING = _MISSING_TYPE()
+# A sentinel object to indicate that following fields are keyword-only by
+# default. Use a class to give it a better repr.
+class _KW_ONLY_TYPE:
+ pass
+KW_ONLY = _KW_ONLY_TYPE()
+
# Since most per-field metadata will be unused, create an empty
# read-only proxy that can be shared among all fields.
_EMPTY_METADATA = types.MappingProxyType({})
@@ -207,7 +229,7 @@
self.type = type
def __repr__(self):
- if isinstance(self.type, type):
+ if isinstance(self.type, type) and not isinstance(self.type, GenericAlias):
type_name = self.type.__name__
else:
# typing objects, e.g. List[int]
@@ -217,7 +239,6 @@
def __class_getitem__(cls, type):
return InitVar(type)
-
# Instances of Field are only ever created from within this module,
# and only from the field() function, although Field instances are
# exposed externally as (conceptually) read-only objects.
@@ -238,11 +259,12 @@
'init',
'compare',
'metadata',
+ 'kw_only',
'_field_type', # Private: not to be used by user code.
)
def __init__(self, default, default_factory, init, repr, hash, compare,
- metadata):
+ metadata, kw_only):
self.name = None
self.type = None
self.default = default
@@ -254,6 +276,7 @@
self.metadata = (_EMPTY_METADATA
if metadata is None else
types.MappingProxyType(metadata))
+ self.kw_only = kw_only
self._field_type = None
def __repr__(self):
@@ -267,6 +290,7 @@
f'hash={self.hash!r},'
f'compare={self.compare!r},'
f'metadata={self.metadata!r},'
+ f'kw_only={self.kw_only!r},'
f'_field_type={self._field_type}'
')')
@@ -320,17 +344,19 @@
# so that a type checker can be told (via overloads) that this is a
# function whose type depends on its parameters.
def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True,
- hash=None, compare=True, metadata=None):
+ hash=None, compare=True, metadata=None, kw_only=MISSING):
"""Return an object to identify dataclass fields.
default is the default value of the field. default_factory is a
0-argument function called to initialize a field's value. If init
- is True, the field will be a parameter to the class's __init__()
- function. If repr is True, the field will be included in the
- object's repr(). If hash is True, the field will be included in
- the object's hash(). If compare is True, the field will be used
- in comparison functions. metadata, if specified, must be a
- mapping which is stored but not otherwise examined by dataclass.
+ is true, the field will be a parameter to the class's __init__()
+ function. If repr is true, the field will be included in the
+ object's repr(). If hash is true, the field will be included in the
+ object's hash(). If compare is true, the field will be used in
+ comparison functions. metadata, if specified, must be a mapping
+ which is stored but not otherwise examined by dataclass. If kw_only
+ is true, the field will become a keyword-only parameter to
+ __init__().
It is an error to specify both default and default_factory.
"""
@@ -338,7 +364,16 @@
if default is not MISSING and default_factory is not MISSING:
raise ValueError('cannot specify both default and default_factory')
return Field(default, default_factory, init, repr, hash, compare,
- metadata)
+ metadata, kw_only)
+
+
+def _fields_in_init_order(fields):
+ # Returns the fields as __init__ will output them. It returns 2 tuples:
+ # the first for normal args, and the second for keyword args.
+
+ return (tuple(f for f in fields if f.init and not f.kw_only),
+ tuple(f for f in fields if f.init and f.kw_only)
+ )
def _tuple_str(obj_name, fields):
@@ -395,7 +430,6 @@
local_vars = ', '.join(locals.keys())
txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}"
-
ns = {}
exec(txt, globals, ns)
return ns['__create_fn__'](**locals)
@@ -413,7 +447,7 @@
return f'{self_name}.{name}={value}'
-def _field_init(f, frozen, globals, self_name):
+def _field_init(f, frozen, globals, self_name, slots):
# Return the text of the line in the body of __init__ that will
# initialize this field.
@@ -453,9 +487,15 @@
globals[default_name] = f.default
value = f.name
else:
- # This field does not need initialization. Signify that
- # to the caller by returning None.
- return None
+ # If the class has slots, then initialize this field.
+ if slots and f.default is not MISSING:
+ globals[default_name] = f.default
+ value = default_name
+ else:
+ # This field does not need initialization: reading from it will
+ # just use the class attribute that contains the default.
+ # Signify that to the caller by returning None.
+ return None
# Only test this now, so that we can create variables for the
# default. However, return None to signify that we're not going
@@ -486,7 +526,8 @@
return f'{f.name}:_type_{f.name}{default}'
-def _init_fn(fields, frozen, has_post_init, self_name, globals):
+def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init,
+ self_name, globals, slots):
# fields contains both real fields and InitVar pseudo-fields.
# Make sure we don't have fields without defaults following fields
@@ -494,9 +535,10 @@
# function source code, but catching it here gives a better error
# message, and future-proofs us in case we build up the function
# using ast.
+
seen_default = False
- for f in fields:
- # Only consider fields in the __init__ call.
+ for f in std_fields:
+ # Only consider the non-kw-only fields in the __init__ call.
if f.init:
if not (f.default is MISSING and f.default_factory is MISSING):
seen_default = True
@@ -512,7 +554,7 @@
body_lines = []
for f in fields:
- line = _field_init(f, frozen, locals, self_name)
+ line = _field_init(f, frozen, locals, self_name, slots)
# line is None means that this field doesn't require
# initialization (it's a pseudo-field). Just skip it.
if line:
@@ -528,8 +570,15 @@
if not body_lines:
body_lines = ['pass']
+ _init_params = [_init_param(f) for f in std_fields]
+ if kw_only_fields:
+ # Add the keyword-only args. Because the * can only be added if
+ # there's at least one keyword-only arg, there needs to be a test here
+ # (instead of just concatenting the lists together).
+ _init_params += ['*']
+ _init_params += [_init_param(f) for f in kw_only_fields]
return _create_fn('__init__',
- [self_name] + [_init_param(f) for f in fields if f.init],
+ [self_name] + _init_params,
body_lines,
locals=locals,
globals=globals,
@@ -608,6 +657,9 @@
return (a_type is dataclasses.InitVar
or type(a_type) is dataclasses.InitVar)
+def _is_kw_only(a_type, dataclasses):
+ return a_type is dataclasses.KW_ONLY
+
def _is_type(annotation, cls, a_module, a_type, is_type_predicate):
# Given a type annotation string, does it refer to a_type in
@@ -668,10 +720,11 @@
return False
-def _get_field(cls, a_name, a_type):
- # Return a Field object for this field name and type. ClassVars
- # and InitVars are also returned, but marked as such (see
- # f._field_type).
+def _get_field(cls, a_name, a_type, default_kw_only):
+ # Return a Field object for this field name and type. ClassVars and
+ # InitVars are also returned, but marked as such (see f._field_type).
+ # default_kw_only is the value of kw_only to use if there isn't a field()
+ # that defines it.
# If the default value isn't derived from Field, then it's only a
# normal default value. Convert it to a Field().
@@ -696,7 +749,7 @@
# In addition to checking for actual types here, also check for
# string annotations. get_type_hints() won't always work for us
# (see https://github.com/python/typing/issues/508 for example),
- # plus it's expensive and would require an eval for every stirng
+ # plus it's expensive and would require an eval for every string
# annotation. So, make a best effort to see if this is a ClassVar
# or InitVar using regex's and checking that the thing referenced
# is actually of the correct type.
@@ -742,6 +795,19 @@
# init=<not-the-default-init-value>)? It makes no sense for
# ClassVar and InitVar to specify init=<anything>.
+ # kw_only validation and assignment.
+ if f._field_type in (_FIELD, _FIELD_INITVAR):
+ # For real and InitVar fields, if kw_only wasn't specified use the
+ # default value.
+ if f.kw_only is MISSING:
+ f.kw_only = default_kw_only
+ else:
+ # Make sure kw_only isn't set for ClassVars
+ assert f._field_type is _FIELD_CLASSVAR
+ if f.kw_only is not MISSING:
+ raise TypeError(f'field {f.name} is a ClassVar but specifies '
+ 'kw_only')
+
# For real fields, disallow mutable defaults for known types.
if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)):
raise ValueError(f'mutable default {type(f.default)} for field '
@@ -749,12 +815,19 @@
return f
+def _set_qualname(cls, value):
+ # Ensure that the functions returned from _create_fn uses the proper
+ # __qualname__ (the class they belong to).
+ if isinstance(value, FunctionType):
+ value.__qualname__ = f"{cls.__qualname__}.{value.__name__}"
+ return value
def _set_new_attribute(cls, name, value):
# Never overwrites an existing attribute. Returns True if the
# attribute already exists.
if name in cls.__dict__:
return True
+ _set_qualname(cls, value)
setattr(cls, name, value)
return False
@@ -769,7 +842,7 @@
def _hash_add(cls, fields, globals):
flds = [f for f in fields if (f.compare if f.hash is None else f.hash)]
- return _hash_fn(flds, globals)
+ return _set_qualname(cls, _hash_fn(flds, globals))
def _hash_exception(cls, fields, globals):
# Raise an exception.
@@ -806,7 +879,8 @@
# version of this table.
-def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen):
+def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen,
+ match_args, kw_only, slots):
# Now that dicts retain insertion order, there's no reason to use
# an ordered dict. I am leveraging that ordering here, because
# derived class fields overwrite base class fields, but the order
@@ -836,7 +910,7 @@
# Only process classes that have been processed by our
# decorator. That is, they have a _FIELDS attribute.
base_fields = getattr(b, _FIELDS, None)
- if base_fields:
+ if base_fields is not None:
has_dataclass_bases = True
for f in base_fields.values():
fields[f.name] = f
@@ -860,8 +934,27 @@
# Now find fields in our class. While doing so, validate some
# things, and set the default values (as class attributes) where
# we can.
- cls_fields = [_get_field(cls, name, type)
- for name, type in cls_annotations.items()]
+ cls_fields = []
+ # Get a reference to this module for the _is_kw_only() test.
+ KW_ONLY_seen = False
+ dataclasses = sys.modules[__name__]
+ for name, type in cls_annotations.items():
+ # See if this is a marker to change the value of kw_only.
+ if (_is_kw_only(type, dataclasses)
+ or (isinstance(type, str)
+ and _is_type(type, cls, dataclasses, dataclasses.KW_ONLY,
+ _is_kw_only))):
+ # Switch the default to kw_only=True, and ignore this
+ # annotation: it's not a real field.
+ if KW_ONLY_seen:
+ raise TypeError(f'{name!r} is KW_ONLY, but KW_ONLY '
+ 'has already been specified')
+ KW_ONLY_seen = True
+ kw_only = True
+ else:
+ # Otherwise it's a field of some type.
+ cls_fields.append(_get_field(cls, name, type, kw_only))
+
for f in cls_fields:
fields[f.name] = f
@@ -916,15 +1009,22 @@
if order and not eq:
raise ValueError('eq must be true if order is true')
+ # Include InitVars and regular fields (so, not ClassVars). This is
+ # initialized here, outside of the "if init:" test, because std_init_fields
+ # is used with match_args, below.
+ all_init_fields = [f for f in fields.values()
+ if f._field_type in (_FIELD, _FIELD_INITVAR)]
+ (std_init_fields,
+ kw_only_init_fields) = _fields_in_init_order(all_init_fields)
+
if init:
# Does this class have a post-init function?
has_post_init = hasattr(cls, _POST_INIT_NAME)
- # Include InitVars and regular fields (so, not ClassVars).
- flds = [f for f in fields.values()
- if f._field_type in (_FIELD, _FIELD_INITVAR)]
_set_new_attribute(cls, '__init__',
- _init_fn(flds,
+ _init_fn(all_init_fields,
+ std_init_fields,
+ kw_only_init_fields,
frozen,
has_post_init,
# The name to use for the "self"
@@ -933,6 +1033,7 @@
'__dataclass_self__' if 'self' in fields
else 'self',
globals,
+ slots,
))
# Get the fields as a list, and include only real fields. This is
@@ -944,7 +1045,7 @@
_set_new_attribute(cls, '__repr__', _repr_fn(flds, globals))
if eq:
- # Create _eq__ method. There's no need for a __ne__ method,
+ # Create __eq__ method. There's no need for a __ne__ method,
# since python will call __eq__ and negate it.
flds = [f for f in field_list if f.compare]
self_tuple = _tuple_str('self', flds)
@@ -992,11 +1093,70 @@
cls.__doc__ = (cls.__name__ +
str(inspect.signature(cls)).replace(' -> None', ''))
+ if match_args:
+ # I could probably compute this once
+ _set_new_attribute(cls, '__match_args__',
+ tuple(f.name for f in std_init_fields))
+
+ if slots:
+ cls = _add_slots(cls, frozen)
+
+ abc.update_abstractmethods(cls)
+
+ return cls
+
+
+# _dataclass_getstate and _dataclass_setstate are needed for pickling frozen
+# classes with slots. These could be slighly more performant if we generated
+# the code instead of iterating over fields. But that can be a project for
+# another day, if performance becomes an issue.
+def _dataclass_getstate(self):
+ return [getattr(self, f.name) for f in fields(self)]
+
+
+def _dataclass_setstate(self, state):
+ for field, value in zip(fields(self), state):
+ # use setattr because dataclass may be frozen
+ object.__setattr__(self, field.name, value)
+
+
+def _add_slots(cls, is_frozen):
+ # Need to create a new class, since we can't set __slots__
+ # after a class has been created.
+
+ # Make sure __slots__ isn't already set.
+ if '__slots__' in cls.__dict__:
+ raise TypeError(f'{cls.__name__} already specifies __slots__')
+
+ # Create a new dict for our new class.
+ cls_dict = dict(cls.__dict__)
+ field_names = tuple(f.name for f in fields(cls))
+ cls_dict['__slots__'] = field_names
+ for field_name in field_names:
+ # Remove our attributes, if present. They'll still be
+ # available in _MARKER.
+ cls_dict.pop(field_name, None)
+
+ # Remove __dict__ itself.
+ cls_dict.pop('__dict__', None)
+
+ # And finally create the class.
+ qualname = getattr(cls, '__qualname__', None)
+ cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
+ if qualname is not None:
+ cls.__qualname__ = qualname
+
+ if is_frozen:
+ # Need this for pickling frozen classes with slots.
+ cls.__getstate__ = _dataclass_getstate
+ cls.__setstate__ = _dataclass_setstate
+
return cls
def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False,
- unsafe_hash=False, frozen=False):
+ unsafe_hash=False, frozen=False, match_args=True,
+ kw_only=False, slots=False):
"""Returns the same class as was passed in, with dunder methods
added based on the fields defined in the class.
@@ -1006,11 +1166,15 @@
repr is true, a __repr__() method is added. If order is true, rich
comparison dunder methods are added. If unsafe_hash is true, a
__hash__() method function is added. If frozen is true, fields may
- not be assigned to after instance creation.
+ not be assigned to after instance creation. If match_args is true,
+ the __match_args__ tuple is added. If kw_only is true, then by
+ default all fields are keyword-only. If slots is true, an
+ __slots__ attribute is added.
"""
def wrap(cls):
- return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen)
+ return _process_class(cls, init, repr, eq, order, unsafe_hash,
+ frozen, match_args, kw_only, slots)
# See if we're being called as @dataclass or @dataclass().
if cls is None:
@@ -1047,7 +1211,7 @@
def is_dataclass(obj):
"""Returns True if obj is a dataclass or an instance of a
dataclass."""
- cls = obj if isinstance(obj, type) else type(obj)
+ cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj)
return hasattr(cls, _FIELDS)
@@ -1169,7 +1333,7 @@
def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True,
repr=True, eq=True, order=False, unsafe_hash=False,
- frozen=False):
+ frozen=False, match_args=True, kw_only=False, slots=False):
"""Return a new dynamically created dataclass.
The dataclass name will be 'cls_name'. 'fields' is an iterable
@@ -1195,14 +1359,12 @@
if namespace is None:
namespace = {}
- else:
- # Copy namespace since we're going to mutate it.
- namespace = namespace.copy()
# While we're looking through the field names, validate that they
# are identifiers, are not keywords, and not duplicates.
seen = set()
- anns = {}
+ annotations = {}
+ defaults = {}
for item in fields:
if isinstance(item, str):
name = item
@@ -1211,7 +1373,7 @@
name, tp, = item
elif len(item) == 3:
name, tp, spec = item
- namespace[name] = spec
+ defaults[name] = spec
else:
raise TypeError(f'Invalid field: {item!r}')
@@ -1223,14 +1385,22 @@
raise TypeError(f'Field name duplicated: {name!r}')
seen.add(name)
- anns[name] = tp
+ annotations[name] = tp
- namespace['__annotations__'] = anns
+ # Update 'ns' with the user-supplied namespace plus our calculated values.
+ def exec_body_callback(ns):
+ ns.update(namespace)
+ ns.update(defaults)
+ ns['__annotations__'] = annotations
+
# We use `types.new_class()` instead of simply `type()` to allow dynamic creation
- # of generic dataclassses.
- cls = types.new_class(cls_name, bases, {}, lambda ns: ns.update(namespace))
+ # of generic dataclasses.
+ cls = types.new_class(cls_name, bases, {}, exec_body_callback)
+
+ # Apply the normal decorator.
return dataclass(cls, init=init, repr=repr, eq=eq, order=order,
- unsafe_hash=unsafe_hash, frozen=frozen)
+ unsafe_hash=unsafe_hash, frozen=frozen,
+ match_args=match_args, kw_only=kw_only, slots=slots)
def replace(obj, /, **changes):
@@ -1271,7 +1441,7 @@
continue
if f.name not in changes:
- if f._field_type is _FIELD_INITVAR:
+ if f._field_type is _FIELD_INITVAR and f.default is MISSING:
raise ValueError(f"InitVar {f.name!r} "
'must be specified with replace()')
changes[f.name] = getattr(obj, f.name)
diff --git a/common/py3-stdlib/datetime.py b/common/py3-stdlib/datetime.py
index e508d99..6bf37cc 100644
--- a/common/py3-stdlib/datetime.py
+++ b/common/py3-stdlib/datetime.py
@@ -11,6 +11,7 @@
import time as _time
import math as _math
import sys
+from operator import index as _index
def _cmp(x, y):
return 0 if x == y else 1 if x > y else -1
@@ -380,42 +381,10 @@
"-timedelta(hours=24) and timedelta(hours=24)" %
(name, offset))
-def _check_int_field(value):
- if isinstance(value, int):
- return value
- if isinstance(value, float):
- raise TypeError('integer argument expected, got float')
- try:
- value = value.__index__()
- except AttributeError:
- pass
- else:
- if not isinstance(value, int):
- raise TypeError('__index__ returned non-int (type %s)' %
- type(value).__name__)
- return value
- orig = value
- try:
- value = value.__int__()
- except AttributeError:
- pass
- else:
- if not isinstance(value, int):
- raise TypeError('__int__ returned non-int (type %s)' %
- type(value).__name__)
- import warnings
- warnings.warn("an integer is required (got type %s)" %
- type(orig).__name__,
- DeprecationWarning,
- stacklevel=2)
- return value
- raise TypeError('an integer is required (got type %s)' %
- type(value).__name__)
-
def _check_date_fields(year, month, day):
- year = _check_int_field(year)
- month = _check_int_field(month)
- day = _check_int_field(day)
+ year = _index(year)
+ month = _index(month)
+ day = _index(day)
if not MINYEAR <= year <= MAXYEAR:
raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
if not 1 <= month <= 12:
@@ -426,10 +395,10 @@
return year, month, day
def _check_time_fields(hour, minute, second, microsecond, fold):
- hour = _check_int_field(hour)
- minute = _check_int_field(minute)
- second = _check_int_field(second)
- microsecond = _check_int_field(microsecond)
+ hour = _index(hour)
+ minute = _index(minute)
+ second = _index(second)
+ microsecond = _index(microsecond)
if not 0 <= hour <= 23:
raise ValueError('hour must be in 0..23', hour)
if not 0 <= minute <= 59:
@@ -2358,7 +2327,7 @@
# This is again a requirement for a sane tzinfo class.
#
# 4. (x+k).s = x.s
-# This follows from #2, and that datimetimetz+timedelta preserves tzinfo.
+# This follows from #2, and that datetime.timetz+timedelta preserves tzinfo.
#
# 5. (x+k).n = x.n + k
# Again follows from how arithmetic is defined.
@@ -2541,10 +2510,10 @@
# Clean up unused names
del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y,
_DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time,
- _check_date_fields, _check_int_field, _check_time_fields,
+ _check_date_fields, _check_time_fields,
_check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror,
_date_class, _days_before_month, _days_before_year, _days_in_month,
- _format_time, _format_offset, _is_leap, _isoweek1monday, _math,
+ _format_time, _format_offset, _index, _is_leap, _isoweek1monday, _math,
_ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord,
_divide_and_round, _parse_isoformat_date, _parse_isoformat_time,
_parse_hh_mm_ss_ff, _IsoCalendarDate)
diff --git a/common/py3-stdlib/difflib.py b/common/py3-stdlib/difflib.py
index 0dda80d..afd8a0c 100644
--- a/common/py3-stdlib/difflib.py
+++ b/common/py3-stdlib/difflib.py
@@ -62,7 +62,7 @@
notion, pairing up elements that appear uniquely in each sequence.
That, and the method here, appear to yield more intuitive difference
reports than does diff. This method appears to be the least vulnerable
- to synching up on blocks of "junk lines", though (like blank lines in
+ to syncing up on blocks of "junk lines", though (like blank lines in
ordinary text files, or maybe "<P>" lines in HTML files). That may be
because this is the only method of the 3 that has a *concept* of
"junk" <wink>.
@@ -115,38 +115,6 @@
case. SequenceMatcher is quadratic time for the worst case and has
expected-case behavior dependent in a complicated way on how many
elements the sequences have in common; best case time is linear.
-
- Methods:
-
- __init__(isjunk=None, a='', b='')
- Construct a SequenceMatcher.
-
- set_seqs(a, b)
- Set the two sequences to be compared.
-
- set_seq1(a)
- Set the first sequence to be compared.
-
- set_seq2(b)
- Set the second sequence to be compared.
-
- find_longest_match(alo=0, ahi=None, blo=0, bhi=None)
- Find longest matching block in a[alo:ahi] and b[blo:bhi].
-
- get_matching_blocks()
- Return list of triples describing matching subsequences.
-
- get_opcodes()
- Return list of 5-tuples describing how to turn a into b.
-
- ratio()
- Return a measure of the sequences' similarity (float in [0,1]).
-
- quick_ratio()
- Return an upper bound on .ratio() relatively quickly.
-
- real_quick_ratio()
- Return an upper bound on ratio() very quickly.
"""
def __init__(self, isjunk=None, a='', b='', autojunk=True):
@@ -837,14 +805,6 @@
+ 4. Complicated is better than complex.
? ++++ ^ ^
+ 5. Flat is better than nested.
-
- Methods:
-
- __init__(linejunk=None, charjunk=None)
- Construct a text differencer, with optional filters.
-
- compare(a, b)
- Compare two sequences of lines; generate the resulting delta.
"""
def __init__(self, linejunk=None, charjunk=None):
diff --git a/common/py3-stdlib/dis.py b/common/py3-stdlib/dis.py
index e289e17..fe5d24e 100644
--- a/common/py3-stdlib/dis.py
+++ b/common/py3-stdlib/dis.py
@@ -338,8 +338,11 @@
argval, argrepr = _get_const_info(arg, constants)
elif op in hasname:
argval, argrepr = _get_name_info(arg, names)
+ elif op in hasjabs:
+ argval = arg*2
+ argrepr = "to " + repr(argval)
elif op in hasjrel:
- argval = offset + 2 + arg
+ argval = offset + 2 + arg*2
argrepr = "to " + repr(argval)
elif op in haslocal:
argval, argrepr = _get_name_info(arg, varnames)
@@ -384,7 +387,7 @@
constants=None, cells=None, linestarts=None,
*, file=None, line_offset=0):
# Omit the line number column entirely if we have no line number info
- show_lineno = linestarts is not None
+ show_lineno = bool(linestarts)
if show_lineno:
maxlineno = max(linestarts.values()) + line_offset
if maxlineno >= 1000:
@@ -425,6 +428,7 @@
extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
else:
arg = None
+ extended_arg = 0
yield (i, op, arg)
def findlabels(code):
@@ -437,9 +441,9 @@
for offset, op, arg in _unpack_opargs(code):
if arg is not None:
if op in hasjrel:
- label = offset + 2 + arg
+ label = offset + 2 + arg*2
elif op in hasjabs:
- label = arg
+ label = arg*2
else:
continue
if label not in labels:
@@ -449,32 +453,15 @@
def findlinestarts(code):
"""Find the offsets in a byte code which are start of lines in the source.
- Generate pairs (offset, lineno) as described in Python/compile.c.
-
+ Generate pairs (offset, lineno)
"""
- byte_increments = code.co_lnotab[0::2]
- line_increments = code.co_lnotab[1::2]
- bytecode_len = len(code.co_code)
+ lastline = None
+ for start, end, line in code.co_lines():
+ if line is not None and line != lastline:
+ lastline = line
+ yield start, line
+ return
- lastlineno = None
- lineno = code.co_firstlineno
- addr = 0
- for byte_incr, line_incr in zip(byte_increments, line_increments):
- if byte_incr:
- if lineno != lastlineno:
- yield (addr, lineno)
- lastlineno = lineno
- addr += byte_incr
- if addr >= bytecode_len:
- # The rest of the lnotab byte offsets are past the end of
- # the bytecode, so the lines were optimized away.
- return
- if line_incr >= 0x80:
- # line_increments is an array of 8-bit signed integers
- line_incr -= 0x100
- lineno += line_incr
- if lineno != lastlineno:
- yield (addr, lineno)
class Bytecode:
"""The bytecode operations of a piece of code
diff --git a/common/py3-stdlib/distutils/__init__.py b/common/py3-stdlib/distutils/__init__.py
index d823d04..fdad6f6 100644
--- a/common/py3-stdlib/distutils/__init__.py
+++ b/common/py3-stdlib/distutils/__init__.py
@@ -9,5 +9,12 @@
"""
import sys
+import warnings
__version__ = sys.version[:sys.version.index(' ')]
+
+_DEPRECATION_MESSAGE = ("The distutils package is deprecated and slated for "
+ "removal in Python 3.12. Use setuptools or check "
+ "PEP 632 for potential alternatives")
+warnings.warn(_DEPRECATION_MESSAGE,
+ DeprecationWarning, 2)
diff --git a/common/py3-stdlib/distutils/ccompiler.py b/common/py3-stdlib/distutils/ccompiler.py
index b5ef143..4c47f2e 100644
--- a/common/py3-stdlib/distutils/ccompiler.py
+++ b/common/py3-stdlib/distutils/ccompiler.py
@@ -392,7 +392,7 @@
return output_dir, macros, include_dirs
def _prep_compile(self, sources, output_dir, depends=None):
- """Decide which souce files must be recompiled.
+ """Decide which source files must be recompiled.
Determine the list of object files corresponding to 'sources',
and figure out which ones really need to be recompiled.
diff --git a/common/py3-stdlib/distutils/command/__init__.py b/common/py3-stdlib/distutils/command/__init__.py
index 481eea9..fd0bfae 100644
--- a/common/py3-stdlib/distutils/command/__init__.py
+++ b/common/py3-stdlib/distutils/command/__init__.py
@@ -19,7 +19,6 @@
'bdist',
'bdist_dumb',
'bdist_rpm',
- 'bdist_wininst',
'check',
'upload',
# These two are reserved for future use:
diff --git a/common/py3-stdlib/distutils/command/bdist.py b/common/py3-stdlib/distutils/command/bdist.py
index 014871d..d580a80 100644
--- a/common/py3-stdlib/distutils/command/bdist.py
+++ b/common/py3-stdlib/distutils/command/bdist.py
@@ -62,7 +62,7 @@
# Establish the preferred order (for the --help-formats option).
format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar',
- 'wininst', 'zip', 'msi']
+ 'zip', 'msi']
# And the real information.
format_command = {'rpm': ('bdist_rpm', "RPM distribution"),
@@ -71,8 +71,6 @@
'xztar': ('bdist_dumb', "xz'ed tar file"),
'ztar': ('bdist_dumb', "compressed tar file"),
'tar': ('bdist_dumb', "tar file"),
- 'wininst': ('bdist_wininst',
- "Windows executable installer"),
'zip': ('bdist_dumb', "ZIP file"),
'msi': ('bdist_msi', "Microsoft Installer")
}
diff --git a/common/py3-stdlib/distutils/command/bdist_msi.py b/common/py3-stdlib/distutils/command/bdist_msi.py
index 0863a18..2ed017b 100644
--- a/common/py3-stdlib/distutils/command/bdist_msi.py
+++ b/common/py3-stdlib/distutils/command/bdist_msi.py
@@ -1,7 +1,5 @@
# Copyright (C) 2005, 2006 Martin von Löwis
# Licensed to PSF under a Contributor Agreement.
-# The bdist_wininst command proper
-# based on bdist_wininst
"""
Implements the bdist_msi command.
"""
diff --git a/common/py3-stdlib/distutils/command/bdist_wininst.py b/common/py3-stdlib/distutils/command/bdist_wininst.py
deleted file mode 100644
index 0e9ddaa..0000000
--- a/common/py3-stdlib/distutils/command/bdist_wininst.py
+++ /dev/null
@@ -1,377 +0,0 @@
-"""distutils.command.bdist_wininst
-
-Implements the Distutils 'bdist_wininst' command: create a windows installer
-exe-program."""
-
-import os
-import sys
-import warnings
-from distutils.core import Command
-from distutils.util import get_platform
-from distutils.dir_util import remove_tree
-from distutils.errors import *
-from distutils.sysconfig import get_python_version
-from distutils import log
-
-class bdist_wininst(Command):
-
- description = "create an executable installer for MS Windows"
-
- user_options = [('bdist-dir=', None,
- "temporary directory for creating the distribution"),
- ('plat-name=', 'p',
- "platform name to embed in generated filenames "
- "(default: %s)" % get_platform()),
- ('keep-temp', 'k',
- "keep the pseudo-installation tree around after " +
- "creating the distribution archive"),
- ('target-version=', None,
- "require a specific python version" +
- " on the target system"),
- ('no-target-compile', 'c',
- "do not compile .py to .pyc on the target system"),
- ('no-target-optimize', 'o',
- "do not compile .py to .pyo (optimized) "
- "on the target system"),
- ('dist-dir=', 'd',
- "directory to put final built distributions in"),
- ('bitmap=', 'b',
- "bitmap to use for the installer instead of python-powered logo"),
- ('title=', 't',
- "title to display on the installer background instead of default"),
- ('skip-build', None,
- "skip rebuilding everything (for testing/debugging)"),
- ('install-script=', None,
- "basename of installation script to be run after "
- "installation or before deinstallation"),
- ('pre-install-script=', None,
- "Fully qualified filename of a script to be run before "
- "any files are installed. This script need not be in the "
- "distribution"),
- ('user-access-control=', None,
- "specify Vista's UAC handling - 'none'/default=no "
- "handling, 'auto'=use UAC if target Python installed for "
- "all users, 'force'=always use UAC"),
- ]
-
- boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize',
- 'skip-build']
-
- # bpo-10945: bdist_wininst requires mbcs encoding only available on Windows
- _unsupported = (sys.platform != "win32")
-
- def __init__(self, *args, **kw):
- super().__init__(*args, **kw)
- warnings.warn("bdist_wininst command is deprecated since Python 3.8, "
- "use bdist_wheel (wheel packages) instead",
- DeprecationWarning, 2)
-
- def initialize_options(self):
- self.bdist_dir = None
- self.plat_name = None
- self.keep_temp = 0
- self.no_target_compile = 0
- self.no_target_optimize = 0
- self.target_version = None
- self.dist_dir = None
- self.bitmap = None
- self.title = None
- self.skip_build = None
- self.install_script = None
- self.pre_install_script = None
- self.user_access_control = None
-
-
- def finalize_options(self):
- self.set_undefined_options('bdist', ('skip_build', 'skip_build'))
-
- if self.bdist_dir is None:
- if self.skip_build and self.plat_name:
- # If build is skipped and plat_name is overridden, bdist will
- # not see the correct 'plat_name' - so set that up manually.
- bdist = self.distribution.get_command_obj('bdist')
- bdist.plat_name = self.plat_name
- # next the command will be initialized using that name
- bdist_base = self.get_finalized_command('bdist').bdist_base
- self.bdist_dir = os.path.join(bdist_base, 'wininst')
-
- if not self.target_version:
- self.target_version = ""
-
- if not self.skip_build and self.distribution.has_ext_modules():
- short_version = get_python_version()
- if self.target_version and self.target_version != short_version:
- raise DistutilsOptionError(
- "target version can only be %s, or the '--skip-build'" \
- " option must be specified" % (short_version,))
- self.target_version = short_version
-
- self.set_undefined_options('bdist',
- ('dist_dir', 'dist_dir'),
- ('plat_name', 'plat_name'),
- )
-
- if self.install_script:
- for script in self.distribution.scripts:
- if self.install_script == os.path.basename(script):
- break
- else:
- raise DistutilsOptionError(
- "install_script '%s' not found in scripts"
- % self.install_script)
-
- def run(self):
- if (sys.platform != "win32" and
- (self.distribution.has_ext_modules() or
- self.distribution.has_c_libraries())):
- raise DistutilsPlatformError \
- ("distribution contains extensions and/or C libraries; "
- "must be compiled on a Windows 32 platform")
-
- if not self.skip_build:
- self.run_command('build')
-
- install = self.reinitialize_command('install', reinit_subcommands=1)
- install.root = self.bdist_dir
- install.skip_build = self.skip_build
- install.warn_dir = 0
- install.plat_name = self.plat_name
-
- install_lib = self.reinitialize_command('install_lib')
- # we do not want to include pyc or pyo files
- install_lib.compile = 0
- install_lib.optimize = 0
-
- if self.distribution.has_ext_modules():
- # If we are building an installer for a Python version other
- # than the one we are currently running, then we need to ensure
- # our build_lib reflects the other Python version rather than ours.
- # Note that for target_version!=sys.version, we must have skipped the
- # build step, so there is no issue with enforcing the build of this
- # version.
- target_version = self.target_version
- if not target_version:
- assert self.skip_build, "Should have already checked this"
- target_version = '%d.%d' % sys.version_info[:2]
- plat_specifier = ".%s-%s" % (self.plat_name, target_version)
- build = self.get_finalized_command('build')
- build.build_lib = os.path.join(build.build_base,
- 'lib' + plat_specifier)
-
- # Use a custom scheme for the zip-file, because we have to decide
- # at installation time which scheme to use.
- for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'):
- value = key.upper()
- if key == 'headers':
- value = value + '/Include/$dist_name'
- setattr(install,
- 'install_' + key,
- value)
-
- log.info("installing to %s", self.bdist_dir)
- install.ensure_finalized()
-
- # avoid warning of 'install_lib' about installing
- # into a directory not in sys.path
- sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))
-
- install.run()
-
- del sys.path[0]
-
- # And make an archive relative to the root of the
- # pseudo-installation tree.
- from tempfile import mktemp
- archive_basename = mktemp()
- fullname = self.distribution.get_fullname()
- arcname = self.make_archive(archive_basename, "zip",
- root_dir=self.bdist_dir)
- # create an exe containing the zip-file
- self.create_exe(arcname, fullname, self.bitmap)
- if self.distribution.has_ext_modules():
- pyversion = get_python_version()
- else:
- pyversion = 'any'
- self.distribution.dist_files.append(('bdist_wininst', pyversion,
- self.get_installer_filename(fullname)))
- # remove the zip-file again
- log.debug("removing temporary file '%s'", arcname)
- os.remove(arcname)
-
- if not self.keep_temp:
- remove_tree(self.bdist_dir, dry_run=self.dry_run)
-
- def get_inidata(self):
- # Return data describing the installation.
- lines = []
- metadata = self.distribution.metadata
-
- # Write the [metadata] section.
- lines.append("[metadata]")
-
- # 'info' will be displayed in the installer's dialog box,
- # describing the items to be installed.
- info = (metadata.long_description or '') + '\n'
-
- # Escape newline characters
- def escape(s):
- return s.replace("\n", "\\n")
-
- for name in ["author", "author_email", "description", "maintainer",
- "maintainer_email", "name", "url", "version"]:
- data = getattr(metadata, name, "")
- if data:
- info = info + ("\n %s: %s" % \
- (name.capitalize(), escape(data)))
- lines.append("%s=%s" % (name, escape(data)))
-
- # The [setup] section contains entries controlling
- # the installer runtime.
- lines.append("\n[Setup]")
- if self.install_script:
- lines.append("install_script=%s" % self.install_script)
- lines.append("info=%s" % escape(info))
- lines.append("target_compile=%d" % (not self.no_target_compile))
- lines.append("target_optimize=%d" % (not self.no_target_optimize))
- if self.target_version:
- lines.append("target_version=%s" % self.target_version)
- if self.user_access_control:
- lines.append("user_access_control=%s" % self.user_access_control)
-
- title = self.title or self.distribution.get_fullname()
- lines.append("title=%s" % escape(title))
- import time
- import distutils
- build_info = "Built %s with distutils-%s" % \
- (time.ctime(time.time()), distutils.__version__)
- lines.append("build_info=%s" % build_info)
- return "\n".join(lines)
-
- def create_exe(self, arcname, fullname, bitmap=None):
- import struct
-
- self.mkpath(self.dist_dir)
-
- cfgdata = self.get_inidata()
-
- installer_name = self.get_installer_filename(fullname)
- self.announce("creating %s" % installer_name)
-
- if bitmap:
- with open(bitmap, "rb") as f:
- bitmapdata = f.read()
- bitmaplen = len(bitmapdata)
- else:
- bitmaplen = 0
-
- with open(installer_name, "wb") as file:
- file.write(self.get_exe_bytes())
- if bitmap:
- file.write(bitmapdata)
-
- # Convert cfgdata from unicode to ascii, mbcs encoded
- if isinstance(cfgdata, str):
- cfgdata = cfgdata.encode("mbcs")
-
- # Append the pre-install script
- cfgdata = cfgdata + b"\0"
- if self.pre_install_script:
- # We need to normalize newlines, so we open in text mode and
- # convert back to bytes. "latin-1" simply avoids any possible
- # failures.
- with open(self.pre_install_script, "r",
- encoding="latin-1") as script:
- script_data = script.read().encode("latin-1")
- cfgdata = cfgdata + script_data + b"\n\0"
- else:
- # empty pre-install script
- cfgdata = cfgdata + b"\0"
- file.write(cfgdata)
-
- # The 'magic number' 0x1234567B is used to make sure that the
- # binary layout of 'cfgdata' is what the wininst.exe binary
- # expects. If the layout changes, increment that number, make
- # the corresponding changes to the wininst.exe sources, and
- # recompile them.
- header = struct.pack("<iii",
- 0x1234567B, # tag
- len(cfgdata), # length
- bitmaplen, # number of bytes in bitmap
- )
- file.write(header)
- with open(arcname, "rb") as f:
- file.write(f.read())
-
- def get_installer_filename(self, fullname):
- # Factored out to allow overriding in subclasses
- if self.target_version:
- # if we create an installer for a specific python version,
- # it's better to include this in the name
- installer_name = os.path.join(self.dist_dir,
- "%s.%s-py%s.exe" %
- (fullname, self.plat_name, self.target_version))
- else:
- installer_name = os.path.join(self.dist_dir,
- "%s.%s.exe" % (fullname, self.plat_name))
- return installer_name
-
- def get_exe_bytes(self):
- # If a target-version other than the current version has been
- # specified, then using the MSVC version from *this* build is no good.
- # Without actually finding and executing the target version and parsing
- # its sys.version, we just hard-code our knowledge of old versions.
- # NOTE: Possible alternative is to allow "--target-version" to
- # specify a Python executable rather than a simple version string.
- # We can then execute this program to obtain any info we need, such
- # as the real sys.version string for the build.
- cur_version = get_python_version()
-
- # If the target version is *later* than us, then we assume they
- # use what we use
- # string compares seem wrong, but are what sysconfig.py itself uses
- if self.target_version and self.target_version < cur_version:
- if self.target_version < "2.4":
- bv = '6.0'
- elif self.target_version == "2.4":
- bv = '7.1'
- elif self.target_version == "2.5":
- bv = '8.0'
- elif self.target_version <= "3.2":
- bv = '9.0'
- elif self.target_version <= "3.4":
- bv = '10.0'
- else:
- bv = '14.0'
- else:
- # for current version - use authoritative check.
- try:
- from msvcrt import CRT_ASSEMBLY_VERSION
- except ImportError:
- # cross-building, so assume the latest version
- bv = '14.0'
- else:
- # as far as we know, CRT is binary compatible based on
- # the first field, so assume 'x.0' until proven otherwise
- major = CRT_ASSEMBLY_VERSION.partition('.')[0]
- bv = major + '.0'
-
-
- # wininst-x.y.exe is in the same directory as this file
- directory = os.path.dirname(__file__)
- # we must use a wininst-x.y.exe built with the same C compiler
- # used for python. XXX What about mingw, borland, and so on?
-
- # if plat_name starts with "win" but is not "win32"
- # we want to strip "win" and leave the rest (e.g. -amd64)
- # for all other cases, we don't want any suffix
- if self.plat_name != 'win32' and self.plat_name[:3] == 'win':
- sfix = self.plat_name[3:]
- else:
- sfix = ''
-
- filename = os.path.join(directory, "wininst-%s%s.exe" % (bv, sfix))
- f = open(filename, "rb")
- try:
- return f.read()
- finally:
- f.close()
diff --git a/common/py3-stdlib/distutils/command/check.py b/common/py3-stdlib/distutils/command/check.py
index ada2500..73a30f3 100644
--- a/common/py3-stdlib/distutils/command/check.py
+++ b/common/py3-stdlib/distutils/command/check.py
@@ -83,7 +83,7 @@
name, version, URL
Recommended fields:
- (author and author_email) or (maintainer and maintainer_email))
+ (author and author_email) or (maintainer and maintainer_email)
Warns if any are missing.
"""
diff --git a/common/py3-stdlib/distutils/command/install.py b/common/py3-stdlib/distutils/command/install.py
index aaa300e..01d5331 100644
--- a/common/py3-stdlib/distutils/command/install.py
+++ b/common/py3-stdlib/distutils/command/install.py
@@ -3,7 +3,9 @@
Implements the Distutils 'install' command."""
import sys
+import sysconfig
import os
+import re
from distutils import log
from distutils.core import Command
@@ -17,35 +19,55 @@
from site import USER_BASE
from site import USER_SITE
-HAS_USER_SITE = True
-WINDOWS_SCHEME = {
- 'purelib': '$base/Lib/site-packages',
- 'platlib': '$base/Lib/site-packages',
- 'headers': '$base/Include/$dist_name',
- 'scripts': '$base/Scripts',
- 'data' : '$base',
-}
+HAS_USER_SITE = (USER_SITE is not None)
-INSTALL_SCHEMES = {
- 'unix_prefix': {
- 'purelib': '$base/lib/python$py_version_short/site-packages',
- 'platlib': '$platbase/$platlibdir/python$py_version_short/site-packages',
- 'headers': '$base/include/python$py_version_short$abiflags/$dist_name',
- 'scripts': '$base/bin',
- 'data' : '$base',
- },
- 'unix_home': {
- 'purelib': '$base/lib/python',
- 'platlib': '$base/$platlibdir/python',
- 'headers': '$base/include/python/$dist_name',
- 'scripts': '$base/bin',
- 'data' : '$base',
- },
- 'nt': WINDOWS_SCHEME,
- }
+# The keys to an installation scheme; if any new types of files are to be
+# installed, be sure to add an entry to every scheme in
+# sysconfig._INSTALL_SCHEMES, and to SCHEME_KEYS here.
+SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data')
-# user site schemes
+# The following code provides backward-compatible INSTALL_SCHEMES
+# while making the sysconfig module the single point of truth.
+# This makes it easier for OS distributions where they need to
+# alter locations for packages installations in a single place.
+# Note that this module is deprecated (PEP 632); all consumers
+# of this information should switch to using sysconfig directly.
+INSTALL_SCHEMES = {"unix_prefix": {}, "unix_home": {}, "nt": {}}
+
+# Copy from sysconfig._INSTALL_SCHEMES
+for key in SCHEME_KEYS:
+ for distutils_scheme_name, sys_scheme_name in (
+ ("unix_prefix", "posix_prefix"), ("unix_home", "posix_home"),
+ ("nt", "nt")):
+ sys_key = key
+ sys_scheme = sysconfig._INSTALL_SCHEMES[sys_scheme_name]
+ if key == "headers" and key not in sys_scheme:
+ # On POSIX-y platforms, Python will:
+ # - Build from .h files in 'headers' (only there when
+ # building CPython)
+ # - Install .h files to 'include'
+ # When 'headers' is missing, fall back to 'include'
+ sys_key = 'include'
+ INSTALL_SCHEMES[distutils_scheme_name][key] = sys_scheme[sys_key]
+
+# Transformation to different template format
+for main_key in INSTALL_SCHEMES:
+ for key, value in INSTALL_SCHEMES[main_key].items():
+ # Change all ocurences of {variable} to $variable
+ value = re.sub(r"\{(.+?)\}", r"$\g<1>", value)
+ value = value.replace("$installed_base", "$base")
+ value = value.replace("$py_version_nodot_plat", "$py_version_nodot")
+ if key == "headers":
+ value += "/$dist_name"
+ if sys.version_info >= (3, 9) and key == "platlib":
+ # platlibdir is available since 3.9: bpo-1294959
+ value = value.replace("/lib/", "/$platlibdir/")
+ INSTALL_SCHEMES[main_key][key] = value
+
+# The following part of INSTALL_SCHEMES has a different definition
+# than the one in sysconfig, but because both depend on the site module,
+# the outcomes should be the same.
if HAS_USER_SITE:
INSTALL_SCHEMES['nt_user'] = {
'purelib': '$usersite',
@@ -64,11 +86,6 @@
'data' : '$userbase',
}
-# The keys to an installation scheme; if any new types of files are to be
-# installed, be sure to add an entry to every installation scheme above,
-# and to SCHEME_KEYS here.
-SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data')
-
class install(Command):
@@ -169,8 +186,9 @@
self.install_lib = None # set to either purelib or platlib
self.install_scripts = None
self.install_data = None
- self.install_userbase = USER_BASE
- self.install_usersite = USER_SITE
+ if HAS_USER_SITE:
+ self.install_userbase = USER_BASE
+ self.install_usersite = USER_SITE
self.compile = None
self.optimize = None
@@ -305,6 +323,9 @@
self.config_vars['userbase'] = self.install_userbase
self.config_vars['usersite'] = self.install_usersite
+ if sysconfig.is_python_build(True):
+ self.config_vars['srcdir'] = sysconfig.get_config_var('srcdir')
+
self.expand_basedirs()
self.dump_dirs("post-expand_basedirs()")
@@ -343,8 +364,9 @@
# Convert directories from Unix /-separated syntax to the local
# convention.
self.convert_paths('lib', 'purelib', 'platlib',
- 'scripts', 'data', 'headers',
- 'userbase', 'usersite')
+ 'scripts', 'data', 'headers')
+ if HAS_USER_SITE:
+ self.convert_paths('userbase', 'usersite')
# Deprecated
# Well, we're not actually fully completely finalized yet: we still
diff --git a/common/py3-stdlib/distutils/command/upload.py b/common/py3-stdlib/distutils/command/upload.py
index 95e9fda..e0ecb65 100644
--- a/common/py3-stdlib/distutils/command/upload.py
+++ b/common/py3-stdlib/distutils/command/upload.py
@@ -9,7 +9,8 @@
import io
import hashlib
from base64 import standard_b64encode
-from urllib.request import urlopen, Request, HTTPError
+from urllib.error import HTTPError
+from urllib.request import urlopen, Request
from urllib.parse import urlparse
from distutils.errors import DistutilsError, DistutilsOptionError
from distutils.core import PyPIRCCommand
diff --git a/common/py3-stdlib/distutils/extension.py b/common/py3-stdlib/distutils/extension.py
index c507da3..e85032e 100644
--- a/common/py3-stdlib/distutils/extension.py
+++ b/common/py3-stdlib/distutils/extension.py
@@ -4,6 +4,7 @@
modules in setup scripts."""
import os
+import re
import warnings
# This class is really only used by the "build_ext" command, so it might
@@ -161,7 +162,7 @@
line = file.readline()
if line is None: # eof
break
- if _variable_rx.match(line): # VAR=VALUE, handled in first pass
+ if re.match(_variable_rx, line): # VAR=VALUE, handled in first pass
continue
if line[0] == line[-1] == "*":
diff --git a/common/py3-stdlib/distutils/msvc9compiler.py b/common/py3-stdlib/distutils/msvc9compiler.py
index 6934e96..a7976fb 100644
--- a/common/py3-stdlib/distutils/msvc9compiler.py
+++ b/common/py3-stdlib/distutils/msvc9compiler.py
@@ -673,7 +673,7 @@
# If a manifest should be embedded, return a tuple of
# (manifest_filename, resource_id). Returns None if no manifest
# should be embedded. See http://bugs.python.org/issue7833 for why
- # we want to avoid any manifest for extension modules if we can)
+ # we want to avoid any manifest for extension modules if we can.
for arg in ld_args:
if arg.startswith("/MANIFESTFILE:"):
temp_manifest = arg.split(":", 1)[1]
diff --git a/common/py3-stdlib/distutils/spawn.py b/common/py3-stdlib/distutils/spawn.py
index f50edd2..31df3f7 100644
--- a/common/py3-stdlib/distutils/spawn.py
+++ b/common/py3-stdlib/distutils/spawn.py
@@ -54,18 +54,22 @@
global _cfg_target, _cfg_target_split
if _cfg_target is None:
from distutils import sysconfig
- _cfg_target = str(sysconfig.get_config_var(
- 'MACOSX_DEPLOYMENT_TARGET') or '')
+ _cfg_target = sysconfig.get_config_var(
+ 'MACOSX_DEPLOYMENT_TARGET') or ''
if _cfg_target:
_cfg_target_split = [int(x) for x in _cfg_target.split('.')]
if _cfg_target:
- # ensure that the deployment target of build process is not less
- # than that used when the interpreter was built. This ensures
- # extension modules are built with correct compatibility values
+ # Ensure that the deployment target of the build process is not
+ # less than 10.3 if the interpreter was built for 10.3 or later.
+ # This ensures extension modules are built with correct
+ # compatibility values, specifically LDSHARED which can use
+ # '-undefined dynamic_lookup' which only works on >= 10.3.
cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target)
- if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
+ cur_target_split = [int(x) for x in cur_target.split('.')]
+ if _cfg_target_split[:2] >= [10, 3] and cur_target_split[:2] < [10, 3]:
my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
- 'now "%s" but "%s" during configure'
+ 'now "%s" but "%s" during configure;'
+ 'must use 10.3 or later'
% (cur_target, _cfg_target))
raise DistutilsPlatformError(my_msg)
env = dict(os.environ,
diff --git a/common/py3-stdlib/distutils/sysconfig.py b/common/py3-stdlib/distutils/sysconfig.py
index 37feae5..3414a76 100644
--- a/common/py3-stdlib/distutils/sysconfig.py
+++ b/common/py3-stdlib/distutils/sysconfig.py
@@ -13,56 +13,174 @@
import os
import re
import sys
+import warnings
+
+from functools import partial
from .errors import DistutilsPlatformError
-# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
-BASE_PREFIX = os.path.normpath(sys.base_prefix)
-BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
+from sysconfig import (
+ _PREFIX as PREFIX,
+ _BASE_PREFIX as BASE_PREFIX,
+ _EXEC_PREFIX as EXEC_PREFIX,
+ _BASE_EXEC_PREFIX as BASE_EXEC_PREFIX,
+ _PROJECT_BASE as project_base,
+ _PYTHON_BUILD as python_build,
+ _init_posix as sysconfig_init_posix,
+ parse_config_h as sysconfig_parse_config_h,
-# Path to the base directory of the project. On Windows the binary may
-# live in project/PCbuild/win32 or project/PCbuild/amd64.
-# set for cross builds
-if "_PYTHON_PROJECT_BASE" in os.environ:
- project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"])
-else:
- if sys.executable:
- project_base = os.path.dirname(os.path.abspath(sys.executable))
- else:
- # sys.executable can be empty if argv[0] has been changed and Python is
- # unable to retrieve the real program name
- project_base = os.getcwd()
+ _init_non_posix,
+ _is_python_source_dir,
+ _sys_home,
+
+ _variable_rx,
+ _findvar1_rx,
+ _findvar2_rx,
+
+ expand_makefile_vars,
+ is_python_build,
+ get_config_h_filename,
+ get_config_var,
+ get_config_vars,
+ get_makefile_filename,
+ get_python_version,
+)
+
+# This is better than
+# from sysconfig import _CONFIG_VARS as _config_vars
+# because it makes sure that the global dictionary is initialized
+# which might not be true in the time of import.
+_config_vars = get_config_vars()
+
+if os.name == "nt":
+ from sysconfig import _fix_pcbuild
+
+warnings.warn(
+ 'The distutils.sysconfig module is deprecated, use sysconfig instead',
+ DeprecationWarning,
+ stacklevel=2
+)
-# python_build: (Boolean) if true, we're either building Python or
-# building an extension with an un-installed Python, so we use
-# different (hard-wired) directories.
-def _is_python_source_dir(d):
- for fn in ("Setup", "Setup.local"):
- if os.path.isfile(os.path.join(d, "Modules", fn)):
- return True
- return False
+# Following functions are the same as in sysconfig but with different API
+def parse_config_h(fp, g=None):
+ return sysconfig_parse_config_h(fp, vars=g)
-_sys_home = getattr(sys, '_home', None)
-if os.name == 'nt':
- def _fix_pcbuild(d):
- if d and os.path.normcase(d).startswith(
- os.path.normcase(os.path.join(PREFIX, "PCbuild"))):
- return PREFIX
- return d
- project_base = _fix_pcbuild(project_base)
- _sys_home = _fix_pcbuild(_sys_home)
+_python_build = partial(is_python_build, check_home=True)
+_init_posix = partial(sysconfig_init_posix, _config_vars)
+_init_nt = partial(_init_non_posix, _config_vars)
-def _python_build():
- if _sys_home:
- return _is_python_source_dir(_sys_home)
- return _is_python_source_dir(project_base)
-python_build = _python_build()
+# Similar function is also implemented in sysconfig as _parse_makefile
+# but without the parsing capabilities of distutils.text_file.TextFile.
+def parse_makefile(fn, g=None):
+ """Parse a Makefile-style file.
+ A dictionary containing name/value pairs is returned. If an
+ optional dictionary is passed in as the second argument, it is
+ used instead of a new dictionary.
+ """
+ from distutils.text_file import TextFile
+ fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1, errors="surrogateescape")
+ if g is None:
+ g = {}
+ done = {}
+ notdone = {}
+
+ while True:
+ line = fp.readline()
+ if line is None: # eof
+ break
+ m = re.match(_variable_rx, line)
+ if m:
+ n, v = m.group(1, 2)
+ v = v.strip()
+ # `$$' is a literal `$' in make
+ tmpv = v.replace('$$', '')
+
+ if "$" in tmpv:
+ notdone[n] = v
+ else:
+ try:
+ v = int(v)
+ except ValueError:
+ # insert literal `$'
+ done[n] = v.replace('$$', '$')
+ else:
+ done[n] = v
+
+ # Variables with a 'PY_' prefix in the makefile. These need to
+ # be made available without that prefix through sysconfig.
+ # Special care is needed to ensure that variable expansion works, even
+ # if the expansion uses the name without a prefix.
+ renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS')
+
+ # do variable interpolation here
+ while notdone:
+ for name in list(notdone):
+ value = notdone[name]
+ m = re.search(_findvar1_rx, value) or re.search(_findvar2_rx, value)
+ if m:
+ n = m.group(1)
+ found = True
+ if n in done:
+ item = str(done[n])
+ elif n in notdone:
+ # get it on a subsequent round
+ found = False
+ elif n in os.environ:
+ # do it like make: fall back to environment
+ item = os.environ[n]
+
+ elif n in renamed_variables:
+ if name.startswith('PY_') and name[3:] in renamed_variables:
+ item = ""
+
+ elif 'PY_' + n in notdone:
+ found = False
+
+ else:
+ item = str(done['PY_' + n])
+ else:
+ done[n] = item = ""
+ if found:
+ after = value[m.end():]
+ value = value[:m.start()] + item + after
+ if "$" in after:
+ notdone[name] = value
+ else:
+ try: value = int(value)
+ except ValueError:
+ done[name] = value.strip()
+ else:
+ done[name] = value
+ del notdone[name]
+
+ if name.startswith('PY_') \
+ and name[3:] in renamed_variables:
+
+ name = name[3:]
+ if name not in done:
+ done[name] = value
+ else:
+ # bogus variable reference; just drop it since we can't deal
+ del notdone[name]
+
+ fp.close()
+
+ # strip spurious spaces
+ for k, v in done.items():
+ if isinstance(v, str):
+ done[k] = v.strip()
+
+ # save the results in the global dictionary
+ g.update(done)
+ return g
+
+
+# Following functions are deprecated together with this module and they
+# have no direct replacement
# Calculate the build qualifier flags if they are defined. Adding the flags
# to the include and lib directories only makes sense for an installation, not
@@ -76,12 +194,76 @@
# this attribute, which is fine.
pass
-def get_python_version():
- """Return a string containing the major and minor Python version,
- leaving off the patchlevel. Sample return values could be '1.5'
- or '2.2'.
+
+def customize_compiler(compiler):
+ """Do any platform-specific customization of a CCompiler instance.
+
+ Mainly needed on Unix, so we can plug in the information that
+ varies across Unices and is stored in Python's Makefile.
"""
- return '%d.%d' % sys.version_info[:2]
+ if compiler.compiler_type == "unix":
+ if sys.platform == "darwin":
+ # Perform first-time customization of compiler-related
+ # config vars on OS X now that we know we need a compiler.
+ # This is primarily to support Pythons from binary
+ # installers. The kind and paths to build tools on
+ # the user system may vary significantly from the system
+ # that Python itself was built on. Also the user OS
+ # version and build tools may not support the same set
+ # of CPU architectures for universal builds.
+ if not _config_vars.get('CUSTOMIZED_OSX_COMPILER'):
+ import _osx_support
+ _osx_support.customize_compiler(_config_vars)
+ _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
+
+ (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
+ get_config_vars('CC', 'CXX', 'CFLAGS',
+ 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
+
+ if 'CC' in os.environ:
+ newcc = os.environ['CC']
+ if (sys.platform == 'darwin'
+ and 'LDSHARED' not in os.environ
+ and ldshared.startswith(cc)):
+ # On OS X, if CC is overridden, use that as the default
+ # command for LDSHARED as well
+ ldshared = newcc + ldshared[len(cc):]
+ cc = newcc
+ if 'CXX' in os.environ:
+ cxx = os.environ['CXX']
+ if 'LDSHARED' in os.environ:
+ ldshared = os.environ['LDSHARED']
+ if 'CPP' in os.environ:
+ cpp = os.environ['CPP']
+ else:
+ cpp = cc + " -E" # not always
+ if 'LDFLAGS' in os.environ:
+ ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+ if 'CFLAGS' in os.environ:
+ cflags = cflags + ' ' + os.environ['CFLAGS']
+ ldshared = ldshared + ' ' + os.environ['CFLAGS']
+ if 'CPPFLAGS' in os.environ:
+ cpp = cpp + ' ' + os.environ['CPPFLAGS']
+ cflags = cflags + ' ' + os.environ['CPPFLAGS']
+ ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+ if 'AR' in os.environ:
+ ar = os.environ['AR']
+ if 'ARFLAGS' in os.environ:
+ archiver = ar + ' ' + os.environ['ARFLAGS']
+ else:
+ archiver = ar + ' ' + ar_flags
+
+ cc_cmd = cc + ' ' + cflags
+ compiler.set_executables(
+ preprocessor=cpp,
+ compiler=cc_cmd,
+ compiler_so=cc_cmd + ' ' + ccshared,
+ compiler_cxx=cxx,
+ linker_so=ldshared,
+ linker_exe=cc,
+ archiver=archiver)
+
+ compiler.shared_lib_extension = shlib_suffix
def get_python_inc(plat_specific=0, prefix=None):
@@ -167,389 +349,3 @@
raise DistutilsPlatformError(
"I don't know where Python installs its library "
"on platform '%s'" % os.name)
-
-
-
-def customize_compiler(compiler):
- """Do any platform-specific customization of a CCompiler instance.
-
- Mainly needed on Unix, so we can plug in the information that
- varies across Unices and is stored in Python's Makefile.
- """
- if compiler.compiler_type == "unix":
- if sys.platform == "darwin":
- # Perform first-time customization of compiler-related
- # config vars on OS X now that we know we need a compiler.
- # This is primarily to support Pythons from binary
- # installers. The kind and paths to build tools on
- # the user system may vary significantly from the system
- # that Python itself was built on. Also the user OS
- # version and build tools may not support the same set
- # of CPU architectures for universal builds.
- global _config_vars
- # Use get_config_var() to ensure _config_vars is initialized.
- if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
- import _osx_support
- _osx_support.customize_compiler(_config_vars)
- _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
-
- (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
- get_config_vars('CC', 'CXX', 'CFLAGS',
- 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
-
- if 'CC' in os.environ:
- newcc = os.environ['CC']
- if (sys.platform == 'darwin'
- and 'LDSHARED' not in os.environ
- and ldshared.startswith(cc)):
- # On OS X, if CC is overridden, use that as the default
- # command for LDSHARED as well
- ldshared = newcc + ldshared[len(cc):]
- cc = newcc
- if 'CXX' in os.environ:
- cxx = os.environ['CXX']
- if 'LDSHARED' in os.environ:
- ldshared = os.environ['LDSHARED']
- if 'CPP' in os.environ:
- cpp = os.environ['CPP']
- else:
- cpp = cc + " -E" # not always
- if 'LDFLAGS' in os.environ:
- ldshared = ldshared + ' ' + os.environ['LDFLAGS']
- if 'CFLAGS' in os.environ:
- cflags = cflags + ' ' + os.environ['CFLAGS']
- ldshared = ldshared + ' ' + os.environ['CFLAGS']
- if 'CPPFLAGS' in os.environ:
- cpp = cpp + ' ' + os.environ['CPPFLAGS']
- cflags = cflags + ' ' + os.environ['CPPFLAGS']
- ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
- if 'AR' in os.environ:
- ar = os.environ['AR']
- if 'ARFLAGS' in os.environ:
- archiver = ar + ' ' + os.environ['ARFLAGS']
- else:
- archiver = ar + ' ' + ar_flags
-
- cc_cmd = cc + ' ' + cflags
- compiler.set_executables(
- preprocessor=cpp,
- compiler=cc_cmd,
- compiler_so=cc_cmd + ' ' + ccshared,
- compiler_cxx=cxx,
- linker_so=ldshared,
- linker_exe=cc,
- archiver=archiver)
-
- compiler.shared_lib_extension = shlib_suffix
-
-
-def get_config_h_filename():
- """Return full pathname of installed pyconfig.h file."""
- if python_build:
- if os.name == "nt":
- inc_dir = os.path.join(_sys_home or project_base, "PC")
- else:
- inc_dir = _sys_home or project_base
- else:
- inc_dir = get_python_inc(plat_specific=1)
-
- return os.path.join(inc_dir, 'pyconfig.h')
-
-
-def get_makefile_filename():
- """Return full pathname of installed Makefile from the Python build."""
- if python_build:
- return os.path.join(_sys_home or project_base, "Makefile")
- lib_dir = get_python_lib(plat_specific=0, standard_lib=1)
- config_file = 'config-{}{}'.format(get_python_version(), build_flags)
- if hasattr(sys.implementation, '_multiarch'):
- config_file += '-%s' % sys.implementation._multiarch
- return os.path.join(lib_dir, config_file, 'Makefile')
-
-
-def parse_config_h(fp, g=None):
- """Parse a config.h-style file.
-
- A dictionary containing name/value pairs is returned. If an
- optional dictionary is passed in as the second argument, it is
- used instead of a new dictionary.
- """
- if g is None:
- g = {}
- define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
- undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
- #
- while True:
- line = fp.readline()
- if not line:
- break
- m = define_rx.match(line)
- if m:
- n, v = m.group(1, 2)
- try: v = int(v)
- except ValueError: pass
- g[n] = v
- else:
- m = undef_rx.match(line)
- if m:
- g[m.group(1)] = 0
- return g
-
-
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-def parse_makefile(fn, g=None):
- """Parse a Makefile-style file.
-
- A dictionary containing name/value pairs is returned. If an
- optional dictionary is passed in as the second argument, it is
- used instead of a new dictionary.
- """
- from distutils.text_file import TextFile
- fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1, errors="surrogateescape")
-
- if g is None:
- g = {}
- done = {}
- notdone = {}
-
- while True:
- line = fp.readline()
- if line is None: # eof
- break
- m = _variable_rx.match(line)
- if m:
- n, v = m.group(1, 2)
- v = v.strip()
- # `$$' is a literal `$' in make
- tmpv = v.replace('$$', '')
-
- if "$" in tmpv:
- notdone[n] = v
- else:
- try:
- v = int(v)
- except ValueError:
- # insert literal `$'
- done[n] = v.replace('$$', '$')
- else:
- done[n] = v
-
- # Variables with a 'PY_' prefix in the makefile. These need to
- # be made available without that prefix through sysconfig.
- # Special care is needed to ensure that variable expansion works, even
- # if the expansion uses the name without a prefix.
- renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS')
-
- # do variable interpolation here
- while notdone:
- for name in list(notdone):
- value = notdone[name]
- m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
- if m:
- n = m.group(1)
- found = True
- if n in done:
- item = str(done[n])
- elif n in notdone:
- # get it on a subsequent round
- found = False
- elif n in os.environ:
- # do it like make: fall back to environment
- item = os.environ[n]
-
- elif n in renamed_variables:
- if name.startswith('PY_') and name[3:] in renamed_variables:
- item = ""
-
- elif 'PY_' + n in notdone:
- found = False
-
- else:
- item = str(done['PY_' + n])
- else:
- done[n] = item = ""
- if found:
- after = value[m.end():]
- value = value[:m.start()] + item + after
- if "$" in after:
- notdone[name] = value
- else:
- try: value = int(value)
- except ValueError:
- done[name] = value.strip()
- else:
- done[name] = value
- del notdone[name]
-
- if name.startswith('PY_') \
- and name[3:] in renamed_variables:
-
- name = name[3:]
- if name not in done:
- done[name] = value
- else:
- # bogus variable reference; just drop it since we can't deal
- del notdone[name]
-
- fp.close()
-
- # strip spurious spaces
- for k, v in done.items():
- if isinstance(v, str):
- done[k] = v.strip()
-
- # save the results in the global dictionary
- g.update(done)
- return g
-
-
-def expand_makefile_vars(s, vars):
- """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
- 'string' according to 'vars' (a dictionary mapping variable names to
- values). Variables not present in 'vars' are silently expanded to the
- empty string. The variable values in 'vars' should not contain further
- variable expansions; if 'vars' is the output of 'parse_makefile()',
- you're fine. Returns a variable-expanded version of 's'.
- """
-
- # This algorithm does multiple expansion, so if vars['foo'] contains
- # "${bar}", it will expand ${foo} to ${bar}, and then expand
- # ${bar}... and so forth. This is fine as long as 'vars' comes from
- # 'parse_makefile()', which takes care of such expansions eagerly,
- # according to make's variable expansion semantics.
-
- while True:
- m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
- if m:
- (beg, end) = m.span()
- s = s[0:beg] + vars.get(m.group(1)) + s[end:]
- else:
- break
- return s
-
-
-_config_vars = None
-
-def _init_posix():
- """Initialize the module as appropriate for POSIX systems."""
- # _sysconfigdata is generated at build time, see the sysconfig module
- name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME',
- '_sysconfigdata_{abi}_{platform}_{multiarch}'.format(
- abi=sys.abiflags,
- platform=sys.platform,
- multiarch=getattr(sys.implementation, '_multiarch', ''),
- ))
- _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
- build_time_vars = _temp.build_time_vars
- global _config_vars
- _config_vars = {}
- _config_vars.update(build_time_vars)
-
-
-def _init_nt():
- """Initialize the module as appropriate for NT"""
- g = {}
- # set basic install directories
- g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
- g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
- # XXX hmmm.. a normal install puts include files here
- g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
- g['EXT_SUFFIX'] = _imp.extension_suffixes()[0]
- g['EXE'] = ".exe"
- g['VERSION'] = get_python_version().replace(".", "")
- g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
-
- global _config_vars
- _config_vars = g
-
-
-def get_config_vars(*args):
- """With no arguments, return a dictionary of all configuration
- variables relevant for the current platform. Generally this includes
- everything needed to build extensions and install both pure modules and
- extensions. On Unix, this means every variable defined in Python's
- installed Makefile; on Windows it's a much smaller set.
-
- With arguments, return a list of values that result from looking up
- each argument in the configuration variable dictionary.
- """
- global _config_vars
- if _config_vars is None:
- func = globals().get("_init_" + os.name)
- if func:
- func()
- else:
- _config_vars = {}
-
- # Normalized versions of prefix and exec_prefix are handy to have;
- # in fact, these are the standard versions used most places in the
- # Distutils.
- _config_vars['prefix'] = PREFIX
- _config_vars['exec_prefix'] = EXEC_PREFIX
-
- # For backward compatibility, see issue19555
- SO = _config_vars.get('EXT_SUFFIX')
- if SO is not None:
- _config_vars['SO'] = SO
-
- # Always convert srcdir to an absolute path
- srcdir = _config_vars.get('srcdir', project_base)
- if os.name == 'posix':
- if python_build:
- # If srcdir is a relative path (typically '.' or '..')
- # then it should be interpreted relative to the directory
- # containing Makefile.
- base = os.path.dirname(get_makefile_filename())
- srcdir = os.path.join(base, srcdir)
- else:
- # srcdir is not meaningful since the installation is
- # spread about the filesystem. We choose the
- # directory containing the Makefile since we know it
- # exists.
- srcdir = os.path.dirname(get_makefile_filename())
- _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir))
-
- # Convert srcdir into an absolute path if it appears necessary.
- # Normally it is relative to the build directory. However, during
- # testing, for example, we might be running a non-installed python
- # from a different directory.
- if python_build and os.name == "posix":
- base = project_base
- if (not os.path.isabs(_config_vars['srcdir']) and
- base != os.getcwd()):
- # srcdir is relative and we are not in the same directory
- # as the executable. Assume executable is in the build
- # directory and make srcdir absolute.
- srcdir = os.path.join(base, _config_vars['srcdir'])
- _config_vars['srcdir'] = os.path.normpath(srcdir)
-
- # OS X platforms require special customization to handle
- # multi-architecture, multi-os-version installers
- if sys.platform == 'darwin':
- import _osx_support
- _osx_support.customize_config_vars(_config_vars)
-
- if args:
- vals = []
- for name in args:
- vals.append(_config_vars.get(name))
- return vals
- else:
- return _config_vars
-
-def get_config_var(name):
- """Return the value of a single variable using the dictionary
- returned by 'get_config_vars()'. Equivalent to
- get_config_vars().get(name)
- """
- if name == 'SO':
- import warnings
- warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2)
- return get_config_vars().get(name)
diff --git a/common/py3-stdlib/distutils/tests/__init__.py b/common/py3-stdlib/distutils/tests/__init__.py
index 6803721..16d011f 100644
--- a/common/py3-stdlib/distutils/tests/__init__.py
+++ b/common/py3-stdlib/distutils/tests/__init__.py
@@ -15,7 +15,8 @@
import os
import sys
import unittest
-from test.support import run_unittest, save_restore_warnings_filters
+from test.support import run_unittest
+from test.support.warnings_helper import save_restore_warnings_filters
here = os.path.dirname(__file__) or os.curdir
diff --git a/common/py3-stdlib/distutils/tests/support.py b/common/py3-stdlib/distutils/tests/support.py
index 259af88..23b907b 100644
--- a/common/py3-stdlib/distutils/tests/support.py
+++ b/common/py3-stdlib/distutils/tests/support.py
@@ -6,7 +6,7 @@
import unittest
import sysconfig
from copy import deepcopy
-import test.support
+from test.support import os_helper
from distutils import log
from distutils.log import DEBUG, INFO, WARN, ERROR, FATAL
@@ -64,7 +64,7 @@
super().tearDown()
while self.tempdirs:
tmpdir = self.tempdirs.pop()
- test.support.rmtree(tmpdir)
+ os_helper.rmtree(tmpdir)
def mkdtemp(self):
"""Create a temporary directory that will be cleaned up.
diff --git a/common/py3-stdlib/distutils/tests/test_archive_util.py b/common/py3-stdlib/distutils/tests/test_archive_util.py
index e9aad0e..edcec25 100644
--- a/common/py3-stdlib/distutils/tests/test_archive_util.py
+++ b/common/py3-stdlib/distutils/tests/test_archive_util.py
@@ -13,7 +13,9 @@
ARCHIVE_FORMATS)
from distutils.spawn import find_executable, spawn
from distutils.tests import support
-from test.support import check_warnings, run_unittest, patch, change_cwd
+from test.support import run_unittest, patch
+from test.support.os_helper import change_cwd
+from test.support.warnings_helper import check_warnings
try:
import grp
diff --git a/common/py3-stdlib/distutils/tests/test_bdist.py b/common/py3-stdlib/distutils/tests/test_bdist.py
index 130d8bf..55fa393 100644
--- a/common/py3-stdlib/distutils/tests/test_bdist.py
+++ b/common/py3-stdlib/distutils/tests/test_bdist.py
@@ -2,10 +2,12 @@
import os
import unittest
from test.support import run_unittest
-import warnings
-from distutils.command.bdist import bdist
-from distutils.tests import support
+import warnings
+with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ from distutils.command.bdist import bdist
+ from distutils.tests import support
class BuildTestCase(support.TempdirManager,
@@ -22,7 +24,7 @@
# what formats does bdist offer?
formats = ['bztar', 'gztar', 'msi', 'rpm', 'tar',
- 'wininst', 'xztar', 'zip', 'ztar']
+ 'xztar', 'zip', 'ztar']
found = sorted(cmd.format_command)
self.assertEqual(found, formats)
@@ -34,15 +36,12 @@
cmd.ensure_finalized()
dist.command_obj['bdist'] = cmd
- names = ['bdist_dumb', 'bdist_wininst'] # bdist_rpm does not support --skip-build
+ names = ['bdist_dumb'] # bdist_rpm does not support --skip-build
if os.name == 'nt':
names.append('bdist_msi')
for name in names:
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', 'bdist_wininst command is deprecated',
- DeprecationWarning)
- subcmd = cmd.get_finalized_command(name)
+ subcmd = cmd.get_finalized_command(name)
if getattr(subcmd, '_unsupported', False):
# command is not supported on this build
continue
diff --git a/common/py3-stdlib/distutils/tests/test_bdist_msi.py b/common/py3-stdlib/distutils/tests/test_bdist_msi.py
index 418e60e..a61266a 100644
--- a/common/py3-stdlib/distutils/tests/test_bdist_msi.py
+++ b/common/py3-stdlib/distutils/tests/test_bdist_msi.py
@@ -1,7 +1,8 @@
"""Tests for distutils.command.bdist_msi."""
import sys
import unittest
-from test.support import run_unittest, check_warnings
+from test.support import run_unittest
+from test.support.warnings_helper import check_warnings
from distutils.tests import support
diff --git a/common/py3-stdlib/distutils/tests/test_bdist_rpm.py b/common/py3-stdlib/distutils/tests/test_bdist_rpm.py
index 6453a02..ba4382f 100644
--- a/common/py3-stdlib/distutils/tests/test_bdist_rpm.py
+++ b/common/py3-stdlib/distutils/tests/test_bdist_rpm.py
@@ -44,7 +44,7 @@
# spurious sdtout/stderr output under Mac OS X
@unittest.skipUnless(sys.platform.startswith('linux'),
'spurious sdtout/stderr output under Mac OS X')
- @requires_zlib
+ @requires_zlib()
@unittest.skipIf(find_executable('rpm') is None,
'the rpm command is not found')
@unittest.skipIf(find_executable('rpmbuild') is None,
@@ -87,7 +87,7 @@
# spurious sdtout/stderr output under Mac OS X
@unittest.skipUnless(sys.platform.startswith('linux'),
'spurious sdtout/stderr output under Mac OS X')
- @requires_zlib
+ @requires_zlib()
# http://bugs.python.org/issue1533164
@unittest.skipIf(find_executable('rpm') is None,
'the rpm command is not found')
diff --git a/common/py3-stdlib/distutils/tests/test_bdist_wininst.py b/common/py3-stdlib/distutils/tests/test_bdist_wininst.py
deleted file mode 100644
index 5c3d025..0000000
--- a/common/py3-stdlib/distutils/tests/test_bdist_wininst.py
+++ /dev/null
@@ -1,38 +0,0 @@
-"""Tests for distutils.command.bdist_wininst."""
-import sys
-import platform
-import unittest
-from test.support import run_unittest, check_warnings
-
-from distutils.command.bdist_wininst import bdist_wininst
-from distutils.tests import support
-
-@unittest.skipIf(sys.platform == 'win32' and platform.machine() == 'ARM64',
- 'bdist_wininst is not supported in this install')
-@unittest.skipIf(getattr(bdist_wininst, '_unsupported', False),
- 'bdist_wininst is not supported in this install')
-class BuildWinInstTestCase(support.TempdirManager,
- support.LoggingSilencer,
- unittest.TestCase):
-
- def test_get_exe_bytes(self):
-
- # issue5731: command was broken on non-windows platforms
- # this test makes sure it works now for every platform
- # let's create a command
- pkg_pth, dist = self.create_dist()
- with check_warnings(("", DeprecationWarning)):
- cmd = bdist_wininst(dist)
- cmd.ensure_finalized()
-
- # let's run the code that finds the right wininst*.exe file
- # and make sure it finds it and returns its content
- # no matter what platform we have
- exe_file = cmd.get_exe_bytes()
- self.assertGreater(len(exe_file), 10)
-
-def test_suite():
- return unittest.makeSuite(BuildWinInstTestCase)
-
-if __name__ == '__main__':
- run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_build_clib.py b/common/py3-stdlib/distutils/tests/test_build_clib.py
index abd8313..19e012a 100644
--- a/common/py3-stdlib/distutils/tests/test_build_clib.py
+++ b/common/py3-stdlib/distutils/tests/test_build_clib.py
@@ -2,6 +2,7 @@
import unittest
import os
import sys
+import sysconfig
from test.support import run_unittest, missing_compiler_executable
@@ -13,6 +14,15 @@
support.LoggingSilencer,
unittest.TestCase):
+ def setUp(self):
+ super().setUp()
+ self._backup_CONFIG_VARS = dict(sysconfig._CONFIG_VARS)
+
+ def tearDown(self):
+ super().tearDown()
+ sysconfig._CONFIG_VARS.clear()
+ sysconfig._CONFIG_VARS.update(self._backup_CONFIG_VARS)
+
def test_check_library_dist(self):
pkg_dir, dist = self.create_dist()
cmd = build_clib(dist)
diff --git a/common/py3-stdlib/distutils/tests/test_build_ext.py b/common/py3-stdlib/distutils/tests/test_build_ext.py
index 1b034c9..8e7364d 100644
--- a/common/py3-stdlib/distutils/tests/test_build_ext.py
+++ b/common/py3-stdlib/distutils/tests/test_build_ext.py
@@ -15,6 +15,7 @@
import unittest
from test import support
+from test.support import os_helper
from test.support.script_helper import assert_python_ok
# http://bugs.python.org/issue4373
@@ -34,11 +35,12 @@
site.USER_BASE = self.mkdtemp()
from distutils.command import build_ext
build_ext.USER_BASE = site.USER_BASE
+ self.old_config_vars = dict(sysconfig._config_vars)
# bpo-30132: On Windows, a .pdb file may be created in the current
# working directory. Create a temporary working directory to cleanup
# everything at the end of the test.
- change_cwd = support.change_cwd(self.tmp_dir)
+ change_cwd = os_helper.change_cwd(self.tmp_dir)
change_cwd.__enter__()
self.addCleanup(change_cwd.__exit__, None, None, None)
@@ -47,6 +49,8 @@
site.USER_BASE = self.old_user_base
from distutils.command import build_ext
build_ext.USER_BASE = self.old_user_base
+ sysconfig._config_vars.clear()
+ sysconfig._config_vars.update(self.old_config_vars)
super(BuildExtTestCase, self).tearDown()
def build_ext(self, *args, **kwargs):
@@ -455,7 +459,7 @@
deptarget = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
if deptarget:
# increment the minor version number (i.e. 10.6 -> 10.7)
- deptarget = [int(x) for x in str(deptarget).split('.')]
+ deptarget = [int(x) for x in deptarget.split('.')]
deptarget[-1] += 1
deptarget = '.'.join(str(i) for i in deptarget)
self._try_compile_deployment_target('<', deptarget)
@@ -488,7 +492,7 @@
# get the deployment target that the interpreter was built with
target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
- target = tuple(map(int, str(target).split('.')[0:2]))
+ target = tuple(map(int, target.split('.')[0:2]))
# format the target value as defined in the Apple
# Availability Macros. We can't use the macro names since
# at least one value we test with will not exist yet.
diff --git a/common/py3-stdlib/distutils/tests/test_config_cmd.py b/common/py3-stdlib/distutils/tests/test_config_cmd.py
index 9aeab07..0127ba7 100644
--- a/common/py3-stdlib/distutils/tests/test_config_cmd.py
+++ b/common/py3-stdlib/distutils/tests/test_config_cmd.py
@@ -2,6 +2,7 @@
import unittest
import os
import sys
+import sysconfig
from test.support import run_unittest, missing_compiler_executable
from distutils.command.config import dump_file, config
@@ -21,9 +22,12 @@
self._logs = []
self.old_log = log.info
log.info = self._info
+ self.old_config_vars = dict(sysconfig._CONFIG_VARS)
def tearDown(self):
log.info = self.old_log
+ sysconfig._CONFIG_VARS.clear()
+ sysconfig._CONFIG_VARS.update(self.old_config_vars)
super(ConfigTestCase, self).tearDown()
def test_dump_file(self):
diff --git a/common/py3-stdlib/distutils/tests/test_core.py b/common/py3-stdlib/distutils/tests/test_core.py
index 27ce732..4e6694a 100644
--- a/common/py3-stdlib/distutils/tests/test_core.py
+++ b/common/py3-stdlib/distutils/tests/test_core.py
@@ -5,8 +5,8 @@
import os
import shutil
import sys
-import test.support
from test.support import captured_stdout, run_unittest
+from test.support import os_helper
import unittest
from distutils.tests import support
from distutils import log
@@ -62,13 +62,13 @@
super(CoreTestCase, self).tearDown()
def cleanup_testfn(self):
- path = test.support.TESTFN
+ path = os_helper.TESTFN
if os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
- def write_setup(self, text, path=test.support.TESTFN):
+ def write_setup(self, text, path=os_helper.TESTFN):
f = open(path, "w")
try:
f.write(text)
@@ -105,8 +105,8 @@
cwd = os.getcwd()
# Create a directory and write the setup.py file there:
- os.mkdir(test.support.TESTFN)
- setup_py = os.path.join(test.support.TESTFN, "setup.py")
+ os.mkdir(os_helper.TESTFN)
+ setup_py = os.path.join(os_helper.TESTFN, "setup.py")
distutils.core.run_setup(
self.write_setup(setup_prints_cwd, path=setup_py))
diff --git a/common/py3-stdlib/distutils/tests/test_dist.py b/common/py3-stdlib/distutils/tests/test_dist.py
index 60956da..f8a9e86 100644
--- a/common/py3-stdlib/distutils/tests/test_dist.py
+++ b/common/py3-stdlib/distutils/tests/test_dist.py
@@ -12,8 +12,9 @@
from distutils.cmd import Command
from test.support import (
- TESTFN, captured_stdout, captured_stderr, run_unittest
+ captured_stdout, captured_stderr, run_unittest
)
+from test.support.os_helper import TESTFN
from distutils.tests import support
from distutils import log
diff --git a/common/py3-stdlib/distutils/tests/test_extension.py b/common/py3-stdlib/distutils/tests/test_extension.py
index e35f273..81fad02 100644
--- a/common/py3-stdlib/distutils/tests/test_extension.py
+++ b/common/py3-stdlib/distutils/tests/test_extension.py
@@ -3,7 +3,8 @@
import os
import warnings
-from test.support import check_warnings, run_unittest
+from test.support import run_unittest
+from test.support.warnings_helper import check_warnings
from distutils.extension import read_setup_file, Extension
class ExtensionTestCase(unittest.TestCase):
diff --git a/common/py3-stdlib/distutils/tests/test_file_util.py b/common/py3-stdlib/distutils/tests/test_file_util.py
index a4e2d02..c7783b8 100644
--- a/common/py3-stdlib/distutils/tests/test_file_util.py
+++ b/common/py3-stdlib/distutils/tests/test_file_util.py
@@ -8,7 +8,9 @@
from distutils import log
from distutils.tests import support
from distutils.errors import DistutilsFileError
-from test.support import run_unittest, unlink
+from test.support import run_unittest
+from test.support.os_helper import unlink
+
class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
diff --git a/common/py3-stdlib/distutils/tests/test_filelist.py b/common/py3-stdlib/distutils/tests/test_filelist.py
index c71342d..cee97d4 100644
--- a/common/py3-stdlib/distutils/tests/test_filelist.py
+++ b/common/py3-stdlib/distutils/tests/test_filelist.py
@@ -8,7 +8,7 @@
from distutils.filelist import glob_to_re, translate_pattern, FileList
from distutils import filelist
-import test.support
+from test.support import os_helper
from test.support import captured_stdout, run_unittest
from distutils.tests import support
@@ -295,9 +295,9 @@
class FindAllTestCase(unittest.TestCase):
- @test.support.skip_unless_symlink
+ @os_helper.skip_unless_symlink
def test_missing_symlink(self):
- with test.support.temp_cwd():
+ with os_helper.temp_cwd():
os.symlink('foo', 'bar')
self.assertEqual(filelist.findall(), [])
@@ -307,13 +307,13 @@
'.' as the parameter, the dot should be omitted from
the results.
"""
- with test.support.temp_cwd():
+ with os_helper.temp_cwd():
os.mkdir('foo')
file1 = os.path.join('foo', 'file1.txt')
- test.support.create_empty_file(file1)
+ os_helper.create_empty_file(file1)
os.mkdir('bar')
file2 = os.path.join('bar', 'file2.txt')
- test.support.create_empty_file(file2)
+ os_helper.create_empty_file(file2)
expected = [file2, file1]
self.assertEqual(sorted(filelist.findall()), expected)
@@ -322,9 +322,9 @@
When findall is called with another path, the full
path name should be returned.
"""
- with test.support.temp_dir() as temp_dir:
+ with os_helper.temp_dir() as temp_dir:
file1 = os.path.join(temp_dir, 'file1.txt')
- test.support.create_empty_file(file1)
+ os_helper.create_empty_file(file1)
expected = [file1]
self.assertEqual(filelist.findall(temp_dir), expected)
diff --git a/common/py3-stdlib/distutils/tests/test_install.py b/common/py3-stdlib/distutils/tests/test_install.py
index 51c80e0..0632024 100644
--- a/common/py3-stdlib/distutils/tests/test_install.py
+++ b/common/py3-stdlib/distutils/tests/test_install.py
@@ -8,7 +8,7 @@
from test.support import captured_stdout, run_unittest
from distutils import sysconfig
-from distutils.command.install import install
+from distutils.command.install import install, HAS_USER_SITE
from distutils.command import install as install_module
from distutils.command.build_ext import build_ext
from distutils.command.install import INSTALL_SCHEMES
@@ -29,6 +29,15 @@
support.LoggingSilencer,
unittest.TestCase):
+ def setUp(self):
+ super().setUp()
+ self._backup_config_vars = dict(sysconfig._config_vars)
+
+ def tearDown(self):
+ super().tearDown()
+ sysconfig._config_vars.clear()
+ sysconfig._config_vars.update(self._backup_config_vars)
+
def test_home_installation_scheme(self):
# This ensure two things:
# - that --home generates the desired set of directory names
@@ -66,6 +75,7 @@
check_path(cmd.install_scripts, os.path.join(destination, "bin"))
check_path(cmd.install_data, destination)
+ @unittest.skipUnless(HAS_USER_SITE, 'need user site')
def test_user_site(self):
# test install with --user
# preparing the environment for the test
@@ -93,8 +103,9 @@
self.addCleanup(cleanup)
- for key in ('nt_user', 'unix_user'):
- self.assertIn(key, INSTALL_SCHEMES)
+ if HAS_USER_SITE:
+ for key in ('nt_user', 'unix_user'):
+ self.assertIn(key, INSTALL_SCHEMES)
dist = Distribution({'name': 'xx'})
cmd = install(dist)
diff --git a/common/py3-stdlib/distutils/tests/test_register.py b/common/py3-stdlib/distutils/tests/test_register.py
index e68b0af..bba4863 100644
--- a/common/py3-stdlib/distutils/tests/test_register.py
+++ b/common/py3-stdlib/distutils/tests/test_register.py
@@ -5,7 +5,8 @@
import urllib
import warnings
-from test.support import check_warnings, run_unittest
+from test.support import run_unittest
+from test.support.warnings_helper import check_warnings
from distutils.command import register as register_module
from distutils.command.register import register
diff --git a/common/py3-stdlib/distutils/tests/test_sdist.py b/common/py3-stdlib/distutils/tests/test_sdist.py
index 23db126..752e9db 100644
--- a/common/py3-stdlib/distutils/tests/test_sdist.py
+++ b/common/py3-stdlib/distutils/tests/test_sdist.py
@@ -6,7 +6,8 @@
import zipfile
from os.path import join
from textwrap import dedent
-from test.support import captured_stdout, check_warnings, run_unittest
+from test.support import captured_stdout, run_unittest
+from test.support.warnings_helper import check_warnings
try:
import zlib
diff --git a/common/py3-stdlib/distutils/tests/test_spawn.py b/common/py3-stdlib/distutils/tests/test_spawn.py
index ad50381..4ec767b 100644
--- a/common/py3-stdlib/distutils/tests/test_spawn.py
+++ b/common/py3-stdlib/distutils/tests/test_spawn.py
@@ -4,7 +4,7 @@
import sys
import unittest.mock
from test.support import run_unittest, unix_shell
-from test import support as test_support
+from test.support import os_helper
from distutils.spawn import find_executable
from distutils.spawn import spawn
@@ -44,9 +44,9 @@
spawn([exe]) # should work without any error
def test_find_executable(self):
- with test_support.temp_dir() as tmp_dir:
+ with os_helper.temp_dir() as tmp_dir:
# use TESTFN to get a pseudo-unique filename
- program_noeext = test_support.TESTFN
+ program_noeext = os_helper.TESTFN
# Give the temporary program an ".exe" suffix for all.
# It's needed on Windows and not harmful on other platforms.
program = program_noeext + ".exe"
@@ -66,7 +66,7 @@
self.assertEqual(rv, filename)
# test find in the current directory
- with test_support.change_cwd(tmp_dir):
+ with os_helper.change_cwd(tmp_dir):
rv = find_executable(program)
self.assertEqual(rv, program)
@@ -76,7 +76,7 @@
self.assertIsNone(rv)
# PATH='': no match, except in the current directory
- with test_support.EnvironmentVarGuard() as env:
+ with os_helper.EnvironmentVarGuard() as env:
env['PATH'] = ''
with unittest.mock.patch('distutils.spawn.os.confstr',
return_value=tmp_dir, create=True), \
@@ -86,12 +86,12 @@
self.assertIsNone(rv)
# look in current directory
- with test_support.change_cwd(tmp_dir):
+ with os_helper.change_cwd(tmp_dir):
rv = find_executable(program)
self.assertEqual(rv, program)
# PATH=':': explicitly looks in the current directory
- with test_support.EnvironmentVarGuard() as env:
+ with os_helper.EnvironmentVarGuard() as env:
env['PATH'] = os.pathsep
with unittest.mock.patch('distutils.spawn.os.confstr',
return_value='', create=True), \
@@ -100,12 +100,12 @@
self.assertIsNone(rv)
# look in current directory
- with test_support.change_cwd(tmp_dir):
+ with os_helper.change_cwd(tmp_dir):
rv = find_executable(program)
self.assertEqual(rv, program)
# missing PATH: test os.confstr("CS_PATH") and os.defpath
- with test_support.EnvironmentVarGuard() as env:
+ with os_helper.EnvironmentVarGuard() as env:
env.pop('PATH', None)
# without confstr
diff --git a/common/py3-stdlib/distutils/tests/test_sysconfig.py b/common/py3-stdlib/distutils/tests/test_sysconfig.py
index 236755d..59676b0 100644
--- a/common/py3-stdlib/distutils/tests/test_sysconfig.py
+++ b/common/py3-stdlib/distutils/tests/test_sysconfig.py
@@ -10,7 +10,10 @@
from distutils import sysconfig
from distutils.ccompiler import get_default_compiler
from distutils.tests import support
-from test.support import TESTFN, run_unittest, check_warnings, swap_item
+from test.support import run_unittest, swap_item
+from test.support.os_helper import TESTFN
+from test.support.warnings_helper import check_warnings
+
class SysconfigTestCase(support.EnvironGuard, unittest.TestCase):
def setUp(self):
diff --git a/common/py3-stdlib/distutils/tests/test_unixccompiler.py b/common/py3-stdlib/distutils/tests/test_unixccompiler.py
index eef702c..24725ea 100644
--- a/common/py3-stdlib/distutils/tests/test_unixccompiler.py
+++ b/common/py3-stdlib/distutils/tests/test_unixccompiler.py
@@ -1,7 +1,8 @@
"""Tests for distutils.unixccompiler."""
import sys
import unittest
-from test.support import EnvironmentVarGuard, run_unittest
+from test.support import run_unittest
+from test.support.os_helper import EnvironmentVarGuard
from distutils import sysconfig
from distutils.unixccompiler import UnixCCompiler
@@ -11,6 +12,7 @@
def setUp(self):
self._backup_platform = sys.platform
self._backup_get_config_var = sysconfig.get_config_var
+ self._backup_config_vars = dict(sysconfig._config_vars)
class CompilerWrapper(UnixCCompiler):
def rpath_foo(self):
return self.runtime_library_dir_option('/foo')
@@ -19,6 +21,8 @@
def tearDown(self):
sys.platform = self._backup_platform
sysconfig.get_config_var = self._backup_get_config_var
+ sysconfig._config_vars.clear()
+ sysconfig._config_vars.update(self._backup_config_vars)
@unittest.skipIf(sys.platform == 'win32', "can't test on Windows")
def test_runtime_libdir_option(self):
diff --git a/common/py3-stdlib/distutils/tests/test_upload.py b/common/py3-stdlib/distutils/tests/test_upload.py
index bca5516..74f0bc0 100644
--- a/common/py3-stdlib/distutils/tests/test_upload.py
+++ b/common/py3-stdlib/distutils/tests/test_upload.py
@@ -2,7 +2,7 @@
import os
import unittest
import unittest.mock as mock
-from urllib.request import HTTPError
+from urllib.error import HTTPError
from test.support import run_unittest
diff --git a/common/py3-stdlib/distutils/tests/test_util.py b/common/py3-stdlib/distutils/tests/test_util.py
index bf0d433..d4a01c6 100644
--- a/common/py3-stdlib/distutils/tests/test_util.py
+++ b/common/py3-stdlib/distutils/tests/test_util.py
@@ -54,7 +54,8 @@
os.uname = self.uname
else:
del os.uname
- sysconfig._config_vars = copy(self._config_vars)
+ sysconfig._config_vars.clear()
+ sysconfig._config_vars.update(self._config_vars)
super(UtilTestCase, self).tearDown()
def _set_uname(self, uname):
diff --git a/common/py3-stdlib/distutils/unixccompiler.py b/common/py3-stdlib/distutils/unixccompiler.py
index f0792de..d00c489 100644
--- a/common/py3-stdlib/distutils/unixccompiler.py
+++ b/common/py3-stdlib/distutils/unixccompiler.py
@@ -215,7 +215,8 @@
return "-L" + dir
def _is_gcc(self, compiler_name):
- return "gcc" in compiler_name or "g++" in compiler_name
+ # clang uses same syntax for rpath as gcc
+ return any(name in compiler_name for name in ("gcc", "g++", "clang"))
def runtime_library_dir_option(self, dir):
# XXX Hackish, at the very least. See Python bug #445902:
diff --git a/common/py3-stdlib/distutils/util.py b/common/py3-stdlib/distutils/util.py
index 4b002ec..2ce5c5b 100644
--- a/common/py3-stdlib/distutils/util.py
+++ b/common/py3-stdlib/distutils/util.py
@@ -9,6 +9,7 @@
import importlib.util
import string
import sys
+import distutils
from distutils.errors import DistutilsPlatformError
from distutils.dep_util import newer
from distutils.spawn import spawn
@@ -419,8 +420,10 @@
direct=1)
""" % (optimize, force, prefix, base_dir, verbose))
+ msg = distutils._DEPRECATION_MESSAGE
cmd = [sys.executable]
cmd.extend(subprocess._optim_args_from_interpreter_flags())
+ cmd.append(f'-Wignore:{msg}:DeprecationWarning')
cmd.append(script_name)
spawn(cmd, dry_run=dry_run)
execute(os.remove, (script_name,), "removing %s" % script_name,
diff --git a/common/py3-stdlib/doctest.py b/common/py3-stdlib/doctest.py
index baa503c..b27cbdf 100644
--- a/common/py3-stdlib/doctest.py
+++ b/common/py3-stdlib/doctest.py
@@ -102,7 +102,7 @@
import sys
import traceback
import unittest
-from io import StringIO
+from io import StringIO, IncrementalNewlineDecoder
from collections import namedtuple
TestResults = namedtuple('TestResults', 'failed attempted')
@@ -212,23 +212,24 @@
raise TypeError("Expected a module, string, or None")
def _newline_convert(data):
- # We have two cases to cover and we need to make sure we do
- # them in the right order
- for newline in ('\r\n', '\r'):
- data = data.replace(newline, '\n')
- return data
+ # The IO module provides a handy decoder for universal newline conversion
+ return IncrementalNewlineDecoder(None, True).decode(data, True)
def _load_testfile(filename, package, module_relative, encoding):
if module_relative:
package = _normalize_module(package, 3)
filename = _module_relative_path(package, filename)
- if getattr(package, '__loader__', None) is not None:
- if hasattr(package.__loader__, 'get_data'):
- file_contents = package.__loader__.get_data(filename)
- file_contents = file_contents.decode(encoding)
- # get_data() opens files as 'rb', so one must do the equivalent
- # conversion as universal newlines would do.
- return _newline_convert(file_contents), filename
+ if (loader := getattr(package, '__loader__', None)) is None:
+ try:
+ loader = package.__spec__.loader
+ except AttributeError:
+ pass
+ if hasattr(loader, 'get_data'):
+ file_contents = loader.get_data(filename)
+ file_contents = file_contents.decode(encoding)
+ # get_data() opens files as 'rb', so one must do the equivalent
+ # conversion as universal newlines would do.
+ return _newline_convert(file_contents), filename
with open(filename, encoding=encoding) as f:
return f.read(), filename
@@ -972,6 +973,17 @@
else:
raise ValueError("object must be a class or function")
+ def _is_routine(self, obj):
+ """
+ Safely unwrap objects and determine if they are functions.
+ """
+ maybe_routine = obj
+ try:
+ maybe_routine = inspect.unwrap(maybe_routine)
+ except ValueError:
+ pass
+ return inspect.isroutine(maybe_routine)
+
def _find(self, tests, obj, name, module, source_lines, globs, seen):
"""
Find tests for the given object and any contained objects, and
@@ -994,9 +1006,9 @@
if inspect.ismodule(obj) and self._recurse:
for valname, val in obj.__dict__.items():
valname = '%s.%s' % (name, valname)
+
# Recurse to functions & classes.
- if ((inspect.isroutine(inspect.unwrap(val))
- or inspect.isclass(val)) and
+ if ((self._is_routine(val) or inspect.isclass(val)) and
self._from_module(module, val)):
self._find(tests, val, valname, module, source_lines,
globs, seen)
@@ -1022,10 +1034,8 @@
if inspect.isclass(obj) and self._recurse:
for valname, val in obj.__dict__.items():
# Special handling for staticmethod/classmethod.
- if isinstance(val, staticmethod):
- val = getattr(obj, valname)
- if isinstance(val, classmethod):
- val = getattr(obj, valname).__func__
+ if isinstance(val, (staticmethod, classmethod)):
+ val = val.__func__
# Recurse to methods, properties, and nested classes.
if ((inspect.isroutine(val) or inspect.isclass(val) or
diff --git a/common/py3-stdlib/email/_parseaddr.py b/common/py3-stdlib/email/_parseaddr.py
index 41ff6f8..ba5ad5a 100644
--- a/common/py3-stdlib/email/_parseaddr.py
+++ b/common/py3-stdlib/email/_parseaddr.py
@@ -65,8 +65,10 @@
"""
if not data:
- return
+ return None
data = data.split()
+ if not data: # This happens for whitespace-only input.
+ return None
# The FWS after the comma after the day-of-week is optional, so search and
# adjust for this.
if data[0].endswith(',') or data[0].lower() in _daynames:
@@ -126,6 +128,8 @@
tss = 0
elif len(tm) == 3:
[thh, tmm, tss] = tm
+ else:
+ return None
else:
return None
try:
diff --git a/common/py3-stdlib/email/base64mime.py b/common/py3-stdlib/email/base64mime.py
index 17f0818..a7cc373 100644
--- a/common/py3-stdlib/email/base64mime.py
+++ b/common/py3-stdlib/email/base64mime.py
@@ -84,7 +84,7 @@
in an email.
"""
if not s:
- return s
+ return ""
encvec = []
max_unencoded = maxlinelen * 3 // 4
diff --git a/common/py3-stdlib/email/contentmanager.py b/common/py3-stdlib/email/contentmanager.py
index b91fb0e..fcf278d 100644
--- a/common/py3-stdlib/email/contentmanager.py
+++ b/common/py3-stdlib/email/contentmanager.py
@@ -144,7 +144,7 @@
linesep = policy.linesep.encode('ascii')
def embedded_body(lines): return linesep.join(lines) + linesep
def normal_body(lines): return b'\n'.join(lines) + b'\n'
- if cte==None:
+ if cte is None:
# Use heuristics to decide on the "best" encoding.
if max((len(x) for x in lines), default=0) <= policy.max_line_length:
try:
@@ -238,9 +238,7 @@
data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True)
data = data.decode('ascii')
elif cte == '7bit':
- # Make sure it really is only ASCII. The early warning here seems
- # worth the overhead...if you care write your own content manager :).
- data.encode('ascii')
+ data = data.decode('ascii')
elif cte in ('8bit', 'binary'):
data = data.decode('ascii', 'surrogateescape')
msg.set_payload(data)
diff --git a/common/py3-stdlib/email/errors.py b/common/py3-stdlib/email/errors.py
index d28a680..3ad0056 100644
--- a/common/py3-stdlib/email/errors.py
+++ b/common/py3-stdlib/email/errors.py
@@ -108,3 +108,6 @@
"""local_part contains non-ASCII characters"""
# This defect only occurs during unicode parsing, not when
# parsing messages decoded from binary.
+
+class InvalidDateDefect(HeaderDefect):
+ """Header has unparsable or invalid date"""
diff --git a/common/py3-stdlib/email/headerregistry.py b/common/py3-stdlib/email/headerregistry.py
index 5d84fc0..b590d69 100644
--- a/common/py3-stdlib/email/headerregistry.py
+++ b/common/py3-stdlib/email/headerregistry.py
@@ -2,10 +2,6 @@
This module provides an implementation of the HeaderRegistry API.
The implementation is designed to flexibly follow RFC5322 rules.
-
-Eventually HeaderRegistry will be a public API, but it isn't yet,
-and will probably change some before that happens.
-
"""
from types import MappingProxyType
@@ -302,7 +298,14 @@
kwds['parse_tree'] = parser.TokenList()
return
if isinstance(value, str):
- value = utils.parsedate_to_datetime(value)
+ kwds['decoded'] = value
+ try:
+ value = utils.parsedate_to_datetime(value)
+ except ValueError:
+ kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format'))
+ kwds['datetime'] = None
+ kwds['parse_tree'] = parser.TokenList()
+ return
kwds['datetime'] = value
kwds['decoded'] = utils.format_datetime(kwds['datetime'])
kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
diff --git a/common/py3-stdlib/email/message.py b/common/py3-stdlib/email/message.py
index 3701b30..6752ce0 100644
--- a/common/py3-stdlib/email/message.py
+++ b/common/py3-stdlib/email/message.py
@@ -948,7 +948,7 @@
if policy is None:
from email.policy import default
policy = default
- Message.__init__(self, policy)
+ super().__init__(policy)
def as_string(self, unixfrom=False, maxheaderlen=None, policy=None):
@@ -965,7 +965,7 @@
policy = self.policy if policy is None else policy
if maxheaderlen is None:
maxheaderlen = policy.max_line_length
- return super().as_string(maxheaderlen=maxheaderlen, policy=policy)
+ return super().as_string(unixfrom, maxheaderlen, policy)
def __str__(self):
return self.as_string(policy=self.policy.clone(utf8=True))
@@ -982,7 +982,7 @@
if subtype in preferencelist:
yield (preferencelist.index(subtype), part)
return
- if maintype != 'multipart':
+ if maintype != 'multipart' or not self.is_multipart():
return
if subtype != 'related':
for subpart in part.iter_parts():
@@ -1087,7 +1087,7 @@
Return an empty iterator for a non-multipart.
"""
- if self.get_content_maintype() == 'multipart':
+ if self.is_multipart():
yield from self.get_payload()
def get_content(self, *args, content_manager=None, **kw):
diff --git a/common/py3-stdlib/email/utils.py b/common/py3-stdlib/email/utils.py
index 1a7719d..cfdfeb3 100644
--- a/common/py3-stdlib/email/utils.py
+++ b/common/py3-stdlib/email/utils.py
@@ -109,7 +109,7 @@
def getaddresses(fieldvalues):
"""Return a list of (REALNAME, EMAIL) for each fieldvalue."""
- all = COMMASPACE.join(fieldvalues)
+ all = COMMASPACE.join(str(v) for v in fieldvalues)
a = _AddressList(all)
return a.addresslist
@@ -195,7 +195,10 @@
def parsedate_to_datetime(data):
- *dtuple, tz = _parsedate_tz(data)
+ parsed_date_tz = _parsedate_tz(data)
+ if parsed_date_tz is None:
+ raise ValueError('Invalid date value or format "%s"' % str(data))
+ *dtuple, tz = parsed_date_tz
if tz is None:
return datetime.datetime(*dtuple[:6])
return datetime.datetime(*dtuple[:6],
diff --git a/common/py3-stdlib/encodings/__init__.py b/common/py3-stdlib/encodings/__init__.py
index ddd5afd..4b37d33 100644
--- a/common/py3-stdlib/encodings/__init__.py
+++ b/common/py3-stdlib/encodings/__init__.py
@@ -61,7 +61,8 @@
if c.isalnum() or c == '.':
if punct and chars:
chars.append('_')
- chars.append(c)
+ if c.isascii():
+ chars.append(c)
punct = False
else:
punct = True
diff --git a/common/py3-stdlib/encodings/raw_unicode_escape.py b/common/py3-stdlib/encodings/raw_unicode_escape.py
index 2b919b4..46c8e07 100644
--- a/common/py3-stdlib/encodings/raw_unicode_escape.py
+++ b/common/py3-stdlib/encodings/raw_unicode_escape.py
@@ -21,15 +21,16 @@
def encode(self, input, final=False):
return codecs.raw_unicode_escape_encode(input, self.errors)[0]
-class IncrementalDecoder(codecs.IncrementalDecoder):
- def decode(self, input, final=False):
- return codecs.raw_unicode_escape_decode(input, self.errors)[0]
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+ def _buffer_decode(self, input, errors, final):
+ return codecs.raw_unicode_escape_decode(input, errors, final)
class StreamWriter(Codec,codecs.StreamWriter):
pass
class StreamReader(Codec,codecs.StreamReader):
- pass
+ def decode(self, input, errors='strict'):
+ return codecs.raw_unicode_escape_decode(input, errors, False)
### encodings module API
diff --git a/common/py3-stdlib/encodings/unicode_escape.py b/common/py3-stdlib/encodings/unicode_escape.py
index 817f932..9b1ce99 100644
--- a/common/py3-stdlib/encodings/unicode_escape.py
+++ b/common/py3-stdlib/encodings/unicode_escape.py
@@ -21,15 +21,16 @@
def encode(self, input, final=False):
return codecs.unicode_escape_encode(input, self.errors)[0]
-class IncrementalDecoder(codecs.IncrementalDecoder):
- def decode(self, input, final=False):
- return codecs.unicode_escape_decode(input, self.errors)[0]
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+ def _buffer_decode(self, input, errors, final):
+ return codecs.unicode_escape_decode(input, errors, final)
class StreamWriter(Codec,codecs.StreamWriter):
pass
class StreamReader(Codec,codecs.StreamReader):
- pass
+ def decode(self, input, errors='strict'):
+ return codecs.unicode_escape_decode(input, errors, False)
### encodings module API
diff --git a/common/py3-stdlib/enum.py b/common/py3-stdlib/enum.py
index ebadd9f..f5657a6 100644
--- a/common/py3-stdlib/enum.py
+++ b/common/py3-stdlib/enum.py
@@ -10,31 +10,55 @@
def _is_descriptor(obj):
- """Returns True if obj is a descriptor, False otherwise."""
+ """
+ Returns True if obj is a descriptor, False otherwise.
+ """
return (
hasattr(obj, '__get__') or
hasattr(obj, '__set__') or
- hasattr(obj, '__delete__'))
-
+ hasattr(obj, '__delete__')
+ )
def _is_dunder(name):
- """Returns True if a __dunder__ name, False otherwise."""
- return (len(name) > 4 and
+ """
+ Returns True if a __dunder__ name, False otherwise.
+ """
+ return (
+ len(name) > 4 and
name[:2] == name[-2:] == '__' and
name[2] != '_' and
- name[-3] != '_')
-
+ name[-3] != '_'
+ )
def _is_sunder(name):
- """Returns True if a _sunder_ name, False otherwise."""
- return (len(name) > 2 and
+ """
+ Returns True if a _sunder_ name, False otherwise.
+ """
+ return (
+ len(name) > 2 and
name[0] == name[-1] == '_' and
name[1:2] != '_' and
- name[-2:-1] != '_')
+ name[-2:-1] != '_'
+ )
+def _is_private(cls_name, name):
+ # do not use `re` as `re` imports `enum`
+ pattern = '_%s__' % (cls_name, )
+ pat_len = len(pattern)
+ if (
+ len(name) > pat_len
+ and name.startswith(pattern)
+ and name[pat_len:pat_len+1] != ['_']
+ and (name[-1] != '_' or name[-2] != '_')
+ ):
+ return True
+ else:
+ return False
def _make_class_unpicklable(cls):
- """Make the given class un-picklable."""
+ """
+ Make the given class un-picklable.
+ """
def _break_on_call_reduce(self, proto):
raise TypeError('%r cannot be pickled' % self)
cls.__reduce_ex__ = _break_on_call_reduce
@@ -49,11 +73,11 @@
class _EnumDict(dict):
- """Track enum member order and ensure member names are not reused.
+ """
+ Track enum member order and ensure member names are not reused.
EnumMeta will use the names found in self._member_names as the
enumeration member names.
-
"""
def __init__(self):
super().__init__()
@@ -63,14 +87,22 @@
self._auto_called = False
def __setitem__(self, key, value):
- """Changes anything not dundered or not a descriptor.
+ """
+ Changes anything not dundered or not a descriptor.
If an enum member name is used twice, an error is raised; duplicate
values are not checked for.
Single underscore (sunder) names are reserved.
-
"""
+ if _is_private(self._cls_name, key):
+ import warnings
+ warnings.warn(
+ "private variables, such as %r, will be normal attributes in 3.11"
+ % (key, ),
+ DeprecationWarning,
+ stacklevel=2,
+ )
if _is_sunder(key):
if key not in (
'_order_', '_create_pseudo_member_',
@@ -90,7 +122,10 @@
self._ignore = value
already = set(value) & set(self._member_names)
if already:
- raise ValueError('_ignore_ cannot specify already set names: %r' % (already, ))
+ raise ValueError(
+ '_ignore_ cannot specify already set names: %r'
+ % (already, )
+ )
elif _is_dunder(key):
if key == '__order__':
key = '_order_'
@@ -105,7 +140,12 @@
raise TypeError('%r already defined as: %r' % (key, self[key]))
if isinstance(value, auto):
if value.value == _auto_null:
- value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:])
+ value.value = self._generate_next_value(
+ key,
+ 1,
+ len(self._member_names),
+ self._last_values[:],
+ )
self._auto_called = True
value = value.value
self._member_names.append(key)
@@ -118,22 +158,26 @@
# This is also why there are checks in EnumMeta like `if Enum is not None`
Enum = None
-
class EnumMeta(type):
- """Metaclass for Enum"""
+ """
+ Metaclass for Enum
+ """
@classmethod
- def __prepare__(metacls, cls, bases):
+ def __prepare__(metacls, cls, bases, **kwds):
# check that previous enum members do not exist
metacls._check_for_existing_members(cls, bases)
# create the namespace dict
enum_dict = _EnumDict()
+ enum_dict._cls_name = cls
# inherit previous flags and _generate_next_value_ function
member_type, first_enum = metacls._get_mixins_(cls, bases)
if first_enum is not None:
- enum_dict['_generate_next_value_'] = getattr(first_enum, '_generate_next_value_', None)
+ enum_dict['_generate_next_value_'] = getattr(
+ first_enum, '_generate_next_value_', None,
+ )
return enum_dict
- def __new__(metacls, cls, bases, classdict):
+ def __new__(metacls, cls, bases, classdict, **kwds):
# an Enum class is final once enumeration items have been defined; it
# cannot be mixed with other types (int, float, etc.) if it has an
# inherited __new__ unless a new __new__ is defined (or the resulting
@@ -145,8 +189,9 @@
for key in ignore:
classdict.pop(key, None)
member_type, first_enum = metacls._get_mixins_(cls, bases)
- __new__, save_new, use_args = metacls._find_new_(classdict, member_type,
- first_enum)
+ __new__, save_new, use_args = metacls._find_new_(
+ classdict, member_type, first_enum,
+ )
# save enum items into separate mapping so they don't get baked into
# the new class
@@ -167,17 +212,18 @@
if '__doc__' not in classdict:
classdict['__doc__'] = 'An enumeration.'
- # create our new Enum type
- enum_class = super().__new__(metacls, cls, bases, classdict)
+ enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
enum_class._member_names_ = [] # names in definition order
enum_class._member_map_ = {} # name->value map
enum_class._member_type_ = member_type
# save DynamicClassAttribute attributes from super classes so we know
# if we can take the shortcut of storing members in the class dict
- dynamic_attributes = {k for c in enum_class.mro()
- for k, v in c.__dict__.items()
- if isinstance(v, DynamicClassAttribute)}
+ dynamic_attributes = {
+ k for c in enum_class.mro()
+ for k, v in c.__dict__.items()
+ if isinstance(v, DynamicClassAttribute)
+ }
# Reverse value->name map for hashable values.
enum_class._value2member_map_ = {}
@@ -197,8 +243,32 @@
methods = ('__getnewargs_ex__', '__getnewargs__',
'__reduce_ex__', '__reduce__')
if not any(m in member_type.__dict__ for m in methods):
- _make_class_unpicklable(enum_class)
-
+ if '__new__' in classdict:
+ # too late, sabotage
+ _make_class_unpicklable(enum_class)
+ else:
+ # final attempt to verify that pickling would work:
+ # travel mro until __new__ is found, checking for
+ # __reduce__ and friends along the way -- if any of them
+ # are found before/when __new__ is found, pickling should
+ # work
+ sabotage = None
+ for chain in bases:
+ for base in chain.__mro__:
+ if base is object:
+ continue
+ elif any(m in base.__dict__ for m in methods):
+ # found one, we're good
+ sabotage = False
+ break
+ elif '__new__' in base.__dict__:
+ # not good
+ sabotage = True
+ break
+ if sabotage is not None:
+ break
+ if sabotage:
+ _make_class_unpicklable(enum_class)
# instantiate them, checking for duplicates as we go
# we instantiate first instead of checking for duplicates first in case
# a custom __new__ is doing something funky with the values -- such as
@@ -287,7 +357,8 @@
return True
def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):
- """Either returns an existing member, or creates a new enum class.
+ """
+ Either returns an existing member, or creates a new enum class.
This method is used both when an enum class is given a value to match
to an enumeration member (i.e. Color(3)) and for the functional API
@@ -309,40 +380,54 @@
not correct, unpickling will fail in some circumstances.
`type`, if set, will be mixed in as the first base class.
-
"""
if names is None: # simple value lookup
return cls.__new__(cls, value)
# otherwise, functional API: we're creating a new Enum type
- return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start)
+ return cls._create_(
+ value,
+ names,
+ module=module,
+ qualname=qualname,
+ type=type,
+ start=start,
+ )
- def __contains__(cls, member):
- if not isinstance(member, Enum):
+ def __contains__(cls, obj):
+ if not isinstance(obj, Enum):
+ import warnings
+ warnings.warn(
+ "in 3.12 __contains__ will no longer raise TypeError, but will return True if\n"
+ "obj is a member or a member's value",
+ DeprecationWarning,
+ stacklevel=2,
+ )
raise TypeError(
"unsupported operand type(s) for 'in': '%s' and '%s'" % (
- type(member).__qualname__, cls.__class__.__qualname__))
- return isinstance(member, cls) and member._name_ in cls._member_map_
+ type(obj).__qualname__, cls.__class__.__qualname__))
+ return isinstance(obj, cls) and obj._name_ in cls._member_map_
def __delattr__(cls, attr):
# nicer error message when someone tries to delete an attribute
# (see issue19025).
if attr in cls._member_map_:
- raise AttributeError(
- "%s: cannot delete Enum member." % cls.__name__)
+ raise AttributeError("%s: cannot delete Enum member." % cls.__name__)
super().__delattr__(attr)
def __dir__(self):
- return (['__class__', '__doc__', '__members__', '__module__'] +
- self._member_names_)
+ return (
+ ['__class__', '__doc__', '__members__', '__module__']
+ + self._member_names_
+ )
def __getattr__(cls, name):
- """Return the enum member matching `name`
+ """
+ Return the enum member matching `name`
We use __getattr__ instead of descriptors or inserting into the enum
class' __dict__ in order to support `name` and `value` being both
properties for enum members (which live in the class' __dict__) and
enum members themselves.
-
"""
if _is_dunder(name):
raise AttributeError(name)
@@ -355,6 +440,9 @@
return cls._member_map_[name]
def __iter__(cls):
+ """
+ Returns members in definition order.
+ """
return (cls._member_map_[name] for name in cls._member_names_)
def __len__(cls):
@@ -362,11 +450,11 @@
@property
def __members__(cls):
- """Returns a mapping of member name->value.
+ """
+ Returns a mapping of member name->value.
This mapping lists all enum members, including aliases. Note that this
is a read-only view of the internal mapping.
-
"""
return MappingProxyType(cls._member_map_)
@@ -374,15 +462,18 @@
return "<enum %r>" % cls.__name__
def __reversed__(cls):
+ """
+ Returns members in reverse definition order.
+ """
return (cls._member_map_[name] for name in reversed(cls._member_names_))
def __setattr__(cls, name, value):
- """Block attempts to reassign Enum members.
+ """
+ Block attempts to reassign Enum members.
A simple assignment to the class namespace only changes one of the
several possible ways to get an Enum member from the Enum class,
resulting in an inconsistent Enumeration.
-
"""
member_map = cls.__dict__.get('_member_map_', {})
if name in member_map:
@@ -390,7 +481,8 @@
super().__setattr__(name, value)
def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1):
- """Convenience method to create a new Enum class.
+ """
+ Convenience method to create a new Enum class.
`names` can be:
@@ -399,7 +491,6 @@
* An iterable of member names. Values are incremented by 1 from `start`.
* An iterable of (member name, value) pairs.
* A mapping of member name -> value pairs.
-
"""
metacls = cls.__class__
bases = (cls, ) if type is None else (type, cls)
@@ -480,37 +571,44 @@
for chain in bases:
for base in chain.__mro__:
if issubclass(base, Enum) and base._member_names_:
- raise TypeError("%s: cannot extend enumeration %r" % (class_name, base.__name__))
+ raise TypeError(
+ "%s: cannot extend enumeration %r"
+ % (class_name, base.__name__)
+ )
@staticmethod
def _get_mixins_(class_name, bases):
- """Returns the type for creating enum members, and the first inherited
+ """
+ Returns the type for creating enum members, and the first inherited
enum class.
bases: the tuple of bases that was given to __new__
-
"""
if not bases:
return object, Enum
def _find_data_type(bases):
- data_types = []
+ data_types = set()
for chain in bases:
candidate = None
for base in chain.__mro__:
if base is object:
continue
+ elif issubclass(base, Enum):
+ if base._member_type_ is not object:
+ data_types.add(base._member_type_)
+ break
elif '__new__' in base.__dict__:
if issubclass(base, Enum):
continue
- data_types.append(candidate or base)
+ data_types.add(candidate or base)
break
- elif not issubclass(base, Enum):
- candidate = base
+ else:
+ candidate = candidate or base
if len(data_types) > 1:
raise TypeError('%r: too many data types: %r' % (class_name, data_types))
elif data_types:
- return data_types[0]
+ return data_types.pop()
else:
return None
@@ -527,12 +625,12 @@
@staticmethod
def _find_new_(classdict, member_type, first_enum):
- """Returns the __new__ to be used for creating the enum members.
+ """
+ Returns the __new__ to be used for creating the enum members.
classdict: the class dictionary given to __new__
member_type: the data type whose __new__ will be used by default
first_enum: enumeration to check for an overriding __new__
-
"""
# now find the correct __new__, checking to see of one was defined
# by the user; also check earlier enum classes in case a __new__ was
@@ -572,10 +670,10 @@
class Enum(metaclass=EnumMeta):
- """Generic enumeration.
+ """
+ Generic enumeration.
Derive from this class to define new enumerations.
-
"""
def __new__(cls, value):
# all enum instances are actually created during class construction
@@ -603,21 +701,35 @@
except Exception as e:
exc = e
result = None
- if isinstance(result, cls):
- return result
- else:
- ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__))
- if result is None and exc is None:
- raise ve_exc
- elif exc is None:
- exc = TypeError(
- 'error in %s._missing_: returned %r instead of None or a valid member'
- % (cls.__name__, result)
- )
- exc.__context__ = ve_exc
- raise exc
+ try:
+ if isinstance(result, cls):
+ return result
+ else:
+ ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__))
+ if result is None and exc is None:
+ raise ve_exc
+ elif exc is None:
+ exc = TypeError(
+ 'error in %s._missing_: returned %r instead of None or a valid member'
+ % (cls.__name__, result)
+ )
+ if not isinstance(exc, ValueError):
+ exc.__context__ = ve_exc
+ raise exc
+ finally:
+ # ensure all variables that could hold an exception are destroyed
+ exc = None
+ ve_exc = None
def _generate_next_value_(name, start, count, last_values):
+ """
+ Generate the next value when not given.
+
+ name: the name of the member
+ start: the initial start value or None
+ count: the number of existing members
+ last_value: the last value assigned or None
+ """
for last_value in reversed(last_values):
try:
return last_value + 1
@@ -638,21 +750,27 @@
return "%s.%s" % (self.__class__.__name__, self._name_)
def __dir__(self):
+ """
+ Returns all members and all public methods
+ """
added_behavior = [
m
for cls in self.__class__.mro()
for m in cls.__dict__
if m[0] != '_' and m not in self._member_map_
- ]
+ ] + [m for m in self.__dict__ if m[0] != '_']
return (['__class__', '__doc__', '__module__'] + added_behavior)
def __format__(self, format_spec):
+ """
+ Returns format using actual value type unless __str__ has been overridden.
+ """
# mixed-in Enums should use the mixed-in type's __format__, otherwise
# we can get strange results with the Enum name showing up instead of
# the value
# pure Enum branch, or branch with __str__ explicitly overridden
- str_overridden = type(self).__str__ != Enum.__str__
+ str_overridden = type(self).__str__ not in (Enum.__str__, Flag.__str__)
if self._member_type_ is object or str_overridden:
cls = str
val = str(self)
@@ -694,7 +812,9 @@
return self.name
class Flag(Enum):
- """Support for flags"""
+ """
+ Support for flags
+ """
def _generate_next_value_(name, start, count, last_values):
"""
@@ -717,6 +837,9 @@
@classmethod
def _missing_(cls, value):
+ """
+ Returns member (possibly creating it) if one can be found for value.
+ """
original_value = value
if value < 0:
value = ~value
@@ -746,6 +869,9 @@
return pseudo_member
def __contains__(self, other):
+ """
+ Returns True if self has at least the same flags set as other.
+ """
if not isinstance(other, self.__class__):
raise TypeError(
"unsupported operand type(s) for 'in': '%s' and '%s'" % (
@@ -804,10 +930,15 @@
class IntFlag(int, Flag):
- """Support for integer-based Flags"""
+ """
+ Support for integer-based Flags
+ """
@classmethod
def _missing_(cls, value):
+ """
+ Returns member (possibly creating it) if one can be found for value.
+ """
if not isinstance(value, int):
raise ValueError("%r is not a valid %s" % (value, cls.__qualname__))
new_member = cls._create_pseudo_member_(value)
@@ -815,6 +946,9 @@
@classmethod
def _create_pseudo_member_(cls, value):
+ """
+ Create a composite member iff value contains only members.
+ """
pseudo_member = cls._value2member_map_.get(value, None)
if pseudo_member is None:
need_to_create = [value]
@@ -869,11 +1003,15 @@
def _high_bit(value):
- """returns index of highest bit, or -1 if value is zero or negative"""
+ """
+ returns index of highest bit, or -1 if value is zero or negative
+ """
return value.bit_length() - 1
def unique(enumeration):
- """Class decorator for enumerations ensuring unique member values."""
+ """
+ Class decorator for enumerations ensuring unique member values.
+ """
duplicates = []
for name, member in enumeration.__members__.items():
if name != member.name:
@@ -886,7 +1024,9 @@
return enumeration
def _decompose(flag, value):
- """Extract all members from the value."""
+ """
+ Extract all members from the value.
+ """
# _decompose is only called if the value is not named
not_covered = value
negative = value < 0
diff --git a/common/py3-stdlib/filecmp.py b/common/py3-stdlib/filecmp.py
index 7a4da6b..70a4b23 100644
--- a/common/py3-stdlib/filecmp.py
+++ b/common/py3-stdlib/filecmp.py
@@ -36,8 +36,9 @@
f2 -- Second file name
- shallow -- Just check stat signature (do not read the files).
- defaults to True.
+ shallow -- treat files as identical if their stat signatures (type, size,
+ mtime) are identical. Otherwise, files are considered different
+ if their sizes or contents differ. [default: True]
Return value:
@@ -115,7 +116,9 @@
same_files: list of identical files.
diff_files: list of filenames which differ.
funny_files: list of files which could not be compared.
- subdirs: a dictionary of dircmp objects, keyed by names in common_dirs.
+ subdirs: a dictionary of dircmp instances (or MyDirCmp instances if this
+ object is of type MyDirCmp, a subclass of dircmp), keyed by names
+ in common_dirs.
"""
def __init__(self, a, b, ignore=None, hide=None): # Initialize
@@ -185,14 +188,15 @@
self.same_files, self.diff_files, self.funny_files = xx
def phase4(self): # Find out differences between common subdirectories
- # A new dircmp object is created for each common subdirectory,
+ # A new dircmp (or MyDirCmp if dircmp was subclassed) object is created
+ # for each common subdirectory,
# these are stored in a dictionary indexed by filename.
# The hide and ignore properties are inherited from the parent
self.subdirs = {}
for x in self.common_dirs:
a_x = os.path.join(self.left, x)
b_x = os.path.join(self.right, x)
- self.subdirs[x] = dircmp(a_x, b_x, self.ignore, self.hide)
+ self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide)
def phase4_closure(self): # Recursively call phase4() on subdirectories
self.phase4()
diff --git a/common/py3-stdlib/fileinput.py b/common/py3-stdlib/fileinput.py
index 0c31f93..3534718 100644
--- a/common/py3-stdlib/fileinput.py
+++ b/common/py3-stdlib/fileinput.py
@@ -3,7 +3,7 @@
Typical use is:
import fileinput
- for line in fileinput.input():
+ for line in fileinput.input(encoding="utf-8"):
process(line)
This iterates over the lines of all files listed in sys.argv[1:],
@@ -63,15 +63,9 @@
deleted when the output file is closed. In-place filtering is
disabled when standard input is read. XXX The current implementation
does not work for MS-DOS 8+3 filesystems.
-
-XXX Possible additions:
-
-- optional getopt argument processing
-- isatty()
-- read(), read(size), even readlines()
-
"""
+import io
import sys, os
from types import GenericAlias
@@ -81,7 +75,8 @@
_state = None
-def input(files=None, inplace=False, backup="", *, mode="r", openhook=None):
+def input(files=None, inplace=False, backup="", *, mode="r", openhook=None,
+ encoding=None, errors=None):
"""Return an instance of the FileInput class, which can be iterated.
The parameters are passed to the constructor of the FileInput class.
@@ -91,7 +86,8 @@
global _state
if _state and _state._file:
raise RuntimeError("input() already active")
- _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook)
+ _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook,
+ encoding=encoding, errors=errors)
return _state
def close():
@@ -186,7 +182,7 @@
"""
def __init__(self, files=None, inplace=False, backup="", *,
- mode="r", openhook=None):
+ mode="r", openhook=None, encoding=None, errors=None):
if isinstance(files, str):
files = (files,)
elif isinstance(files, os.PathLike):
@@ -209,6 +205,17 @@
self._file = None
self._isstdin = False
self._backupfilename = None
+ self._encoding = encoding
+ self._errors = errors
+
+ # We can not use io.text_encoding() here because old openhook doesn't
+ # take encoding parameter.
+ if (sys.flags.warn_default_encoding and
+ "b" not in mode and encoding is None and openhook is None):
+ import warnings
+ warnings.warn("'encoding' argument not specified.",
+ EncodingWarning, 2)
+
# restrict mode argument to reading modes
if mode not in ('r', 'rU', 'U', 'rb'):
raise ValueError("FileInput opening mode must be one of "
@@ -324,6 +331,13 @@
self._file = None
self._isstdin = False
self._backupfilename = 0
+
+ # EncodingWarning is emitted in __init__() already
+ if "b" not in self._mode:
+ encoding = self._encoding or "locale"
+ else:
+ encoding = None
+
if self._filename == '-':
self._filename = '<stdin>'
if 'b' in self._mode:
@@ -341,18 +355,18 @@
pass
# The next few lines may raise OSError
os.rename(self._filename, self._backupfilename)
- self._file = open(self._backupfilename, self._mode)
+ self._file = open(self._backupfilename, self._mode, encoding=encoding)
try:
perm = os.fstat(self._file.fileno()).st_mode
except OSError:
- self._output = open(self._filename, self._write_mode)
+ self._output = open(self._filename, self._write_mode, encoding=encoding)
else:
mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC
if hasattr(os, 'O_BINARY'):
mode |= os.O_BINARY
fd = os.open(self._filename, mode, perm)
- self._output = os.fdopen(fd, self._write_mode)
+ self._output = os.fdopen(fd, self._write_mode, encoding=encoding)
try:
os.chmod(self._filename, perm)
except OSError:
@@ -362,9 +376,15 @@
else:
# This may raise OSError
if self._openhook:
- self._file = self._openhook(self._filename, self._mode)
+ # Custom hooks made previous to Python 3.10 didn't have
+ # encoding argument
+ if self._encoding is None:
+ self._file = self._openhook(self._filename, self._mode)
+ else:
+ self._file = self._openhook(
+ self._filename, self._mode, encoding=self._encoding, errors=self._errors)
else:
- self._file = open(self._filename, self._mode)
+ self._file = open(self._filename, self._mode, encoding=encoding, errors=self._errors)
self._readline = self._file.readline # hide FileInput._readline
return self._readline()
@@ -395,16 +415,23 @@
__class_getitem__ = classmethod(GenericAlias)
-def hook_compressed(filename, mode):
+def hook_compressed(filename, mode, *, encoding=None, errors=None):
+ if encoding is None: # EncodingWarning is emitted in FileInput() already.
+ encoding = "locale"
ext = os.path.splitext(filename)[1]
if ext == '.gz':
import gzip
- return gzip.open(filename, mode)
+ stream = gzip.open(filename, mode)
elif ext == '.bz2':
import bz2
- return bz2.BZ2File(filename, mode)
+ stream = bz2.BZ2File(filename, mode)
else:
- return open(filename, mode)
+ return open(filename, mode, encoding=encoding, errors=errors)
+
+ # gzip and bz2 are binary mode by default.
+ if "b" not in mode:
+ stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors)
+ return stream
def hook_encoded(encoding, errors=None):
diff --git a/common/py3-stdlib/fnmatch.py b/common/py3-stdlib/fnmatch.py
index 0eb1802..7c52c23 100644
--- a/common/py3-stdlib/fnmatch.py
+++ b/common/py3-stdlib/fnmatch.py
@@ -52,7 +52,7 @@
return re.compile(res).match
def filter(names, pat):
- """Return the subset of the list NAMES that match PAT."""
+ """Construct a list from those elements of the iterable NAMES that match PAT."""
result = []
pat = os.path.normcase(pat)
match = _compile_pattern(pat)
diff --git a/common/py3-stdlib/formatter.py b/common/py3-stdlib/formatter.py
deleted file mode 100644
index e2394de..0000000
--- a/common/py3-stdlib/formatter.py
+++ /dev/null
@@ -1,452 +0,0 @@
-"""Generic output formatting.
-
-Formatter objects transform an abstract flow of formatting events into
-specific output events on writer objects. Formatters manage several stack
-structures to allow various properties of a writer object to be changed and
-restored; writers need not be able to handle relative changes nor any sort
-of ``change back'' operation. Specific writer properties which may be
-controlled via formatter objects are horizontal alignment, font, and left
-margin indentations. A mechanism is provided which supports providing
-arbitrary, non-exclusive style settings to a writer as well. Additional
-interfaces facilitate formatting events which are not reversible, such as
-paragraph separation.
-
-Writer objects encapsulate device interfaces. Abstract devices, such as
-file formats, are supported as well as physical devices. The provided
-implementations all work with abstract devices. The interface makes
-available mechanisms for setting the properties which formatter objects
-manage and inserting data into the output.
-"""
-
-import sys
-import warnings
-warnings.warn('the formatter module is deprecated', DeprecationWarning,
- stacklevel=2)
-
-
-AS_IS = None
-
-
-class NullFormatter:
- """A formatter which does nothing.
-
- If the writer parameter is omitted, a NullWriter instance is created.
- No methods of the writer are called by NullFormatter instances.
-
- Implementations should inherit from this class if implementing a writer
- interface but don't need to inherit any implementation.
-
- """
-
- def __init__(self, writer=None):
- if writer is None:
- writer = NullWriter()
- self.writer = writer
- def end_paragraph(self, blankline): pass
- def add_line_break(self): pass
- def add_hor_rule(self, *args, **kw): pass
- def add_label_data(self, format, counter, blankline=None): pass
- def add_flowing_data(self, data): pass
- def add_literal_data(self, data): pass
- def flush_softspace(self): pass
- def push_alignment(self, align): pass
- def pop_alignment(self): pass
- def push_font(self, x): pass
- def pop_font(self): pass
- def push_margin(self, margin): pass
- def pop_margin(self): pass
- def set_spacing(self, spacing): pass
- def push_style(self, *styles): pass
- def pop_style(self, n=1): pass
- def assert_line_data(self, flag=1): pass
-
-
-class AbstractFormatter:
- """The standard formatter.
-
- This implementation has demonstrated wide applicability to many writers,
- and may be used directly in most circumstances. It has been used to
- implement a full-featured World Wide Web browser.
-
- """
-
- # Space handling policy: blank spaces at the boundary between elements
- # are handled by the outermost context. "Literal" data is not checked
- # to determine context, so spaces in literal data are handled directly
- # in all circumstances.
-
- def __init__(self, writer):
- self.writer = writer # Output device
- self.align = None # Current alignment
- self.align_stack = [] # Alignment stack
- self.font_stack = [] # Font state
- self.margin_stack = [] # Margin state
- self.spacing = None # Vertical spacing state
- self.style_stack = [] # Other state, e.g. color
- self.nospace = 1 # Should leading space be suppressed
- self.softspace = 0 # Should a space be inserted
- self.para_end = 1 # Just ended a paragraph
- self.parskip = 0 # Skipped space between paragraphs?
- self.hard_break = 1 # Have a hard break
- self.have_label = 0
-
- def end_paragraph(self, blankline):
- if not self.hard_break:
- self.writer.send_line_break()
- self.have_label = 0
- if self.parskip < blankline and not self.have_label:
- self.writer.send_paragraph(blankline - self.parskip)
- self.parskip = blankline
- self.have_label = 0
- self.hard_break = self.nospace = self.para_end = 1
- self.softspace = 0
-
- def add_line_break(self):
- if not (self.hard_break or self.para_end):
- self.writer.send_line_break()
- self.have_label = self.parskip = 0
- self.hard_break = self.nospace = 1
- self.softspace = 0
-
- def add_hor_rule(self, *args, **kw):
- if not self.hard_break:
- self.writer.send_line_break()
- self.writer.send_hor_rule(*args, **kw)
- self.hard_break = self.nospace = 1
- self.have_label = self.para_end = self.softspace = self.parskip = 0
-
- def add_label_data(self, format, counter, blankline = None):
- if self.have_label or not self.hard_break:
- self.writer.send_line_break()
- if not self.para_end:
- self.writer.send_paragraph((blankline and 1) or 0)
- if isinstance(format, str):
- self.writer.send_label_data(self.format_counter(format, counter))
- else:
- self.writer.send_label_data(format)
- self.nospace = self.have_label = self.hard_break = self.para_end = 1
- self.softspace = self.parskip = 0
-
- def format_counter(self, format, counter):
- label = ''
- for c in format:
- if c == '1':
- label = label + ('%d' % counter)
- elif c in 'aA':
- if counter > 0:
- label = label + self.format_letter(c, counter)
- elif c in 'iI':
- if counter > 0:
- label = label + self.format_roman(c, counter)
- else:
- label = label + c
- return label
-
- def format_letter(self, case, counter):
- label = ''
- while counter > 0:
- counter, x = divmod(counter-1, 26)
- # This makes a strong assumption that lowercase letters
- # and uppercase letters form two contiguous blocks, with
- # letters in order!
- s = chr(ord(case) + x)
- label = s + label
- return label
-
- def format_roman(self, case, counter):
- ones = ['i', 'x', 'c', 'm']
- fives = ['v', 'l', 'd']
- label, index = '', 0
- # This will die of IndexError when counter is too big
- while counter > 0:
- counter, x = divmod(counter, 10)
- if x == 9:
- label = ones[index] + ones[index+1] + label
- elif x == 4:
- label = ones[index] + fives[index] + label
- else:
- if x >= 5:
- s = fives[index]
- x = x-5
- else:
- s = ''
- s = s + ones[index]*x
- label = s + label
- index = index + 1
- if case == 'I':
- return label.upper()
- return label
-
- def add_flowing_data(self, data):
- if not data: return
- prespace = data[:1].isspace()
- postspace = data[-1:].isspace()
- data = " ".join(data.split())
- if self.nospace and not data:
- return
- elif prespace or self.softspace:
- if not data:
- if not self.nospace:
- self.softspace = 1
- self.parskip = 0
- return
- if not self.nospace:
- data = ' ' + data
- self.hard_break = self.nospace = self.para_end = \
- self.parskip = self.have_label = 0
- self.softspace = postspace
- self.writer.send_flowing_data(data)
-
- def add_literal_data(self, data):
- if not data: return
- if self.softspace:
- self.writer.send_flowing_data(" ")
- self.hard_break = data[-1:] == '\n'
- self.nospace = self.para_end = self.softspace = \
- self.parskip = self.have_label = 0
- self.writer.send_literal_data(data)
-
- def flush_softspace(self):
- if self.softspace:
- self.hard_break = self.para_end = self.parskip = \
- self.have_label = self.softspace = 0
- self.nospace = 1
- self.writer.send_flowing_data(' ')
-
- def push_alignment(self, align):
- if align and align != self.align:
- self.writer.new_alignment(align)
- self.align = align
- self.align_stack.append(align)
- else:
- self.align_stack.append(self.align)
-
- def pop_alignment(self):
- if self.align_stack:
- del self.align_stack[-1]
- if self.align_stack:
- self.align = align = self.align_stack[-1]
- self.writer.new_alignment(align)
- else:
- self.align = None
- self.writer.new_alignment(None)
-
- def push_font(self, font):
- size, i, b, tt = font
- if self.softspace:
- self.hard_break = self.para_end = self.softspace = 0
- self.nospace = 1
- self.writer.send_flowing_data(' ')
- if self.font_stack:
- csize, ci, cb, ctt = self.font_stack[-1]
- if size is AS_IS: size = csize
- if i is AS_IS: i = ci
- if b is AS_IS: b = cb
- if tt is AS_IS: tt = ctt
- font = (size, i, b, tt)
- self.font_stack.append(font)
- self.writer.new_font(font)
-
- def pop_font(self):
- if self.font_stack:
- del self.font_stack[-1]
- if self.font_stack:
- font = self.font_stack[-1]
- else:
- font = None
- self.writer.new_font(font)
-
- def push_margin(self, margin):
- self.margin_stack.append(margin)
- fstack = [m for m in self.margin_stack if m]
- if not margin and fstack:
- margin = fstack[-1]
- self.writer.new_margin(margin, len(fstack))
-
- def pop_margin(self):
- if self.margin_stack:
- del self.margin_stack[-1]
- fstack = [m for m in self.margin_stack if m]
- if fstack:
- margin = fstack[-1]
- else:
- margin = None
- self.writer.new_margin(margin, len(fstack))
-
- def set_spacing(self, spacing):
- self.spacing = spacing
- self.writer.new_spacing(spacing)
-
- def push_style(self, *styles):
- if self.softspace:
- self.hard_break = self.para_end = self.softspace = 0
- self.nospace = 1
- self.writer.send_flowing_data(' ')
- for style in styles:
- self.style_stack.append(style)
- self.writer.new_styles(tuple(self.style_stack))
-
- def pop_style(self, n=1):
- del self.style_stack[-n:]
- self.writer.new_styles(tuple(self.style_stack))
-
- def assert_line_data(self, flag=1):
- self.nospace = self.hard_break = not flag
- self.para_end = self.parskip = self.have_label = 0
-
-
-class NullWriter:
- """Minimal writer interface to use in testing & inheritance.
-
- A writer which only provides the interface definition; no actions are
- taken on any methods. This should be the base class for all writers
- which do not need to inherit any implementation methods.
-
- """
- def __init__(self): pass
- def flush(self): pass
- def new_alignment(self, align): pass
- def new_font(self, font): pass
- def new_margin(self, margin, level): pass
- def new_spacing(self, spacing): pass
- def new_styles(self, styles): pass
- def send_paragraph(self, blankline): pass
- def send_line_break(self): pass
- def send_hor_rule(self, *args, **kw): pass
- def send_label_data(self, data): pass
- def send_flowing_data(self, data): pass
- def send_literal_data(self, data): pass
-
-
-class AbstractWriter(NullWriter):
- """A writer which can be used in debugging formatters, but not much else.
-
- Each method simply announces itself by printing its name and
- arguments on standard output.
-
- """
-
- def new_alignment(self, align):
- print("new_alignment(%r)" % (align,))
-
- def new_font(self, font):
- print("new_font(%r)" % (font,))
-
- def new_margin(self, margin, level):
- print("new_margin(%r, %d)" % (margin, level))
-
- def new_spacing(self, spacing):
- print("new_spacing(%r)" % (spacing,))
-
- def new_styles(self, styles):
- print("new_styles(%r)" % (styles,))
-
- def send_paragraph(self, blankline):
- print("send_paragraph(%r)" % (blankline,))
-
- def send_line_break(self):
- print("send_line_break()")
-
- def send_hor_rule(self, *args, **kw):
- print("send_hor_rule()")
-
- def send_label_data(self, data):
- print("send_label_data(%r)" % (data,))
-
- def send_flowing_data(self, data):
- print("send_flowing_data(%r)" % (data,))
-
- def send_literal_data(self, data):
- print("send_literal_data(%r)" % (data,))
-
-
-class DumbWriter(NullWriter):
- """Simple writer class which writes output on the file object passed in
- as the file parameter or, if file is omitted, on standard output. The
- output is simply word-wrapped to the number of columns specified by
- the maxcol parameter. This class is suitable for reflowing a sequence
- of paragraphs.
-
- """
-
- def __init__(self, file=None, maxcol=72):
- self.file = file or sys.stdout
- self.maxcol = maxcol
- NullWriter.__init__(self)
- self.reset()
-
- def reset(self):
- self.col = 0
- self.atbreak = 0
-
- def send_paragraph(self, blankline):
- self.file.write('\n'*blankline)
- self.col = 0
- self.atbreak = 0
-
- def send_line_break(self):
- self.file.write('\n')
- self.col = 0
- self.atbreak = 0
-
- def send_hor_rule(self, *args, **kw):
- self.file.write('\n')
- self.file.write('-'*self.maxcol)
- self.file.write('\n')
- self.col = 0
- self.atbreak = 0
-
- def send_literal_data(self, data):
- self.file.write(data)
- i = data.rfind('\n')
- if i >= 0:
- self.col = 0
- data = data[i+1:]
- data = data.expandtabs()
- self.col = self.col + len(data)
- self.atbreak = 0
-
- def send_flowing_data(self, data):
- if not data: return
- atbreak = self.atbreak or data[0].isspace()
- col = self.col
- maxcol = self.maxcol
- write = self.file.write
- for word in data.split():
- if atbreak:
- if col + len(word) >= maxcol:
- write('\n')
- col = 0
- else:
- write(' ')
- col = col + 1
- write(word)
- col = col + len(word)
- atbreak = 1
- self.col = col
- self.atbreak = data[-1].isspace()
-
-
-def test(file = None):
- w = DumbWriter()
- f = AbstractFormatter(w)
- if file is not None:
- fp = open(file)
- elif sys.argv[1:]:
- fp = open(sys.argv[1])
- else:
- fp = sys.stdin
- try:
- for line in fp:
- if line == '\n':
- f.end_paragraph(1)
- else:
- f.add_flowing_data(line)
- finally:
- if fp is not sys.stdin:
- fp.close()
- f.end_paragraph(0)
-
-
-if __name__ == '__main__':
- test()
diff --git a/common/py3-stdlib/fractions.py b/common/py3-stdlib/fractions.py
index de3e23b..96047be 100644
--- a/common/py3-stdlib/fractions.py
+++ b/common/py3-stdlib/fractions.py
@@ -380,32 +380,139 @@
return forward, reverse
+ # Rational arithmetic algorithms: Knuth, TAOCP, Volume 2, 4.5.1.
+ #
+ # Assume input fractions a and b are normalized.
+ #
+ # 1) Consider addition/subtraction.
+ #
+ # Let g = gcd(da, db). Then
+ #
+ # na nb na*db ± nb*da
+ # a ± b == -- ± -- == ------------- ==
+ # da db da*db
+ #
+ # na*(db//g) ± nb*(da//g) t
+ # == ----------------------- == -
+ # (da*db)//g d
+ #
+ # Now, if g > 1, we're working with smaller integers.
+ #
+ # Note, that t, (da//g) and (db//g) are pairwise coprime.
+ #
+ # Indeed, (da//g) and (db//g) share no common factors (they were
+ # removed) and da is coprime with na (since input fractions are
+ # normalized), hence (da//g) and na are coprime. By symmetry,
+ # (db//g) and nb are coprime too. Then,
+ #
+ # gcd(t, da//g) == gcd(na*(db//g), da//g) == 1
+ # gcd(t, db//g) == gcd(nb*(da//g), db//g) == 1
+ #
+ # Above allows us optimize reduction of the result to lowest
+ # terms. Indeed,
+ #
+ # g2 = gcd(t, d) == gcd(t, (da//g)*(db//g)*g) == gcd(t, g)
+ #
+ # t//g2 t//g2
+ # a ± b == ----------------------- == ----------------
+ # (da//g)*(db//g)*(g//g2) (da//g)*(db//g2)
+ #
+ # is a normalized fraction. This is useful because the unnormalized
+ # denominator d could be much larger than g.
+ #
+ # We should special-case g == 1 (and g2 == 1), since 60.8% of
+ # randomly-chosen integers are coprime:
+ # https://en.wikipedia.org/wiki/Coprime_integers#Probability_of_coprimality
+ # Note, that g2 == 1 always for fractions, obtained from floats: here
+ # g is a power of 2 and the unnormalized numerator t is an odd integer.
+ #
+ # 2) Consider multiplication
+ #
+ # Let g1 = gcd(na, db) and g2 = gcd(nb, da), then
+ #
+ # na*nb na*nb (na//g1)*(nb//g2)
+ # a*b == ----- == ----- == -----------------
+ # da*db db*da (db//g1)*(da//g2)
+ #
+ # Note, that after divisions we're multiplying smaller integers.
+ #
+ # Also, the resulting fraction is normalized, because each of
+ # two factors in the numerator is coprime to each of the two factors
+ # in the denominator.
+ #
+ # Indeed, pick (na//g1). It's coprime with (da//g2), because input
+ # fractions are normalized. It's also coprime with (db//g1), because
+ # common factors are removed by g1 == gcd(na, db).
+ #
+ # As for addition/subtraction, we should special-case g1 == 1
+ # and g2 == 1 for same reason. That happens also for multiplying
+ # rationals, obtained from floats.
+
def _add(a, b):
"""a + b"""
- da, db = a.denominator, b.denominator
- return Fraction(a.numerator * db + b.numerator * da,
- da * db)
+ na, da = a.numerator, a.denominator
+ nb, db = b.numerator, b.denominator
+ g = math.gcd(da, db)
+ if g == 1:
+ return Fraction(na * db + da * nb, da * db, _normalize=False)
+ s = da // g
+ t = na * (db // g) + nb * s
+ g2 = math.gcd(t, g)
+ if g2 == 1:
+ return Fraction(t, s * db, _normalize=False)
+ return Fraction(t // g2, s * (db // g2), _normalize=False)
__add__, __radd__ = _operator_fallbacks(_add, operator.add)
def _sub(a, b):
"""a - b"""
- da, db = a.denominator, b.denominator
- return Fraction(a.numerator * db - b.numerator * da,
- da * db)
+ na, da = a.numerator, a.denominator
+ nb, db = b.numerator, b.denominator
+ g = math.gcd(da, db)
+ if g == 1:
+ return Fraction(na * db - da * nb, da * db, _normalize=False)
+ s = da // g
+ t = na * (db // g) - nb * s
+ g2 = math.gcd(t, g)
+ if g2 == 1:
+ return Fraction(t, s * db, _normalize=False)
+ return Fraction(t // g2, s * (db // g2), _normalize=False)
__sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub)
def _mul(a, b):
"""a * b"""
- return Fraction(a.numerator * b.numerator, a.denominator * b.denominator)
+ na, da = a.numerator, a.denominator
+ nb, db = b.numerator, b.denominator
+ g1 = math.gcd(na, db)
+ if g1 > 1:
+ na //= g1
+ db //= g1
+ g2 = math.gcd(nb, da)
+ if g2 > 1:
+ nb //= g2
+ da //= g2
+ return Fraction(na * nb, db * da, _normalize=False)
__mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul)
def _div(a, b):
"""a / b"""
- return Fraction(a.numerator * b.denominator,
- a.denominator * b.numerator)
+ # Same as _mul(), with inversed b.
+ na, da = a.numerator, a.denominator
+ nb, db = b.numerator, b.denominator
+ g1 = math.gcd(na, nb)
+ if g1 > 1:
+ na //= g1
+ nb //= g1
+ g2 = math.gcd(db, da)
+ if g2 > 1:
+ da //= g2
+ db //= g2
+ n, d = na * db, nb * da
+ if d < 0:
+ n, d = -n, -d
+ return Fraction(n, d, _normalize=False)
__truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv)
diff --git a/common/py3-stdlib/ftplib.py b/common/py3-stdlib/ftplib.py
index 1f760ed..7c5a507 100644
--- a/common/py3-stdlib/ftplib.py
+++ b/common/py3-stdlib/ftplib.py
@@ -102,7 +102,9 @@
sock = None
file = None
welcome = None
- passiveserver = 1
+ passiveserver = True
+ # Disables https://bugs.python.org/issue43285 security if set to True.
+ trust_server_pasv_ipv4_address = False
def __init__(self, host='', user='', passwd='', acct='',
timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None, *,
@@ -320,8 +322,13 @@
return sock
def makepasv(self):
+ """Internal: Does the PASV or EPSV handshake -> (address, port)"""
if self.af == socket.AF_INET:
- host, port = parse227(self.sendcmd('PASV'))
+ untrusted_host, port = parse227(self.sendcmd('PASV'))
+ if self.trust_server_pasv_ipv4_address:
+ host = untrusted_host
+ else:
+ host = self.sock.getpeername()[0]
else:
host, port = parse229(self.sendcmd('EPSV'), self.sock.getpeername())
return host, port
diff --git a/common/py3-stdlib/functools.py b/common/py3-stdlib/functools.py
index 5cab497..305ceb4 100644
--- a/common/py3-stdlib/functools.py
+++ b/common/py3-stdlib/functools.py
@@ -88,84 +88,84 @@
def _gt_from_lt(self, other, NotImplemented=NotImplemented):
'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).'
- op_result = self.__lt__(other)
+ op_result = type(self).__lt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result and self != other
def _le_from_lt(self, other, NotImplemented=NotImplemented):
'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).'
- op_result = self.__lt__(other)
+ op_result = type(self).__lt__(self, other)
if op_result is NotImplemented:
return op_result
return op_result or self == other
def _ge_from_lt(self, other, NotImplemented=NotImplemented):
'Return a >= b. Computed by @total_ordering from (not a < b).'
- op_result = self.__lt__(other)
+ op_result = type(self).__lt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result
def _ge_from_le(self, other, NotImplemented=NotImplemented):
'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).'
- op_result = self.__le__(other)
+ op_result = type(self).__le__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result or self == other
def _lt_from_le(self, other, NotImplemented=NotImplemented):
'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).'
- op_result = self.__le__(other)
+ op_result = type(self).__le__(self, other)
if op_result is NotImplemented:
return op_result
return op_result and self != other
def _gt_from_le(self, other, NotImplemented=NotImplemented):
'Return a > b. Computed by @total_ordering from (not a <= b).'
- op_result = self.__le__(other)
+ op_result = type(self).__le__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result
def _lt_from_gt(self, other, NotImplemented=NotImplemented):
'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).'
- op_result = self.__gt__(other)
+ op_result = type(self).__gt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result and self != other
def _ge_from_gt(self, other, NotImplemented=NotImplemented):
'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).'
- op_result = self.__gt__(other)
+ op_result = type(self).__gt__(self, other)
if op_result is NotImplemented:
return op_result
return op_result or self == other
def _le_from_gt(self, other, NotImplemented=NotImplemented):
'Return a <= b. Computed by @total_ordering from (not a > b).'
- op_result = self.__gt__(other)
+ op_result = type(self).__gt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result
def _le_from_ge(self, other, NotImplemented=NotImplemented):
'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).'
- op_result = self.__ge__(other)
+ op_result = type(self).__ge__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result or self == other
def _gt_from_ge(self, other, NotImplemented=NotImplemented):
'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).'
- op_result = self.__ge__(other)
+ op_result = type(self).__ge__(self, other)
if op_result is NotImplemented:
return op_result
return op_result and self != other
def _lt_from_ge(self, other, NotImplemented=NotImplemented):
'Return a < b. Computed by @total_ordering from (not a >= b).'
- op_result = self.__ge__(other)
+ op_result = type(self).__ge__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result
@@ -236,14 +236,14 @@
def reduce(function, sequence, initial=_initial_missing):
"""
- reduce(function, sequence[, initial]) -> value
+ reduce(function, iterable[, initial]) -> value
- Apply a function of two arguments cumulatively to the items of a sequence,
- from left to right, so as to reduce the sequence to a single value.
- For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+ Apply a function of two arguments cumulatively to the items of a sequence
+ or iterable, from left to right, so as to reduce the iterable to a single
+ value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
((((1+2)+3)+4)+5). If initial is present, it is placed before the items
- of the sequence in the calculation, and serves as a default when the
- sequence is empty.
+ of the iterable in the calculation, and serves as a default when the
+ iterable is empty.
"""
it = iter(sequence)
@@ -252,7 +252,8 @@
try:
value = next(it)
except StopIteration:
- raise TypeError("reduce() of empty sequence with no initial value") from None
+ raise TypeError(
+ "reduce() of empty iterable with no initial value") from None
else:
value = initial
@@ -491,7 +492,7 @@
with f.cache_info(). Clear the cache and statistics with f.cache_clear().
Access the underlying function with f.__wrapped__.
- See: http://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU)
+ See: https://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU)
"""
@@ -659,7 +660,7 @@
def _c3_merge(sequences):
"""Merges MROs in *sequences* to a single MRO using the C3 algorithm.
- Adapted from http://www.python.org/download/releases/2.3/mro/.
+ Adapted from https://www.python.org/download/releases/2.3/mro/.
"""
result = []
@@ -739,6 +740,7 @@
# Remove entries which are already present in the __mro__ or unrelated.
def is_related(typ):
return (typ not in bases and hasattr(typ, '__mro__')
+ and not isinstance(typ, GenericAlias)
and issubclass(cls, typ))
types = [n for n in types if is_related(n)]
# Remove entries which are strict bases of other entries (they will end up
@@ -836,6 +838,9 @@
dispatch_cache[cls] = impl
return impl
+ def _is_valid_dispatch_type(cls):
+ return isinstance(cls, type) and not isinstance(cls, GenericAlias)
+
def register(cls, func=None):
"""generic_func.register(cls, func) -> func
@@ -843,9 +848,15 @@
"""
nonlocal cache_token
- if func is None:
- if isinstance(cls, type):
+ if _is_valid_dispatch_type(cls):
+ if func is None:
return lambda f: register(cls, f)
+ else:
+ if func is not None:
+ raise TypeError(
+ f"Invalid first argument to `register()`. "
+ f"{cls!r} is not a class."
+ )
ann = getattr(cls, '__annotations__', {})
if not ann:
raise TypeError(
@@ -858,11 +869,12 @@
# only import typing if annotation parsing is necessary
from typing import get_type_hints
argname, cls = next(iter(get_type_hints(func).items()))
- if not isinstance(cls, type):
+ if not _is_valid_dispatch_type(cls):
raise TypeError(
f"Invalid annotation for {argname!r}. "
f"{cls!r} is not a class."
)
+
registry[cls] = func
if cache_token is None and hasattr(cls, '__abstractmethods__'):
cache_token = get_cache_token()
diff --git a/common/py3-stdlib/getpass.py b/common/py3-stdlib/getpass.py
index 6911f41..6970d8a 100644
--- a/common/py3-stdlib/getpass.py
+++ b/common/py3-stdlib/getpass.py
@@ -95,7 +95,7 @@
def win_getpass(prompt='Password: ', stream=None):
- """Prompt for password with echo off, using Windows getch()."""
+ """Prompt for password with echo off, using Windows getwch()."""
if sys.stdin is not sys.__stdin__:
return fallback_getpass(prompt, stream)
diff --git a/common/py3-stdlib/glob.py b/common/py3-stdlib/glob.py
index 0dd2f8b..9fc08f4 100644
--- a/common/py3-stdlib/glob.py
+++ b/common/py3-stdlib/glob.py
@@ -1,13 +1,16 @@
"""Filename globbing utility."""
+import contextlib
import os
import re
import fnmatch
+import itertools
+import stat
import sys
__all__ = ["glob", "iglob", "escape"]
-def glob(pathname, *, recursive=False):
+def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
"""Return a list of paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
@@ -18,9 +21,9 @@
If recursive is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
"""
- return list(iglob(pathname, recursive=recursive))
+ return list(iglob(pathname, root_dir=root_dir, dir_fd=dir_fd, recursive=recursive))
-def iglob(pathname, *, recursive=False):
+def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
"""Return an iterator which yields the paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
@@ -32,35 +35,44 @@
zero or more directories and subdirectories.
"""
sys.audit("glob.glob", pathname, recursive)
- it = _iglob(pathname, recursive, False)
- if recursive and _isrecursive(pathname):
- s = next(it) # skip empty string
- assert not s
+ sys.audit("glob.glob/2", pathname, recursive, root_dir, dir_fd)
+ if root_dir is not None:
+ root_dir = os.fspath(root_dir)
+ else:
+ root_dir = pathname[:0]
+ it = _iglob(pathname, root_dir, dir_fd, recursive, False)
+ if not pathname or recursive and _isrecursive(pathname[:2]):
+ try:
+ s = next(it) # skip empty string
+ if s:
+ it = itertools.chain((s,), it)
+ except StopIteration:
+ pass
return it
-def _iglob(pathname, recursive, dironly):
+def _iglob(pathname, root_dir, dir_fd, recursive, dironly):
dirname, basename = os.path.split(pathname)
if not has_magic(pathname):
assert not dironly
if basename:
- if os.path.lexists(pathname):
+ if _lexists(_join(root_dir, pathname), dir_fd):
yield pathname
else:
# Patterns ending with a slash should match only directories
- if os.path.isdir(dirname):
+ if _isdir(_join(root_dir, dirname), dir_fd):
yield pathname
return
if not dirname:
if recursive and _isrecursive(basename):
- yield from _glob2(dirname, basename, dironly)
+ yield from _glob2(root_dir, basename, dir_fd, dironly)
else:
- yield from _glob1(dirname, basename, dironly)
+ yield from _glob1(root_dir, basename, dir_fd, dironly)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
# contains magic characters (i.e. r'\\?\C:').
if dirname != pathname and has_magic(dirname):
- dirs = _iglob(dirname, recursive, True)
+ dirs = _iglob(dirname, root_dir, dir_fd, recursive, True)
else:
dirs = [dirname]
if has_magic(basename):
@@ -71,76 +83,125 @@
else:
glob_in_dir = _glob0
for dirname in dirs:
- for name in glob_in_dir(dirname, basename, dironly):
+ for name in glob_in_dir(_join(root_dir, dirname), basename, dir_fd, dironly):
yield os.path.join(dirname, name)
# These 2 helper functions non-recursively glob inside a literal directory.
# They return a list of basenames. _glob1 accepts a pattern while _glob0
# takes a literal basename (so it only has to check for its existence).
-def _glob1(dirname, pattern, dironly):
- names = list(_iterdir(dirname, dironly))
+def _glob1(dirname, pattern, dir_fd, dironly):
+ names = _listdir(dirname, dir_fd, dironly)
if not _ishidden(pattern):
names = (x for x in names if not _ishidden(x))
return fnmatch.filter(names, pattern)
-def _glob0(dirname, basename, dironly):
- if not basename:
- # `os.path.split()` returns an empty basename for paths ending with a
- # directory separator. 'q*x/' should match only directories.
- if os.path.isdir(dirname):
+def _glob0(dirname, basename, dir_fd, dironly):
+ if basename:
+ if _lexists(_join(dirname, basename), dir_fd):
return [basename]
else:
- if os.path.lexists(os.path.join(dirname, basename)):
+ # `os.path.split()` returns an empty basename for paths ending with a
+ # directory separator. 'q*x/' should match only directories.
+ if _isdir(dirname, dir_fd):
return [basename]
return []
# Following functions are not public but can be used by third-party code.
def glob0(dirname, pattern):
- return _glob0(dirname, pattern, False)
+ return _glob0(dirname, pattern, None, False)
def glob1(dirname, pattern):
- return _glob1(dirname, pattern, False)
+ return _glob1(dirname, pattern, None, False)
# This helper function recursively yields relative pathnames inside a literal
# directory.
-def _glob2(dirname, pattern, dironly):
+def _glob2(dirname, pattern, dir_fd, dironly):
assert _isrecursive(pattern)
yield pattern[:0]
- yield from _rlistdir(dirname, dironly)
+ yield from _rlistdir(dirname, dir_fd, dironly)
# If dironly is false, yields all file names inside a directory.
# If dironly is true, yields only directory names.
-def _iterdir(dirname, dironly):
- if not dirname:
- if isinstance(dirname, bytes):
- dirname = bytes(os.curdir, 'ASCII')
- else:
- dirname = os.curdir
+def _iterdir(dirname, dir_fd, dironly):
try:
- with os.scandir(dirname) as it:
- for entry in it:
- try:
- if not dironly or entry.is_dir():
- yield entry.name
- except OSError:
- pass
+ fd = None
+ fsencode = None
+ if dir_fd is not None:
+ if dirname:
+ fd = arg = os.open(dirname, _dir_open_flags, dir_fd=dir_fd)
+ else:
+ arg = dir_fd
+ if isinstance(dirname, bytes):
+ fsencode = os.fsencode
+ elif dirname:
+ arg = dirname
+ elif isinstance(dirname, bytes):
+ arg = bytes(os.curdir, 'ASCII')
+ else:
+ arg = os.curdir
+ try:
+ with os.scandir(arg) as it:
+ for entry in it:
+ try:
+ if not dironly or entry.is_dir():
+ if fsencode is not None:
+ yield fsencode(entry.name)
+ else:
+ yield entry.name
+ except OSError:
+ pass
+ finally:
+ if fd is not None:
+ os.close(fd)
except OSError:
return
+def _listdir(dirname, dir_fd, dironly):
+ with contextlib.closing(_iterdir(dirname, dir_fd, dironly)) as it:
+ return list(it)
+
# Recursively yields relative pathnames inside a literal directory.
-def _rlistdir(dirname, dironly):
- names = list(_iterdir(dirname, dironly))
+def _rlistdir(dirname, dir_fd, dironly):
+ names = _listdir(dirname, dir_fd, dironly)
for x in names:
if not _ishidden(x):
yield x
- path = os.path.join(dirname, x) if dirname else x
- for y in _rlistdir(path, dironly):
- yield os.path.join(x, y)
+ path = _join(dirname, x) if dirname else x
+ for y in _rlistdir(path, dir_fd, dironly):
+ yield _join(x, y)
+def _lexists(pathname, dir_fd):
+ # Same as os.path.lexists(), but with dir_fd
+ if dir_fd is None:
+ return os.path.lexists(pathname)
+ try:
+ os.lstat(pathname, dir_fd=dir_fd)
+ except (OSError, ValueError):
+ return False
+ else:
+ return True
+
+def _isdir(pathname, dir_fd):
+ # Same as os.path.isdir(), but with dir_fd
+ if dir_fd is None:
+ return os.path.isdir(pathname)
+ try:
+ st = os.stat(pathname, dir_fd=dir_fd)
+ except (OSError, ValueError):
+ return False
+ else:
+ return stat.S_ISDIR(st.st_mode)
+
+def _join(dirname, basename):
+ # It is common if dirname or basename is empty
+ if not dirname or not basename:
+ return dirname or basename
+ return os.path.join(dirname, basename)
+
magic_check = re.compile('([*?[])')
magic_check_bytes = re.compile(b'([*?[])')
@@ -171,3 +232,6 @@
else:
pathname = magic_check.sub(r'[\1]', pathname)
return drive + pathname
+
+
+_dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)
diff --git a/common/py3-stdlib/graphlib.py b/common/py3-stdlib/graphlib.py
index d0e7a48..1c5d9a4 100644
--- a/common/py3-stdlib/graphlib.py
+++ b/common/py3-stdlib/graphlib.py
@@ -17,7 +17,7 @@
self.npredecessors = 0
# List of successor nodes. The list can contain duplicated elements as
- # long as they're all reflected in the successor's npredecessors attribute).
+ # long as they're all reflected in the successor's npredecessors attribute.
self.successors = []
diff --git a/common/py3-stdlib/gzip.py b/common/py3-stdlib/gzip.py
index e422773..475ec32 100644
--- a/common/py3-stdlib/gzip.py
+++ b/common/py3-stdlib/gzip.py
@@ -62,6 +62,7 @@
raise TypeError("filename must be a str or bytes object, or a file")
if "t" in mode:
+ encoding = io.text_encoding(encoding)
return io.TextIOWrapper(binary_file, encoding, errors, newline)
else:
return binary_file
@@ -277,7 +278,7 @@
if self.fileobj is None:
raise ValueError("write() on closed GzipFile object")
- if isinstance(data, bytes):
+ if isinstance(data, (bytes, bytearray)):
length = len(data)
else:
# accept any data that supports the buffer protocol
@@ -516,7 +517,7 @@
def _read_eof(self):
# We've read to the end of the file
- # We check the that the computed CRC and size of the
+ # We check that the computed CRC and size of the
# uncompressed data matches the stored values. Note that the size
# stored is the true file size mod 2**32.
crc32, isize = struct.unpack("<II", self._read_exact(8))
@@ -583,8 +584,7 @@
g = sys.stdout.buffer
else:
if arg[-3:] != ".gz":
- print("filename doesn't end in .gz:", repr(arg))
- continue
+ sys.exit(f"filename doesn't end in .gz: {arg!r}")
f = open(arg, "rb")
g = builtins.open(arg[:-3], "wb")
else:
@@ -596,7 +596,7 @@
f = builtins.open(arg, "rb")
g = open(arg + ".gz", "wb")
while True:
- chunk = f.read(1024)
+ chunk = f.read(io.DEFAULT_BUFFER_SIZE)
if not chunk:
break
g.write(chunk)
diff --git a/common/py3-stdlib/hashlib.py b/common/py3-stdlib/hashlib.py
index 58c340d..21a73f3 100644
--- a/common/py3-stdlib/hashlib.py
+++ b/common/py3-stdlib/hashlib.py
@@ -173,6 +173,7 @@
algorithms_available = algorithms_available.union(
_hashlib.openssl_md_meth_names)
except ImportError:
+ _hashlib = None
new = __py_new
__get_hash = __get_builtin_constructor
@@ -180,6 +181,7 @@
# OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
from _hashlib import pbkdf2_hmac
except ImportError:
+ from warnings import warn as _warn
_trans_5C = bytes((x ^ 0x5C) for x in range(256))
_trans_36 = bytes((x ^ 0x36) for x in range(256))
@@ -190,6 +192,11 @@
as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster
for long passwords.
"""
+ _warn(
+ "Python implementation of pbkdf2_hmac() is deprecated.",
+ category=DeprecationWarning,
+ stacklevel=2
+ )
if not isinstance(hash_name, str):
raise TypeError(hash_name)
diff --git a/common/py3-stdlib/hmac.py b/common/py3-stdlib/hmac.py
index 180bc37..8b4f920 100644
--- a/common/py3-stdlib/hmac.py
+++ b/common/py3-stdlib/hmac.py
@@ -8,11 +8,12 @@
import _hashlib as _hashopenssl
except ImportError:
_hashopenssl = None
- _openssl_md_meths = None
+ _functype = None
from _operator import _compare_digest as compare_digest
else:
- _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names)
compare_digest = _hashopenssl.compare_digest
+ _functype = type(_hashopenssl.openssl_sha256) # builtin type
+
import hashlib as _hashlib
trans_5C = bytes((x ^ 0x5C) for x in range(256))
@@ -23,7 +24,6 @@
digest_size = None
-
class HMAC:
"""RFC 2104 HMAC class. Also complies with RFC 4231.
@@ -32,7 +32,7 @@
blocksize = 64 # 512-bit HMAC; can be changed in subclasses.
__slots__ = (
- "_digest_cons", "_inner", "_outer", "block_size", "digest_size"
+ "_hmac", "_inner", "_outer", "block_size", "digest_size"
)
def __init__(self, key, msg=None, digestmod=''):
@@ -55,15 +55,30 @@
if not digestmod:
raise TypeError("Missing required parameter 'digestmod'.")
- if callable(digestmod):
- self._digest_cons = digestmod
- elif isinstance(digestmod, str):
- self._digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
+ if _hashopenssl and isinstance(digestmod, (str, _functype)):
+ try:
+ self._init_hmac(key, msg, digestmod)
+ except _hashopenssl.UnsupportedDigestmodError:
+ self._init_old(key, msg, digestmod)
else:
- self._digest_cons = lambda d=b'': digestmod.new(d)
+ self._init_old(key, msg, digestmod)
- self._outer = self._digest_cons()
- self._inner = self._digest_cons()
+ def _init_hmac(self, key, msg, digestmod):
+ self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod)
+ self.digest_size = self._hmac.digest_size
+ self.block_size = self._hmac.block_size
+
+ def _init_old(self, key, msg, digestmod):
+ if callable(digestmod):
+ digest_cons = digestmod
+ elif isinstance(digestmod, str):
+ digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
+ else:
+ digest_cons = lambda d=b'': digestmod.new(d)
+
+ self._hmac = None
+ self._outer = digest_cons()
+ self._inner = digest_cons()
self.digest_size = self._inner.digest_size
if hasattr(self._inner, 'block_size'):
@@ -79,13 +94,13 @@
RuntimeWarning, 2)
blocksize = self.blocksize
+ if len(key) > blocksize:
+ key = digest_cons(key).digest()
+
# self.blocksize is the default blocksize. self.block_size is
# effective block size as well as the public API attribute.
self.block_size = blocksize
- if len(key) > blocksize:
- key = self._digest_cons(key).digest()
-
key = key.ljust(blocksize, b'\0')
self._outer.update(key.translate(trans_5C))
self._inner.update(key.translate(trans_36))
@@ -94,23 +109,15 @@
@property
def name(self):
- return "hmac-" + self._inner.name
-
- @property
- def digest_cons(self):
- return self._digest_cons
-
- @property
- def inner(self):
- return self._inner
-
- @property
- def outer(self):
- return self._outer
+ if self._hmac:
+ return self._hmac.name
+ else:
+ return f"hmac-{self._inner.name}"
def update(self, msg):
"""Feed data from msg into this hashing object."""
- self._inner.update(msg)
+ inst = self._hmac or self._inner
+ inst.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
@@ -119,10 +126,14 @@
"""
# Call __new__ directly to avoid the expensive __init__.
other = self.__class__.__new__(self.__class__)
- other._digest_cons = self._digest_cons
other.digest_size = self.digest_size
- other._inner = self._inner.copy()
- other._outer = self._outer.copy()
+ if self._hmac:
+ other._hmac = self._hmac.copy()
+ other._inner = other._outer = None
+ else:
+ other._hmac = None
+ other._inner = self._inner.copy()
+ other._outer = self._outer.copy()
return other
def _current(self):
@@ -130,9 +141,12 @@
To be used only internally with digest() and hexdigest().
"""
- h = self._outer.copy()
- h.update(self._inner.digest())
- return h
+ if self._hmac:
+ return self._hmac
+ else:
+ h = self._outer.copy()
+ h.update(self._inner.digest())
+ return h
def digest(self):
"""Return the hash value of this hashing object.
@@ -179,9 +193,11 @@
A hashlib constructor returning a new hash object. *OR*
A module supporting PEP 247.
"""
- if (_hashopenssl is not None and
- isinstance(digest, str) and digest in _openssl_md_meths):
- return _hashopenssl.hmac_digest(key, msg, digest)
+ if _hashopenssl is not None and isinstance(digest, (str, _functype)):
+ try:
+ return _hashopenssl.hmac_digest(key, msg, digest)
+ except _hashopenssl.UnsupportedDigestmodError:
+ pass
if callable(digest):
digest_cons = digest
diff --git a/common/py3-stdlib/html/parser.py b/common/py3-stdlib/html/parser.py
index 6083077..58f6bb3 100644
--- a/common/py3-stdlib/html/parser.py
+++ b/common/py3-stdlib/html/parser.py
@@ -46,7 +46,7 @@
|"[^"]*" # LIT-enclosed value
|(?!['"])[^>\s]* # bare value
)
- (?:\s*,)* # possibly followed by a comma
+ \s* # possibly followed by a space
)?(?:\s|/(?!>))*
)*
)?
@@ -405,7 +405,7 @@
tagname = namematch.group(1).lower()
# consume and ignore other stuff between the name and the >
# Note: this is not 100% correct, since we might have things like
- # </tag attr=">">, but looking for > after tha name should cover
+ # </tag attr=">">, but looking for > after the name should cover
# most of the cases and is much simpler
gtpos = rawdata.find('>', namematch.end())
self.handle_endtag(tagname)
diff --git a/common/py3-stdlib/http/__init__.py b/common/py3-stdlib/http/__init__.py
index 37be765..bf8d7d6 100644
--- a/common/py3-stdlib/http/__init__.py
+++ b/common/py3-stdlib/http/__init__.py
@@ -2,6 +2,7 @@
__all__ = ['HTTPStatus']
+
class HTTPStatus(IntEnum):
"""HTTP status codes and reason phrases
diff --git a/common/py3-stdlib/http/client.py b/common/py3-stdlib/http/client.py
index c2ad047..a6ab135 100644
--- a/common/py3-stdlib/http/client.py
+++ b/common/py3-stdlib/http/client.py
@@ -70,10 +70,12 @@
import email.parser
import email.message
+import errno
import http
import io
import re
import socket
+import sys
import collections.abc
from urllib.parse import urlsplit
@@ -201,15 +203,11 @@
lst.append(line)
return lst
-def parse_headers(fp, _class=HTTPMessage):
- """Parses only RFC2822 headers from a file pointer.
+def _read_headers(fp):
+ """Reads potential header lines into a list from a file pointer.
- email Parser wants to see strings rather than bytes.
- But a TextIOWrapper around self.rfile would buffer too many bytes
- from the stream, bytes which we later need to read as bytes.
- So we read the correct bytes here, as bytes, for email Parser
- to parse.
-
+ Length of line is limited by _MAXLINE, and number of
+ headers is limited by _MAXHEADERS.
"""
headers = []
while True:
@@ -221,6 +219,19 @@
raise HTTPException("got more than %d headers" % _MAXHEADERS)
if line in (b'\r\n', b'\n', b''):
break
+ return headers
+
+def parse_headers(fp, _class=HTTPMessage):
+ """Parses only RFC2822 headers from a file pointer.
+
+ email Parser wants to see strings rather than bytes.
+ But a TextIOWrapper around self.rfile would buffer too many bytes
+ from the stream, bytes which we later need to read as bytes.
+ So we read the correct bytes here, as bytes, for email Parser
+ to parse.
+
+ """
+ headers = _read_headers(fp)
hstring = b''.join(headers).decode('iso-8859-1')
return email.parser.Parser(_class=_class).parsestr(hstring)
@@ -308,15 +319,10 @@
if status != CONTINUE:
break
# skip the header from the 100 response
- while True:
- skip = self.fp.readline(_MAXLINE + 1)
- if len(skip) > _MAXLINE:
- raise LineTooLong("header line")
- skip = skip.strip()
- if not skip:
- break
- if self.debuglevel > 0:
- print("header:", skip)
+ skipped_headers = _read_headers(self.fp)
+ if self.debuglevel > 0:
+ print("headers:", skipped_headers)
+ del skipped_headers
self.code = self.status = status
self.reason = reason.strip()
@@ -349,9 +355,6 @@
# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
self.length = None
length = self.headers.get("content-length")
-
- # are we using the chunked-style of transfer encoding?
- tr_enc = self.headers.get("transfer-encoding")
if length and not self.chunked:
try:
self.length = int(length)
@@ -452,18 +455,25 @@
self._close_conn()
return b""
+ if self.chunked:
+ return self._read_chunked(amt)
+
if amt is not None:
- # Amount is given, implement using readinto
- b = bytearray(amt)
- n = self.readinto(b)
- return memoryview(b)[:n].tobytes()
+ if self.length is not None and amt > self.length:
+ # clip the read to the "end of response"
+ amt = self.length
+ s = self.fp.read(amt)
+ if not s and amt:
+ # Ideally, we would raise IncompleteRead if the content-length
+ # wasn't satisfied, but it might break compatibility.
+ self._close_conn()
+ elif self.length is not None:
+ self.length -= len(s)
+ if not self.length:
+ self._close_conn()
+ return s
else:
# Amount is not given (unbounded read) so we must check self.length
- # and self.chunked
-
- if self.chunked:
- return self._readall_chunked()
-
if self.length is None:
s = self.fp.read()
else:
@@ -564,7 +574,7 @@
self.chunk_left = chunk_left
return chunk_left
- def _readall_chunked(self):
+ def _read_chunked(self, amt=None):
assert self.chunked != _UNKNOWN
value = []
try:
@@ -572,7 +582,15 @@
chunk_left = self._get_chunk_left()
if chunk_left is None:
break
+
+ if amt is not None and amt <= chunk_left:
+ value.append(self._safe_read(amt))
+ self.chunk_left = chunk_left - amt
+ break
+
value.append(self._safe_read(chunk_left))
+ if amt is not None:
+ amt -= chunk_left
self.chunk_left = 0
return b''.join(value)
except IncompleteRead:
@@ -846,7 +864,7 @@
the endpoint passed to `set_tunnel`. This done by sending an HTTP
CONNECT request to the proxy server when the connection is established.
- This method must be called before the HTML connection has been
+ This method must be called before the HTTP connection has been
established.
The headers argument should be a mapping of extra HTTP headers to send
@@ -886,23 +904,24 @@
self.debuglevel = level
def _tunnel(self):
- connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
- self._tunnel_port)
- connect_bytes = connect_str.encode("ascii")
- self.send(connect_bytes)
+ connect = b"CONNECT %s:%d HTTP/1.0\r\n" % (
+ self._tunnel_host.encode("ascii"), self._tunnel_port)
+ headers = [connect]
for header, value in self._tunnel_headers.items():
- header_str = "%s: %s\r\n" % (header, value)
- header_bytes = header_str.encode("latin-1")
- self.send(header_bytes)
- self.send(b'\r\n')
+ headers.append(f"{header}: {value}\r\n".encode("latin-1"))
+ headers.append(b"\r\n")
+ # Making a single send() call instead of one per line encourages
+ # the host OS to use a more optimal packet size instead of
+ # potentially emitting a series of small packets.
+ self.send(b"".join(headers))
+ del headers
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != http.HTTPStatus.OK:
self.close()
- raise OSError("Tunnel connection failed: %d %s" % (code,
- message.strip()))
+ raise OSError(f"Tunnel connection failed: {code} {message.strip()}")
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
@@ -918,9 +937,15 @@
def connect(self):
"""Connect to the host and port specified in __init__."""
+ sys.audit("http.client.connect", self, self.host, self.port)
self.sock = self._create_connection(
(self.host,self.port), self.timeout, self.source_address)
- self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ # Might fail in OSs that don't implement TCP_NODELAY
+ try:
+ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ except OSError as e:
+ if e.errno != errno.ENOPROTOOPT:
+ raise
if self._tunnel_host:
self._tunnel()
@@ -965,8 +990,10 @@
break
if encode:
datablock = datablock.encode("iso-8859-1")
+ sys.audit("http.client.send", self, datablock)
self.sock.sendall(datablock)
return
+ sys.audit("http.client.send", self, data)
try:
self.sock.sendall(data)
except TypeError:
@@ -1392,6 +1419,9 @@
self.cert_file = cert_file
if context is None:
context = ssl._create_default_https_context()
+ # send ALPN extension to indicate HTTP/1.1 protocol
+ if self._http_vsn == 11:
+ context.set_alpn_protocols(['http/1.1'])
# enable PHA for TLS 1.3 connections if available
if context.post_handshake_auth is not None:
context.post_handshake_auth = True
diff --git a/common/py3-stdlib/http/cookiejar.py b/common/py3-stdlib/http/cookiejar.py
index 47ed5c3..eaa76c2 100644
--- a/common/py3-stdlib/http/cookiejar.py
+++ b/common/py3-stdlib/http/cookiejar.py
@@ -50,10 +50,18 @@
logger = logging.getLogger("http.cookiejar")
return logger.debug(*args)
-
+HTTPONLY_ATTR = "HTTPOnly"
+HTTPONLY_PREFIX = "#HttpOnly_"
DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
+NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File")
MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
"instance initialised with one)")
+NETSCAPE_HEADER_TEXT = """\
+# Netscape HTTP Cookie File
+# http://curl.haxx.se/rfc/cookie_spec.html
+# This is a generated file! Do not edit.
+
+"""
def _warn_unhandled_exception():
# There are a few catch-all except: statements in this module, for
@@ -2004,19 +2012,11 @@
header by default (Mozilla can cope with that).
"""
- magic_re = re.compile("#( Netscape)? HTTP Cookie File")
- header = """\
-# Netscape HTTP Cookie File
-# http://curl.haxx.se/rfc/cookie_spec.html
-# This is a generated file! Do not edit.
-
-"""
def _really_load(self, f, filename, ignore_discard, ignore_expires):
now = time.time()
- magic = f.readline()
- if not self.magic_re.search(magic):
+ if not NETSCAPE_MAGIC_RGX.match(f.readline()):
raise LoadError(
"%r does not look like a Netscape format cookies file" %
filename)
@@ -2024,8 +2024,17 @@
try:
while 1:
line = f.readline()
+ rest = {}
+
if line == "": break
+ # httponly is a cookie flag as defined in rfc6265
+ # when encoded in a netscape cookie file,
+ # the line is prepended with "#HttpOnly_"
+ if line.startswith(HTTPONLY_PREFIX):
+ rest[HTTPONLY_ATTR] = ""
+ line = line[len(HTTPONLY_PREFIX):]
+
# last field may be absent, so keep any trailing tab
if line.endswith("\n"): line = line[:-1]
@@ -2063,7 +2072,7 @@
discard,
None,
None,
- {})
+ rest)
if not ignore_discard and c.discard:
continue
if not ignore_expires and c.is_expired(now):
@@ -2083,16 +2092,17 @@
else: raise ValueError(MISSING_FILENAME_TEXT)
with open(filename, "w") as f:
- f.write(self.header)
+ f.write(NETSCAPE_HEADER_TEXT)
now = time.time()
for cookie in self:
+ domain = cookie.domain
if not ignore_discard and cookie.discard:
continue
if not ignore_expires and cookie.is_expired(now):
continue
if cookie.secure: secure = "TRUE"
else: secure = "FALSE"
- if cookie.domain.startswith("."): initial_dot = "TRUE"
+ if domain.startswith("."): initial_dot = "TRUE"
else: initial_dot = "FALSE"
if cookie.expires is not None:
expires = str(cookie.expires)
@@ -2107,7 +2117,9 @@
else:
name = cookie.name
value = cookie.value
+ if cookie.has_nonstandard_attr(HTTPONLY_ATTR):
+ domain = HTTPONLY_PREFIX + domain
f.write(
- "\t".join([cookie.domain, initial_dot, cookie.path,
+ "\t".join([domain, initial_dot, cookie.path,
secure, expires, name, value])+
"\n")
diff --git a/common/py3-stdlib/http/server.py b/common/py3-stdlib/http/server.py
index def05f4..58abadf 100644
--- a/common/py3-stdlib/http/server.py
+++ b/common/py3-stdlib/http/server.py
@@ -103,8 +103,6 @@
import sys
import time
import urllib.parse
-import contextlib
-from functools import partial
from http import HTTPStatus
@@ -414,7 +412,7 @@
method = getattr(self, mname)
method()
self.wfile.flush() #actually send the response if not already done.
- except socket.timeout as e:
+ except TimeoutError as e:
#a read or a write timed out. Discard this connection
self.log_error("Request timed out: %r", e)
self.close_connection = True
@@ -689,6 +687,7 @@
parts[3], parts[4])
new_url = urllib.parse.urlunsplit(new_parts)
self.send_header("Location", new_url)
+ self.send_header("Content-Length", "0")
self.end_headers()
return None
for index in "index.html", "index.htm":
@@ -1092,8 +1091,7 @@
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
- if query:
- env['QUERY_STRING'] = query
+ env['QUERY_STRING'] = query
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.get("authorization")
if authorization:
@@ -1239,7 +1237,6 @@
"""
ServerClass.address_family, addr = _get_best_family(bind, port)
-
HandlerClass.protocol_version = protocol
with ServerClass(addr, HandlerClass) as httpd:
host, port = httpd.socket.getsockname()[:2]
@@ -1256,29 +1253,29 @@
if __name__ == '__main__':
import argparse
+ import contextlib
parser = argparse.ArgumentParser()
parser.add_argument('--cgi', action='store_true',
- help='Run as CGI Server')
+ help='run as CGI server')
parser.add_argument('--bind', '-b', metavar='ADDRESS',
- help='Specify alternate bind address '
- '[default: all interfaces]')
+ help='specify alternate bind address '
+ '(default: all interfaces)')
parser.add_argument('--directory', '-d', default=os.getcwd(),
- help='Specify alternative directory '
- '[default:current directory]')
- parser.add_argument('port', action='store',
- default=8000, type=int,
+ help='specify alternate directory '
+ '(default: current directory)')
+ parser.add_argument('port', action='store', default=8000, type=int,
nargs='?',
- help='Specify alternate port [default: 8000]')
+ help='specify alternate port (default: 8000)')
args = parser.parse_args()
if args.cgi:
handler_class = CGIHTTPRequestHandler
else:
- handler_class = partial(SimpleHTTPRequestHandler,
- directory=args.directory)
+ handler_class = SimpleHTTPRequestHandler
# ensure dual-stack is not disabled; ref #38907
class DualStackServer(ThreadingHTTPServer):
+
def server_bind(self):
# suppress exception when protocol is IPv4
with contextlib.suppress(Exception):
@@ -1286,6 +1283,10 @@
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
return super().server_bind()
+ def finish_request(self, request, client_address):
+ self.RequestHandlerClass(request, client_address, self,
+ directory=args.directory)
+
test(
HandlerClass=handler_class,
ServerClass=DualStackServer,
diff --git a/common/py3-stdlib/imaplib.py b/common/py3-stdlib/imaplib.py
index d9720f2..7318439 100644
--- a/common/py3-stdlib/imaplib.py
+++ b/common/py3-stdlib/imaplib.py
@@ -1251,13 +1251,12 @@
sys.stderr.write(' %s.%02d %s\n' % (tm, (secs*100)%100, s))
sys.stderr.flush()
- def _dump_ur(self, dict):
- # Dump untagged responses (in `dict').
- l = dict.items()
- if not l: return
- t = '\n\t\t'
- l = map(lambda x:'%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or ''), l)
- self._mesg('untagged responses dump:%s%s' % (t, t.join(l)))
+ def _dump_ur(self, untagged_resp_dict):
+ if not untagged_resp_dict:
+ return
+ items = (f'{key}: {value!r}'
+ for key, value in untagged_resp_dict.items())
+ self._mesg('untagged responses dump:' + '\n\t\t'.join(items))
def _log(self, line):
# Keep log of last `_cmd_log_len' interactions for debugging.
diff --git a/common/py3-stdlib/imp.py b/common/py3-stdlib/imp.py
index 31f8c76..e02aaef 100644
--- a/common/py3-stdlib/imp.py
+++ b/common/py3-stdlib/imp.py
@@ -28,7 +28,8 @@
import types
import warnings
-warnings.warn("the imp module is deprecated in favour of importlib; "
+warnings.warn("the imp module is deprecated in favour of importlib and slated "
+ "for removal in Python 3.12; "
"see the module's documentation for alternative uses",
DeprecationWarning, stacklevel=2)
diff --git a/common/py3-stdlib/importlib/__init__.py b/common/py3-stdlib/importlib/__init__.py
index 0c73c50..ce61883 100644
--- a/common/py3-stdlib/importlib/__init__.py
+++ b/common/py3-stdlib/importlib/__init__.py
@@ -34,7 +34,7 @@
import _frozen_importlib_external as _bootstrap_external
except ImportError:
from . import _bootstrap_external
- _bootstrap_external._setup(_bootstrap)
+ _bootstrap_external._set_bootstrap_module(_bootstrap)
_bootstrap._bootstrap_external = _bootstrap_external
else:
_bootstrap_external.__name__ = 'importlib._bootstrap_external'
@@ -54,7 +54,6 @@
# Fully bootstrapped at this point, import whatever you like, circular
# dependencies and startup overhead minimisation permitting :)
-import types
import warnings
@@ -79,8 +78,8 @@
This function is deprecated in favor of importlib.util.find_spec().
"""
- warnings.warn('Deprecated since Python 3.4. '
- 'Use importlib.util.find_spec() instead.',
+ warnings.warn('Deprecated since Python 3.4 and slated for removal in '
+ 'Python 3.12; use importlib.util.find_spec() instead',
DeprecationWarning, stacklevel=2)
try:
loader = sys.modules[name].__loader__
@@ -136,12 +135,13 @@
The module must have been successfully imported before.
"""
- if not module or not isinstance(module, types.ModuleType):
- raise TypeError("reload() argument must be a module")
try:
name = module.__spec__.name
except AttributeError:
- name = module.__name__
+ try:
+ name = module.__name__
+ except AttributeError:
+ raise TypeError("reload() argument must be a module")
if sys.modules.get(name) is not module:
msg = "module {} not in sys.modules"
diff --git a/common/py3-stdlib/importlib/_abc.py b/common/py3-stdlib/importlib/_abc.py
new file mode 100644
index 0000000..f80348f
--- /dev/null
+++ b/common/py3-stdlib/importlib/_abc.py
@@ -0,0 +1,54 @@
+"""Subset of importlib.abc used to reduce importlib.util imports."""
+from . import _bootstrap
+import abc
+import warnings
+
+
+class Loader(metaclass=abc.ABCMeta):
+
+ """Abstract base class for import loaders."""
+
+ def create_module(self, spec):
+ """Return a module to initialize and into which to load.
+
+ This method should raise ImportError if anything prevents it
+ from creating a new module. It may return None to indicate
+ that the spec should create the new module.
+ """
+ # By default, defer to default semantics for the new module.
+ return None
+
+ # We don't define exec_module() here since that would break
+ # hasattr checks we do to support backward compatibility.
+
+ def load_module(self, fullname):
+ """Return the loaded module.
+
+ The module must be added to sys.modules and have import-related
+ attributes set properly. The fullname is a str.
+
+ ImportError is raised on failure.
+
+ This method is deprecated in favor of loader.exec_module(). If
+ exec_module() exists then it is used to provide a backwards-compatible
+ functionality for this method.
+
+ """
+ if not hasattr(self, 'exec_module'):
+ raise ImportError
+ # Warning implemented in _load_module_shim().
+ return _bootstrap._load_module_shim(self, fullname)
+
+ def module_repr(self, module):
+ """Return a module's repr.
+
+ Used by the module type when the method does not raise
+ NotImplementedError.
+
+ This method is deprecated.
+
+ """
+ warnings.warn("importlib.abc.Loader.module_repr() is deprecated and "
+ "slated for removal in Python 3.12", DeprecationWarning)
+ # The exception will cause ModuleType.__repr__ to ignore this method.
+ raise NotImplementedError
diff --git a/common/py3-stdlib/importlib/_adapters.py b/common/py3-stdlib/importlib/_adapters.py
new file mode 100644
index 0000000..e72edd1
--- /dev/null
+++ b/common/py3-stdlib/importlib/_adapters.py
@@ -0,0 +1,83 @@
+from contextlib import suppress
+
+from . import abc
+
+
+class SpecLoaderAdapter:
+ """
+ Adapt a package spec to adapt the underlying loader.
+ """
+
+ def __init__(self, spec, adapter=lambda spec: spec.loader):
+ self.spec = spec
+ self.loader = adapter(spec)
+
+ def __getattr__(self, name):
+ return getattr(self.spec, name)
+
+
+class TraversableResourcesLoader:
+ """
+ Adapt a loader to provide TraversableResources.
+ """
+
+ def __init__(self, spec):
+ self.spec = spec
+
+ def get_resource_reader(self, name):
+ return DegenerateFiles(self.spec)._native()
+
+
+class DegenerateFiles:
+ """
+ Adapter for an existing or non-existant resource reader
+ to provide a degenerate .files().
+ """
+
+ class Path(abc.Traversable):
+ def iterdir(self):
+ return iter(())
+
+ def is_dir(self):
+ return False
+
+ is_file = exists = is_dir # type: ignore
+
+ def joinpath(self, other):
+ return DegenerateFiles.Path()
+
+ @property
+ def name(self):
+ return ''
+
+ def open(self, mode='rb', *args, **kwargs):
+ raise ValueError()
+
+ def __init__(self, spec):
+ self.spec = spec
+
+ @property
+ def _reader(self):
+ with suppress(AttributeError):
+ return self.spec.loader.get_resource_reader(self.spec.name)
+
+ def _native(self):
+ """
+ Return the native reader if it supports files().
+ """
+ reader = self._reader
+ return reader if hasattr(reader, 'files') else self
+
+ def __getattr__(self, attr):
+ return getattr(self._reader, attr)
+
+ def files(self):
+ return DegenerateFiles.Path()
+
+
+def wrap_spec(package):
+ """
+ Construct a package spec with traversable compatibility
+ on the spec/loader/reader.
+ """
+ return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)
diff --git a/common/py3-stdlib/importlib/_bootstrap.py b/common/py3-stdlib/importlib/_bootstrap.py
index e00b27e..527bc9c 100644
--- a/common/py3-stdlib/importlib/_bootstrap.py
+++ b/common/py3-stdlib/importlib/_bootstrap.py
@@ -20,10 +20,23 @@
# reference any injected objects! This includes not only global code but also
# anything specified at the class level.
+def _object_name(obj):
+ try:
+ return obj.__qualname__
+ except AttributeError:
+ return type(obj).__qualname__
+
# Bootstrap-related code ######################################################
+# Modules injected manually by _setup()
+_thread = None
+_warnings = None
+_weakref = None
+
+# Import done by _install_external_importers()
_bootstrap_external = None
+
def _wrap(new, old):
"""Simple substitute for functools.update_wrapper."""
for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
@@ -262,9 +275,12 @@
def _load_module_shim(self, fullname):
"""Load the specified module into sys.modules and return it.
- This method is deprecated. Use loader.exec_module instead.
+ This method is deprecated. Use loader.exec_module() instead.
"""
+ msg = ("the load_module() method is deprecated and slated for removal in "
+ "Python 3.12; use exec_module() instead")
+ _warnings.warn(msg, DeprecationWarning)
spec = spec_from_loader(fullname, self)
if fullname in sys.modules:
module = sys.modules[fullname]
@@ -276,26 +292,16 @@
# Module specifications #######################################################
def _module_repr(module):
- # The implementation of ModuleType.__repr__().
+ """The implementation of ModuleType.__repr__()."""
loader = getattr(module, '__loader__', None)
- if hasattr(loader, 'module_repr'):
- # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader
- # drop their implementations for module_repr. we can add a
- # deprecation warning here.
+ if spec := getattr(module, "__spec__", None):
+ return _module_repr_from_spec(spec)
+ elif hasattr(loader, 'module_repr'):
try:
return loader.module_repr(module)
except Exception:
pass
- try:
- spec = module.__spec__
- except AttributeError:
- pass
- else:
- if spec is not None:
- return _module_repr_from_spec(spec)
-
- # We could use module.__class__.__name__ instead of 'module' in the
- # various repr permutations.
+ # Fall through to a catch-all which always succeeds.
try:
name = module.__name__
except AttributeError:
@@ -605,9 +611,9 @@
else:
_init_module_attrs(spec, module, override=True)
if not hasattr(spec.loader, 'exec_module'):
- # (issue19713) Once BuiltinImporter and ExtensionFileLoader
- # have exec_module() implemented, we can add a deprecation
- # warning here.
+ msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
+ "falling back to load_module()")
+ _warnings.warn(msg, ImportWarning)
spec.loader.load_module(name)
else:
spec.loader.exec_module(module)
@@ -620,9 +626,8 @@
def _load_backward_compatible(spec):
- # (issue19713) Once BuiltinImporter and ExtensionFileLoader
- # have exec_module() implemented, we can add a deprecation
- # warning here.
+ # It is assumed that all callers have been warned about using load_module()
+ # appropriately before calling this function.
try:
spec.loader.load_module(spec.name)
except:
@@ -661,6 +666,9 @@
if spec.loader is not None:
# Not a namespace package.
if not hasattr(spec.loader, 'exec_module'):
+ msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
+ "falling back to load_module()")
+ _warnings.warn(msg, ImportWarning)
return _load_backward_compatible(spec)
module = module_from_spec(spec)
@@ -731,6 +739,8 @@
The method is deprecated. The import machinery does the job itself.
"""
+ _warnings.warn("BuiltinImporter.module_repr() is deprecated and "
+ "slated for removal in Python 3.12", DeprecationWarning)
return f'<module {module.__name__!r} ({BuiltinImporter._ORIGIN})>'
@classmethod
@@ -751,19 +761,22 @@
This method is deprecated. Use find_spec() instead.
"""
+ _warnings.warn("BuiltinImporter.find_module() is deprecated and "
+ "slated for removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
spec = cls.find_spec(fullname, path)
return spec.loader if spec is not None else None
- @classmethod
- def create_module(self, spec):
+ @staticmethod
+ def create_module(spec):
"""Create a built-in module"""
if spec.name not in sys.builtin_module_names:
raise ImportError('{!r} is not a built-in module'.format(spec.name),
name=spec.name)
return _call_with_frames_removed(_imp.create_builtin, spec)
- @classmethod
- def exec_module(self, module):
+ @staticmethod
+ def exec_module(module):
"""Exec a built-in module"""
_call_with_frames_removed(_imp.exec_builtin, module)
@@ -806,6 +819,8 @@
The method is deprecated. The import machinery does the job itself.
"""
+ _warnings.warn("FrozenImporter.module_repr() is deprecated and "
+ "slated for removal in Python 3.12", DeprecationWarning)
return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
@classmethod
@@ -822,10 +837,13 @@
This method is deprecated. Use find_spec() instead.
"""
+ _warnings.warn("FrozenImporter.find_module() is deprecated and "
+ "slated for removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
return cls if _imp.is_frozen(fullname) else None
- @classmethod
- def create_module(cls, spec):
+ @staticmethod
+ def create_module(spec):
"""Use default semantics for module creation."""
@staticmethod
@@ -844,6 +862,7 @@
This method is deprecated. Use exec_module() instead.
"""
+ # Warning about deprecation implemented in _load_module_shim().
return _load_module_shim(cls, fullname)
@classmethod
@@ -890,8 +909,9 @@
def _find_spec_legacy(finder, name, path):
- # This would be a good place for a DeprecationWarning if
- # we ended up going that route.
+ msg = (f"{_object_name(finder)}.find_spec() not found; "
+ "falling back to find_module()")
+ _warnings.warn(msg, ImportWarning)
loader = finder.find_module(name, path)
if loader is None:
return None
diff --git a/common/py3-stdlib/importlib/_bootstrap_external.py b/common/py3-stdlib/importlib/_bootstrap_external.py
index 25a3f8c..49bcaea 100644
--- a/common/py3-stdlib/importlib/_bootstrap_external.py
+++ b/common/py3-stdlib/importlib/_bootstrap_external.py
@@ -19,6 +19,37 @@
# reference any injected objects! This includes not only global code but also
# anything specified at the class level.
+# Module injected manually by _set_bootstrap_module()
+_bootstrap = None
+
+# Import builtin modules
+import _imp
+import _io
+import sys
+import _warnings
+import marshal
+
+
+_MS_WINDOWS = (sys.platform == 'win32')
+if _MS_WINDOWS:
+ import nt as _os
+ import winreg
+else:
+ import posix as _os
+
+
+if _MS_WINDOWS:
+ path_separators = ['\\', '/']
+else:
+ path_separators = ['/']
+# Assumption made in _path_join()
+assert all(len(sep) == 1 for sep in path_separators)
+path_sep = path_separators[0]
+path_sep_tuple = tuple(path_separators)
+path_separators = ''.join(path_separators)
+_pathseps_with_colon = {f':{s}' for s in path_separators}
+
+
# Bootstrap-related code ######################################################
_CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win',
_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin'
@@ -42,6 +73,8 @@
return False
return _relax_case
+_relax_case = _make_relax_case()
+
def _pack_uint32(x):
"""Convert a 32-bit integer to little-endian."""
@@ -59,22 +92,49 @@
return int.from_bytes(data, 'little')
-def _path_join(*path_parts):
- """Replacement for os.path.join()."""
- return path_sep.join([part.rstrip(path_separators)
- for part in path_parts if part])
+if _MS_WINDOWS:
+ def _path_join(*path_parts):
+ """Replacement for os.path.join()."""
+ if not path_parts:
+ return ""
+ if len(path_parts) == 1:
+ return path_parts[0]
+ root = ""
+ path = []
+ for new_root, tail in map(_os._path_splitroot, path_parts):
+ if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple):
+ root = new_root.rstrip(path_separators) or root
+ path = [path_sep + tail]
+ elif new_root.endswith(':'):
+ if root.casefold() != new_root.casefold():
+ # Drive relative paths have to be resolved by the OS, so we reset the
+ # tail but do not add a path_sep prefix.
+ root = new_root
+ path = [tail]
+ else:
+ path.append(tail)
+ else:
+ root = new_root or root
+ path.append(tail)
+ path = [p.rstrip(path_separators) for p in path if p]
+ if len(path) == 1 and not path[0]:
+ # Avoid losing the root's trailing separator when joining with nothing
+ return root + path_sep
+ return root + path_sep.join(path)
+
+else:
+ def _path_join(*path_parts):
+ """Replacement for os.path.join()."""
+ return path_sep.join([part.rstrip(path_separators)
+ for part in path_parts if part])
def _path_split(path):
"""Replacement for os.path.split()."""
- if len(path_separators) == 1:
- front, _, tail = path.rpartition(path_sep)
- return front, tail
- for x in reversed(path):
- if x in path_separators:
- front, tail = path.rsplit(x, maxsplit=1)
- return front, tail
- return '', path
+ i = max(path.rfind(p) for p in path_separators)
+ if i < 0:
+ return '', path
+ return path[:i], path[i + 1:]
def _path_stat(path):
@@ -108,13 +168,18 @@
return _path_is_mode_type(path, 0o040000)
-def _path_isabs(path):
- """Replacement for os.path.isabs.
+if _MS_WINDOWS:
+ def _path_isabs(path):
+ """Replacement for os.path.isabs."""
+ if not path:
+ return False
+ root = _os._path_splitroot(path)[0].replace('/', '\\')
+ return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\'))
- Considers a Windows drive-relative path (no drive, but starts with slash) to
- still be "absolute".
- """
- return path.startswith(path_separators) or path[1:3] in _pathseps_with_colon
+else:
+ def _path_isabs(path):
+ """Replacement for os.path.isabs."""
+ return path.startswith(path_separators)
def _write_atomic(path, data, mode=0o666):
@@ -277,6 +342,16 @@
# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156)
# Python 3.9a2 3424 (simplify bytecodes for *value unpacking)
# Python 3.9a2 3425 (simplify bytecodes for **value unpacking)
+# Python 3.10a1 3430 (Make 'annotations' future by default)
+# Python 3.10a1 3431 (New line number table format -- PEP 626)
+# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202)
+# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0)
+# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching)
+# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets).
+# Python 3.10b1 3436 (Add GEN_START bytecode #43683)
+# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
+# Python 3.10b1 3438 Safer line number table handling.
+# Python 3.10b1 3439 (Add ROT_N)
#
# MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -286,13 +361,17 @@
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.
-MAGIC_NUMBER = (3425).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3439).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
_PYCACHE = '__pycache__'
_OPT = 'opt-'
-SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed.
+SOURCE_SUFFIXES = ['.py']
+if _MS_WINDOWS:
+ SOURCE_SUFFIXES.append('.pyw')
+
+EXTENSION_SUFFIXES = _imp.extension_suffixes()
BYTECODE_SUFFIXES = ['.pyc']
# Deprecated.
@@ -467,15 +546,18 @@
raise ImportError('loader for %s cannot handle %s' %
(self.name, name), name=name)
return method(self, name, *args, **kwargs)
- try:
+
+ # FIXME: @_check_name is used to define class methods before the
+ # _bootstrap module is set by _set_bootstrap_module().
+ if _bootstrap is not None:
_wrap = _bootstrap._wrap
- except NameError:
- # XXX yuck
+ else:
def _wrap(new, old):
for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
if hasattr(old, replace):
setattr(new, replace, getattr(old, replace))
new.__dict__.update(old.__dict__)
+
_wrap(_check_name_wrapper, method)
return _check_name_wrapper
@@ -487,6 +569,9 @@
This method is deprecated in favor of finder.find_spec().
"""
+ _warnings.warn("find_module() is deprecated and "
+ "slated for removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
# Call find_loader(). If it returns a string (indicating this
# is a namespace package portion), generate a warning and
# return None.
@@ -658,6 +743,11 @@
pass
else:
location = _os.fspath(location)
+ if not _path_isabs(location):
+ try:
+ location = _path_join(_os.getcwd(), location)
+ except OSError:
+ pass
# If the location is on the filesystem, but doesn't actually exist,
# we could return None here, indicating that the location is not
@@ -711,10 +801,10 @@
REGISTRY_KEY_DEBUG = (
'Software\\Python\\PythonCore\\{sys_version}'
'\\Modules\\{fullname}\\Debug')
- DEBUG_BUILD = False # Changed in _setup()
+ DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES)
- @classmethod
- def _open_registry(cls, key):
+ @staticmethod
+ def _open_registry(key):
try:
return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key)
except OSError:
@@ -755,9 +845,12 @@
def find_module(cls, fullname, path=None):
"""Find module named in the registry.
- This method is deprecated. Use exec_module() instead.
+ This method is deprecated. Use find_spec() instead.
"""
+ _warnings.warn("WindowsRegistryFinder.find_module() is deprecated and "
+ "slated for removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
spec = cls.find_spec(fullname, path)
if spec is not None:
return spec.loader
@@ -790,7 +883,8 @@
_bootstrap._call_with_frames_removed(exec, code, module.__dict__)
def load_module(self, fullname):
- """This module is deprecated."""
+ """This method is deprecated."""
+ # Warning implemented in _load_module_shim().
return _bootstrap._load_module_shim(self, fullname)
@@ -965,7 +1059,7 @@
"""
# The only reason for this method is for the name check.
# Issue #14857: Avoid the zero-argument form of super so the implementation
- # of that form can be updated without breaking the frozen module
+ # of that form can be updated without breaking the frozen module.
return super(FileLoader, self).load_module(fullname)
@_check_name
@@ -982,32 +1076,10 @@
with _io.FileIO(path, 'r') as file:
return file.read()
- # ResourceReader ABC API.
-
@_check_name
def get_resource_reader(self, module):
- if self.is_package(module):
- return self
- return None
-
- def open_resource(self, resource):
- path = _path_join(_path_split(self.path)[0], resource)
- return _io.FileIO(path, 'r')
-
- def resource_path(self, resource):
- if not self.is_resource(resource):
- raise FileNotFoundError
- path = _path_join(_path_split(self.path)[0], resource)
- return path
-
- def is_resource(self, name):
- if path_sep in name:
- return False
- path = _path_join(_path_split(self.path)[0], name)
- return _path_isfile(path)
-
- def contents(self):
- return iter(_os.listdir(_path_split(self.path)[0]))
+ from importlib.readers import FileReader
+ return FileReader(self)
class SourceFileLoader(FileLoader, SourceLoader):
@@ -1080,10 +1152,6 @@
return None
-# Filled in by _setup().
-EXTENSION_SUFFIXES = []
-
-
class ExtensionFileLoader(FileLoader, _LoaderBasics):
"""Loader for extension modules.
@@ -1144,10 +1212,15 @@
using path_finder. For top-level modules, the parent module's path
is sys.path."""
+ # When invalidate_caches() is called, this epoch is incremented
+ # https://bugs.python.org/issue45703
+ _epoch = 0
+
def __init__(self, name, path, path_finder):
self._name = name
self._path = path
self._last_parent_path = tuple(self._get_parent_path())
+ self._last_epoch = self._epoch
self._path_finder = path_finder
def _find_parent_path_names(self):
@@ -1167,7 +1240,7 @@
def _recalculate(self):
# If the parent's path has changed, recalculate _path
parent_path = tuple(self._get_parent_path()) # Make a copy
- if parent_path != self._last_parent_path:
+ if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
spec = self._path_finder(self._name, parent_path)
# Note that no changes are made if a loader is returned, but we
# do remember the new parent path
@@ -1175,6 +1248,7 @@
if spec.submodule_search_locations:
self._path = spec.submodule_search_locations
self._last_parent_path = parent_path # Save the copy
+ self._last_epoch = self._epoch
return self._path
def __iter__(self):
@@ -1204,13 +1278,15 @@
def __init__(self, name, path, path_finder):
self._path = _NamespacePath(name, path, path_finder)
- @classmethod
- def module_repr(cls, module):
+ @staticmethod
+ def module_repr(module):
"""Return repr for the module.
The method is deprecated. The import machinery does the job itself.
"""
+ _warnings.warn("_NamespaceLoader.module_repr() is deprecated and "
+ "slated for removal in Python 3.12", DeprecationWarning)
return '<module {!r} (namespace)>'.format(module.__name__)
def is_package(self, fullname):
@@ -1237,8 +1313,13 @@
# The import system never calls this method.
_bootstrap._verbose_message('namespace module loaded with path {!r}',
self._path)
+ # Warning implemented in _load_module_shim().
return _bootstrap._load_module_shim(self, fullname)
+ def get_resource_reader(self, module):
+ from importlib.readers import NamespaceReader
+ return NamespaceReader(self._path)
+
# Finders #####################################################################
@@ -1246,8 +1327,8 @@
"""Meta path finder for sys.path and package __path__ attributes."""
- @classmethod
- def invalidate_caches(cls):
+ @staticmethod
+ def invalidate_caches():
"""Call the invalidate_caches() method on all path entry finders
stored in sys.path_importer_caches (where implemented)."""
for name, finder in list(sys.path_importer_cache.items()):
@@ -1255,9 +1336,12 @@
del sys.path_importer_cache[name]
elif hasattr(finder, 'invalidate_caches'):
finder.invalidate_caches()
+ # Also invalidate the caches of _NamespacePaths
+ # https://bugs.python.org/issue45703
+ _NamespacePath._epoch += 1
- @classmethod
- def _path_hooks(cls, path):
+ @staticmethod
+ def _path_hooks(path):
"""Search sys.path_hooks for a finder for 'path'."""
if sys.path_hooks is not None and not sys.path_hooks:
_warnings.warn('sys.path_hooks is empty', ImportWarning)
@@ -1296,8 +1380,14 @@
# This would be a good place for a DeprecationWarning if
# we ended up going that route.
if hasattr(finder, 'find_loader'):
+ msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; "
+ "falling back to find_loader()")
+ _warnings.warn(msg, ImportWarning)
loader, portions = finder.find_loader(fullname)
else:
+ msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; "
+ "falling back to find_module()")
+ _warnings.warn(msg, ImportWarning)
loader = finder.find_module(fullname)
portions = []
if loader is not None:
@@ -1370,13 +1460,16 @@
This method is deprecated. Use find_spec() instead.
"""
+ _warnings.warn("PathFinder.find_module() is deprecated and "
+ "slated for removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
spec = cls.find_spec(fullname, path)
if spec is None:
return None
return spec.loader
- @classmethod
- def find_distributions(cls, *args, **kwargs):
+ @staticmethod
+ def find_distributions(*args, **kwargs):
"""
Find distributions.
@@ -1408,6 +1501,8 @@
self._loaders = loaders
# Base (directory) path
self.path = path or '.'
+ if not _path_isabs(self.path):
+ self.path = _path_join(_os.getcwd(), self.path)
self._path_mtime = -1
self._path_cache = set()
self._relaxed_path_cache = set()
@@ -1425,6 +1520,9 @@
This method is deprecated. Use find_spec() instead.
"""
+ _warnings.warn("FileFinder.find_loader() is deprecated and "
+ "slated for removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
spec = self.find_spec(fullname)
if spec is None:
return None, []
@@ -1470,7 +1568,10 @@
is_namespace = _path_isdir(base_path)
# Check for a file w/ a proper suffix exists.
for suffix, loader_class in self._loaders:
- full_path = _path_join(self.path, tail_module + suffix)
+ try:
+ full_path = _path_join(self.path, tail_module + suffix)
+ except ValueError:
+ return None
_bootstrap._verbose_message('trying {}', full_path, verbosity=2)
if cache_module + suffix in cache:
if _path_isfile(full_path):
@@ -1572,66 +1673,14 @@
return [extensions, source, bytecode]
-def _setup(_bootstrap_module):
- """Setup the path-based importers for importlib by importing needed
- built-in modules and injecting them into the global namespace.
-
- Other components are extracted from the core bootstrap module.
-
- """
- global sys, _imp, _bootstrap
+def _set_bootstrap_module(_bootstrap_module):
+ global _bootstrap
_bootstrap = _bootstrap_module
- sys = _bootstrap.sys
- _imp = _bootstrap._imp
-
- self_module = sys.modules[__name__]
-
- # Directly load the os module (needed during bootstrap).
- os_details = ('posix', ['/']), ('nt', ['\\', '/'])
- for builtin_os, path_separators in os_details:
- # Assumption made in _path_join()
- assert all(len(sep) == 1 for sep in path_separators)
- path_sep = path_separators[0]
- if builtin_os in sys.modules:
- os_module = sys.modules[builtin_os]
- break
- else:
- try:
- os_module = _bootstrap._builtin_from_name(builtin_os)
- break
- except ImportError:
- continue
- else:
- raise ImportError('importlib requires posix or nt')
-
- setattr(self_module, '_os', os_module)
- setattr(self_module, 'path_sep', path_sep)
- setattr(self_module, 'path_separators', ''.join(path_separators))
- setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators})
-
- # Directly load built-in modules needed during bootstrap.
- builtin_names = ['_io', '_warnings', 'marshal']
- if builtin_os == 'nt':
- builtin_names.append('winreg')
- for builtin_name in builtin_names:
- if builtin_name not in sys.modules:
- builtin_module = _bootstrap._builtin_from_name(builtin_name)
- else:
- builtin_module = sys.modules[builtin_name]
- setattr(self_module, builtin_name, builtin_module)
-
- # Constants
- setattr(self_module, '_relax_case', _make_relax_case())
- EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
- if builtin_os == 'nt':
- SOURCE_SUFFIXES.append('.pyw')
- if '_d.pyd' in EXTENSION_SUFFIXES:
- WindowsRegistryFinder.DEBUG_BUILD = True
def _install(_bootstrap_module):
"""Install the path-based import components."""
- _setup(_bootstrap_module)
+ _set_bootstrap_module(_bootstrap_module)
supported_loaders = _get_supported_file_loaders()
sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
sys.meta_path.append(PathFinder)
diff --git a/common/py3-stdlib/importlib/_common.py b/common/py3-stdlib/importlib/_common.py
index c1204f0..549fee3 100644
--- a/common/py3-stdlib/importlib/_common.py
+++ b/common/py3-stdlib/importlib/_common.py
@@ -1,9 +1,72 @@
import os
import pathlib
-import zipfile
import tempfile
import functools
import contextlib
+import types
+import importlib
+
+from typing import Union, Any, Optional
+from .abc import ResourceReader, Traversable
+
+from ._adapters import wrap_spec
+
+Package = Union[types.ModuleType, str]
+
+
+def files(package):
+ # type: (Package) -> Traversable
+ """
+ Get a Traversable resource from a package
+ """
+ return from_package(get_package(package))
+
+
+def normalize_path(path):
+ # type: (Any) -> str
+ """Normalize a path by ensuring it is a string.
+
+ If the resulting string contains path separators, an exception is raised.
+ """
+ str_path = str(path)
+ parent, file_name = os.path.split(str_path)
+ if parent:
+ raise ValueError(f'{path!r} must be only a file name')
+ return file_name
+
+
+def get_resource_reader(package):
+ # type: (types.ModuleType) -> Optional[ResourceReader]
+ """
+ Return the package's loader if it's a ResourceReader.
+ """
+ # We can't use
+ # a issubclass() check here because apparently abc.'s __subclasscheck__()
+ # hook wants to create a weak reference to the object, but
+ # zipimport.zipimporter does not support weak references, resulting in a
+ # TypeError. That seems terrible.
+ spec = package.__spec__
+ reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore
+ if reader is None:
+ return None
+ return reader(spec.name) # type: ignore
+
+
+def resolve(cand):
+ # type: (Package) -> types.ModuleType
+ return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand)
+
+
+def get_package(package):
+ # type: (Package) -> types.ModuleType
+ """Take a package name or module object and return the module.
+
+ Raise an exception if the resolved module is not a package.
+ """
+ resolved = resolve(package)
+ if wrap_spec(resolved).submodule_search_locations is None:
+ raise TypeError(f'{package!r} is not a package')
+ return resolved
def from_package(package):
@@ -11,18 +74,9 @@
Return a Traversable object for the given package.
"""
- return fallback_resources(package.__spec__)
-
-
-def fallback_resources(spec):
- package_directory = pathlib.Path(spec.origin).parent
- try:
- archive_path = spec.loader.archive
- rel_path = package_directory.relative_to(archive_path)
- return zipfile.Path(archive_path, str(rel_path) + '/')
- except Exception:
- pass
- return package_directory
+ spec = wrap_spec(package)
+ reader = spec.loader.get_resource_reader(spec.name)
+ return reader.files()
@contextlib.contextmanager
@@ -34,6 +88,7 @@
try:
os.write(fd, reader())
os.close(fd)
+ del reader
yield pathlib.Path(raw_path)
finally:
try:
@@ -43,14 +98,12 @@
@functools.singledispatch
-@contextlib.contextmanager
def as_file(path):
"""
Given a Traversable object, return that object as a
path on the local file system in a context manager.
"""
- with _tempfile(path.read_bytes, suffix=path.name) as local:
- yield local
+ return _tempfile(path.read_bytes, suffix=path.name)
@as_file.register(pathlib.Path)
diff --git a/common/py3-stdlib/importlib/abc.py b/common/py3-stdlib/importlib/abc.py
index b8a9bb1..0b4a3f8 100644
--- a/common/py3-stdlib/importlib/abc.py
+++ b/common/py3-stdlib/importlib/abc.py
@@ -1,5 +1,4 @@
"""Abstract base classes related to import."""
-from . import _bootstrap
from . import _bootstrap_external
from . import machinery
try:
@@ -12,8 +11,10 @@
import _frozen_importlib_external
except ImportError:
_frozen_importlib_external = _bootstrap_external
+from ._abc import Loader
import abc
import warnings
+from typing import BinaryIO, Iterable, Text
from typing import Protocol, runtime_checkable
@@ -40,15 +41,27 @@
Deprecated since Python 3.3
"""
+ def __init__(self):
+ warnings.warn("the Finder ABC is deprecated and "
+ "slated for removal in Python 3.12; use MetaPathFinder "
+ "or PathEntryFinder instead",
+ DeprecationWarning)
+
@abc.abstractmethod
def find_module(self, fullname, path=None):
"""An abstract method that should find a module.
The fullname is a str and the optional path is a str or None.
Returns a Loader object or None.
"""
+ warnings.warn("importlib.abc.Finder along with its find_module() "
+ "method are deprecated and "
+ "slated for removal in Python 3.12; use "
+ "MetaPathFinder.find_spec() or "
+ "PathEntryFinder.find_spec() instead",
+ DeprecationWarning)
-class MetaPathFinder(Finder):
+class MetaPathFinder(metaclass=abc.ABCMeta):
"""Abstract base class for import finders on sys.meta_path."""
@@ -67,8 +80,8 @@
"""
warnings.warn("MetaPathFinder.find_module() is deprecated since Python "
- "3.4 in favor of MetaPathFinder.find_spec() "
- "(available since 3.4)",
+ "3.4 in favor of MetaPathFinder.find_spec() and is "
+ "slated for removal in Python 3.12",
DeprecationWarning,
stacklevel=2)
if not hasattr(self, 'find_spec'):
@@ -85,7 +98,7 @@
machinery.PathFinder, machinery.WindowsRegistryFinder)
-class PathEntryFinder(Finder):
+class PathEntryFinder(metaclass=abc.ABCMeta):
"""Abstract base class for path entry finders used by PathFinder."""
@@ -134,53 +147,6 @@
_register(PathEntryFinder, machinery.FileFinder)
-class Loader(metaclass=abc.ABCMeta):
-
- """Abstract base class for import loaders."""
-
- def create_module(self, spec):
- """Return a module to initialize and into which to load.
-
- This method should raise ImportError if anything prevents it
- from creating a new module. It may return None to indicate
- that the spec should create the new module.
- """
- # By default, defer to default semantics for the new module.
- return None
-
- # We don't define exec_module() here since that would break
- # hasattr checks we do to support backward compatibility.
-
- def load_module(self, fullname):
- """Return the loaded module.
-
- The module must be added to sys.modules and have import-related
- attributes set properly. The fullname is a str.
-
- ImportError is raised on failure.
-
- This method is deprecated in favor of loader.exec_module(). If
- exec_module() exists then it is used to provide a backwards-compatible
- functionality for this method.
-
- """
- if not hasattr(self, 'exec_module'):
- raise ImportError
- return _bootstrap._load_module_shim(self, fullname)
-
- def module_repr(self, module):
- """Return a module's repr.
-
- Used by the module type when the method does not raise
- NotImplementedError.
-
- This method is deprecated.
-
- """
- # The exception will cause ModuleType.__repr__ to ignore this method.
- raise NotImplementedError
-
-
class ResourceLoader(Loader):
"""Abstract base class for loaders which can return data from their
@@ -344,49 +310,45 @@
class ResourceReader(metaclass=abc.ABCMeta):
-
- """Abstract base class to provide resource-reading support.
-
- Loaders that support resource reading are expected to implement
- the ``get_resource_reader(fullname)`` method and have it either return None
- or an object compatible with this ABC.
- """
+ """Abstract base class for loaders to provide resource reading support."""
@abc.abstractmethod
- def open_resource(self, resource):
+ def open_resource(self, resource: Text) -> BinaryIO:
"""Return an opened, file-like object for binary reading.
- The 'resource' argument is expected to represent only a file name
- and thus not contain any subdirectory components.
-
+ The 'resource' argument is expected to represent only a file name.
If the resource cannot be found, FileNotFoundError is raised.
"""
+ # This deliberately raises FileNotFoundError instead of
+ # NotImplementedError so that if this method is accidentally called,
+ # it'll still do the right thing.
raise FileNotFoundError
@abc.abstractmethod
- def resource_path(self, resource):
+ def resource_path(self, resource: Text) -> Text:
"""Return the file system path to the specified resource.
- The 'resource' argument is expected to represent only a file name
- and thus not contain any subdirectory components.
-
+ The 'resource' argument is expected to represent only a file name.
If the resource does not exist on the file system, raise
FileNotFoundError.
"""
+ # This deliberately raises FileNotFoundError instead of
+ # NotImplementedError so that if this method is accidentally called,
+ # it'll still do the right thing.
raise FileNotFoundError
@abc.abstractmethod
- def is_resource(self, name):
- """Return True if the named 'name' is consider a resource."""
+ def is_resource(self, path: Text) -> bool:
+ """Return True if the named 'path' is a resource.
+
+ Files are resources, directories are not.
+ """
raise FileNotFoundError
@abc.abstractmethod
- def contents(self):
- """Return an iterable of strings over the contents of the package."""
- return []
-
-
-_register(ResourceReader, machinery.SourceFileLoader)
+ def contents(self) -> Iterable[str]:
+ """Return an iterable of entries in `package`."""
+ raise FileNotFoundError
@runtime_checkable
@@ -402,26 +364,28 @@
Yield Traversable objects in self
"""
- @abc.abstractmethod
def read_bytes(self):
"""
Read contents of self as bytes
"""
+ with self.open('rb') as strm:
+ return strm.read()
- @abc.abstractmethod
def read_text(self, encoding=None):
"""
- Read contents of self as bytes
+ Read contents of self as text
"""
+ with self.open(encoding=encoding) as strm:
+ return strm.read()
@abc.abstractmethod
- def is_dir(self):
+ def is_dir(self) -> bool:
"""
Return True if self is a dir
"""
@abc.abstractmethod
- def is_file(self):
+ def is_file(self) -> bool:
"""
Return True if self is a file
"""
@@ -432,11 +396,11 @@
Return Traversable child in self
"""
- @abc.abstractmethod
def __truediv__(self, child):
"""
Return Traversable child in self
"""
+ return self.joinpath(child)
@abc.abstractmethod
def open(self, mode='r', *args, **kwargs):
@@ -449,14 +413,18 @@
"""
@abc.abstractproperty
- def name(self):
- # type: () -> str
+ def name(self) -> str:
"""
The base name of this object without any parent references.
"""
class TraversableResources(ResourceReader):
+ """
+ The required interface for providing traversable
+ resources.
+ """
+
@abc.abstractmethod
def files(self):
"""Return a Traversable object for the loaded package."""
@@ -468,7 +436,7 @@
raise FileNotFoundError(resource)
def is_resource(self, path):
- return self.files().joinpath(path).isfile()
+ return self.files().joinpath(path).is_file()
def contents(self):
return (item.name for item in self.files().iterdir())
diff --git a/common/py3-stdlib/importlib/machinery.py b/common/py3-stdlib/importlib/machinery.py
index 1b2b5c9..9a7757f 100644
--- a/common/py3-stdlib/importlib/machinery.py
+++ b/common/py3-stdlib/importlib/machinery.py
@@ -1,7 +1,5 @@
"""The machinery of importlib: finders, loaders, hooks, etc."""
-import _imp
-
from ._bootstrap import ModuleSpec
from ._bootstrap import BuiltinImporter
from ._bootstrap import FrozenImporter
diff --git a/common/py3-stdlib/importlib/metadata.py b/common/py3-stdlib/importlib/metadata.py
deleted file mode 100644
index ffa0cba..0000000
--- a/common/py3-stdlib/importlib/metadata.py
+++ /dev/null
@@ -1,586 +0,0 @@
-import io
-import os
-import re
-import abc
-import csv
-import sys
-import email
-import pathlib
-import zipfile
-import operator
-import functools
-import itertools
-import posixpath
-import collections
-
-from configparser import ConfigParser
-from contextlib import suppress
-from importlib import import_module
-from importlib.abc import MetaPathFinder
-from itertools import starmap
-
-
-__all__ = [
- 'Distribution',
- 'DistributionFinder',
- 'PackageNotFoundError',
- 'distribution',
- 'distributions',
- 'entry_points',
- 'files',
- 'metadata',
- 'requires',
- 'version',
- ]
-
-
-class PackageNotFoundError(ModuleNotFoundError):
- """The package was not found."""
-
-
-class EntryPoint(
- collections.namedtuple('EntryPointBase', 'name value group')):
- """An entry point as defined by Python packaging conventions.
-
- See `the packaging docs on entry points
- <https://packaging.python.org/specifications/entry-points/>`_
- for more information.
- """
-
- pattern = re.compile(
- r'(?P<module>[\w.]+)\s*'
- r'(:\s*(?P<attr>[\w.]+))?\s*'
- r'(?P<extras>\[.*\])?\s*$'
- )
- """
- A regular expression describing the syntax for an entry point,
- which might look like:
-
- - module
- - package.module
- - package.module:attribute
- - package.module:object.attribute
- - package.module:attr [extra1, extra2]
-
- Other combinations are possible as well.
-
- The expression is lenient about whitespace around the ':',
- following the attr, and following any extras.
- """
-
- def load(self):
- """Load the entry point from its definition. If only a module
- is indicated by the value, return that module. Otherwise,
- return the named object.
- """
- match = self.pattern.match(self.value)
- module = import_module(match.group('module'))
- attrs = filter(None, (match.group('attr') or '').split('.'))
- return functools.reduce(getattr, attrs, module)
-
- @property
- def module(self):
- match = self.pattern.match(self.value)
- return match.group('module')
-
- @property
- def attr(self):
- match = self.pattern.match(self.value)
- return match.group('attr')
-
- @property
- def extras(self):
- match = self.pattern.match(self.value)
- return list(re.finditer(r'\w+', match.group('extras') or ''))
-
- @classmethod
- def _from_config(cls, config):
- return [
- cls(name, value, group)
- for group in config.sections()
- for name, value in config.items(group)
- ]
-
- @classmethod
- def _from_text(cls, text):
- config = ConfigParser(delimiters='=')
- # case sensitive: https://stackoverflow.com/q/1611799/812183
- config.optionxform = str
- try:
- config.read_string(text)
- except AttributeError: # pragma: nocover
- # Python 2 has no read_string
- config.readfp(io.StringIO(text))
- return EntryPoint._from_config(config)
-
- def __iter__(self):
- """
- Supply iter so one may construct dicts of EntryPoints easily.
- """
- return iter((self.name, self))
-
- def __reduce__(self):
- return (
- self.__class__,
- (self.name, self.value, self.group),
- )
-
-
-class PackagePath(pathlib.PurePosixPath):
- """A reference to a path in a package"""
-
- def read_text(self, encoding='utf-8'):
- with self.locate().open(encoding=encoding) as stream:
- return stream.read()
-
- def read_binary(self):
- with self.locate().open('rb') as stream:
- return stream.read()
-
- def locate(self):
- """Return a path-like object for this path"""
- return self.dist.locate_file(self)
-
-
-class FileHash:
- def __init__(self, spec):
- self.mode, _, self.value = spec.partition('=')
-
- def __repr__(self):
- return '<FileHash mode: {} value: {}>'.format(self.mode, self.value)
-
-
-class Distribution:
- """A Python distribution package."""
-
- @abc.abstractmethod
- def read_text(self, filename):
- """Attempt to load metadata file given by the name.
-
- :param filename: The name of the file in the distribution info.
- :return: The text if found, otherwise None.
- """
-
- @abc.abstractmethod
- def locate_file(self, path):
- """
- Given a path to a file in this distribution, return a path
- to it.
- """
-
- @classmethod
- def from_name(cls, name):
- """Return the Distribution for the given package name.
-
- :param name: The name of the distribution package to search for.
- :return: The Distribution instance (or subclass thereof) for the named
- package, if found.
- :raises PackageNotFoundError: When the named package's distribution
- metadata cannot be found.
- """
- for resolver in cls._discover_resolvers():
- dists = resolver(DistributionFinder.Context(name=name))
- dist = next(iter(dists), None)
- if dist is not None:
- return dist
- else:
- raise PackageNotFoundError(name)
-
- @classmethod
- def discover(cls, **kwargs):
- """Return an iterable of Distribution objects for all packages.
-
- Pass a ``context`` or pass keyword arguments for constructing
- a context.
-
- :context: A ``DistributionFinder.Context`` object.
- :return: Iterable of Distribution objects for all packages.
- """
- context = kwargs.pop('context', None)
- if context and kwargs:
- raise ValueError("cannot accept context and kwargs")
- context = context or DistributionFinder.Context(**kwargs)
- return itertools.chain.from_iterable(
- resolver(context)
- for resolver in cls._discover_resolvers()
- )
-
- @staticmethod
- def at(path):
- """Return a Distribution for the indicated metadata path
-
- :param path: a string or path-like object
- :return: a concrete Distribution instance for the path
- """
- return PathDistribution(pathlib.Path(path))
-
- @staticmethod
- def _discover_resolvers():
- """Search the meta_path for resolvers."""
- declared = (
- getattr(finder, 'find_distributions', None)
- for finder in sys.meta_path
- )
- return filter(None, declared)
-
- @classmethod
- def _local(cls, root='.'):
- from pep517 import build, meta
- system = build.compat_system(root)
- builder = functools.partial(
- meta.build,
- source_dir=root,
- system=system,
- )
- return PathDistribution(zipfile.Path(meta.build_as_zip(builder)))
-
- @property
- def metadata(self):
- """Return the parsed metadata for this Distribution.
-
- The returned object will have keys that name the various bits of
- metadata. See PEP 566 for details.
- """
- text = (
- self.read_text('METADATA')
- or self.read_text('PKG-INFO')
- # This last clause is here to support old egg-info files. Its
- # effect is to just end up using the PathDistribution's self._path
- # (which points to the egg-info file) attribute unchanged.
- or self.read_text('')
- )
- return email.message_from_string(text)
-
- @property
- def version(self):
- """Return the 'Version' metadata for the distribution package."""
- return self.metadata['Version']
-
- @property
- def entry_points(self):
- return EntryPoint._from_text(self.read_text('entry_points.txt'))
-
- @property
- def files(self):
- """Files in this distribution.
-
- :return: List of PackagePath for this distribution or None
-
- Result is `None` if the metadata file that enumerates files
- (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
- missing.
- Result may be empty if the metadata exists but is empty.
- """
- file_lines = self._read_files_distinfo() or self._read_files_egginfo()
-
- def make_file(name, hash=None, size_str=None):
- result = PackagePath(name)
- result.hash = FileHash(hash) if hash else None
- result.size = int(size_str) if size_str else None
- result.dist = self
- return result
-
- return file_lines and list(starmap(make_file, csv.reader(file_lines)))
-
- def _read_files_distinfo(self):
- """
- Read the lines of RECORD
- """
- text = self.read_text('RECORD')
- return text and text.splitlines()
-
- def _read_files_egginfo(self):
- """
- SOURCES.txt might contain literal commas, so wrap each line
- in quotes.
- """
- text = self.read_text('SOURCES.txt')
- return text and map('"{}"'.format, text.splitlines())
-
- @property
- def requires(self):
- """Generated requirements specified for this Distribution"""
- reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
- return reqs and list(reqs)
-
- def _read_dist_info_reqs(self):
- return self.metadata.get_all('Requires-Dist')
-
- def _read_egg_info_reqs(self):
- source = self.read_text('requires.txt')
- return source and self._deps_from_requires_text(source)
-
- @classmethod
- def _deps_from_requires_text(cls, source):
- section_pairs = cls._read_sections(source.splitlines())
- sections = {
- section: list(map(operator.itemgetter('line'), results))
- for section, results in
- itertools.groupby(section_pairs, operator.itemgetter('section'))
- }
- return cls._convert_egg_info_reqs_to_simple_reqs(sections)
-
- @staticmethod
- def _read_sections(lines):
- section = None
- for line in filter(None, lines):
- section_match = re.match(r'\[(.*)\]$', line)
- if section_match:
- section = section_match.group(1)
- continue
- yield locals()
-
- @staticmethod
- def _convert_egg_info_reqs_to_simple_reqs(sections):
- """
- Historically, setuptools would solicit and store 'extra'
- requirements, including those with environment markers,
- in separate sections. More modern tools expect each
- dependency to be defined separately, with any relevant
- extras and environment markers attached directly to that
- requirement. This method converts the former to the
- latter. See _test_deps_from_requires_text for an example.
- """
- def make_condition(name):
- return name and 'extra == "{name}"'.format(name=name)
-
- def parse_condition(section):
- section = section or ''
- extra, sep, markers = section.partition(':')
- if extra and markers:
- markers = '({markers})'.format(markers=markers)
- conditions = list(filter(None, [markers, make_condition(extra)]))
- return '; ' + ' and '.join(conditions) if conditions else ''
-
- for section, deps in sections.items():
- for dep in deps:
- yield dep + parse_condition(section)
-
-
-class DistributionFinder(MetaPathFinder):
- """
- A MetaPathFinder capable of discovering installed distributions.
- """
-
- class Context:
- """
- Keyword arguments presented by the caller to
- ``distributions()`` or ``Distribution.discover()``
- to narrow the scope of a search for distributions
- in all DistributionFinders.
-
- Each DistributionFinder may expect any parameters
- and should attempt to honor the canonical
- parameters defined below when appropriate.
- """
-
- name = None
- """
- Specific name for which a distribution finder should match.
- A name of ``None`` matches all distributions.
- """
-
- def __init__(self, **kwargs):
- vars(self).update(kwargs)
-
- @property
- def path(self):
- """
- The path that a distribution finder should search.
-
- Typically refers to Python package paths and defaults
- to ``sys.path``.
- """
- return vars(self).get('path', sys.path)
-
- @abc.abstractmethod
- def find_distributions(self, context=Context()):
- """
- Find distributions.
-
- Return an iterable of all Distribution instances capable of
- loading the metadata for packages matching the ``context``,
- a DistributionFinder.Context instance.
- """
-
-
-class FastPath:
- """
- Micro-optimized class for searching a path for
- children.
- """
-
- def __init__(self, root):
- self.root = root
- self.base = os.path.basename(self.root).lower()
-
- def joinpath(self, child):
- return pathlib.Path(self.root, child)
-
- def children(self):
- with suppress(Exception):
- return os.listdir(self.root or '')
- with suppress(Exception):
- return self.zip_children()
- return []
-
- def zip_children(self):
- zip_path = zipfile.Path(self.root)
- names = zip_path.root.namelist()
- self.joinpath = zip_path.joinpath
-
- return dict.fromkeys(
- child.split(posixpath.sep, 1)[0]
- for child in names
- )
-
- def is_egg(self, search):
- base = self.base
- return (
- base == search.versionless_egg_name
- or base.startswith(search.prefix)
- and base.endswith('.egg'))
-
- def search(self, name):
- for child in self.children():
- n_low = child.lower()
- if (n_low in name.exact_matches
- or n_low.startswith(name.prefix)
- and n_low.endswith(name.suffixes)
- # legacy case:
- or self.is_egg(name) and n_low == 'egg-info'):
- yield self.joinpath(child)
-
-
-class Prepared:
- """
- A prepared search for metadata on a possibly-named package.
- """
- normalized = ''
- prefix = ''
- suffixes = '.dist-info', '.egg-info'
- exact_matches = [''][:0]
- versionless_egg_name = ''
-
- def __init__(self, name):
- self.name = name
- if name is None:
- return
- self.normalized = name.lower().replace('-', '_')
- self.prefix = self.normalized + '-'
- self.exact_matches = [
- self.normalized + suffix for suffix in self.suffixes]
- self.versionless_egg_name = self.normalized + '.egg'
-
-
-class MetadataPathFinder(DistributionFinder):
- @classmethod
- def find_distributions(cls, context=DistributionFinder.Context()):
- """
- Find distributions.
-
- Return an iterable of all Distribution instances capable of
- loading the metadata for packages matching ``context.name``
- (or all names if ``None`` indicated) along the paths in the list
- of directories ``context.path``.
- """
- found = cls._search_paths(context.name, context.path)
- return map(PathDistribution, found)
-
- @classmethod
- def _search_paths(cls, name, paths):
- """Find metadata directories in paths heuristically."""
- return itertools.chain.from_iterable(
- path.search(Prepared(name))
- for path in map(FastPath, paths)
- )
-
-
-class PathDistribution(Distribution):
- def __init__(self, path):
- """Construct a distribution from a path to the metadata directory.
-
- :param path: A pathlib.Path or similar object supporting
- .joinpath(), __div__, .parent, and .read_text().
- """
- self._path = path
-
- def read_text(self, filename):
- with suppress(FileNotFoundError, IsADirectoryError, KeyError,
- NotADirectoryError, PermissionError):
- return self._path.joinpath(filename).read_text(encoding='utf-8')
- read_text.__doc__ = Distribution.read_text.__doc__
-
- def locate_file(self, path):
- return self._path.parent / path
-
-
-def distribution(distribution_name):
- """Get the ``Distribution`` instance for the named package.
-
- :param distribution_name: The name of the distribution package as a string.
- :return: A ``Distribution`` instance (or subclass thereof).
- """
- return Distribution.from_name(distribution_name)
-
-
-def distributions(**kwargs):
- """Get all ``Distribution`` instances in the current environment.
-
- :return: An iterable of ``Distribution`` instances.
- """
- return Distribution.discover(**kwargs)
-
-
-def metadata(distribution_name):
- """Get the metadata for the named package.
-
- :param distribution_name: The name of the distribution package to query.
- :return: An email.Message containing the parsed metadata.
- """
- return Distribution.from_name(distribution_name).metadata
-
-
-def version(distribution_name):
- """Get the version string for the named package.
-
- :param distribution_name: The name of the distribution package to query.
- :return: The version string for the package as defined in the package's
- "Version" metadata key.
- """
- return distribution(distribution_name).version
-
-
-def entry_points():
- """Return EntryPoint objects for all installed packages.
-
- :return: EntryPoint objects for all installed packages.
- """
- eps = itertools.chain.from_iterable(
- dist.entry_points for dist in distributions())
- by_group = operator.attrgetter('group')
- ordered = sorted(eps, key=by_group)
- grouped = itertools.groupby(ordered, by_group)
- return {
- group: tuple(eps)
- for group, eps in grouped
- }
-
-
-def files(distribution_name):
- """Return a list of files for the named package.
-
- :param distribution_name: The name of the distribution package to query.
- :return: List of files composing the distribution.
- """
- return distribution(distribution_name).files
-
-
-def requires(distribution_name):
- """
- Return a list of requirements for the named package.
-
- :return: An iterator of requirements, suitable for
- packaging.requirement.Requirement.
- """
- return distribution(distribution_name).requires
diff --git a/common/py3-stdlib/importlib/metadata/__init__.py b/common/py3-stdlib/importlib/metadata/__init__.py
new file mode 100644
index 0000000..b3063cd
--- /dev/null
+++ b/common/py3-stdlib/importlib/metadata/__init__.py
@@ -0,0 +1,1045 @@
+import os
+import re
+import abc
+import csv
+import sys
+import email
+import pathlib
+import zipfile
+import operator
+import textwrap
+import warnings
+import functools
+import itertools
+import posixpath
+import collections
+
+from . import _adapters, _meta
+from ._meta import PackageMetadata
+from ._collections import FreezableDefaultDict, Pair
+from ._functools import method_cache
+from ._itertools import unique_everseen
+from ._meta import PackageMetadata, SimplePath
+
+from contextlib import suppress
+from importlib import import_module
+from importlib.abc import MetaPathFinder
+from itertools import starmap
+from typing import List, Mapping, Optional, Union
+
+
+__all__ = [
+ 'Distribution',
+ 'DistributionFinder',
+ 'PackageMetadata',
+ 'PackageNotFoundError',
+ 'distribution',
+ 'distributions',
+ 'entry_points',
+ 'files',
+ 'metadata',
+ 'packages_distributions',
+ 'requires',
+ 'version',
+]
+
+
+class PackageNotFoundError(ModuleNotFoundError):
+ """The package was not found."""
+
+ def __str__(self):
+ return f"No package metadata was found for {self.name}"
+
+ @property
+ def name(self):
+ (name,) = self.args
+ return name
+
+
+class Sectioned:
+ """
+ A simple entry point config parser for performance
+
+ >>> for item in Sectioned.read(Sectioned._sample):
+ ... print(item)
+ Pair(name='sec1', value='# comments ignored')
+ Pair(name='sec1', value='a = 1')
+ Pair(name='sec1', value='b = 2')
+ Pair(name='sec2', value='a = 2')
+
+ >>> res = Sectioned.section_pairs(Sectioned._sample)
+ >>> item = next(res)
+ >>> item.name
+ 'sec1'
+ >>> item.value
+ Pair(name='a', value='1')
+ >>> item = next(res)
+ >>> item.value
+ Pair(name='b', value='2')
+ >>> item = next(res)
+ >>> item.name
+ 'sec2'
+ >>> item.value
+ Pair(name='a', value='2')
+ >>> list(res)
+ []
+ """
+
+ _sample = textwrap.dedent(
+ """
+ [sec1]
+ # comments ignored
+ a = 1
+ b = 2
+
+ [sec2]
+ a = 2
+ """
+ ).lstrip()
+
+ @classmethod
+ def section_pairs(cls, text):
+ return (
+ section._replace(value=Pair.parse(section.value))
+ for section in cls.read(text, filter_=cls.valid)
+ if section.name is not None
+ )
+
+ @staticmethod
+ def read(text, filter_=None):
+ lines = filter(filter_, map(str.strip, text.splitlines()))
+ name = None
+ for value in lines:
+ section_match = value.startswith('[') and value.endswith(']')
+ if section_match:
+ name = value.strip('[]')
+ continue
+ yield Pair(name, value)
+
+ @staticmethod
+ def valid(line):
+ return line and not line.startswith('#')
+
+
+class EntryPoint(
+ collections.namedtuple('EntryPointBase', 'name value group')):
+ """An entry point as defined by Python packaging conventions.
+
+ See `the packaging docs on entry points
+ <https://packaging.python.org/specifications/entry-points/>`_
+ for more information.
+
+ >>> ep = EntryPoint(
+ ... name=None, group=None, value='package.module:attr [extra1, extra2]')
+ >>> ep.module
+ 'package.module'
+ >>> ep.attr
+ 'attr'
+ >>> ep.extras
+ ['extra1', 'extra2']
+ """
+
+ pattern = re.compile(
+ r'(?P<module>[\w.]+)\s*'
+ r'(:\s*(?P<attr>[\w.]+)\s*)?'
+ r'((?P<extras>\[.*\])\s*)?$'
+ )
+ """
+ A regular expression describing the syntax for an entry point,
+ which might look like:
+
+ - module
+ - package.module
+ - package.module:attribute
+ - package.module:object.attribute
+ - package.module:attr [extra1, extra2]
+
+ Other combinations are possible as well.
+
+ The expression is lenient about whitespace around the ':',
+ following the attr, and following any extras.
+ """
+
+ dist: Optional['Distribution'] = None
+
+ def load(self):
+ """Load the entry point from its definition. If only a module
+ is indicated by the value, return that module. Otherwise,
+ return the named object.
+ """
+ match = self.pattern.match(self.value)
+ module = import_module(match.group('module'))
+ attrs = filter(None, (match.group('attr') or '').split('.'))
+ return functools.reduce(getattr, attrs, module)
+
+ @property
+ def module(self):
+ match = self.pattern.match(self.value)
+ return match.group('module')
+
+ @property
+ def attr(self):
+ match = self.pattern.match(self.value)
+ return match.group('attr')
+
+ @property
+ def extras(self):
+ match = self.pattern.match(self.value)
+ return re.findall(r'\w+', match.group('extras') or '')
+
+ def _for(self, dist):
+ self.dist = dist
+ return self
+
+ def __iter__(self):
+ """
+ Supply iter so one may construct dicts of EntryPoints by name.
+ """
+ msg = (
+ "Construction of dict of EntryPoints is deprecated in "
+ "favor of EntryPoints."
+ )
+ warnings.warn(msg, DeprecationWarning)
+ return iter((self.name, self))
+
+ def __reduce__(self):
+ return (
+ self.__class__,
+ (self.name, self.value, self.group),
+ )
+
+ def matches(self, **params):
+ """
+ EntryPoint matches the given parameters.
+
+ >>> ep = EntryPoint(group='foo', name='bar', value='bing:bong [extra1, extra2]')
+ >>> ep.matches(group='foo')
+ True
+ >>> ep.matches(name='bar', value='bing:bong [extra1, extra2]')
+ True
+ >>> ep.matches(group='foo', name='other')
+ False
+ >>> ep.matches()
+ True
+ >>> ep.matches(extras=['extra1', 'extra2'])
+ True
+ >>> ep.matches(module='bing')
+ True
+ >>> ep.matches(attr='bong')
+ True
+ """
+ attrs = (getattr(self, param) for param in params)
+ return all(map(operator.eq, params.values(), attrs))
+
+
+class DeprecatedList(list):
+ """
+ Allow an otherwise immutable object to implement mutability
+ for compatibility.
+
+ >>> recwarn = getfixture('recwarn')
+ >>> dl = DeprecatedList(range(3))
+ >>> dl[0] = 1
+ >>> dl.append(3)
+ >>> del dl[3]
+ >>> dl.reverse()
+ >>> dl.sort()
+ >>> dl.extend([4])
+ >>> dl.pop(-1)
+ 4
+ >>> dl.remove(1)
+ >>> dl += [5]
+ >>> dl + [6]
+ [1, 2, 5, 6]
+ >>> dl + (6,)
+ [1, 2, 5, 6]
+ >>> dl.insert(0, 0)
+ >>> dl
+ [0, 1, 2, 5]
+ >>> dl == [0, 1, 2, 5]
+ True
+ >>> dl == (0, 1, 2, 5)
+ True
+ >>> len(recwarn)
+ 1
+ """
+
+ __slots__ = ()
+
+ _warn = functools.partial(
+ warnings.warn,
+ "EntryPoints list interface is deprecated. Cast to list if needed.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ def __setitem__(self, *args, **kwargs):
+ self._warn()
+ return super().__setitem__(*args, **kwargs)
+
+ def __delitem__(self, *args, **kwargs):
+ self._warn()
+ return super().__delitem__(*args, **kwargs)
+
+ def append(self, *args, **kwargs):
+ self._warn()
+ return super().append(*args, **kwargs)
+
+ def reverse(self, *args, **kwargs):
+ self._warn()
+ return super().reverse(*args, **kwargs)
+
+ def extend(self, *args, **kwargs):
+ self._warn()
+ return super().extend(*args, **kwargs)
+
+ def pop(self, *args, **kwargs):
+ self._warn()
+ return super().pop(*args, **kwargs)
+
+ def remove(self, *args, **kwargs):
+ self._warn()
+ return super().remove(*args, **kwargs)
+
+ def __iadd__(self, *args, **kwargs):
+ self._warn()
+ return super().__iadd__(*args, **kwargs)
+
+ def __add__(self, other):
+ if not isinstance(other, tuple):
+ self._warn()
+ other = tuple(other)
+ return self.__class__(tuple(self) + other)
+
+ def insert(self, *args, **kwargs):
+ self._warn()
+ return super().insert(*args, **kwargs)
+
+ def sort(self, *args, **kwargs):
+ self._warn()
+ return super().sort(*args, **kwargs)
+
+ def __eq__(self, other):
+ if not isinstance(other, tuple):
+ self._warn()
+ other = tuple(other)
+
+ return tuple(self).__eq__(other)
+
+
+class EntryPoints(DeprecatedList):
+ """
+ An immutable collection of selectable EntryPoint objects.
+ """
+
+ __slots__ = ()
+
+ def __getitem__(self, name): # -> EntryPoint:
+ """
+ Get the EntryPoint in self matching name.
+ """
+ if isinstance(name, int):
+ warnings.warn(
+ "Accessing entry points by index is deprecated. "
+ "Cast to tuple if needed.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return super().__getitem__(name)
+ try:
+ return next(iter(self.select(name=name)))
+ except StopIteration:
+ raise KeyError(name)
+
+ def select(self, **params):
+ """
+ Select entry points from self that match the
+ given parameters (typically group and/or name).
+ """
+ return EntryPoints(ep for ep in self if ep.matches(**params))
+
+ @property
+ def names(self):
+ """
+ Return the set of all names of all entry points.
+ """
+ return set(ep.name for ep in self)
+
+ @property
+ def groups(self):
+ """
+ Return the set of all groups of all entry points.
+
+ For coverage while SelectableGroups is present.
+ >>> EntryPoints().groups
+ set()
+ """
+ return set(ep.group for ep in self)
+
+ @classmethod
+ def _from_text_for(cls, text, dist):
+ return cls(ep._for(dist) for ep in cls._from_text(text))
+
+ @classmethod
+ def _from_text(cls, text):
+ return itertools.starmap(EntryPoint, cls._parse_groups(text or ''))
+
+ @staticmethod
+ def _parse_groups(text):
+ return (
+ (item.value.name, item.value.value, item.name)
+ for item in Sectioned.section_pairs(text)
+ )
+
+
+class Deprecated:
+ """
+ Compatibility add-in for mapping to indicate that
+ mapping behavior is deprecated.
+
+ >>> recwarn = getfixture('recwarn')
+ >>> class DeprecatedDict(Deprecated, dict): pass
+ >>> dd = DeprecatedDict(foo='bar')
+ >>> dd.get('baz', None)
+ >>> dd['foo']
+ 'bar'
+ >>> list(dd)
+ ['foo']
+ >>> list(dd.keys())
+ ['foo']
+ >>> 'foo' in dd
+ True
+ >>> list(dd.values())
+ ['bar']
+ >>> len(recwarn)
+ 1
+ """
+
+ _warn = functools.partial(
+ warnings.warn,
+ "SelectableGroups dict interface is deprecated. Use select.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ def __getitem__(self, name):
+ self._warn()
+ return super().__getitem__(name)
+
+ def get(self, name, default=None):
+ self._warn()
+ return super().get(name, default)
+
+ def __iter__(self):
+ self._warn()
+ return super().__iter__()
+
+ def __contains__(self, *args):
+ self._warn()
+ return super().__contains__(*args)
+
+ def keys(self):
+ self._warn()
+ return super().keys()
+
+ def values(self):
+ self._warn()
+ return super().values()
+
+
+class SelectableGroups(Deprecated, dict):
+ """
+ A backward- and forward-compatible result from
+ entry_points that fully implements the dict interface.
+ """
+
+ @classmethod
+ def load(cls, eps):
+ by_group = operator.attrgetter('group')
+ ordered = sorted(eps, key=by_group)
+ grouped = itertools.groupby(ordered, by_group)
+ return cls((group, EntryPoints(eps)) for group, eps in grouped)
+
+ @property
+ def _all(self):
+ """
+ Reconstruct a list of all entrypoints from the groups.
+ """
+ groups = super(Deprecated, self).values()
+ return EntryPoints(itertools.chain.from_iterable(groups))
+
+ @property
+ def groups(self):
+ return self._all.groups
+
+ @property
+ def names(self):
+ """
+ for coverage:
+ >>> SelectableGroups().names
+ set()
+ """
+ return self._all.names
+
+ def select(self, **params):
+ if not params:
+ return self
+ return self._all.select(**params)
+
+
+class PackagePath(pathlib.PurePosixPath):
+ """A reference to a path in a package"""
+
+ def read_text(self, encoding='utf-8'):
+ with self.locate().open(encoding=encoding) as stream:
+ return stream.read()
+
+ def read_binary(self):
+ with self.locate().open('rb') as stream:
+ return stream.read()
+
+ def locate(self):
+ """Return a path-like object for this path"""
+ return self.dist.locate_file(self)
+
+
+class FileHash:
+ def __init__(self, spec):
+ self.mode, _, self.value = spec.partition('=')
+
+ def __repr__(self):
+ return f'<FileHash mode: {self.mode} value: {self.value}>'
+
+
+class Distribution:
+ """A Python distribution package."""
+
+ @abc.abstractmethod
+ def read_text(self, filename):
+ """Attempt to load metadata file given by the name.
+
+ :param filename: The name of the file in the distribution info.
+ :return: The text if found, otherwise None.
+ """
+
+ @abc.abstractmethod
+ def locate_file(self, path):
+ """
+ Given a path to a file in this distribution, return a path
+ to it.
+ """
+
+ @classmethod
+ def from_name(cls, name):
+ """Return the Distribution for the given package name.
+
+ :param name: The name of the distribution package to search for.
+ :return: The Distribution instance (or subclass thereof) for the named
+ package, if found.
+ :raises PackageNotFoundError: When the named package's distribution
+ metadata cannot be found.
+ """
+ for resolver in cls._discover_resolvers():
+ dists = resolver(DistributionFinder.Context(name=name))
+ dist = next(iter(dists), None)
+ if dist is not None:
+ return dist
+ else:
+ raise PackageNotFoundError(name)
+
+ @classmethod
+ def discover(cls, **kwargs):
+ """Return an iterable of Distribution objects for all packages.
+
+ Pass a ``context`` or pass keyword arguments for constructing
+ a context.
+
+ :context: A ``DistributionFinder.Context`` object.
+ :return: Iterable of Distribution objects for all packages.
+ """
+ context = kwargs.pop('context', None)
+ if context and kwargs:
+ raise ValueError("cannot accept context and kwargs")
+ context = context or DistributionFinder.Context(**kwargs)
+ return itertools.chain.from_iterable(
+ resolver(context) for resolver in cls._discover_resolvers()
+ )
+
+ @staticmethod
+ def at(path):
+ """Return a Distribution for the indicated metadata path
+
+ :param path: a string or path-like object
+ :return: a concrete Distribution instance for the path
+ """
+ return PathDistribution(pathlib.Path(path))
+
+ @staticmethod
+ def _discover_resolvers():
+ """Search the meta_path for resolvers."""
+ declared = (
+ getattr(finder, 'find_distributions', None) for finder in sys.meta_path
+ )
+ return filter(None, declared)
+
+ @classmethod
+ def _local(cls, root='.'):
+ from pep517 import build, meta
+
+ system = build.compat_system(root)
+ builder = functools.partial(
+ meta.build,
+ source_dir=root,
+ system=system,
+ )
+ return PathDistribution(zipfile.Path(meta.build_as_zip(builder)))
+
+ @property
+ def metadata(self) -> _meta.PackageMetadata:
+ """Return the parsed metadata for this Distribution.
+
+ The returned object will have keys that name the various bits of
+ metadata. See PEP 566 for details.
+ """
+ text = (
+ self.read_text('METADATA')
+ or self.read_text('PKG-INFO')
+ # This last clause is here to support old egg-info files. Its
+ # effect is to just end up using the PathDistribution's self._path
+ # (which points to the egg-info file) attribute unchanged.
+ or self.read_text('')
+ )
+ return _adapters.Message(email.message_from_string(text))
+
+ @property
+ def name(self):
+ """Return the 'Name' metadata for the distribution package."""
+ return self.metadata['Name']
+
+ @property
+ def _normalized_name(self):
+ """Return a normalized version of the name."""
+ return Prepared.normalize(self.name)
+
+ @property
+ def version(self):
+ """Return the 'Version' metadata for the distribution package."""
+ return self.metadata['Version']
+
+ @property
+ def entry_points(self):
+ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self)
+
+ @property
+ def files(self):
+ """Files in this distribution.
+
+ :return: List of PackagePath for this distribution or None
+
+ Result is `None` if the metadata file that enumerates files
+ (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
+ missing.
+ Result may be empty if the metadata exists but is empty.
+ """
+ file_lines = self._read_files_distinfo() or self._read_files_egginfo()
+
+ def make_file(name, hash=None, size_str=None):
+ result = PackagePath(name)
+ result.hash = FileHash(hash) if hash else None
+ result.size = int(size_str) if size_str else None
+ result.dist = self
+ return result
+
+ return file_lines and list(starmap(make_file, csv.reader(file_lines)))
+
+ def _read_files_distinfo(self):
+ """
+ Read the lines of RECORD
+ """
+ text = self.read_text('RECORD')
+ return text and text.splitlines()
+
+ def _read_files_egginfo(self):
+ """
+ SOURCES.txt might contain literal commas, so wrap each line
+ in quotes.
+ """
+ text = self.read_text('SOURCES.txt')
+ return text and map('"{}"'.format, text.splitlines())
+
+ @property
+ def requires(self):
+ """Generated requirements specified for this Distribution"""
+ reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
+ return reqs and list(reqs)
+
+ def _read_dist_info_reqs(self):
+ return self.metadata.get_all('Requires-Dist')
+
+ def _read_egg_info_reqs(self):
+ source = self.read_text('requires.txt')
+ return None if source is None else self._deps_from_requires_text(source)
+
+ @classmethod
+ def _deps_from_requires_text(cls, source):
+ return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source))
+
+ @staticmethod
+ def _convert_egg_info_reqs_to_simple_reqs(sections):
+ """
+ Historically, setuptools would solicit and store 'extra'
+ requirements, including those with environment markers,
+ in separate sections. More modern tools expect each
+ dependency to be defined separately, with any relevant
+ extras and environment markers attached directly to that
+ requirement. This method converts the former to the
+ latter. See _test_deps_from_requires_text for an example.
+ """
+
+ def make_condition(name):
+ return name and f'extra == "{name}"'
+
+ def quoted_marker(section):
+ section = section or ''
+ extra, sep, markers = section.partition(':')
+ if extra and markers:
+ markers = f'({markers})'
+ conditions = list(filter(None, [markers, make_condition(extra)]))
+ return '; ' + ' and '.join(conditions) if conditions else ''
+
+ def url_req_space(req):
+ """
+ PEP 508 requires a space between the url_spec and the quoted_marker.
+ Ref python/importlib_metadata#357.
+ """
+ # '@' is uniquely indicative of a url_req.
+ return ' ' * ('@' in req)
+
+ for section in sections:
+ space = url_req_space(section.value)
+ yield section.value + space + quoted_marker(section.name)
+
+
+class DistributionFinder(MetaPathFinder):
+ """
+ A MetaPathFinder capable of discovering installed distributions.
+ """
+
+ class Context:
+ """
+ Keyword arguments presented by the caller to
+ ``distributions()`` or ``Distribution.discover()``
+ to narrow the scope of a search for distributions
+ in all DistributionFinders.
+
+ Each DistributionFinder may expect any parameters
+ and should attempt to honor the canonical
+ parameters defined below when appropriate.
+ """
+
+ name = None
+ """
+ Specific name for which a distribution finder should match.
+ A name of ``None`` matches all distributions.
+ """
+
+ def __init__(self, **kwargs):
+ vars(self).update(kwargs)
+
+ @property
+ def path(self):
+ """
+ The sequence of directory path that a distribution finder
+ should search.
+
+ Typically refers to Python installed package paths such as
+ "site-packages" directories and defaults to ``sys.path``.
+ """
+ return vars(self).get('path', sys.path)
+
+ @abc.abstractmethod
+ def find_distributions(self, context=Context()):
+ """
+ Find distributions.
+
+ Return an iterable of all Distribution instances capable of
+ loading the metadata for packages matching the ``context``,
+ a DistributionFinder.Context instance.
+ """
+
+
+class FastPath:
+ """
+ Micro-optimized class for searching a path for
+ children.
+ """
+
+ @functools.lru_cache() # type: ignore
+ def __new__(cls, root):
+ return super().__new__(cls)
+
+ def __init__(self, root):
+ self.root = root
+
+ def joinpath(self, child):
+ return pathlib.Path(self.root, child)
+
+ def children(self):
+ with suppress(Exception):
+ return os.listdir(self.root or '.')
+ with suppress(Exception):
+ return self.zip_children()
+ return []
+
+ def zip_children(self):
+ zip_path = zipfile.Path(self.root)
+ names = zip_path.root.namelist()
+ self.joinpath = zip_path.joinpath
+
+ return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names)
+
+ def search(self, name):
+ return self.lookup(self.mtime).search(name)
+
+ @property
+ def mtime(self):
+ with suppress(OSError):
+ return os.stat(self.root).st_mtime
+ self.lookup.cache_clear()
+
+ @method_cache
+ def lookup(self, mtime):
+ return Lookup(self)
+
+
+class Lookup:
+ def __init__(self, path: FastPath):
+ base = os.path.basename(path.root).lower()
+ base_is_egg = base.endswith(".egg")
+ self.infos = FreezableDefaultDict(list)
+ self.eggs = FreezableDefaultDict(list)
+
+ for child in path.children():
+ low = child.lower()
+ if low.endswith((".dist-info", ".egg-info")):
+ # rpartition is faster than splitext and suitable for this purpose.
+ name = low.rpartition(".")[0].partition("-")[0]
+ normalized = Prepared.normalize(name)
+ self.infos[normalized].append(path.joinpath(child))
+ elif base_is_egg and low == "egg-info":
+ name = base.rpartition(".")[0].partition("-")[0]
+ legacy_normalized = Prepared.legacy_normalize(name)
+ self.eggs[legacy_normalized].append(path.joinpath(child))
+
+ self.infos.freeze()
+ self.eggs.freeze()
+
+ def search(self, prepared):
+ infos = (
+ self.infos[prepared.normalized]
+ if prepared
+ else itertools.chain.from_iterable(self.infos.values())
+ )
+ eggs = (
+ self.eggs[prepared.legacy_normalized]
+ if prepared
+ else itertools.chain.from_iterable(self.eggs.values())
+ )
+ return itertools.chain(infos, eggs)
+
+
+class Prepared:
+ """
+ A prepared search for metadata on a possibly-named package.
+ """
+
+ normalized = None
+ legacy_normalized = None
+
+ def __init__(self, name):
+ self.name = name
+ if name is None:
+ return
+ self.normalized = self.normalize(name)
+ self.legacy_normalized = self.legacy_normalize(name)
+
+ @staticmethod
+ def normalize(name):
+ """
+ PEP 503 normalization plus dashes as underscores.
+ """
+ return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_')
+
+ @staticmethod
+ def legacy_normalize(name):
+ """
+ Normalize the package name as found in the convention in
+ older packaging tools versions and specs.
+ """
+ return name.lower().replace('-', '_')
+
+ def __bool__(self):
+ return bool(self.name)
+
+
+class MetadataPathFinder(DistributionFinder):
+ @classmethod
+ def find_distributions(cls, context=DistributionFinder.Context()):
+ """
+ Find distributions.
+
+ Return an iterable of all Distribution instances capable of
+ loading the metadata for packages matching ``context.name``
+ (or all names if ``None`` indicated) along the paths in the list
+ of directories ``context.path``.
+ """
+ found = cls._search_paths(context.name, context.path)
+ return map(PathDistribution, found)
+
+ @classmethod
+ def _search_paths(cls, name, paths):
+ """Find metadata directories in paths heuristically."""
+ prepared = Prepared(name)
+ return itertools.chain.from_iterable(
+ path.search(prepared) for path in map(FastPath, paths)
+ )
+
+ def invalidate_caches(cls):
+ FastPath.__new__.cache_clear()
+
+
+class PathDistribution(Distribution):
+ def __init__(self, path: SimplePath):
+ """Construct a distribution.
+
+ :param path: SimplePath indicating the metadata directory.
+ """
+ self._path = path
+
+ def read_text(self, filename):
+ with suppress(
+ FileNotFoundError,
+ IsADirectoryError,
+ KeyError,
+ NotADirectoryError,
+ PermissionError,
+ ):
+ return self._path.joinpath(filename).read_text(encoding='utf-8')
+
+ read_text.__doc__ = Distribution.read_text.__doc__
+
+ def locate_file(self, path):
+ return self._path.parent / path
+
+ @property
+ def _normalized_name(self):
+ """
+ Performance optimization: where possible, resolve the
+ normalized name from the file system path.
+ """
+ stem = os.path.basename(str(self._path))
+ return self._name_from_stem(stem) or super()._normalized_name
+
+ def _name_from_stem(self, stem):
+ name, ext = os.path.splitext(stem)
+ if ext not in ('.dist-info', '.egg-info'):
+ return
+ name, sep, rest = stem.partition('-')
+ return name
+
+
+def distribution(distribution_name):
+ """Get the ``Distribution`` instance for the named package.
+
+ :param distribution_name: The name of the distribution package as a string.
+ :return: A ``Distribution`` instance (or subclass thereof).
+ """
+ return Distribution.from_name(distribution_name)
+
+
+def distributions(**kwargs):
+ """Get all ``Distribution`` instances in the current environment.
+
+ :return: An iterable of ``Distribution`` instances.
+ """
+ return Distribution.discover(**kwargs)
+
+
+def metadata(distribution_name) -> _meta.PackageMetadata:
+ """Get the metadata for the named package.
+
+ :param distribution_name: The name of the distribution package to query.
+ :return: A PackageMetadata containing the parsed metadata.
+ """
+ return Distribution.from_name(distribution_name).metadata
+
+
+def version(distribution_name):
+ """Get the version string for the named package.
+
+ :param distribution_name: The name of the distribution package to query.
+ :return: The version string for the package as defined in the package's
+ "Version" metadata key.
+ """
+ return distribution(distribution_name).version
+
+
+def entry_points(**params) -> Union[EntryPoints, SelectableGroups]:
+ """Return EntryPoint objects for all installed packages.
+
+ Pass selection parameters (group or name) to filter the
+ result to entry points matching those properties (see
+ EntryPoints.select()).
+
+ For compatibility, returns ``SelectableGroups`` object unless
+ selection parameters are supplied. In the future, this function
+ will return ``EntryPoints`` instead of ``SelectableGroups``
+ even when no selection parameters are supplied.
+
+ For maximum future compatibility, pass selection parameters
+ or invoke ``.select`` with parameters on the result.
+
+ :return: EntryPoints or SelectableGroups for all installed packages.
+ """
+ norm_name = operator.attrgetter('_normalized_name')
+ unique = functools.partial(unique_everseen, key=norm_name)
+ eps = itertools.chain.from_iterable(
+ dist.entry_points for dist in unique(distributions())
+ )
+ return SelectableGroups.load(eps).select(**params)
+
+
+def files(distribution_name):
+ """Return a list of files for the named package.
+
+ :param distribution_name: The name of the distribution package to query.
+ :return: List of files composing the distribution.
+ """
+ return distribution(distribution_name).files
+
+
+def requires(distribution_name):
+ """
+ Return a list of requirements for the named package.
+
+ :return: An iterator of requirements, suitable for
+ packaging.requirement.Requirement.
+ """
+ return distribution(distribution_name).requires
+
+
+def packages_distributions() -> Mapping[str, List[str]]:
+ """
+ Return a mapping of top-level packages to their
+ distributions.
+
+ >>> import collections.abc
+ >>> pkgs = packages_distributions()
+ >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values())
+ True
+ """
+ pkg_to_dist = collections.defaultdict(list)
+ for dist in distributions():
+ for pkg in (dist.read_text('top_level.txt') or '').split():
+ pkg_to_dist[pkg].append(dist.metadata['Name'])
+ return dict(pkg_to_dist)
diff --git a/common/py3-stdlib/importlib/metadata/_adapters.py b/common/py3-stdlib/importlib/metadata/_adapters.py
new file mode 100644
index 0000000..aa460d3
--- /dev/null
+++ b/common/py3-stdlib/importlib/metadata/_adapters.py
@@ -0,0 +1,68 @@
+import re
+import textwrap
+import email.message
+
+from ._text import FoldedCase
+
+
+class Message(email.message.Message):
+ multiple_use_keys = set(
+ map(
+ FoldedCase,
+ [
+ 'Classifier',
+ 'Obsoletes-Dist',
+ 'Platform',
+ 'Project-URL',
+ 'Provides-Dist',
+ 'Provides-Extra',
+ 'Requires-Dist',
+ 'Requires-External',
+ 'Supported-Platform',
+ 'Dynamic',
+ ],
+ )
+ )
+ """
+ Keys that may be indicated multiple times per PEP 566.
+ """
+
+ def __new__(cls, orig: email.message.Message):
+ res = super().__new__(cls)
+ vars(res).update(vars(orig))
+ return res
+
+ def __init__(self, *args, **kwargs):
+ self._headers = self._repair_headers()
+
+ # suppress spurious error from mypy
+ def __iter__(self):
+ return super().__iter__()
+
+ def _repair_headers(self):
+ def redent(value):
+ "Correct for RFC822 indentation"
+ if not value or '\n' not in value:
+ return value
+ return textwrap.dedent(' ' * 8 + value)
+
+ headers = [(key, redent(value)) for key, value in vars(self)['_headers']]
+ if self._payload:
+ headers.append(('Description', self.get_payload()))
+ return headers
+
+ @property
+ def json(self):
+ """
+ Convert PackageMetadata to a JSON-compatible format
+ per PEP 0566.
+ """
+
+ def transform(key):
+ value = self.get_all(key) if key in self.multiple_use_keys else self[key]
+ if key == 'Keywords':
+ value = re.split(r'\s+', value)
+ tk = key.lower().replace('-', '_')
+ return tk, value
+
+ return dict(map(transform, map(FoldedCase, self)))
diff --git a/common/py3-stdlib/importlib/metadata/_collections.py b/common/py3-stdlib/importlib/metadata/_collections.py
new file mode 100644
index 0000000..cf0954e
--- /dev/null
+++ b/common/py3-stdlib/importlib/metadata/_collections.py
@@ -0,0 +1,30 @@
+import collections
+
+
+# from jaraco.collections 3.3
+class FreezableDefaultDict(collections.defaultdict):
+ """
+ Often it is desirable to prevent the mutation of
+ a default dict after its initial construction, such
+ as to prevent mutation during iteration.
+
+ >>> dd = FreezableDefaultDict(list)
+ >>> dd[0].append('1')
+ >>> dd.freeze()
+ >>> dd[1]
+ []
+ >>> len(dd)
+ 1
+ """
+
+ def __missing__(self, key):
+ return getattr(self, '_frozen', super().__missing__)(key)
+
+ def freeze(self):
+ self._frozen = lambda key: self.default_factory()
+
+
+class Pair(collections.namedtuple('Pair', 'name value')):
+ @classmethod
+ def parse(cls, text):
+ return cls(*map(str.strip, text.split("=", 1)))
diff --git a/common/py3-stdlib/importlib/metadata/_functools.py b/common/py3-stdlib/importlib/metadata/_functools.py
new file mode 100644
index 0000000..73f50d0
--- /dev/null
+++ b/common/py3-stdlib/importlib/metadata/_functools.py
@@ -0,0 +1,85 @@
+import types
+import functools
+
+
+# from jaraco.functools 3.3
+def method_cache(method, cache_wrapper=None):
+ """
+ Wrap lru_cache to support storing the cache data in the object instances.
+
+ Abstracts the common paradigm where the method explicitly saves an
+ underscore-prefixed protected property on first call and returns that
+ subsequently.
+
+ >>> class MyClass:
+ ... calls = 0
+ ...
+ ... @method_cache
+ ... def method(self, value):
+ ... self.calls += 1
+ ... return value
+
+ >>> a = MyClass()
+ >>> a.method(3)
+ 3
+ >>> for x in range(75):
+ ... res = a.method(x)
+ >>> a.calls
+ 75
+
+ Note that the apparent behavior will be exactly like that of lru_cache
+ except that the cache is stored on each instance, so values in one
+ instance will not flush values from another, and when an instance is
+ deleted, so are the cached values for that instance.
+
+ >>> b = MyClass()
+ >>> for x in range(35):
+ ... res = b.method(x)
+ >>> b.calls
+ 35
+ >>> a.method(0)
+ 0
+ >>> a.calls
+ 75
+
+ Note that if method had been decorated with ``functools.lru_cache()``,
+ a.calls would have been 76 (due to the cached value of 0 having been
+ flushed by the 'b' instance).
+
+ Clear the cache with ``.cache_clear()``
+
+ >>> a.method.cache_clear()
+
+ Same for a method that hasn't yet been called.
+
+ >>> c = MyClass()
+ >>> c.method.cache_clear()
+
+ Another cache wrapper may be supplied:
+
+ >>> cache = functools.lru_cache(maxsize=2)
+ >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache)
+ >>> a = MyClass()
+ >>> a.method2()
+ 3
+
+ Caution - do not subsequently wrap the method with another decorator, such
+ as ``@property``, which changes the semantics of the function.
+
+ See also
+ http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/
+ for another implementation and additional justification.
+ """
+ cache_wrapper = cache_wrapper or functools.lru_cache()
+
+ def wrapper(self, *args, **kwargs):
+ # it's the first call, replace the method with a cached, bound method
+ bound_method = types.MethodType(method, self)
+ cached_method = cache_wrapper(bound_method)
+ setattr(self, method.__name__, cached_method)
+ return cached_method(*args, **kwargs)
+
+ # Support cache clear even before cache has been created.
+ wrapper.cache_clear = lambda: None
+
+ return wrapper
diff --git a/common/py3-stdlib/importlib/metadata/_itertools.py b/common/py3-stdlib/importlib/metadata/_itertools.py
new file mode 100644
index 0000000..dd45f2f
--- /dev/null
+++ b/common/py3-stdlib/importlib/metadata/_itertools.py
@@ -0,0 +1,19 @@
+from itertools import filterfalse
+
+
+def unique_everseen(iterable, key=None):
+ "List unique elements, preserving order. Remember all elements ever seen."
+ # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+ # unique_everseen('ABBCcAD', str.lower) --> A B C D
+ seen = set()
+ seen_add = seen.add
+ if key is None:
+ for element in filterfalse(seen.__contains__, iterable):
+ seen_add(element)
+ yield element
+ else:
+ for element in iterable:
+ k = key(element)
+ if k not in seen:
+ seen_add(k)
+ yield element
diff --git a/common/py3-stdlib/importlib/metadata/_meta.py b/common/py3-stdlib/importlib/metadata/_meta.py
new file mode 100644
index 0000000..1a6edbf
--- /dev/null
+++ b/common/py3-stdlib/importlib/metadata/_meta.py
@@ -0,0 +1,47 @@
+from typing import Any, Dict, Iterator, List, Protocol, TypeVar, Union
+
+
+_T = TypeVar("_T")
+
+
+class PackageMetadata(Protocol):
+ def __len__(self) -> int:
+ ... # pragma: no cover
+
+ def __contains__(self, item: str) -> bool:
+ ... # pragma: no cover
+
+ def __getitem__(self, key: str) -> str:
+ ... # pragma: no cover
+
+ def __iter__(self) -> Iterator[str]:
+ ... # pragma: no cover
+
+ def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]:
+ """
+ Return all values associated with a possibly multi-valued key.
+ """
+
+ @property
+ def json(self) -> Dict[str, Union[str, List[str]]]:
+ """
+ A JSON-compatible form of the metadata.
+ """
+
+
+class SimplePath(Protocol):
+ """
+ A minimal subset of pathlib.Path required by PathDistribution.
+ """
+
+ def joinpath(self) -> 'SimplePath':
+ ... # pragma: no cover
+
+ def __div__(self) -> 'SimplePath':
+ ... # pragma: no cover
+
+ def parent(self) -> 'SimplePath':
+ ... # pragma: no cover
+
+ def read_text(self) -> str:
+ ... # pragma: no cover
diff --git a/common/py3-stdlib/importlib/metadata/_text.py b/common/py3-stdlib/importlib/metadata/_text.py
new file mode 100644
index 0000000..766979d
--- /dev/null
+++ b/common/py3-stdlib/importlib/metadata/_text.py
@@ -0,0 +1,99 @@
+import re
+
+from ._functools import method_cache
+
+
+# from jaraco.text 3.5
+class FoldedCase(str):
+ """
+ A case insensitive string class; behaves just like str
+ except compares equal when the only variation is case.
+
+ >>> s = FoldedCase('hello world')
+
+ >>> s == 'Hello World'
+ True
+
+ >>> 'Hello World' == s
+ True
+
+ >>> s != 'Hello World'
+ False
+
+ >>> s.index('O')
+ 4
+
+ >>> s.split('O')
+ ['hell', ' w', 'rld']
+
+ >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta']))
+ ['alpha', 'Beta', 'GAMMA']
+
+ Sequence membership is straightforward.
+
+ >>> "Hello World" in [s]
+ True
+ >>> s in ["Hello World"]
+ True
+
+ You may test for set inclusion, but candidate and elements
+ must both be folded.
+
+ >>> FoldedCase("Hello World") in {s}
+ True
+ >>> s in {FoldedCase("Hello World")}
+ True
+
+ String inclusion works as long as the FoldedCase object
+ is on the right.
+
+ >>> "hello" in FoldedCase("Hello World")
+ True
+
+ But not if the FoldedCase object is on the left:
+
+ >>> FoldedCase('hello') in 'Hello World'
+ False
+
+ In that case, use in_:
+
+ >>> FoldedCase('hello').in_('Hello World')
+ True
+
+ >>> FoldedCase('hello') > FoldedCase('Hello')
+ False
+ """
+
+ def __lt__(self, other):
+ return self.lower() < other.lower()
+
+ def __gt__(self, other):
+ return self.lower() > other.lower()
+
+ def __eq__(self, other):
+ return self.lower() == other.lower()
+
+ def __ne__(self, other):
+ return self.lower() != other.lower()
+
+ def __hash__(self):
+ return hash(self.lower())
+
+ def __contains__(self, other):
+ return super(FoldedCase, self).lower().__contains__(other.lower())
+
+ def in_(self, other):
+ "Does self appear in other?"
+ return self in FoldedCase(other)
+
+ # cache lower since it's likely to be called frequently.
+ @method_cache
+ def lower(self):
+ return super(FoldedCase, self).lower()
+
+ def index(self, sub):
+ return self.lower().index(sub.lower())
+
+ def split(self, splitter=' ', maxsplit=0):
+ pattern = re.compile(re.escape(splitter), re.I)
+ return pattern.split(self, maxsplit)
diff --git a/common/py3-stdlib/importlib/readers.py b/common/py3-stdlib/importlib/readers.py
new file mode 100644
index 0000000..41089c0
--- /dev/null
+++ b/common/py3-stdlib/importlib/readers.py
@@ -0,0 +1,123 @@
+import collections
+import zipfile
+import pathlib
+from . import abc
+
+
+def remove_duplicates(items):
+ return iter(collections.OrderedDict.fromkeys(items))
+
+
+class FileReader(abc.TraversableResources):
+ def __init__(self, loader):
+ self.path = pathlib.Path(loader.path).parent
+
+ def resource_path(self, resource):
+ """
+ Return the file system path to prevent
+ `resources.path()` from creating a temporary
+ copy.
+ """
+ return str(self.path.joinpath(resource))
+
+ def files(self):
+ return self.path
+
+
+class ZipReader(abc.TraversableResources):
+ def __init__(self, loader, module):
+ _, _, name = module.rpartition('.')
+ self.prefix = loader.prefix.replace('\\', '/') + name + '/'
+ self.archive = loader.archive
+
+ def open_resource(self, resource):
+ try:
+ return super().open_resource(resource)
+ except KeyError as exc:
+ raise FileNotFoundError(exc.args[0])
+
+ def is_resource(self, path):
+ # workaround for `zipfile.Path.is_file` returning true
+ # for non-existent paths.
+ target = self.files().joinpath(path)
+ return target.is_file() and target.exists()
+
+ def files(self):
+ return zipfile.Path(self.archive, self.prefix)
+
+
+class MultiplexedPath(abc.Traversable):
+ """
+ Given a series of Traversable objects, implement a merged
+ version of the interface across all objects. Useful for
+ namespace packages which may be multihomed at a single
+ name.
+ """
+
+ def __init__(self, *paths):
+ self._paths = list(map(pathlib.Path, remove_duplicates(paths)))
+ if not self._paths:
+ message = 'MultiplexedPath must contain at least one path'
+ raise FileNotFoundError(message)
+ if not all(path.is_dir() for path in self._paths):
+ raise NotADirectoryError('MultiplexedPath only supports directories')
+
+ def iterdir(self):
+ visited = []
+ for path in self._paths:
+ for file in path.iterdir():
+ if file.name in visited:
+ continue
+ visited.append(file.name)
+ yield file
+
+ def read_bytes(self):
+ raise FileNotFoundError(f'{self} is not a file')
+
+ def read_text(self, *args, **kwargs):
+ raise FileNotFoundError(f'{self} is not a file')
+
+ def is_dir(self):
+ return True
+
+ def is_file(self):
+ return False
+
+ def joinpath(self, child):
+ # first try to find child in current paths
+ for file in self.iterdir():
+ if file.name == child:
+ return file
+ # if it does not exist, construct it with the first path
+ return self._paths[0] / child
+
+ __truediv__ = joinpath
+
+ def open(self, *args, **kwargs):
+ raise FileNotFoundError(f'{self} is not a file')
+
+ @property
+ def name(self):
+ return self._paths[0].name
+
+ def __repr__(self):
+ paths = ', '.join(f"'{path}'" for path in self._paths)
+ return f'MultiplexedPath({paths})'
+
+
+class NamespaceReader(abc.TraversableResources):
+ def __init__(self, namespace_path):
+ if 'NamespacePath' not in str(namespace_path):
+ raise ValueError('Invalid path')
+ self.path = MultiplexedPath(*list(namespace_path))
+
+ def resource_path(self, resource):
+ """
+ Return the file system path to prevent
+ `resources.path()` from creating a temporary
+ copy.
+ """
+ return str(self.path.joinpath(resource))
+
+ def files(self):
+ return self.path
diff --git a/common/py3-stdlib/importlib/resources.py b/common/py3-stdlib/importlib/resources.py
index b803a01..8a98663 100644
--- a/common/py3-stdlib/importlib/resources.py
+++ b/common/py3-stdlib/importlib/resources.py
@@ -1,22 +1,26 @@
import os
+import io
-from . import abc as resources_abc
from . import _common
-from ._common import as_file
-from contextlib import contextmanager, suppress
-from importlib import import_module
+from ._common import as_file, files
+from .abc import ResourceReader
+from contextlib import suppress
from importlib.abc import ResourceLoader
+from importlib.machinery import ModuleSpec
from io import BytesIO, TextIOWrapper
from pathlib import Path
from types import ModuleType
-from typing import ContextManager, Iterable, Optional, Union
+from typing import ContextManager, Iterable, Union
from typing import cast
from typing.io import BinaryIO, TextIO
+from collections.abc import Sequence
+from functools import singledispatch
__all__ = [
'Package',
'Resource',
+ 'ResourceReader',
'as_file',
'contents',
'files',
@@ -26,99 +30,57 @@
'path',
'read_binary',
'read_text',
- ]
+]
Package = Union[str, ModuleType]
Resource = Union[str, os.PathLike]
-def _resolve(name) -> ModuleType:
- """If name is a string, resolve to a module."""
- if hasattr(name, '__spec__'):
- return name
- return import_module(name)
-
-
-def _get_package(package) -> ModuleType:
- """Take a package name or module object and return the module.
-
- If a name, the module is imported. If the resolved module
- object is not a package, raise an exception.
- """
- module = _resolve(package)
- if module.__spec__.submodule_search_locations is None:
- raise TypeError('{!r} is not a package'.format(package))
- return module
-
-
-def _normalize_path(path) -> str:
- """Normalize a path by ensuring it is a string.
-
- If the resulting string contains path separators, an exception is raised.
- """
- parent, file_name = os.path.split(path)
- if parent:
- raise ValueError('{!r} must be only a file name'.format(path))
- return file_name
-
-
-def _get_resource_reader(
- package: ModuleType) -> Optional[resources_abc.ResourceReader]:
- # Return the package's loader if it's a ResourceReader. We can't use
- # a issubclass() check here because apparently abc.'s __subclasscheck__()
- # hook wants to create a weak reference to the object, but
- # zipimport.zipimporter does not support weak references, resulting in a
- # TypeError. That seems terrible.
- spec = package.__spec__
- if hasattr(spec.loader, 'get_resource_reader'):
- return cast(resources_abc.ResourceReader,
- spec.loader.get_resource_reader(spec.name))
- return None
-
-
-def _check_location(package):
- if package.__spec__.origin is None or not package.__spec__.has_location:
- raise FileNotFoundError(f'Package has no location {package!r}')
-
-
def open_binary(package: Package, resource: Resource) -> BinaryIO:
"""Return a file-like object opened for binary reading of the resource."""
- resource = _normalize_path(resource)
- package = _get_package(package)
- reader = _get_resource_reader(package)
+ resource = _common.normalize_path(resource)
+ package = _common.get_package(package)
+ reader = _common.get_resource_reader(package)
if reader is not None:
return reader.open_resource(resource)
- absolute_package_path = os.path.abspath(
- package.__spec__.origin or 'non-existent file')
- package_path = os.path.dirname(absolute_package_path)
- full_path = os.path.join(package_path, resource)
- try:
- return open(full_path, mode='rb')
- except OSError:
- # Just assume the loader is a resource loader; all the relevant
- # importlib.machinery loaders are and an AttributeError for
- # get_data() will make it clear what is needed from the loader.
- loader = cast(ResourceLoader, package.__spec__.loader)
- data = None
- if hasattr(package.__spec__.loader, 'get_data'):
- with suppress(OSError):
- data = loader.get_data(full_path)
- if data is None:
- package_name = package.__spec__.name
- message = '{!r} resource not found in {!r}'.format(
- resource, package_name)
- raise FileNotFoundError(message)
- return BytesIO(data)
+ spec = cast(ModuleSpec, package.__spec__)
+ # Using pathlib doesn't work well here due to the lack of 'strict'
+ # argument for pathlib.Path.resolve() prior to Python 3.6.
+ if spec.submodule_search_locations is not None:
+ paths = spec.submodule_search_locations
+ elif spec.origin is not None:
+ paths = [os.path.dirname(os.path.abspath(spec.origin))]
+
+ for package_path in paths:
+ full_path = os.path.join(package_path, resource)
+ try:
+ return open(full_path, mode='rb')
+ except OSError:
+ # Just assume the loader is a resource loader; all the relevant
+ # importlib.machinery loaders are and an AttributeError for
+ # get_data() will make it clear what is needed from the loader.
+ loader = cast(ResourceLoader, spec.loader)
+ data = None
+ if hasattr(spec.loader, 'get_data'):
+ with suppress(OSError):
+ data = loader.get_data(full_path)
+ if data is not None:
+ return BytesIO(data)
+
+ raise FileNotFoundError(f'{resource!r} resource not found in {spec.name!r}')
-def open_text(package: Package,
- resource: Resource,
- encoding: str = 'utf-8',
- errors: str = 'strict') -> TextIO:
+def open_text(
+ package: Package,
+ resource: Resource,
+ encoding: str = 'utf-8',
+ errors: str = 'strict',
+) -> TextIO:
"""Return a file-like object opened for text reading of the resource."""
return TextIOWrapper(
- open_binary(package, resource), encoding=encoding, errors=errors)
+ open_binary(package, resource), encoding=encoding, errors=errors
+ )
def read_binary(package: Package, resource: Resource) -> bytes:
@@ -127,10 +89,12 @@
return fp.read()
-def read_text(package: Package,
- resource: Resource,
- encoding: str = 'utf-8',
- errors: str = 'strict') -> str:
+def read_text(
+ package: Package,
+ resource: Resource,
+ encoding: str = 'utf-8',
+ errors: str = 'strict',
+) -> str:
"""Return the decoded string of the resource.
The decoding-related arguments have the same semantics as those of
@@ -140,16 +104,10 @@
return fp.read()
-def files(package: Package) -> resources_abc.Traversable:
- """
- Get a Traversable resource from a package
- """
- return _common.from_package(_get_package(package))
-
-
def path(
- package: Package, resource: Resource,
- ) -> 'ContextManager[Path]':
+ package: Package,
+ resource: Resource,
+) -> 'ContextManager[Path]':
"""A context manager providing a file path object to the resource.
If the resource does not already exist on its own on the file system,
@@ -158,23 +116,30 @@
raised if the file was deleted prior to the context manager
exiting).
"""
- reader = _get_resource_reader(_get_package(package))
+ reader = _common.get_resource_reader(_common.get_package(package))
return (
- _path_from_reader(reader, resource)
- if reader else
- _common.as_file(files(package).joinpath(_normalize_path(resource)))
+ _path_from_reader(reader, _common.normalize_path(resource))
+ if reader
+ else _common.as_file(
+ _common.files(package).joinpath(_common.normalize_path(resource))
)
+ )
-@contextmanager
def _path_from_reader(reader, resource):
- norm_resource = _normalize_path(resource)
+ return _path_from_resource_path(reader, resource) or _path_from_open_resource(
+ reader, resource
+ )
+
+
+def _path_from_resource_path(reader, resource):
with suppress(FileNotFoundError):
- yield Path(reader.resource_path(norm_resource))
- return
- opener_reader = reader.open_resource(norm_resource)
- with _common._tempfile(opener_reader.read, suffix=norm_resource) as res:
- yield res
+ return Path(reader.resource_path(resource))
+
+
+def _path_from_open_resource(reader, resource):
+ saved = io.BytesIO(reader.open_resource(resource).read())
+ return _common._tempfile(saved.read, suffix=resource)
def is_resource(package: Package, name: str) -> bool:
@@ -182,9 +147,9 @@
Directories are *not* resources.
"""
- package = _get_package(package)
- _normalize_path(name)
- reader = _get_resource_reader(package)
+ package = _common.get_package(package)
+ _common.normalize_path(name)
+ reader = _common.get_resource_reader(package)
if reader is not None:
return reader.is_resource(name)
package_contents = set(contents(package))
@@ -200,16 +165,21 @@
not considered resources. Use `is_resource()` on each entry returned here
to check if it is a resource or not.
"""
- package = _get_package(package)
- reader = _get_resource_reader(package)
+ package = _common.get_package(package)
+ reader = _common.get_resource_reader(package)
if reader is not None:
- return reader.contents()
- # Is the package a namespace package? By definition, namespace packages
- # cannot have resources.
- namespace = (
- package.__spec__.origin is None or
- package.__spec__.origin == 'namespace'
- )
- if namespace or not package.__spec__.has_location:
- return ()
- return list(item.name for item in _common.from_package(package).iterdir())
+ return _ensure_sequence(reader.contents())
+ transversable = _common.from_package(package)
+ if transversable.is_dir():
+ return list(item.name for item in transversable.iterdir())
+ return []
+
+
+@singledispatch
+def _ensure_sequence(iterable):
+ return list(iterable)
+
+
+@_ensure_sequence.register(Sequence)
+def _(iterable):
+ return iterable
diff --git a/common/py3-stdlib/importlib/util.py b/common/py3-stdlib/importlib/util.py
index 269a6fa..8623c89 100644
--- a/common/py3-stdlib/importlib/util.py
+++ b/common/py3-stdlib/importlib/util.py
@@ -1,5 +1,5 @@
"""Utility code for constructing importers, etc."""
-from . import abc
+from ._abc import Loader
from ._bootstrap import module_from_spec
from ._bootstrap import _resolve_name
from ._bootstrap import spec_from_loader
@@ -149,7 +149,8 @@
"""
@functools.wraps(fxn)
def set_package_wrapper(*args, **kwargs):
- warnings.warn('The import system now takes care of this automatically.',
+ warnings.warn('The import system now takes care of this automatically; '
+ 'this decorator is slated for removal in Python 3.12',
DeprecationWarning, stacklevel=2)
module = fxn(*args, **kwargs)
if getattr(module, '__package__', None) is None:
@@ -168,7 +169,8 @@
"""
@functools.wraps(fxn)
def set_loader_wrapper(self, *args, **kwargs):
- warnings.warn('The import system now takes care of this automatically.',
+ warnings.warn('The import system now takes care of this automatically; '
+ 'this decorator is slated for removal in Python 3.12',
DeprecationWarning, stacklevel=2)
module = fxn(self, *args, **kwargs)
if getattr(module, '__loader__', None) is None:
@@ -195,7 +197,8 @@
the second argument.
"""
- warnings.warn('The import system now takes care of this automatically.',
+ warnings.warn('The import system now takes care of this automatically; '
+ 'this decorator is slated for removal in Python 3.12',
DeprecationWarning, stacklevel=2)
@functools.wraps(fxn)
def module_for_loader_wrapper(self, fullname, *args, **kwargs):
@@ -232,7 +235,6 @@
# Figure out exactly what attributes were mutated between the creation
# of the module and now.
attrs_then = self.__spec__.loader_state['__dict__']
- original_type = self.__spec__.loader_state['__class__']
attrs_now = self.__dict__
attrs_updated = {}
for key, value in attrs_now.items():
@@ -263,7 +265,7 @@
delattr(self, attr)
-class LazyLoader(abc.Loader):
+class LazyLoader(Loader):
"""A loader that creates a module which defers loading until attribute access."""
diff --git a/common/py3-stdlib/inspect.py b/common/py3-stdlib/inspect.py
index 18bed90..c5881cc 100644
--- a/common/py3-stdlib/inspect.py
+++ b/common/py3-stdlib/inspect.py
@@ -24,6 +24,8 @@
stack(), trace() - get info about frames on the stack or in a traceback
signature() - get a Signature object for the callable
+
+ get_annotations() - safely compute an object's annotations
"""
# This module is in the public domain. No warranties.
@@ -60,6 +62,122 @@
# See Include/object.h
TPFLAGS_IS_ABSTRACT = 1 << 20
+
+def get_annotations(obj, *, globals=None, locals=None, eval_str=False):
+ """Compute the annotations dict for an object.
+
+ obj may be a callable, class, or module.
+ Passing in an object of any other type raises TypeError.
+
+ Returns a dict. get_annotations() returns a new dict every time
+ it's called; calling it twice on the same object will return two
+ different but equivalent dicts.
+
+ This function handles several details for you:
+
+ * If eval_str is true, values of type str will
+ be un-stringized using eval(). This is intended
+ for use with stringized annotations
+ ("from __future__ import annotations").
+ * If obj doesn't have an annotations dict, returns an
+ empty dict. (Functions and methods always have an
+ annotations dict; classes, modules, and other types of
+ callables may not.)
+ * Ignores inherited annotations on classes. If a class
+ doesn't have its own annotations dict, returns an empty dict.
+ * All accesses to object members and dict values are done
+ using getattr() and dict.get() for safety.
+ * Always, always, always returns a freshly-created dict.
+
+ eval_str controls whether or not values of type str are replaced
+ with the result of calling eval() on those values:
+
+ * If eval_str is true, eval() is called on values of type str.
+ * If eval_str is false (the default), values of type str are unchanged.
+
+ globals and locals are passed in to eval(); see the documentation
+ for eval() for more information. If either globals or locals is
+ None, this function may replace that value with a context-specific
+ default, contingent on type(obj):
+
+ * If obj is a module, globals defaults to obj.__dict__.
+ * If obj is a class, globals defaults to
+ sys.modules[obj.__module__].__dict__ and locals
+ defaults to the obj class namespace.
+ * If obj is a callable, globals defaults to obj.__globals__,
+ although if obj is a wrapped function (using
+ functools.update_wrapper()) it is first unwrapped.
+ """
+ if isinstance(obj, type):
+ # class
+ obj_dict = getattr(obj, '__dict__', None)
+ if obj_dict and hasattr(obj_dict, 'get'):
+ ann = obj_dict.get('__annotations__', None)
+ if isinstance(ann, types.GetSetDescriptorType):
+ ann = None
+ else:
+ ann = None
+
+ obj_globals = None
+ module_name = getattr(obj, '__module__', None)
+ if module_name:
+ module = sys.modules.get(module_name, None)
+ if module:
+ obj_globals = getattr(module, '__dict__', None)
+ obj_locals = dict(vars(obj))
+ unwrap = obj
+ elif isinstance(obj, types.ModuleType):
+ # module
+ ann = getattr(obj, '__annotations__', None)
+ obj_globals = getattr(obj, '__dict__')
+ obj_locals = None
+ unwrap = None
+ elif callable(obj):
+ # this includes types.Function, types.BuiltinFunctionType,
+ # types.BuiltinMethodType, functools.partial, functools.singledispatch,
+ # "class funclike" from Lib/test/test_inspect... on and on it goes.
+ ann = getattr(obj, '__annotations__', None)
+ obj_globals = getattr(obj, '__globals__', None)
+ obj_locals = None
+ unwrap = obj
+ else:
+ raise TypeError(f"{obj!r} is not a module, class, or callable.")
+
+ if ann is None:
+ return {}
+
+ if not isinstance(ann, dict):
+ raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None")
+
+ if not ann:
+ return {}
+
+ if not eval_str:
+ return dict(ann)
+
+ if unwrap is not None:
+ while True:
+ if hasattr(unwrap, '__wrapped__'):
+ unwrap = unwrap.__wrapped__
+ continue
+ if isinstance(unwrap, functools.partial):
+ unwrap = unwrap.func
+ continue
+ break
+ if hasattr(unwrap, "__globals__"):
+ obj_globals = unwrap.__globals__
+
+ if globals is None:
+ globals = obj_globals
+ if locals is None:
+ locals = obj_locals
+
+ return_value = {key:
+ value if not isinstance(value, str) else eval(value, globals, locals)
+ for key, value in ann.items() }
+ return return_value
+
+
# ----------------------------------------------------------- type-checking
def ismodule(object):
"""Return true if the object is a module.
@@ -277,7 +395,7 @@
co_kwonlyargcount number of keyword only arguments (not including ** arg)
co_lnotab encoded mapping of line numbers to bytecode indices
co_name name with which this code object was defined
- co_names tuple of names of local variables
+ co_names tuple of names other than arguments and function locals
co_nlocals number of local variables
co_stacksize virtual machine stack space required
co_varnames tuple of names of arguments and local variables"""
@@ -407,7 +525,7 @@
# attribute with the same name as a DynamicClassAttribute exists.
for base in mro:
for k, v in base.__dict__.items():
- if isinstance(v, types.DynamicClassAttribute):
+ if isinstance(v, types.DynamicClassAttribute) and v.fget is not None:
names.append(k)
result = []
processed = set()
@@ -663,6 +781,8 @@
module = sys.modules.get(object.__module__)
if getattr(module, '__file__', None):
return module.__file__
+ if object.__module__ == '__main__':
+ raise OSError('source code not available')
raise TypeError('{!r} is a built-in class'.format(object))
if ismethod(object):
object = object.__func__
@@ -706,10 +826,13 @@
if os.path.exists(filename):
return filename
# only return a non-existent filename if the module has a PEP 302 loader
- if getattr(getmodule(object, filename), '__loader__', None) is not None:
+ module = getmodule(object, filename)
+ if getattr(module, '__loader__', None) is not None:
+ return filename
+ elif getattr(getattr(module, "__spec__", None), "loader", None) is not None:
return filename
# or it is in the linecache
- if filename in linecache.cache:
+ elif filename in linecache.cache:
return filename
def getabsfile(object, _filename=None):
@@ -736,7 +859,7 @@
# Try the cache again with the absolute file name
try:
file = getabsfile(object, _filename)
- except TypeError:
+ except (TypeError, FileNotFoundError):
return None
if file in modulesbyfile:
return sys.modules.get(modulesbyfile[file])
@@ -1162,7 +1285,8 @@
sig = _signature_from_callable(func,
follow_wrapper_chains=False,
skip_bound_arg=False,
- sigcls=Signature)
+ sigcls=Signature,
+ eval_str=False)
except Exception as ex:
# Most of the times 'signature' will raise ValueError.
# But, it can also raise AttributeError, and, maybe something
@@ -1233,6 +1357,8 @@
def formatannotation(annotation, base_module=None):
if getattr(annotation, '__module__', None) == 'typing':
return repr(annotation).replace('typing.', '')
+ if isinstance(annotation, types.GenericAlias):
+ return str(annotation)
if isinstance(annotation, type):
if annotation.__module__ in ('builtins', base_module):
return annotation.__qualname__
@@ -1895,7 +2021,7 @@
isinstance(name, str) and
(defaults is None or isinstance(defaults, tuple)) and
(kwdefaults is None or isinstance(kwdefaults, dict)) and
- isinstance(annotations, dict))
+ (isinstance(annotations, (dict)) or annotations is None) )
def _signature_get_bound_param(spec):
@@ -2147,7 +2273,8 @@
return _signature_fromstr(cls, func, s, skip_bound_arg)
-def _signature_from_function(cls, func, skip_bound_arg=True):
+def _signature_from_function(cls, func, skip_bound_arg=True,
+ globals=None, locals=None, eval_str=False):
"""Private helper: constructs Signature for the given python function."""
is_duck_function = False
@@ -2173,7 +2300,7 @@
positional = arg_names[:pos_count]
keyword_only_count = func_code.co_kwonlyargcount
keyword_only = arg_names[pos_count:pos_count + keyword_only_count]
- annotations = func.__annotations__
+ annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str)
defaults = func.__defaults__
kwdefaults = func.__kwdefaults__
@@ -2244,23 +2371,30 @@
def _signature_from_callable(obj, *,
follow_wrapper_chains=True,
skip_bound_arg=True,
+ globals=None,
+ locals=None,
+ eval_str=False,
sigcls):
"""Private helper function to get signature for arbitrary
callable objects.
"""
+ _get_signature_of = functools.partial(_signature_from_callable,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ globals=globals,
+ locals=locals,
+ sigcls=sigcls,
+ eval_str=eval_str)
+
if not callable(obj):
raise TypeError('{!r} is not a callable object'.format(obj))
if isinstance(obj, types.MethodType):
# In this case we skip the first parameter of the underlying
# function (usually `self` or `cls`).
- sig = _signature_from_callable(
- obj.__func__,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
+ sig = _get_signature_of(obj.__func__)
if skip_bound_arg:
return _signature_bound_method(sig)
@@ -2274,11 +2408,7 @@
# If the unwrapped object is a *method*, we might want to
# skip its first parameter (self).
# See test_signature_wrapped_bound_method for details.
- return _signature_from_callable(
- obj,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
+ return _get_signature_of(obj)
try:
sig = obj.__signature__
@@ -2305,11 +2435,7 @@
# (usually `self`, or `cls`) will not be passed
# automatically (as for boundmethods)
- wrapped_sig = _signature_from_callable(
- partialmethod.func,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
+ wrapped_sig = _get_signature_of(partialmethod.func)
sig = _signature_get_partial(wrapped_sig, partialmethod, (None,))
first_wrapped_param = tuple(wrapped_sig.parameters.values())[0]
@@ -2328,18 +2454,15 @@
# If it's a pure Python function, or an object that is duck type
# of a Python function (Cython functions, for instance), then:
return _signature_from_function(sigcls, obj,
- skip_bound_arg=skip_bound_arg)
+ skip_bound_arg=skip_bound_arg,
+ globals=globals, locals=locals, eval_str=eval_str)
if _signature_is_builtin(obj):
return _signature_from_builtin(sigcls, obj,
skip_bound_arg=skip_bound_arg)
if isinstance(obj, functools.partial):
- wrapped_sig = _signature_from_callable(
- obj.func,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
+ wrapped_sig = _get_signature_of(obj.func)
return _signature_get_partial(wrapped_sig, obj)
sig = None
@@ -2350,29 +2473,25 @@
# in its metaclass
call = _signature_get_user_defined_method(type(obj), '__call__')
if call is not None:
- sig = _signature_from_callable(
- call,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
+ sig = _get_signature_of(call)
else:
- # Now we check if the 'obj' class has a '__new__' method
+ factory_method = None
new = _signature_get_user_defined_method(obj, '__new__')
- if new is not None:
- sig = _signature_from_callable(
- new,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
- else:
- # Finally, we should have at least __init__ implemented
- init = _signature_get_user_defined_method(obj, '__init__')
- if init is not None:
- sig = _signature_from_callable(
- init,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
+ init = _signature_get_user_defined_method(obj, '__init__')
+ # Now we check if the 'obj' class has an own '__new__' method
+ if '__new__' in obj.__dict__:
+ factory_method = new
+ # or an own '__init__' method
+ elif '__init__' in obj.__dict__:
+ factory_method = init
+ # If not, we take inherited '__new__' or '__init__', if present
+ elif new is not None:
+ factory_method = new
+ elif init is not None:
+ factory_method = init
+
+ if factory_method is not None:
+ sig = _get_signature_of(factory_method)
if sig is None:
# At this point we know, that `obj` is a class, with no user-
@@ -2392,9 +2511,9 @@
pass
else:
if text_sig:
- # If 'obj' class has a __text_signature__ attribute:
+ # If 'base' class has a __text_signature__ attribute:
# return a signature based on it
- return _signature_fromstr(sigcls, obj, text_sig)
+ return _signature_fromstr(sigcls, base, text_sig)
# No '__text_signature__' was found for the 'obj' class.
# Last option is to check if its '__init__' is
@@ -2418,11 +2537,7 @@
call = _signature_get_user_defined_method(type(obj), '__call__')
if call is not None:
try:
- sig = _signature_from_callable(
- call,
- follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg,
- sigcls=sigcls)
+ sig = _get_signature_of(call)
except ValueError as ex:
msg = 'no signature found for {!r}'.format(obj)
raise ValueError(msg) from ex
@@ -2874,10 +2989,12 @@
return _signature_from_builtin(cls, func)
@classmethod
- def from_callable(cls, obj, *, follow_wrapped=True):
+ def from_callable(cls, obj, *,
+ follow_wrapped=True, globals=None, locals=None, eval_str=False):
"""Constructs Signature for the given callable object."""
return _signature_from_callable(obj, sigcls=cls,
- follow_wrapper_chains=follow_wrapped)
+ follow_wrapper_chains=follow_wrapped,
+ globals=globals, locals=locals, eval_str=eval_str)
@property
def parameters(self):
@@ -3125,9 +3242,10 @@
return rendered
-def signature(obj, *, follow_wrapped=True):
+def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False):
"""Get a signature object for the passed callable."""
- return Signature.from_callable(obj, follow_wrapped=follow_wrapped)
+ return Signature.from_callable(obj, follow_wrapped=follow_wrapped,
+ globals=globals, locals=locals, eval_str=eval_str)
def _main():
diff --git a/common/py3-stdlib/io.py b/common/py3-stdlib/io.py
index fbce6ef..2a6140c 100644
--- a/common/py3-stdlib/io.py
+++ b/common/py3-stdlib/io.py
@@ -54,9 +54,24 @@
from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
BufferedWriter, BufferedRWPair, BufferedRandom,
- IncrementalNewlineDecoder, TextIOWrapper)
+ IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
-OpenWrapper = _io.open # for compatibility with _pyio
+
+def __getattr__(name):
+ if name == "OpenWrapper":
+ # bpo-43680: Until Python 3.9, _pyio.open was not a static method and
+ # builtins.open was set to OpenWrapper to not become a bound method
+ # when set to a class variable. _io.open is a built-in function whereas
+ # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now
+ # a static method, and builtins.open() is now io.open().
+ import warnings
+ warnings.warn('OpenWrapper is deprecated, use open instead',
+ DeprecationWarning, stacklevel=2)
+ global OpenWrapper
+ OpenWrapper = open
+ return OpenWrapper
+ raise AttributeError(name)
+
# Pretend this exception was created here.
UnsupportedOperation.__module__ = "io"
diff --git a/common/py3-stdlib/ipaddress.py b/common/py3-stdlib/ipaddress.py
index bc662c4..4a6496a 100644
--- a/common/py3-stdlib/ipaddress.py
+++ b/common/py3-stdlib/ipaddress.py
@@ -16,6 +16,7 @@
IPV4LENGTH = 32
IPV6LENGTH = 128
+
class AddressValueError(ValueError):
"""A Value Error related to the address."""
@@ -1214,7 +1215,7 @@
"""
if not octet_str:
raise ValueError("Empty octet not permitted")
- # Whitelist the characters, since int() allows a lot of bizarre stuff.
+ # Reject non-ASCII digits.
if not (octet_str.isascii() and octet_str.isdigit()):
msg = "Only decimal digits permitted in %r"
raise ValueError(msg % octet_str)
@@ -1223,6 +1224,11 @@
if len(octet_str) > 3:
msg = "At most 3 characters permitted in %r"
raise ValueError(msg % octet_str)
+ # Handle leading zeros as strict as glibc's inet_pton()
+ # See security bug bpo-36384
+ if octet_str != '0' and octet_str[0] == '0':
+ msg = "Leading zeros are not permitted in %r"
+ raise ValueError(msg % octet_str)
# Convert to integer (we know digits are legal)
octet_int = int(octet_str, 10)
if octet_int > 255:
@@ -1719,7 +1725,7 @@
[0..FFFF].
"""
- # Whitelist the characters, since int() allows a lot of bizarre stuff.
+ # Reject non-ASCII digits.
if not cls._HEX_DIGITS.issuperset(hextet_str):
raise ValueError("Only hex digits permitted in %r" % hextet_str)
# We do the length check second, since the invalid character error
@@ -1997,9 +2003,13 @@
Returns:
A boolean, True if the address is reserved per
- iana-ipv6-special-registry.
+ iana-ipv6-special-registry, or is ipv4_mapped and is
+ reserved in the iana-ipv4-special-registry.
"""
+ ipv4_mapped = self.ipv4_mapped
+ if ipv4_mapped is not None:
+ return ipv4_mapped.is_private
return any(self in net for net in self._constants._private_networks)
@property
diff --git a/common/py3-stdlib/json/__init__.py b/common/py3-stdlib/json/__init__.py
index 2c52bde..e4c21da 100644
--- a/common/py3-stdlib/json/__init__.py
+++ b/common/py3-stdlib/json/__init__.py
@@ -133,7 +133,7 @@
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
- result in an ``OverflowError`` (or worse).
+ result in an ``RecursionError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
@@ -195,7 +195,7 @@
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
- result in an ``OverflowError`` (or worse).
+ result in an ``RecursionError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
diff --git a/common/py3-stdlib/json/encoder.py b/common/py3-stdlib/json/encoder.py
index c8c78b9..21bff2c 100644
--- a/common/py3-stdlib/json/encoder.py
+++ b/common/py3-stdlib/json/encoder.py
@@ -116,7 +116,7 @@
If check_circular is true, then lists, dicts, and custom encoded
objects will be checked for circular references during encoding to
- prevent an infinite recursion (which would cause an OverflowError).
+ prevent an infinite recursion (which would cause an RecursionError).
Otherwise, no such check takes place.
If allow_nan is true, then NaN, Infinity, and -Infinity will be
diff --git a/common/py3-stdlib/json/tool.py b/common/py3-stdlib/json/tool.py
index 5dee0a7..0490b8c 100644
--- a/common/py3-stdlib/json/tool.py
+++ b/common/py3-stdlib/json/tool.py
@@ -13,6 +13,7 @@
import argparse
import json
import sys
+from pathlib import Path
def main():
@@ -25,9 +26,9 @@
help='a JSON file to be validated or pretty-printed',
default=sys.stdin)
parser.add_argument('outfile', nargs='?',
- type=argparse.FileType('w', encoding="utf-8"),
+ type=Path,
help='write the output of infile to outfile',
- default=sys.stdout)
+ default=None)
parser.add_argument('--sort-keys', action='store_true', default=False,
help='sort the output of dictionaries alphabetically by key')
parser.add_argument('--no-ensure-ascii', dest='ensure_ascii', action='store_false',
@@ -58,15 +59,21 @@
dump_args['indent'] = None
dump_args['separators'] = ',', ':'
- with options.infile as infile, options.outfile as outfile:
+ with options.infile as infile:
try:
if options.json_lines:
objs = (json.loads(line) for line in infile)
else:
- objs = (json.load(infile), )
- for obj in objs:
- json.dump(obj, outfile, **dump_args)
- outfile.write('\n')
+ objs = (json.load(infile),)
+
+ if options.outfile is None:
+ out = sys.stdout
+ else:
+ out = options.outfile.open('w', encoding='utf-8')
+ with out as outfile:
+ for obj in objs:
+ json.dump(obj, outfile, **dump_args)
+ outfile.write('\n')
except ValueError as e:
raise SystemExit(e)
diff --git a/common/py3-stdlib/keyword.py b/common/py3-stdlib/keyword.py
index 59fcfb0..cc2b46b 100644
--- a/common/py3-stdlib/keyword.py
+++ b/common/py3-stdlib/keyword.py
@@ -6,7 +6,7 @@
the python source tree and run:
PYTHONPATH=Tools/peg_generator python3 -m pegen.keywordgen \
- Grammar/Grammar \
+ Grammar/python.gram \
Grammar/Tokens \
Lib/keyword.py
@@ -19,7 +19,6 @@
'False',
'None',
'True',
- '__peg_parser__',
'and',
'as',
'assert',
@@ -55,7 +54,9 @@
]
softkwlist = [
-
+ '_',
+ 'case',
+ 'match'
]
iskeyword = frozenset(kwlist).__contains__
diff --git a/common/py3-stdlib/linecache.py b/common/py3-stdlib/linecache.py
index fa5dbd0..23191d6 100644
--- a/common/py3-stdlib/linecache.py
+++ b/common/py3-stdlib/linecache.py
@@ -154,7 +154,7 @@
:return: True if a lazy load is registered in the cache,
otherwise False. To register such a load a module loader with a
- get_source method must be found, the filename must be a cachable
+ get_source method must be found, the filename must be a cacheable
filename, and the filename must not be already cached.
"""
if filename in cache:
@@ -165,9 +165,14 @@
if not filename or (filename.startswith('<') and filename.endswith('>')):
return False
# Try for a __loader__, if available
- if module_globals and '__loader__' in module_globals:
- name = module_globals.get('__name__')
- loader = module_globals['__loader__']
+ if module_globals and '__name__' in module_globals:
+ name = module_globals['__name__']
+ if (loader := module_globals.get('__loader__')) is None:
+ if spec := module_globals.get('__spec__'):
+ try:
+ loader = spec.loader
+ except AttributeError:
+ pass
get_source = getattr(loader, 'get_source', None)
if name and get_source:
diff --git a/common/py3-stdlib/locale.py b/common/py3-stdlib/locale.py
index 1a4e9f6..6d4f519 100644
--- a/common/py3-stdlib/locale.py
+++ b/common/py3-stdlib/locale.py
@@ -185,8 +185,14 @@
formatted = percent % ((value,) + additional)
else:
formatted = percent % value
+ if percent[-1] in 'eEfFgGdiu':
+ formatted = _localize(formatted, grouping, monetary)
+ return formatted
+
+# Transform formatted as locale number according to the locale settings
+def _localize(formatted, grouping=False, monetary=False):
# floats and decimal ints need special action!
- if percent[-1] in 'eEfFgG':
+ if '.' in formatted:
seps = 0
parts = formatted.split('.')
if grouping:
@@ -196,7 +202,7 @@
formatted = decimal_point.join(parts)
if seps:
formatted = _strip_padding(formatted, seps)
- elif percent[-1] in 'diu':
+ else:
seps = 0
if grouping:
formatted, seps = _group(formatted, monetary=monetary)
@@ -267,7 +273,7 @@
raise ValueError("Currency formatting is not possible using "
"the 'C' locale.")
- s = _format('%%.%if' % digits, abs(val), grouping, monetary=True)
+ s = _localize(f'{abs(val):.{digits}f}', grouping, monetary=True)
# '<' and '>' are markers if the sign must be inserted between symbol and value
s = '<' + s + '>'
@@ -323,6 +329,10 @@
string = string.replace(dd, '.')
return string
+def localize(string, grouping=False, monetary=False):
+ """Parses a string as locale number according to the locale settings."""
+ return _localize(string, grouping, monetary)
+
def atof(string, func=float):
"Parses a string as a float according to the locale settings."
return func(delocalize(string))
@@ -619,53 +629,49 @@
"""
_setlocale(category, _build_localename(getdefaultlocale()))
-if sys.platform.startswith("win"):
- # On Win32, this will return the ANSI code page
- def getpreferredencoding(do_setlocale = True):
- """Return the charset that the user is likely using."""
- if sys.flags.utf8_mode:
- return 'UTF-8'
- import _bootlocale
- return _bootlocale.getpreferredencoding(False)
-else:
- # On Unix, if CODESET is available, use that.
- try:
- CODESET
- except NameError:
+
+try:
+ from _locale import _get_locale_encoding
+except ImportError:
+ def _get_locale_encoding():
if hasattr(sys, 'getandroidapilevel'):
# On Android langinfo.h and CODESET are missing, and UTF-8 is
# always used in mbstowcs() and wcstombs().
- def getpreferredencoding(do_setlocale = True):
- return 'UTF-8'
- else:
- # Fall back to parsing environment variables :-(
- def getpreferredencoding(do_setlocale = True):
- """Return the charset that the user is likely using,
- by looking at environment variables."""
- if sys.flags.utf8_mode:
- return 'UTF-8'
- res = getdefaultlocale()[1]
- if res is None:
- # LANG not set, default conservatively to ASCII
- res = 'ascii'
- return res
- else:
- def getpreferredencoding(do_setlocale = True):
- """Return the charset that the user is likely using,
- according to the system configuration."""
- if sys.flags.utf8_mode:
- return 'UTF-8'
- import _bootlocale
- if do_setlocale:
- oldloc = setlocale(LC_CTYPE)
- try:
- setlocale(LC_CTYPE, "")
- except Error:
- pass
- result = _bootlocale.getpreferredencoding(False)
- if do_setlocale:
- setlocale(LC_CTYPE, oldloc)
- return result
+ return 'UTF-8'
+ if sys.flags.utf8_mode:
+ return 'UTF-8'
+ encoding = getdefaultlocale()[1]
+ if encoding is None:
+ # LANG not set, default conservatively to ASCII
+ encoding = 'ascii'
+ return encoding
+
+try:
+ CODESET
+except NameError:
+ def getpreferredencoding(do_setlocale=True):
+ """Return the charset that the user is likely using."""
+ return _get_locale_encoding()
+else:
+ # On Unix, if CODESET is available, use that.
+ def getpreferredencoding(do_setlocale=True):
+ """Return the charset that the user is likely using,
+ according to the system configuration."""
+ if sys.flags.utf8_mode:
+ return 'UTF-8'
+
+ if not do_setlocale:
+ return _get_locale_encoding()
+
+ old_loc = setlocale(LC_CTYPE)
+ try:
+ try:
+ setlocale(LC_CTYPE, "")
+ except Error:
+ pass
+ return _get_locale_encoding()
+ finally:
+ setlocale(LC_CTYPE, old_loc)
### Database
diff --git a/common/py3-stdlib/logging/__init__.py b/common/py3-stdlib/logging/__init__.py
index 7b169a1..19bd2bc 100644
--- a/common/py3-stdlib/logging/__init__.py
+++ b/common/py3-stdlib/logging/__init__.py
@@ -118,7 +118,7 @@
def getLevelName(level):
"""
- Return the textual representation of logging level 'level'.
+ Return the textual or numeric representation of logging level 'level'.
If the level is one of the predefined levels (CRITICAL, ERROR, WARNING,
INFO, DEBUG) then you get the corresponding string. If you have
@@ -128,7 +128,11 @@
If a numeric value corresponding to one of the defined levels is passed
in, the corresponding string representation is returned.
- Otherwise, the string "Level %s" % level is returned.
+ If a string representation of the level is passed in, the corresponding
+ numeric value is returned.
+
+ If no matching numeric or string value is passed in, the string
+ 'Level %s' % level is returned.
"""
# See Issues #22386, #27937 and #29220 for why it's this way
result = _levelToName.get(level)
@@ -194,7 +198,8 @@
raise ValueError("Unknown level: %r" % level)
rv = _nameToLevel[level]
else:
- raise TypeError("Level not an integer or a valid string: %r" % level)
+ raise TypeError("Level not an integer or a valid string: %r"
+ % (level,))
return rv
#---------------------------------------------------------------------------
@@ -411,8 +416,9 @@
asctime_search = '%(asctime)'
validation_pattern = re.compile(r'%\(\w+\)[#0+ -]*(\*|\d+)?(\.(\*|\d+))?[diouxefgcrsa%]', re.I)
- def __init__(self, fmt):
+ def __init__(self, fmt, *, defaults=None):
self._fmt = fmt or self.default_format
+ self._defaults = defaults
def usesTime(self):
return self._fmt.find(self.asctime_search) >= 0
@@ -423,7 +429,11 @@
raise ValueError("Invalid format '%s' for '%s' style" % (self._fmt, self.default_format[0]))
def _format(self, record):
- return self._fmt % record.__dict__
+ if defaults := self._defaults:
+ values = defaults | record.__dict__
+ else:
+ values = record.__dict__
+ return self._fmt % values
def format(self, record):
try:
@@ -441,7 +451,11 @@
field_spec = re.compile(r'^(\d+|\w+)(\.\w+|\[[^]]+\])*$')
def _format(self, record):
- return self._fmt.format(**record.__dict__)
+ if defaults := self._defaults:
+ values = defaults | record.__dict__
+ else:
+ values = record.__dict__
+ return self._fmt.format(**values)
def validate(self):
"""Validate the input format, ensure it is the correct string formatting style"""
@@ -467,8 +481,8 @@
asctime_format = '${asctime}'
asctime_search = '${asctime}'
- def __init__(self, fmt):
- self._fmt = fmt or self.default_format
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
self._tpl = Template(self._fmt)
def usesTime(self):
@@ -490,7 +504,11 @@
raise ValueError('invalid format: no fields')
def _format(self, record):
- return self._tpl.substitute(**record.__dict__)
+ if defaults := self._defaults:
+ values = defaults | record.__dict__
+ else:
+ values = record.__dict__
+ return self._tpl.substitute(**values)
BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s"
@@ -546,7 +564,8 @@
converter = time.localtime
- def __init__(self, fmt=None, datefmt=None, style='%', validate=True):
+ def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *,
+ defaults=None):
"""
Initialize the formatter with specified format strings.
@@ -565,7 +584,7 @@
if style not in _STYLES:
raise ValueError('Style must be one of: %s' % ','.join(
_STYLES.keys()))
- self._style = _STYLES[style][0](fmt)
+ self._style = _STYLES[style][0](fmt, defaults=defaults)
if validate:
self._style.validate()
@@ -859,6 +878,7 @@
self._name = None
self.level = _checkLevel(level)
self.formatter = None
+ self._closed = False
# Add the handler to the global _handlerList (for cleanup on shutdown)
_addHandlerRef(self)
self.createLock()
@@ -977,6 +997,7 @@
#get the module data lock, as we're updating a shared structure.
_acquireLock()
try: #unlikely to raise an exception, but you never know...
+ self._closed = True
if self._name and self._name in _handlers:
del _handlers[self._name]
finally:
@@ -1131,8 +1152,14 @@
self.baseFilename = os.path.abspath(filename)
self.mode = mode
self.encoding = encoding
+ if "b" not in mode:
+ self.encoding = io.text_encoding(encoding)
self.errors = errors
self.delay = delay
+ # bpo-26789: FileHandler keeps a reference to the builtin open()
+ # function to be able to open or reopen the file during Python
+ # finalization.
+ self._builtin_open = open
if delay:
#We don't open the stream, but we still need to call the
#Handler constructor to set level, formatter, lock etc.
@@ -1159,6 +1186,8 @@
finally:
# Issue #19523: call unconditionally to
# prevent a handler leak when delay is set
+ # Also see Issue #42378: we also rely on
+ # self._closed being set to True there
StreamHandler.close(self)
finally:
self.release()
@@ -1168,8 +1197,9 @@
Open the current base file with the (original) mode and encoding.
Return the resulting stream.
"""
- return open(self.baseFilename, self.mode, encoding=self.encoding,
- errors=self.errors)
+ open_func = self._builtin_open
+ return open_func(self.baseFilename, self.mode,
+ encoding=self.encoding, errors=self.errors)
def emit(self, record):
"""
@@ -1177,10 +1207,15 @@
If the stream was not opened because 'delay' was specified in the
constructor, open it before calling the superclass's emit.
+
+ If stream is not open, current mode is 'w' and `_closed=True`, record
+ will not be emitted (see Issue #42378).
"""
if self.stream is None:
- self.stream = self._open()
- StreamHandler.emit(self, record)
+ if self.mode != 'w' or not self._closed:
+ self.stream = self._open()
+ if self.stream:
+ StreamHandler.emit(self, record)
def __repr__(self):
level = getLevelName(self.level)
@@ -1269,6 +1304,14 @@
self.loggerClass = None
self.logRecordFactory = None
+ @property
+ def disable(self):
+ return self._disable
+
+ @disable.setter
+ def disable(self, value):
+ self._disable = _checkLevel(value)
+
def getLogger(self, name):
"""
Get a logger with the specified name (channel name), creating it
@@ -1480,7 +1523,11 @@
if self.isEnabledFor(CRITICAL):
self._log(CRITICAL, msg, args, **kwargs)
- fatal = critical
+ def fatal(self, msg, *args, **kwargs):
+ """
+ Don't use this method, use critical() instead.
+ """
+ self.critical(msg, *args, **kwargs)
def log(self, level, msg, *args, **kwargs):
"""
@@ -1751,7 +1798,7 @@
information in logging output.
"""
- def __init__(self, logger, extra):
+ def __init__(self, logger, extra=None):
"""
Initialize the adapter with a logger and a dict-like object which
provides contextual information. This constructor signature allows
@@ -1986,8 +2033,10 @@
filename = kwargs.pop("filename", None)
mode = kwargs.pop("filemode", 'a')
if filename:
- if 'b'in mode:
+ if 'b' in mode:
errors = None
+ else:
+ encoding = io.text_encoding(encoding)
h = FileHandler(filename, mode,
encoding=encoding, errors=errors)
else:
@@ -2039,7 +2088,11 @@
basicConfig()
root.critical(msg, *args, **kwargs)
-fatal = critical
+def fatal(msg, *args, **kwargs):
+ """
+ Don't use this function, use critical() instead.
+ """
+ critical(msg, *args, **kwargs)
def error(msg, *args, **kwargs):
"""
diff --git a/common/py3-stdlib/logging/config.py b/common/py3-stdlib/logging/config.py
index fd3aded..3bc63b7 100644
--- a/common/py3-stdlib/logging/config.py
+++ b/common/py3-stdlib/logging/config.py
@@ -48,7 +48,7 @@
# _listener holds the server object doing the listening
_listener = None
-def fileConfig(fname, defaults=None, disable_existing_loggers=True):
+def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=None):
"""
Read the logging configuration from a ConfigParser-format file.
@@ -66,7 +66,8 @@
if hasattr(fname, 'readline'):
cp.read_file(fname)
else:
- cp.read(fname)
+ encoding = io.text_encoding(encoding)
+ cp.read(fname, encoding=encoding)
formatters = _create_formatters(cp)
diff --git a/common/py3-stdlib/logging/handlers.py b/common/py3-stdlib/logging/handlers.py
index 867ef4e..61a3995 100644
--- a/common/py3-stdlib/logging/handlers.py
+++ b/common/py3-stdlib/logging/handlers.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2021 by Vinay Sajip. All Rights Reserved.
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted,
@@ -18,12 +18,12 @@
Additional handlers for the logging package for Python. The core package is
based on PEP 282 and comments thereto in comp.lang.python.
-Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2021 Vinay Sajip. All Rights Reserved.
To use, simply 'import logging.handlers' and log away!
"""
-import logging, socket, os, pickle, struct, time, re
+import io, logging, socket, os, pickle, struct, time, re
from stat import ST_DEV, ST_INO, ST_MTIME
import queue
import threading
@@ -150,6 +150,8 @@
# on each run.
if maxBytes > 0:
mode = 'a'
+ if "b" not in mode:
+ encoding = io.text_encoding(encoding)
BaseRotatingHandler.__init__(self, filename, mode, encoding=encoding,
delay=delay, errors=errors)
self.maxBytes = maxBytes
@@ -185,14 +187,17 @@
Basically, see if the supplied record would cause the file to exceed
the size limit we have.
"""
+ # See bpo-45401: Never rollover anything other than regular files
+ if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+ return False
if self.stream is None: # delay was set...
self.stream = self._open()
if self.maxBytes > 0: # are we rolling over?
msg = "%s\n" % self.format(record)
self.stream.seek(0, 2) #due to non-posix-compliant Windows feature
if self.stream.tell() + len(msg) >= self.maxBytes:
- return 1
- return 0
+ return True
+ return False
class TimedRotatingFileHandler(BaseRotatingHandler):
"""
@@ -205,6 +210,7 @@
def __init__(self, filename, when='h', interval=1, backupCount=0,
encoding=None, delay=False, utc=False, atTime=None,
errors=None):
+ encoding = io.text_encoding(encoding)
BaseRotatingHandler.__init__(self, filename, 'a', encoding=encoding,
delay=delay, errors=errors)
self.when = when.upper()
@@ -342,10 +348,13 @@
record is not used, as we are just comparing times, but it is needed so
the method signatures are the same
"""
+ # See bpo-45401: Never rollover anything other than regular files
+ if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+ return False
t = int(time.time())
if t >= self.rolloverAt:
- return 1
- return 0
+ return True
+ return False
def getFilesToDelete(self):
"""
@@ -356,13 +365,32 @@
dirName, baseName = os.path.split(self.baseFilename)
fileNames = os.listdir(dirName)
result = []
- prefix = baseName + "."
+ # See bpo-44753: Don't use the extension when computing the prefix.
+ n, e = os.path.splitext(baseName)
+ prefix = n + '.'
plen = len(prefix)
for fileName in fileNames:
+ if self.namer is None:
+ # Our files will always start with baseName
+ if not fileName.startswith(baseName):
+ continue
+ else:
+ # Our files could be just about anything after custom naming, but
+ # likely candidates are of the form
+ # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log
+ if (not fileName.startswith(baseName) and fileName.endswith(e) and
+ len(fileName) > (plen + 1) and not fileName[plen+1].isdigit()):
+ continue
+
if fileName[:plen] == prefix:
suffix = fileName[plen:]
- if self.extMatch.match(suffix):
- result.append(os.path.join(dirName, fileName))
+ # See bpo-45628: The date/time suffix could be anywhere in the
+ # filename
+ parts = suffix.split('.')
+ for part in parts:
+ if self.extMatch.match(part):
+ result.append(os.path.join(dirName, fileName))
+ break
if len(result) < self.backupCount:
result = []
else:
@@ -442,6 +470,8 @@
"""
def __init__(self, filename, mode='a', encoding=None, delay=False,
errors=None):
+ if "b" not in mode:
+ encoding = io.text_encoding(encoding)
logging.FileHandler.__init__(self, filename, mode=mode,
encoding=encoding, delay=delay,
errors=errors)
@@ -1142,7 +1172,7 @@
class HTTPHandler(logging.Handler):
"""
- A class which sends records to a Web server, using either GET or
+ A class which sends records to a web server, using either GET or
POST semantics.
"""
def __init__(self, host, url, method="GET", secure=False, credentials=None,
@@ -1191,7 +1221,7 @@
"""
Emit a record.
- Send the record to the Web server as a percent-encoded dictionary
+ Send the record to the web server as a percent-encoded dictionary
"""
try:
import urllib.parse
diff --git a/common/py3-stdlib/lzma.py b/common/py3-stdlib/lzma.py
index 0817b87..800f521 100644
--- a/common/py3-stdlib/lzma.py
+++ b/common/py3-stdlib/lzma.py
@@ -225,14 +225,22 @@
"""Write a bytes object to the file.
Returns the number of uncompressed bytes written, which is
- always len(data). Note that due to buffering, the file on disk
- may not reflect the data written until close() is called.
+ always the length of data in bytes. Note that due to buffering,
+ the file on disk may not reflect the data written until close()
+ is called.
"""
self._check_can_write()
+ if isinstance(data, (bytes, bytearray)):
+ length = len(data)
+ else:
+ # accept any data that supports the buffer protocol
+ data = memoryview(data)
+ length = data.nbytes
+
compressed = self._compressor.compress(data)
self._fp.write(compressed)
- self._pos += len(data)
- return len(data)
+ self._pos += length
+ return length
def seek(self, offset, whence=io.SEEK_SET):
"""Change the file position.
@@ -302,6 +310,7 @@
preset=preset, filters=filters)
if "t" in mode:
+ encoding = io.text_encoding(encoding)
return io.TextIOWrapper(binary_file, encoding, errors, newline)
else:
return binary_file
diff --git a/common/py3-stdlib/mimetypes.py b/common/py3-stdlib/mimetypes.py
index 92c2a47..b72ce08 100644
--- a/common/py3-stdlib/mimetypes.py
+++ b/common/py3-stdlib/mimetypes.py
@@ -27,6 +27,12 @@
import sys
import posixpath
import urllib.parse
+
+try:
+ from _winapi import _mimetypes_read_windows_registry
+except ImportError:
+ _mimetypes_read_windows_registry = None
+
try:
import winreg as _winreg
except ImportError:
@@ -135,25 +141,23 @@
type = 'text/plain'
return type, None # never compressed, so encoding is None
base, ext = posixpath.splitext(url)
- while ext in self.suffix_map:
- base, ext = posixpath.splitext(base + self.suffix_map[ext])
+ while (ext_lower := ext.lower()) in self.suffix_map:
+ base, ext = posixpath.splitext(base + self.suffix_map[ext_lower])
+ # encodings_map is case sensitive
if ext in self.encodings_map:
encoding = self.encodings_map[ext]
base, ext = posixpath.splitext(base)
else:
encoding = None
+ ext = ext.lower()
types_map = self.types_map[True]
if ext in types_map:
return types_map[ext], encoding
- elif ext.lower() in types_map:
- return types_map[ext.lower()], encoding
elif strict:
return None, encoding
types_map = self.types_map[False]
if ext in types_map:
return types_map[ext], encoding
- elif ext.lower() in types_map:
- return types_map[ext.lower()], encoding
else:
return None, encoding
@@ -169,7 +173,7 @@
but non-standard types.
"""
type = type.lower()
- extensions = self.types_map_inv[True].get(type, [])
+ extensions = list(self.types_map_inv[True].get(type, []))
if not strict:
for ext in self.types_map_inv[False].get(type, []):
if ext not in extensions:
@@ -237,10 +241,21 @@
types.
"""
- # Windows only
- if not _winreg:
+ if not _mimetypes_read_windows_registry and not _winreg:
return
+ add_type = self.add_type
+ if strict:
+ add_type = lambda type, ext: self.add_type(type, ext, True)
+
+ # Accelerated function if it is available
+ if _mimetypes_read_windows_registry:
+ _mimetypes_read_windows_registry(add_type)
+ elif _winreg:
+ self._read_windows_registry(add_type)
+
+ @classmethod
+ def _read_windows_registry(cls, add_type):
def enum_types(mimedb):
i = 0
while True:
@@ -265,7 +280,7 @@
subkey, 'Content Type')
if datatype != _winreg.REG_SZ:
continue
- self.add_type(mimetype, subkeyname, strict)
+ add_type(mimetype, subkeyname)
except OSError:
continue
@@ -349,8 +364,8 @@
if files is None or _db is None:
db = MimeTypes()
- if _winreg:
- db.read_windows_registry()
+ # Quick return if not supported
+ db.read_windows_registry()
if files is None:
files = knownfiles
@@ -448,6 +463,7 @@
'.dvi' : 'application/x-dvi',
'.gtar' : 'application/x-gtar',
'.hdf' : 'application/x-hdf',
+ '.h5' : 'application/x-hdf5',
'.latex' : 'application/x-latex',
'.mif' : 'application/x-mif',
'.cdf' : 'application/x-netcdf',
@@ -480,10 +496,19 @@
'.wsdl' : 'application/xml',
'.xpdl' : 'application/xml',
'.zip' : 'application/zip',
+ '.3gp' : 'audio/3gpp',
+ '.3gpp' : 'audio/3gpp',
+ '.3g2' : 'audio/3gpp2',
+ '.3gpp2' : 'audio/3gpp2',
+ '.aac' : 'audio/aac',
+ '.adts' : 'audio/aac',
+ '.loas' : 'audio/aac',
+ '.ass' : 'audio/aac',
'.au' : 'audio/basic',
'.snd' : 'audio/basic',
'.mp3' : 'audio/mpeg',
'.mp2' : 'audio/mpeg',
+ '.opus' : 'audio/opus',
'.aif' : 'audio/x-aiff',
'.aifc' : 'audio/x-aiff',
'.aiff' : 'audio/x-aiff',
@@ -495,6 +520,8 @@
'.jpg' : 'image/jpeg',
'.jpe' : 'image/jpeg',
'.jpeg' : 'image/jpeg',
+ '.heic' : 'image/heic',
+ '.heif' : 'image/heif',
'.png' : 'image/png',
'.svg' : 'image/svg+xml',
'.tiff' : 'image/tiff',
diff --git a/common/py3-stdlib/multiprocessing/managers.py b/common/py3-stdlib/multiprocessing/managers.py
index 0eb16c6..b6b4cdd 100644
--- a/common/py3-stdlib/multiprocessing/managers.py
+++ b/common/py3-stdlib/multiprocessing/managers.py
@@ -8,8 +8,7 @@
# Licensed to PSF under a Contributor Agreement.
#
-__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token',
- 'SharedMemoryManager' ]
+__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
#
# Imports
@@ -35,9 +34,11 @@
from . import get_context
try:
from . import shared_memory
- HAS_SHMEM = True
except ImportError:
HAS_SHMEM = False
+else:
+ HAS_SHMEM = True
+ __all__.append('SharedMemoryManager')
#
# Register some things for pickling
@@ -192,11 +193,8 @@
t.daemon = True
t.start()
- def handle_request(self, c):
- '''
- Handle a new connection
- '''
- funcname = result = request = None
+ def _handle_request(self, c):
+ request = None
try:
connection.deliver_challenge(c, self.authkey)
connection.answer_challenge(c, self.authkey)
@@ -213,6 +211,7 @@
msg = ('#TRACEBACK', format_exc())
else:
msg = ('#RETURN', result)
+
try:
c.send(msg)
except Exception as e:
@@ -224,7 +223,17 @@
util.info(' ... request was %r', request)
util.info(' ... exception was %r', e)
- c.close()
+ def handle_request(self, conn):
+ '''
+ Handle a new connection
+ '''
+ try:
+ self._handle_request(conn)
+ except SystemExit:
+ # Server.serve_client() calls sys.exit(0) on EOF
+ pass
+ finally:
+ conn.close()
def serve_client(self, conn):
'''
@@ -959,7 +968,7 @@
def AutoProxy(token, serializer, manager=None, authkey=None,
- exposed=None, incref=True):
+ exposed=None, incref=True, manager_owned=False):
'''
Return an auto-proxy for `token`
'''
@@ -979,7 +988,7 @@
ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
- incref=incref)
+ incref=incref, manager_owned=manager_owned)
proxy._isauto = True
return proxy
diff --git a/common/py3-stdlib/multiprocessing/resource_tracker.py b/common/py3-stdlib/multiprocessing/resource_tracker.py
index c9bfa9b..cc42dbd 100644
--- a/common/py3-stdlib/multiprocessing/resource_tracker.py
+++ b/common/py3-stdlib/multiprocessing/resource_tracker.py
@@ -37,8 +37,16 @@
import _multiprocessing
import _posixshmem
+ # Use sem_unlink() to clean up named semaphores.
+ #
+ # sem_unlink() may be missing if the Python build process detected the
+ # absence of POSIX named semaphores. In that case, no named semaphores were
+ # ever opened, so no cleanup would be necessary.
+ if hasattr(_multiprocessing, 'sem_unlink'):
+ _CLEANUP_FUNCS.update({
+ 'semaphore': _multiprocessing.sem_unlink,
+ })
_CLEANUP_FUNCS.update({
- 'semaphore': _multiprocessing.sem_unlink,
'shared_memory': _posixshmem.shm_unlink,
})
diff --git a/common/py3-stdlib/multiprocessing/util.py b/common/py3-stdlib/multiprocessing/util.py
index 21f2a7e..a468333 100644
--- a/common/py3-stdlib/multiprocessing/util.py
+++ b/common/py3-stdlib/multiprocessing/util.py
@@ -419,7 +419,7 @@
try:
fd = os.open(os.devnull, os.O_RDONLY)
try:
- sys.stdin = open(fd, closefd=False)
+ sys.stdin = open(fd, encoding="utf-8", closefd=False)
except:
os.close(fd)
raise
diff --git a/common/py3-stdlib/netrc.py b/common/py3-stdlib/netrc.py
index f0ae48c..734d94c 100644
--- a/common/py3-stdlib/netrc.py
+++ b/common/py3-stdlib/netrc.py
@@ -26,8 +26,12 @@
file = os.path.join(os.path.expanduser("~"), ".netrc")
self.hosts = {}
self.macros = {}
- with open(file) as fp:
- self._parse(file, fp, default_netrc)
+ try:
+ with open(file, encoding="utf-8") as fp:
+ self._parse(file, fp, default_netrc)
+ except UnicodeDecodeError:
+ with open(file, encoding="locale") as fp:
+ self._parse(file, fp, default_netrc)
def _parse(self, file, fp, default_netrc):
lexer = shlex.shlex(fp)
diff --git a/common/py3-stdlib/ntpath.py b/common/py3-stdlib/ntpath.py
index 6f77177..527c7ae 100644
--- a/common/py3-stdlib/ntpath.py
+++ b/common/py3-stdlib/ntpath.py
@@ -312,12 +312,25 @@
drive = ''
userhome = join(drive, os.environ['HOMEPATH'])
+ if i != 1: #~user
+ target_user = path[1:i]
+ if isinstance(target_user, bytes):
+ target_user = os.fsdecode(target_user)
+ current_user = os.environ.get('USERNAME')
+
+ if target_user != current_user:
+ # Try to guess user home directory. By default all user
+ # profile directories are located in the same place and are
+ # named by corresponding usernames. If userhome isn't a
+ # normal profile directory, this guess is likely wrong,
+ # so we bail out.
+ if current_user != basename(userhome):
+ return path
+ userhome = join(dirname(userhome), target_user)
+
if isinstance(path, bytes):
userhome = os.fsencode(userhome)
- if i != 1: #~user
- userhome = join(dirname(userhome), path[1:i])
-
return userhome + path[i:]
@@ -622,7 +635,7 @@
tail = join(name, tail) if tail else name
return tail
- def realpath(path):
+ def realpath(path, *, strict=False):
path = normpath(path)
if isinstance(path, bytes):
prefix = b'\\\\?\\'
@@ -647,6 +660,8 @@
path = _getfinalpathname(path)
initial_winerror = 0
except OSError as ex:
+ if strict:
+ raise
initial_winerror = ex.winerror
path = _getfinalpathname_nonstrict(path)
# The path returned by _getfinalpathname will always start with \\?\ -
diff --git a/common/py3-stdlib/nturl2path.py b/common/py3-stdlib/nturl2path.py
index 853e660..61852af 100644
--- a/common/py3-stdlib/nturl2path.py
+++ b/common/py3-stdlib/nturl2path.py
@@ -50,6 +50,14 @@
# becomes
# ///C:/foo/bar/spam.foo
import urllib.parse
+ # First, clean up some special forms. We are going to sacrifice
+ # the additional information anyway
+ if p[:4] == '\\\\?\\':
+ p = p[4:]
+ if p[:4].upper() == 'UNC\\':
+ p = '\\' + p[4:]
+ elif p[1:2] != ':':
+ raise OSError('Bad path: ' + p)
if not ':' in p:
# No drive specifier, just convert slashes and quote the name
if p[:2] == '\\\\':
@@ -59,7 +67,7 @@
p = '\\\\' + p
components = p.split('\\')
return urllib.parse.quote('/'.join(components))
- comp = p.split(':')
+ comp = p.split(':', maxsplit=2)
if len(comp) != 2 or len(comp[0]) > 1:
error = 'Bad path: ' + p
raise OSError(error)
diff --git a/common/py3-stdlib/numbers.py b/common/py3-stdlib/numbers.py
index ed815ef..5b98e64 100644
--- a/common/py3-stdlib/numbers.py
+++ b/common/py3-stdlib/numbers.py
@@ -33,7 +33,7 @@
"""Complex defines the operations that work on the builtin complex type.
In short, those are: a conversion to complex, .real, .imag, +, -,
- *, /, abs(), .conjugate, ==, and !=.
+ *, /, **, abs(), .conjugate, ==, and !=.
If it is given heterogeneous arguments, and doesn't have special
knowledge about them, it should fall back to the builtin complex
@@ -292,7 +292,11 @@
class Integral(Rational):
- """Integral adds a conversion to int and the bit-string operations."""
+ """Integral adds methods that work on integral numbers.
+
+ In short, these are conversion to int, pow with modulus, and the
+ bit-string operations.
+ """
__slots__ = ()
diff --git a/common/py3-stdlib/opcode.py b/common/py3-stdlib/opcode.py
index ac1aa53..37e88e9 100644
--- a/common/py3-stdlib/opcode.py
+++ b/common/py3-stdlib/opcode.py
@@ -67,7 +67,6 @@
def_op('UNARY_NOT', 12)
def_op('UNARY_INVERT', 15)
-
def_op('BINARY_MATRIX_MULTIPLY', 16)
def_op('INPLACE_MATRIX_MULTIPLY', 17)
@@ -82,8 +81,12 @@
def_op('BINARY_TRUE_DIVIDE', 27)
def_op('INPLACE_FLOOR_DIVIDE', 28)
def_op('INPLACE_TRUE_DIVIDE', 29)
+def_op('GET_LEN', 30)
+def_op('MATCH_MAPPING', 31)
+def_op('MATCH_SEQUENCE', 32)
+def_op('MATCH_KEYS', 33)
+def_op('COPY_DICT_WITHOUT_KEYS', 34)
-def_op('RERAISE', 48)
def_op('WITH_EXCEPT_START', 49)
def_op('GET_AITER', 50)
def_op('GET_ANEXT', 51)
@@ -105,7 +108,6 @@
def_op('INPLACE_POWER', 67)
def_op('GET_ITER', 68)
def_op('GET_YIELD_FROM_ITER', 69)
-
def_op('PRINT_EXPR', 70)
def_op('LOAD_BUILD_CLASS', 71)
def_op('YIELD_FROM', 72)
@@ -137,6 +139,7 @@
name_op('DELETE_ATTR', 96) # ""
name_op('STORE_GLOBAL', 97) # ""
name_op('DELETE_GLOBAL', 98) # ""
+def_op('ROT_N', 99)
def_op('LOAD_CONST', 100) # Index in const list
hasconst.append(100)
name_op('LOAD_NAME', 101) # Index in name list
@@ -149,18 +152,16 @@
hascompare.append(107)
name_op('IMPORT_NAME', 108) # Index in name list
name_op('IMPORT_FROM', 109) # Index in name list
-
jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip
jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code
jabs_op('JUMP_IF_TRUE_OR_POP', 112) # ""
jabs_op('JUMP_ABSOLUTE', 113) # ""
jabs_op('POP_JUMP_IF_FALSE', 114) # ""
jabs_op('POP_JUMP_IF_TRUE', 115) # ""
-
name_op('LOAD_GLOBAL', 116) # Index in name list
-
def_op('IS_OP', 117)
def_op('CONTAINS_OP', 118)
+def_op('RERAISE', 119)
jabs_op('JUMP_IF_NOT_EXC_MATCH', 121)
jrel_op('SETUP_FINALLY', 122) # Distance to target address
@@ -172,10 +173,12 @@
def_op('DELETE_FAST', 126) # Local variable number
haslocal.append(126)
+def_op('GEN_START', 129) # Kind of generator/coroutine
def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3)
def_op('CALL_FUNCTION', 131) # #args
def_op('MAKE_FUNCTION', 132) # Flags
def_op('BUILD_SLICE', 133) # Number of items
+
def_op('LOAD_CLOSURE', 135)
hasfree.append(135)
def_op('LOAD_DEREF', 136)
@@ -187,28 +190,24 @@
def_op('CALL_FUNCTION_KW', 141) # #args + #kwargs
def_op('CALL_FUNCTION_EX', 142) # Flags
-
jrel_op('SETUP_WITH', 143)
-
+def_op('EXTENDED_ARG', 144)
+EXTENDED_ARG = 144
def_op('LIST_APPEND', 145)
def_op('SET_ADD', 146)
def_op('MAP_ADD', 147)
-
def_op('LOAD_CLASSDEREF', 148)
hasfree.append(148)
-def_op('EXTENDED_ARG', 144)
-EXTENDED_ARG = 144
+def_op('MATCH_CLASS', 152)
jrel_op('SETUP_ASYNC_WITH', 154)
-
def_op('FORMAT_VALUE', 155)
def_op('BUILD_CONST_KEY_MAP', 156)
def_op('BUILD_STRING', 157)
name_op('LOAD_METHOD', 160)
def_op('CALL_METHOD', 161)
-
def_op('LIST_EXTEND', 162)
def_op('SET_UPDATE', 163)
def_op('DICT_MERGE', 164)
diff --git a/common/py3-stdlib/operator.py b/common/py3-stdlib/operator.py
index fb58851..241fdbb 100644
--- a/common/py3-stdlib/operator.py
+++ b/common/py3-stdlib/operator.py
@@ -155,10 +155,10 @@
return b in a
def countOf(a, b):
- "Return the number of times b occurs in a."
+ "Return the number of items in a which are, or which equal, b."
count = 0
for i in a:
- if i == b:
+ if i is b or i == b:
count += 1
return count
@@ -173,7 +173,7 @@
def indexOf(a, b):
"Return the first index of b in a."
for i, j in enumerate(a):
- if j == b:
+ if j is b or j == b:
return i
else:
raise ValueError('sequence.index(x): x not in sequence')
diff --git a/common/py3-stdlib/os.py b/common/py3-stdlib/os.py
index b794159..d26cfc9 100644
--- a/common/py3-stdlib/os.py
+++ b/common/py3-stdlib/os.py
@@ -36,7 +36,7 @@
__all__ = ["altsep", "curdir", "pardir", "sep", "pathsep", "linesep",
"defpath", "name", "path", "devnull", "SEEK_SET", "SEEK_CUR",
"SEEK_END", "fsencode", "fsdecode", "get_exec_path", "fdopen",
- "popen", "extsep"]
+ "extsep"]
def _exists(name):
return name in globals()
@@ -969,58 +969,64 @@
__all__.extend(["spawnlp", "spawnlpe"])
-
-# Supply os.popen()
-def popen(cmd, mode="r", buffering=-1):
- if not isinstance(cmd, str):
- raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
- if mode not in ("r", "w"):
- raise ValueError("invalid mode %r" % mode)
- if buffering == 0 or buffering is None:
- raise ValueError("popen() does not support unbuffered streams")
- import subprocess, io
- if mode == "r":
- proc = subprocess.Popen(cmd,
- shell=True,
- stdout=subprocess.PIPE,
- bufsize=buffering)
- return _wrap_close(io.TextIOWrapper(proc.stdout), proc)
- else:
- proc = subprocess.Popen(cmd,
- shell=True,
- stdin=subprocess.PIPE,
- bufsize=buffering)
- return _wrap_close(io.TextIOWrapper(proc.stdin), proc)
-
-# Helper for popen() -- a proxy for a file whose close waits for the process
-class _wrap_close:
- def __init__(self, stream, proc):
- self._stream = stream
- self._proc = proc
- def close(self):
- self._stream.close()
- returncode = self._proc.wait()
- if returncode == 0:
- return None
- if name == 'nt':
- return returncode
+# VxWorks has no user space shell provided. As a result, running
+# command in a shell can't be supported.
+if sys.platform != 'vxworks':
+ # Supply os.popen()
+ def popen(cmd, mode="r", buffering=-1):
+ if not isinstance(cmd, str):
+ raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
+ if mode not in ("r", "w"):
+ raise ValueError("invalid mode %r" % mode)
+ if buffering == 0 or buffering is None:
+ raise ValueError("popen() does not support unbuffered streams")
+ import subprocess, io
+ if mode == "r":
+ proc = subprocess.Popen(cmd,
+ shell=True, text=True,
+ stdout=subprocess.PIPE,
+ bufsize=buffering)
+ return _wrap_close(proc.stdout, proc)
else:
- return returncode << 8 # Shift left to match old behavior
- def __enter__(self):
- return self
- def __exit__(self, *args):
- self.close()
- def __getattr__(self, name):
- return getattr(self._stream, name)
- def __iter__(self):
- return iter(self._stream)
+ proc = subprocess.Popen(cmd,
+ shell=True, text=True,
+ stdin=subprocess.PIPE,
+ bufsize=buffering)
+ return _wrap_close(proc.stdin, proc)
+
+ # Helper for popen() -- a proxy for a file whose close waits for the process
+ class _wrap_close:
+ def __init__(self, stream, proc):
+ self._stream = stream
+ self._proc = proc
+ def close(self):
+ self._stream.close()
+ returncode = self._proc.wait()
+ if returncode == 0:
+ return None
+ if name == 'nt':
+ return returncode
+ else:
+ return returncode << 8 # Shift left to match old behavior
+ def __enter__(self):
+ return self
+ def __exit__(self, *args):
+ self.close()
+ def __getattr__(self, name):
+ return getattr(self._stream, name)
+ def __iter__(self):
+ return iter(self._stream)
+
+ __all__.append("popen")
# Supply os.fdopen()
-def fdopen(fd, *args, **kwargs):
+def fdopen(fd, mode="r", buffering=-1, encoding=None, *args, **kwargs):
if not isinstance(fd, int):
raise TypeError("invalid fd type (%s, expected integer)" % type(fd))
import io
- return io.open(fd, *args, **kwargs)
+ if "b" not in mode:
+ encoding = io.text_encoding(encoding)
+ return io.open(fd, mode, buffering, encoding, *args, **kwargs)
# For testing purposes, make sure the function is available when the C
diff --git a/common/py3-stdlib/pathlib.py b/common/py3-stdlib/pathlib.py
index 147be2f..621fba0 100644
--- a/common/py3-stdlib/pathlib.py
+++ b/common/py3-stdlib/pathlib.py
@@ -6,6 +6,7 @@
import posixpath
import re
import sys
+import warnings
from _collections_abc import Sequence
from errno import EINVAL, ENOENT, ENOTDIR, EBADF, ELOOP
from operator import attrgetter
@@ -13,18 +14,6 @@
from urllib.parse import quote_from_bytes as urlquote_from_bytes
-supports_symlinks = True
-if os.name == 'nt':
- import nt
- if sys.getwindowsversion()[:2] >= (6, 0):
- from nt import _getfinalpathname
- else:
- supports_symlinks = False
- _getfinalpathname = None
-else:
- nt = None
-
-
__all__ = [
"PurePath", "PurePosixPath", "PureWindowsPath",
"Path", "PosixPath", "WindowsPath",
@@ -34,13 +23,17 @@
# Internals
#
+_WINERROR_NOT_READY = 21 # drive exists but is not accessible
+_WINERROR_INVALID_NAME = 123 # fix for bpo-35306
+_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself
+
# EBADF - guard against macOS `stat` throwing EBADF
_IGNORED_ERROS = (ENOENT, ENOTDIR, EBADF, ELOOP)
_IGNORED_WINERRORS = (
- 21, # ERROR_NOT_READY - drive exists but is not accessible
- 1921, # ERROR_CANT_RESOLVE_FILENAME - fix for broken symlink pointing to itself
-)
+ _WINERROR_NOT_READY,
+ _WINERROR_INVALID_NAME,
+ _WINERROR_CANT_RESOLVE_FILENAME)
def _ignore_error(exception):
return (getattr(exception, 'errno', None) in _IGNORED_ERROS or
@@ -131,16 +124,25 @@
ext_namespace_prefix = '\\\\?\\'
reserved_names = (
- {'CON', 'PRN', 'AUX', 'NUL'} |
- {'COM%d' % i for i in range(1, 10)} |
- {'LPT%d' % i for i in range(1, 10)}
+ {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} |
+ {'COM%s' % c for c in '123456789\xb9\xb2\xb3'} |
+ {'LPT%s' % c for c in '123456789\xb9\xb2\xb3'}
)
# Interesting findings about extended paths:
- # - '\\?\c:\a', '//?/c:\a' and '//?/c:/a' are all supported
- # but '\\?\c:/a' is not
- # - extended paths are always absolute; "relative" extended paths will
- # fail.
+ # * '\\?\c:\a' is an extended path, which bypasses normal Windows API
+ # path processing. Thus relative paths are not resolved and slash is not
+ # translated to backslash. It has the native NT path limit of 32767
+ # characters, but a bit less after resolving device symbolic links,
+ # such as '\??\C:' => '\Device\HarddiskVolume2'.
+ # * '\\?\c:/a' looks for a device named 'C:/a' because slash is a
+ # regular name character in the object namespace.
+ # * '\\?\c:\foo/bar' is invalid because '/' is illegal in NT filesystems.
+ # The only path separator at the filesystem level is backslash.
+ # * '//?/c:\a' and '//?/c:/a' are effectively equivalent to '\\.\c:\a' and
+ # thus limited to MAX_PATH.
+ # * Prior to Windows 8, ANSI API bytes paths are limited to MAX_PATH,
+ # even with the '\\?\' prefix.
def splitroot(self, part, sep=sep):
first = part[0:1]
@@ -190,30 +192,6 @@
def compile_pattern(self, pattern):
return re.compile(fnmatch.translate(pattern), re.IGNORECASE).fullmatch
- def resolve(self, path, strict=False):
- s = str(path)
- if not s:
- return os.getcwd()
- previous_s = None
- if _getfinalpathname is not None:
- if strict:
- return self._ext_to_normal(_getfinalpathname(s))
- else:
- tail_parts = [] # End of the path after the first one not found
- while True:
- try:
- s = self._ext_to_normal(_getfinalpathname(s))
- except FileNotFoundError:
- previous_s = s
- s, tail = os.path.split(s)
- tail_parts.append(tail)
- if previous_s == s:
- return path
- else:
- return os.path.join(s, *reversed(tail_parts))
- # Means fallback on absolute
- return None
-
def _split_extended_path(self, s, ext_prefix=ext_namespace_prefix):
prefix = ''
if s.startswith(ext_prefix):
@@ -224,21 +202,18 @@
s = '\\' + s[3:]
return prefix, s
- def _ext_to_normal(self, s):
- # Turn back an extended path into a normal DOS-like path
- return self._split_extended_path(s)[1]
-
def is_reserved(self, parts):
# NOTE: the rules for reserved names seem somewhat complicated
- # (e.g. r"..\NUL" is reserved but not r"foo\NUL").
- # We err on the side of caution and return True for paths which are
- # not considered reserved by Windows.
+ # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not
+ # exist). We err on the side of caution and return True for paths
+ # which are not considered reserved by Windows.
if not parts:
return False
if parts[0].startswith('\\\\'):
# UNC paths are never reserved
return False
- return parts[-1].partition('.')[0].upper() in self.reserved_names
+ name = parts[-1].partition('.')[0].partition(':')[0].rstrip(' ')
+ return name.upper() in self.reserved_names
def make_uri(self, path):
# Under Windows, file URIs use the UTF-8 encoding.
@@ -252,34 +227,6 @@
# It's a path on a network drive => 'file://host/share/a/b'
return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8'))
- def gethomedir(self, username):
- if 'USERPROFILE' in os.environ:
- userhome = os.environ['USERPROFILE']
- elif 'HOMEPATH' in os.environ:
- try:
- drv = os.environ['HOMEDRIVE']
- except KeyError:
- drv = ''
- userhome = drv + os.environ['HOMEPATH']
- else:
- raise RuntimeError("Can't determine home directory")
-
- if username:
- # Try to guess user home directory. By default all users
- # directories are located in the same place and are named by
- # corresponding usernames. If current user home directory points
- # to nonstandard place, this guess is likely wrong.
- if os.environ['USERNAME'] != username:
- drv, root, parts = self.parse_parts((userhome,))
- if parts[-1] != os.environ['USERNAME']:
- raise RuntimeError("Can't determine home directory "
- "for %r" % username)
- parts[-1] = username
- if drv or root:
- userhome = drv + root + self.join(parts[1:])
- else:
- userhome = self.join(parts)
- return userhome
class _PosixFlavour(_Flavour):
sep = '/'
@@ -313,54 +260,6 @@
def compile_pattern(self, pattern):
return re.compile(fnmatch.translate(pattern)).fullmatch
- def resolve(self, path, strict=False):
- sep = self.sep
- accessor = path._accessor
- seen = {}
- def _resolve(path, rest):
- if rest.startswith(sep):
- path = ''
-
- for name in rest.split(sep):
- if not name or name == '.':
- # current dir
- continue
- if name == '..':
- # parent dir
- path, _, _ = path.rpartition(sep)
- continue
- if path.endswith(sep):
- newpath = path + name
- else:
- newpath = path + sep + name
- if newpath in seen:
- # Already seen this path
- path = seen[newpath]
- if path is not None:
- # use cached value
- continue
- # The symlink is not resolved, so we must have a symlink loop.
- raise RuntimeError("Symlink loop from %r" % newpath)
- # Resolve the symbolic link
- try:
- target = accessor.readlink(newpath)
- except OSError as e:
- if e.errno != EINVAL and strict:
- raise
- # Not a symlink, or non-strict mode. We just leave the path
- # untouched.
- path = newpath
- else:
- seen[newpath] = None # not resolved symlink
- path = _resolve(path, target)
- seen[newpath] = path # resolved symlink
-
- return path
- # NOTE: according to POSIX, getcwd() cannot contain path components
- # which are symlinks.
- base = '' if path.is_absolute() else os.getcwd()
- return _resolve(base, str(path)) or sep
-
def is_reserved(self, parts):
return False
@@ -370,21 +269,6 @@
bpath = bytes(path)
return 'file://' + urlquote_from_bytes(bpath)
- def gethomedir(self, username):
- if not username:
- try:
- return os.environ['HOME']
- except KeyError:
- import pwd
- return pwd.getpwuid(os.getuid()).pw_dir
- else:
- import pwd
- try:
- return pwd.getpwnam(username).pw_dir
- except KeyError:
- raise RuntimeError("Can't determine home directory "
- "for %r" % username)
-
_windows_flavour = _WindowsFlavour()
_posix_flavour = _PosixFlavour()
@@ -399,9 +283,7 @@
stat = os.stat
- lstat = os.lstat
-
- open = os.open
+ open = io.open
listdir = os.listdir
@@ -409,21 +291,14 @@
chmod = os.chmod
- if hasattr(os, "lchmod"):
- lchmod = os.lchmod
- else:
- def lchmod(self, pathobj, mode):
- raise NotImplementedError("lchmod() not available on this system")
-
mkdir = os.mkdir
unlink = os.unlink
if hasattr(os, "link"):
- link_to = os.link
+ link = os.link
else:
- @staticmethod
- def link_to(self, target):
+ def link(self, src, dst):
raise NotImplementedError("os.link() not available on this system")
rmdir = os.rmdir
@@ -432,23 +307,35 @@
replace = os.replace
- if nt:
- if supports_symlinks:
- symlink = os.symlink
- else:
- def symlink(a, b, target_is_directory):
- raise NotImplementedError("symlink() not available on this system")
+ if hasattr(os, "symlink"):
+ symlink = os.symlink
else:
- # Under POSIX, os.symlink() takes two args
- @staticmethod
- def symlink(a, b, target_is_directory):
- return os.symlink(a, b)
+ def symlink(self, src, dst, target_is_directory=False):
+ raise NotImplementedError("os.symlink() not available on this system")
- utime = os.utime
+ def touch(self, path, mode=0o666, exist_ok=True):
+ if exist_ok:
+ # First try to bump modification time
+ # Implementation note: GNU touch uses the UTIME_NOW option of
+ # the utimensat() / futimens() functions.
+ try:
+ os.utime(path, None)
+ except OSError:
+ # Avoid exception chaining
+ pass
+ else:
+ return
+ flags = os.O_CREAT | os.O_WRONLY
+ if not exist_ok:
+ flags |= os.O_EXCL
+ fd = os.open(path, flags, mode)
+ os.close(fd)
- # Helper for resolve()
- def readlink(self, path):
- return os.readlink(path)
+ if hasattr(os, "readlink"):
+ readlink = os.readlink
+ else:
+ def readlink(self, path):
+ raise NotImplementedError("os.readlink() not available on this system")
def owner(self, path):
try:
@@ -464,6 +351,12 @@
except ImportError:
raise NotImplementedError("Path.group() is unsupported on this system")
+ getcwd = os.getcwd
+
+ expanduser = staticmethod(os.path.expanduser)
+
+ realpath = staticmethod(os.path.realpath)
+
_normal_accessor = _NormalAccessor()
@@ -630,7 +523,10 @@
return len(self._parts)
def __getitem__(self, idx):
- if idx < 0 or idx >= len(self):
+ if isinstance(idx, slice):
+ return tuple(self[i] for i in range(*idx.indices(len(self))))
+
+ if idx >= len(self) or idx < -len(self):
raise IndexError(idx)
return self._pathcls._from_parsed_parts(self._drv, self._root,
self._parts[:-idx - 1])
@@ -689,7 +585,7 @@
return cls._flavour.parse_parts(parts)
@classmethod
- def _from_parts(cls, args, init=True):
+ def _from_parts(cls, args):
# We need to call _parse_args on the instance, so as to get the
# right flavour.
self = object.__new__(cls)
@@ -697,18 +593,14 @@
self._drv = drv
self._root = root
self._parts = parts
- if init:
- self._init()
return self
@classmethod
- def _from_parsed_parts(cls, drv, root, parts, init=True):
+ def _from_parsed_parts(cls, drv, root, parts):
self = object.__new__(cls)
self._drv = drv
self._root = root
self._parts = parts
- if init:
- self._init()
return self
@classmethod
@@ -718,10 +610,6 @@
else:
return cls._flavour.join(parts)
- def _init(self):
- # Overridden in concrete Path
- pass
-
def _make_child(self, args):
drv, root, parts = self._parse_args(args)
drv, root, parts = self._flavour.join_parsed_parts(
@@ -1061,29 +949,18 @@
object. You can also instantiate a PosixPath or WindowsPath directly,
but cannot instantiate a WindowsPath on a POSIX system or vice versa.
"""
- __slots__ = (
- '_accessor',
- )
+ _accessor = _normal_accessor
+ __slots__ = ()
def __new__(cls, *args, **kwargs):
if cls is Path:
cls = WindowsPath if os.name == 'nt' else PosixPath
- self = cls._from_parts(args, init=False)
+ self = cls._from_parts(args)
if not self._flavour.is_supported:
raise NotImplementedError("cannot instantiate %r on your system"
% (cls.__name__,))
- self._init()
return self
- def _init(self,
- # Private non-constructor arguments
- template=None,
- ):
- if template is not None:
- self._accessor = template._accessor
- else:
- self._accessor = _normal_accessor
-
def _make_child_relpath(self, part):
# This is an optimization used for dir walking. `part` must be
# a single part relative to this path.
@@ -1104,17 +981,6 @@
# removed in the future.
pass
- def _opener(self, name, flags, mode=0o666):
- # A stub for the opener argument to built-in open()
- return self._accessor.open(self, flags, mode)
-
- def _raw_open(self, flags, mode=0o777):
- """
- Open the file pointed by this path and return a file descriptor,
- as os.open() does.
- """
- return self._accessor.open(self, flags, mode)
-
# Public API
@classmethod
@@ -1122,14 +988,14 @@
"""Return a new path pointing to the current working directory
(as returned by os.getcwd()).
"""
- return cls(os.getcwd())
+ return cls(cls._accessor.getcwd())
@classmethod
def home(cls):
"""Return a new path pointing to the user's home directory (as
returned by os.path.expanduser('~')).
"""
- return cls(cls()._flavour.gethomedir(None))
+ return cls("~").expanduser()
def samefile(self, other_path):
"""Return whether other_path is the same or not as this file
@@ -1191,9 +1057,7 @@
return self
# FIXME this must defer to the specific flavour (and, under Windows,
# use nt._getfullpathname())
- obj = self._from_parts([os.getcwd()] + self._parts, init=False)
- obj._init(template=self)
- return obj
+ return self._from_parts([self._accessor.getcwd()] + self._parts)
def resolve(self, strict=False):
"""
@@ -1201,24 +1065,34 @@
normalizing it (for example turning slashes into backslashes under
Windows).
"""
- s = self._flavour.resolve(self, strict=strict)
- if s is None:
- # No symlink resolution => for consistency, raise an error if
- # the path doesn't exist or is forbidden
- self.stat()
- s = str(self.absolute())
- # Now we have no symlinks in the path, it's safe to normalize it.
- normed = self._flavour.pathmod.normpath(s)
- obj = self._from_parts((normed,), init=False)
- obj._init(template=self)
- return obj
- def stat(self):
+ def check_eloop(e):
+ winerror = getattr(e, 'winerror', 0)
+ if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME:
+ raise RuntimeError("Symlink loop from %r" % e.filename)
+
+ try:
+ s = self._accessor.realpath(self, strict=strict)
+ except OSError as e:
+ check_eloop(e)
+ raise
+ p = self._from_parts((s,))
+
+ # In non-strict mode, realpath() doesn't raise on symlink loops.
+ # Ensure we get an exception by calling stat()
+ if not strict:
+ try:
+ p.stat()
+ except OSError as e:
+ check_eloop(e)
+ return p
+
+ def stat(self, *, follow_symlinks=True):
"""
Return the result of the stat() system call on this path, like
os.stat() does.
"""
- return self._accessor.stat(self)
+ return self._accessor.stat(self, follow_symlinks=follow_symlinks)
def owner(self):
"""
@@ -1238,8 +1112,10 @@
Open the file pointed by this path and return a file object, as
the built-in open() function does.
"""
- return io.open(self, mode, buffering, encoding, errors, newline,
- opener=self._opener)
+ if "b" not in mode:
+ encoding = io.text_encoding(encoding)
+ return self._accessor.open(self, mode, buffering, encoding, errors,
+ newline)
def read_bytes(self):
"""
@@ -1252,6 +1128,7 @@
"""
Open the file in text mode, read it, and close the file.
"""
+ encoding = io.text_encoding(encoding)
with self.open(mode='r', encoding=encoding, errors=errors) as f:
return f.read()
@@ -1264,14 +1141,15 @@
with self.open(mode='wb') as f:
return f.write(view)
- def write_text(self, data, encoding=None, errors=None):
+ def write_text(self, data, encoding=None, errors=None, newline=None):
"""
Open the file in text mode, write to it, and close the file.
"""
if not isinstance(data, str):
raise TypeError('data must be str, not %s' %
data.__class__.__name__)
- with self.open(mode='w', encoding=encoding, errors=errors) as f:
+ encoding = io.text_encoding(encoding)
+ with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f:
return f.write(data)
def readlink(self):
@@ -1279,30 +1157,13 @@
Return the path to which the symbolic link points.
"""
path = self._accessor.readlink(self)
- obj = self._from_parts((path,), init=False)
- obj._init(template=self)
- return obj
+ return self._from_parts((path,))
def touch(self, mode=0o666, exist_ok=True):
"""
Create this file with the given access mode, if it doesn't exist.
"""
- if exist_ok:
- # First try to bump modification time
- # Implementation note: GNU touch uses the UTIME_NOW option of
- # the utimensat() / futimens() functions.
- try:
- self._accessor.utime(self, None)
- except OSError:
- # Avoid exception chaining
- pass
- else:
- return
- flags = os.O_CREAT | os.O_WRONLY
- if not exist_ok:
- flags |= os.O_EXCL
- fd = self._raw_open(flags, mode)
- os.close(fd)
+ self._accessor.touch(self, mode, exist_ok)
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
"""
@@ -1321,18 +1182,18 @@
if not exist_ok or not self.is_dir():
raise
- def chmod(self, mode):
+ def chmod(self, mode, *, follow_symlinks=True):
"""
Change the permissions of the path, like os.chmod().
"""
- self._accessor.chmod(self, mode)
+ self._accessor.chmod(self, mode, follow_symlinks=follow_symlinks)
def lchmod(self, mode):
"""
Like chmod(), except if the path points to a symlink, the symlink's
permissions are changed, rather than its target's.
"""
- self._accessor.lchmod(self, mode)
+ self.chmod(mode, follow_symlinks=False)
def unlink(self, missing_ok=False):
"""
@@ -1356,13 +1217,7 @@
Like stat(), except if the path points to a symlink, the symlink's
status information is returned, rather than its target's.
"""
- return self._accessor.lstat(self)
-
- def link_to(self, target):
- """
- Create a hard link pointing to a path named target.
- """
- self._accessor.link_to(self, target)
+ return self.stat(follow_symlinks=False)
def rename(self, target):
"""
@@ -1392,11 +1247,37 @@
def symlink_to(self, target, target_is_directory=False):
"""
- Make this path a symlink pointing to the given path.
- Note the order of arguments (self, target) is the reverse of os.symlink's.
+ Make this path a symlink pointing to the target path.
+ Note the order of arguments (link, target) is the reverse of os.symlink.
"""
self._accessor.symlink(target, self, target_is_directory)
+ def hardlink_to(self, target):
+ """
+ Make this path a hard link pointing to the same file as *target*.
+
+ Note the order of arguments (self, target) is the reverse of os.link's.
+ """
+ self._accessor.link(target, self)
+
+ def link_to(self, target):
+ """
+ Make the target path a hard link pointing to this path.
+
+ Note this function does not make this path a hard link to *target*,
+ despite the implication of the function and argument names. The order
+ of arguments (target, link) is the reverse of Path.symlink_to, but
+ matches that of os.link.
+
+ Deprecated since Python 3.10 and scheduled for removal in Python 3.12.
+ Use `hardlink_to()` instead.
+ """
+ warnings.warn("pathlib.Path.link_to() is deprecated and is scheduled "
+ "for removal in Python 3.12. "
+ "Use pathlib.Path.hardlink_to() instead.",
+ DeprecationWarning, stacklevel=2)
+ self._accessor.link(self, target)
+
# Convenience functions for querying the stat results
def exists(self):
@@ -1424,7 +1305,7 @@
if not _ignore_error(e):
raise
# Path doesn't exist or is a broken symlink
- # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ )
return False
except ValueError:
# Non-encodable path
@@ -1441,7 +1322,7 @@
if not _ignore_error(e):
raise
# Path doesn't exist or is a broken symlink
- # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ )
return False
except ValueError:
# Non-encodable path
@@ -1492,7 +1373,7 @@
if not _ignore_error(e):
raise
# Path doesn't exist or is a broken symlink
- # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ )
return False
except ValueError:
# Non-encodable path
@@ -1508,7 +1389,7 @@
if not _ignore_error(e):
raise
# Path doesn't exist or is a broken symlink
- # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ )
return False
except ValueError:
# Non-encodable path
@@ -1524,7 +1405,7 @@
if not _ignore_error(e):
raise
# Path doesn't exist or is a broken symlink
- # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ )
return False
except ValueError:
# Non-encodable path
@@ -1540,7 +1421,7 @@
if not _ignore_error(e):
raise
# Path doesn't exist or is a broken symlink
- # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ )
return False
except ValueError:
# Non-encodable path
@@ -1552,7 +1433,9 @@
"""
if (not (self._drv or self._root) and
self._parts and self._parts[0][:1] == '~'):
- homedir = self._flavour.gethomedir(self._parts[0][1:])
+ homedir = self._accessor.expanduser(self._parts[0])
+ if homedir[:1] == "~":
+ raise RuntimeError("Could not determine home directory.")
return self._from_parts([homedir] + self._parts[1:])
return self
diff --git a/common/py3-stdlib/pdb.py b/common/py3-stdlib/pdb.py
index d7d9571..7ab50b4 100755
--- a/common/py3-stdlib/pdb.py
+++ b/common/py3-stdlib/pdb.py
@@ -384,8 +384,7 @@
sys.stdin = save_stdin
sys.displayhook = save_displayhook
except:
- exc_info = sys.exc_info()[:2]
- self.error(traceback.format_exception_only(*exc_info)[-1].strip())
+ self._error_exc()
def precmd(self, line):
"""Handle alias expansion and ';;' separator."""
@@ -752,7 +751,8 @@
"""
# this method should be callable before starting debugging, so default
# to "no globals" if there is no current frame
- globs = self.curframe.f_globals if hasattr(self, 'curframe') else None
+ frame = getattr(self, 'curframe', None)
+ globs = frame.f_globals if frame else None
line = linecache.getline(filename, lineno, globs)
if not line:
self.message('End of file')
@@ -893,7 +893,7 @@
except ValueError:
err = "Invalid line number (%s)" % arg
else:
- bplist = self.get_breaks(filename, lineno)
+ bplist = self.get_breaks(filename, lineno)[:]
err = self.clear_break(filename, lineno)
if err:
self.error(err)
@@ -1026,7 +1026,11 @@
if arg:
import shlex
argv0 = sys.argv[0:1]
- sys.argv = shlex.split(arg)
+ try:
+ sys.argv = shlex.split(arg)
+ except ValueError as e:
+ self.error('Cannot run %s: %s' % (arg, e))
+ return
sys.argv[:0] = argv0
# this is caught in the main debugger loop
raise Restart
@@ -1103,8 +1107,7 @@
try:
sys.call_tracing(p.run, (arg, globals, locals))
except Exception:
- exc_info = sys.exc_info()[:2]
- self.error(traceback.format_exception_only(*exc_info)[-1].strip())
+ self._error_exc()
self.message("LEAVING RECURSIVE DEBUGGER")
sys.settrace(self.trace_dispatch)
self.lastcmd = p.lastcmd
@@ -1162,8 +1165,7 @@
try:
return eval(arg, self.curframe.f_globals, self.curframe_locals)
except:
- exc_info = sys.exc_info()[:2]
- self.error(traceback.format_exception_only(*exc_info)[-1].strip())
+ self._error_exc()
raise
def _getval_except(self, arg, frame=None):
@@ -1177,23 +1179,31 @@
err = traceback.format_exception_only(*exc_info)[-1].strip()
return _rstr('** raised %s **' % err)
+ def _error_exc(self):
+ exc_info = sys.exc_info()[:2]
+ self.error(traceback.format_exception_only(*exc_info)[-1].strip())
+
+ def _msg_val_func(self, arg, func):
+ try:
+ val = self._getval(arg)
+ except:
+ return # _getval() has displayed the error
+ try:
+ self.message(func(val))
+ except:
+ self._error_exc()
+
def do_p(self, arg):
"""p expression
Print the value of the expression.
"""
- try:
- self.message(repr(self._getval(arg)))
- except:
- pass
+ self._msg_val_func(arg, repr)
def do_pp(self, arg):
"""pp expression
Pretty-print the value of the expression.
"""
- try:
- self.message(pprint.pformat(self._getval(arg)))
- except:
- pass
+ self._msg_val_func(arg, pprint.pformat)
complete_print = _complete_expression
complete_p = _complete_expression
@@ -1483,6 +1493,9 @@
self.error('No help for %r; please do not run Python with -OO '
'if you need command help' % arg)
return
+ if command.__doc__ is None:
+ self.error('No help for %r; __doc__ string missing' % arg)
+ return
self.message(command.__doc__.rstrip())
do_h = do_help
@@ -1684,10 +1697,19 @@
print('Error:', mainpyfile, 'does not exist')
sys.exit(1)
+ if run_as_module:
+ import runpy
+ try:
+ runpy._get_module_details(mainpyfile)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list
- # Replace pdb's dir with script's dir in front of module search path.
if not run_as_module:
+ mainpyfile = os.path.realpath(mainpyfile)
+ # Replace pdb's dir with script's dir in front of module search path.
sys.path[0] = os.path.dirname(mainpyfile)
# Note on saving/restoring sys.argv: it's a good idea when sys.argv was
@@ -1707,7 +1729,7 @@
print("The program finished and will be restarted")
except Restart:
print("Restarting", mainpyfile, "with arguments:")
- print("\t" + " ".join(args))
+ print("\t" + " ".join(sys.argv[1:]))
except SystemExit:
# In most cases SystemExit does not warrant a post-mortem session.
print("The program exited via sys.exit(). Exit status:", end=' ')
diff --git a/common/py3-stdlib/pickle.py b/common/py3-stdlib/pickle.py
index e63a8b6..e7f30f2 100644
--- a/common/py3-stdlib/pickle.py
+++ b/common/py3-stdlib/pickle.py
@@ -818,6 +818,7 @@
self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
else:
self.write(BYTEARRAY8 + pack("<Q", n) + obj)
+ self.memoize(obj)
dispatch[bytearray] = save_bytearray
if _HAVE_PICKLE_BUFFER:
@@ -1172,7 +1173,7 @@
used in Python 3. The *encoding* and *errors* tell pickle how
to decode 8-bit string instances pickled by Python 2; these
default to 'ASCII' and 'strict', respectively. *encoding* can be
- 'bytes' to read theses 8-bit string instances as bytes objects.
+ 'bytes' to read these 8-bit string instances as bytes objects.
"""
self._buffers = iter(buffers) if buffers is not None else None
self._file_readline = file.readline
diff --git a/common/py3-stdlib/pipes.py b/common/py3-stdlib/pipes.py
index f1a16f6..8cc74b0 100644
--- a/common/py3-stdlib/pipes.py
+++ b/common/py3-stdlib/pipes.py
@@ -109,7 +109,7 @@
def append(self, cmd, kind):
"""t.append(cmd, kind) adds a new step at the end."""
- if type(cmd) is not type(''):
+ if not isinstance(cmd, str):
raise TypeError('Template.append: cmd must be a string')
if kind not in stepkinds:
raise ValueError('Template.append: bad kind %r' % (kind,))
@@ -125,7 +125,7 @@
def prepend(self, cmd, kind):
"""t.prepend(cmd, kind) adds a new step at the front."""
- if type(cmd) is not type(''):
+ if not isinstance(cmd, str):
raise TypeError('Template.prepend: cmd must be a string')
if kind not in stepkinds:
raise ValueError('Template.prepend: bad kind %r' % (kind,))
diff --git a/common/py3-stdlib/pkgutil.py b/common/py3-stdlib/pkgutil.py
index 4c18467..8e010c7 100644
--- a/common/py3-stdlib/pkgutil.py
+++ b/common/py3-stdlib/pkgutil.py
@@ -7,7 +7,6 @@
import importlib.machinery
import os
import os.path
-import re
import sys
from types import ModuleType
import warnings
@@ -205,7 +204,8 @@
def __init__(self, path=None):
global imp
- warnings.warn("This emulation is deprecated, use 'importlib' instead",
+ warnings.warn("This emulation is deprecated and slated for removal "
+ "in Python 3.12; use 'importlib' instead",
DeprecationWarning)
_import_imp()
self.path = path
@@ -272,7 +272,8 @@
code = source = None
def __init__(self, fullname, file, filename, etc):
- warnings.warn("This emulation is deprecated, use 'importlib' instead",
+ warnings.warn("This emulation is deprecated and slated for removal in "
+ "Python 3.12; use 'importlib' instead",
DeprecationWarning)
_import_imp()
self.file = file
@@ -638,9 +639,7 @@
return loader.get_data(resource_name)
-_DOTTED_WORDS = r'(?!\d)(\w+)(\.(?!\d)(\w+))*'
-_NAME_PATTERN = re.compile(f'^(?P<pkg>{_DOTTED_WORDS})(?P<cln>:(?P<obj>{_DOTTED_WORDS})?)?$', re.U)
-del _DOTTED_WORDS
+_NAME_PATTERN = None
def resolve_name(name):
"""
@@ -672,8 +671,17 @@
ValueError - if `name` isn't in a recognised format
ImportError - if an import failed when it shouldn't have
AttributeError - if a failure occurred when traversing the object hierarchy
- within the imported package to get to the desired object)
+ within the imported package to get to the desired object.
"""
+ global _NAME_PATTERN
+ if _NAME_PATTERN is None:
+ # Lazy import to speedup Python startup time
+ import re
+ dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*'
+ _NAME_PATTERN = re.compile(f'^(?P<pkg>{dotted_words})'
+ f'(?P<cln>:(?P<obj>{dotted_words})?)?$',
+ re.UNICODE)
+
m = _NAME_PATTERN.match(name)
if not m:
raise ValueError(f'invalid format: {name!r}')
diff --git a/common/py3-stdlib/platform.py b/common/py3-stdlib/platform.py
index e9f50ab..e32f9c1 100755
--- a/common/py3-stdlib/platform.py
+++ b/common/py3-stdlib/platform.py
@@ -174,7 +174,7 @@
The file is read and scanned in chunks of chunksize bytes.
"""
- if executable is None:
+ if not executable:
try:
ver = os.confstr('CS_GNU_LIBC_VERSION')
# parse 'glibc 2.28' as ('glibc', '2.28')
@@ -239,11 +239,9 @@
if build:
l.append(build)
try:
- ints = map(int, l)
+ strings = list(map(str, map(int, l)))
except ValueError:
strings = l
- else:
- strings = list(map(str, ints))
version = '.'.join(strings[:3])
return version
@@ -282,6 +280,7 @@
for cmd in ('ver', 'command /c ver', 'cmd /c ver'):
try:
info = subprocess.check_output(cmd,
+ stdin=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
text=True,
shell=True)
@@ -365,17 +364,20 @@
return release, version, csd, ptype
winver = getwindowsversion()
- maj, min, build = winver.platform_version or winver[:3]
- version = '{0}.{1}.{2}'.format(maj, min, build)
+ try:
+ major, minor, build = map(int, _syscmd_ver()[2].split('.'))
+ except ValueError:
+ major, minor, build = winver.platform_version or winver[:3]
+ version = '{0}.{1}.{2}'.format(major, minor, build)
- release = (_WIN32_CLIENT_RELEASES.get((maj, min)) or
- _WIN32_CLIENT_RELEASES.get((maj, None)) or
+ release = (_WIN32_CLIENT_RELEASES.get((major, minor)) or
+ _WIN32_CLIENT_RELEASES.get((major, None)) or
release)
# getwindowsversion() reflect the compatibility mode Python is
# running under, and so the service pack value is only going to be
# valid if the versions match.
- if winver[:2] == (maj, min):
+ if winver[:2] == (major, minor):
try:
csd = 'SP{}'.format(winver.service_pack_major)
except AttributeError:
@@ -384,8 +386,8 @@
# VER_NT_SERVER = 3
if getattr(winver, 'product_type', None) == 3:
- release = (_WIN32_SERVER_RELEASES.get((maj, min)) or
- _WIN32_SERVER_RELEASES.get((maj, None)) or
+ release = (_WIN32_SERVER_RELEASES.get((major, minor)) or
+ _WIN32_SERVER_RELEASES.get((major, None)) or
release)
try:
@@ -524,16 +526,6 @@
# XXX Whatever the new SunOS marketing name is...
system = 'Solaris'
- elif system == 'IRIX64':
- # IRIX reports IRIX64 on platforms with 64-bit support; yet it
- # is really a version and not a different platform, since 32-bit
- # apps are also supported..
- system = 'IRIX'
- if version:
- version = version + ' (64bit)'
- else:
- version = '64bit'
-
elif system in ('win32', 'win16'):
# In case one of the other tricks
system = 'Windows'
@@ -698,9 +690,6 @@
# Bits
if '32-bit' in fileout:
bits = '32bit'
- elif 'N32' in fileout:
- # On Irix only
- bits = 'n32bit'
elif '64-bit' in fileout:
bits = '64bit'
@@ -782,7 +771,7 @@
):
"""
A uname_result that's largely compatible with a
- simple namedtuple except that 'platform' is
+ simple namedtuple except that 'processor' is
resolved late and cached to avoid calling "uname"
except when needed.
"""
@@ -797,12 +786,25 @@
(self.processor,)
)
+ @classmethod
+ def _make(cls, iterable):
+ # override factory to affect length check
+ num_fields = len(cls._fields)
+ result = cls.__new__(cls, *iterable)
+ if len(result) != num_fields + 1:
+ msg = f'Expected {num_fields} arguments, got {len(result)}'
+ raise TypeError(msg)
+ return result
+
def __getitem__(self, key):
- return tuple(iter(self))[key]
+ return tuple(self)[key]
def __len__(self):
return len(tuple(iter(self)))
+ def __reduce__(self):
+ return uname_result, tuple(self)[:len(self._fields)]
+
_uname_cache = None
@@ -1243,6 +1245,63 @@
_platform_cache[(aliased, terse)] = platform
return platform
+### freedesktop.org os-release standard
+# https://www.freedesktop.org/software/systemd/man/os-release.html
+
+# NAME=value with optional quotes (' or "). The regular expression is less
+# strict than shell lexer, but that's ok.
+_os_release_line = re.compile(
+ "^(?P<name>[a-zA-Z0-9_]+)=(?P<quote>[\"\']?)(?P<value>.*)(?P=quote)$"
+)
+# unescape five special characters mentioned in the standard
+_os_release_unescape = re.compile(r"\\([\\\$\"\'`])")
+# /etc takes precedence over /usr/lib
+_os_release_candidates = ("/etc/os-release", "/usr/lib/os-release")
+_os_release_cache = None
+
+
+def _parse_os_release(lines):
+ # These fields are mandatory fields with well-known defaults
+ # in practice all Linux distributions override NAME, ID, and PRETTY_NAME.
+ info = {
+ "NAME": "Linux",
+ "ID": "linux",
+ "PRETTY_NAME": "Linux",
+ }
+
+ for line in lines:
+ mo = _os_release_line.match(line)
+ if mo is not None:
+ info[mo.group('name')] = _os_release_unescape.sub(
+ r"\1", mo.group('value')
+ )
+
+ return info
+
+
+def freedesktop_os_release():
+ """Return operation system identification from freedesktop.org os-release
+ """
+ global _os_release_cache
+
+ if _os_release_cache is None:
+ errno = None
+ for candidate in _os_release_candidates:
+ try:
+ with open(candidate, encoding="utf-8") as f:
+ _os_release_cache = _parse_os_release(f)
+ break
+ except OSError as e:
+ errno = e.errno
+ else:
+ raise OSError(
+ errno,
+ f"Unable to read files {', '.join(_os_release_candidates)}"
+ )
+
+ return _os_release_cache.copy()
+
+
### Command line interface
if __name__ == '__main__':
diff --git a/common/py3-stdlib/posixpath.py b/common/py3-stdlib/posixpath.py
index ecb4e5a..1953746 100644
--- a/common/py3-stdlib/posixpath.py
+++ b/common/py3-stdlib/posixpath.py
@@ -262,6 +262,9 @@
# password database, return the path unchanged
return path
userhome = pwent.pw_dir
+ # if no user home, return the path unchanged on VxWorks
+ if userhome is None and sys.platform == "vxworks":
+ return path
if isinstance(path, bytes):
userhome = os.fsencode(userhome)
root = b'/'
@@ -349,6 +352,7 @@
initial_slashes = path.startswith(sep)
# POSIX allows one or two initial slashes, but treats three or more
# as single slash.
+ # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
if (initial_slashes and
path.startswith(sep*2) and not path.startswith(sep*3)):
initial_slashes = 2
@@ -384,16 +388,16 @@
# Return a canonical path (i.e. the absolute location of a file on the
# filesystem).
-def realpath(filename):
+def realpath(filename, *, strict=False):
"""Return the canonical path of the specified filename, eliminating any
symbolic links encountered in the path."""
filename = os.fspath(filename)
- path, ok = _joinrealpath(filename[:0], filename, {})
+ path, ok = _joinrealpath(filename[:0], filename, strict, {})
return abspath(path)
# Join two paths, normalizing and eliminating any symbolic links
# encountered in the second path.
-def _joinrealpath(path, rest, seen):
+def _joinrealpath(path, rest, strict, seen):
if isinstance(path, bytes):
sep = b'/'
curdir = b'.'
@@ -422,7 +426,15 @@
path = pardir
continue
newpath = join(path, name)
- if not islink(newpath):
+ try:
+ st = os.lstat(newpath)
+ except OSError:
+ if strict:
+ raise
+ is_link = False
+ else:
+ is_link = stat.S_ISLNK(st.st_mode)
+ if not is_link:
path = newpath
continue
# Resolve the symbolic link
@@ -433,10 +445,14 @@
# use cached value
continue
# The symlink is not resolved, so we must have a symlink loop.
- # Return already resolved part + rest of the path unchanged.
- return join(newpath, rest), False
+ if strict:
+ # Raise OSError(errno.ELOOP)
+ os.stat(newpath)
+ else:
+ # Return already resolved part + rest of the path unchanged.
+ return join(newpath, rest), False
seen[newpath] = None # not resolved symlink
- path, ok = _joinrealpath(path, os.readlink(newpath), seen)
+ path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen)
if not ok:
return join(path, rest), False
seen[newpath] = path # resolved symlink
diff --git a/common/py3-stdlib/pprint.py b/common/py3-stdlib/pprint.py
index 7c1118a..d91421f 100644
--- a/common/py3-stdlib/pprint.py
+++ b/common/py3-stdlib/pprint.py
@@ -35,6 +35,7 @@
"""
import collections as _collections
+import dataclasses as _dataclasses
import re
import sys as _sys
import types as _types
@@ -45,18 +46,20 @@
def pprint(object, stream=None, indent=1, width=80, depth=None, *,
- compact=False, sort_dicts=True):
+ compact=False, sort_dicts=True, underscore_numbers=False):
"""Pretty-print a Python object to a stream [default is sys.stdout]."""
printer = PrettyPrinter(
stream=stream, indent=indent, width=width, depth=depth,
- compact=compact, sort_dicts=sort_dicts)
+ compact=compact, sort_dicts=sort_dicts,
+ underscore_numbers=underscore_numbers)
printer.pprint(object)
def pformat(object, indent=1, width=80, depth=None, *,
- compact=False, sort_dicts=True):
+ compact=False, sort_dicts=True, underscore_numbers=False):
"""Format a Python object into a pretty-printed representation."""
return PrettyPrinter(indent=indent, width=width, depth=depth,
- compact=compact, sort_dicts=sort_dicts).pformat(object)
+ compact=compact, sort_dicts=sort_dicts,
+ underscore_numbers=underscore_numbers).pformat(object)
def pp(object, *args, sort_dicts=False, **kwargs):
"""Pretty-print a Python object"""
@@ -64,15 +67,15 @@
def saferepr(object):
"""Version of repr() which can handle recursive data structures."""
- return _safe_repr(object, {}, None, 0, True)[0]
+ return PrettyPrinter()._safe_repr(object, {}, None, 0)[0]
def isreadable(object):
"""Determine if saferepr(object) is readable by eval()."""
- return _safe_repr(object, {}, None, 0, True)[1]
+ return PrettyPrinter()._safe_repr(object, {}, None, 0)[1]
def isrecursive(object):
"""Determine if object requires a recursive representation."""
- return _safe_repr(object, {}, None, 0, True)[2]
+ return PrettyPrinter()._safe_repr(object, {}, None, 0)[2]
class _safe_key:
"""Helper function for key functions when sorting unorderable objects.
@@ -102,7 +105,7 @@
class PrettyPrinter:
def __init__(self, indent=1, width=80, depth=None, stream=None, *,
- compact=False, sort_dicts=True):
+ compact=False, sort_dicts=True, underscore_numbers=False):
"""Handle pretty printing operations onto a stream using a set of
configured parameters.
@@ -143,6 +146,7 @@
self._stream = _sys.stdout
self._compact = bool(compact)
self._sort_dicts = sort_dicts
+ self._underscore_numbers = underscore_numbers
def pprint(self, object):
self._format(object, self._stream, 0, 0, {}, 0)
@@ -176,14 +180,26 @@
p(self, object, stream, indent, allowance, context, level + 1)
del context[objid]
return
- elif isinstance(object, dict):
+ elif (_dataclasses.is_dataclass(object) and
+ not isinstance(object, type) and
+ object.__dataclass_params__.repr and
+ # Check dataclass has generated repr method.
+ hasattr(object.__repr__, "__wrapped__") and
+ "__create_fn__" in object.__repr__.__wrapped__.__qualname__):
context[objid] = 1
- self._pprint_dict(object, stream, indent, allowance,
- context, level + 1)
+ self._pprint_dataclass(object, stream, indent, allowance, context, level + 1)
del context[objid]
return
stream.write(rep)
+ def _pprint_dataclass(self, object, stream, indent, allowance, context, level):
+ cls_name = object.__class__.__name__
+ indent += len(cls_name) + 1
+ items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr]
+ stream.write(cls_name + '(')
+ self._format_namespace_items(items, stream, indent, allowance, context, level)
+ stream.write(')')
+
_dispatch = {}
def _pprint_dict(self, object, stream, indent, allowance, context, level):
@@ -350,21 +366,9 @@
else:
cls_name = object.__class__.__name__
indent += len(cls_name) + 1
- delimnl = ',\n' + ' ' * indent
items = object.__dict__.items()
- last_index = len(items) - 1
-
stream.write(cls_name + '(')
- for i, (key, ent) in enumerate(items):
- stream.write(key)
- stream.write('=')
-
- last = i == last_index
- self._format(ent, stream, indent + len(key) + 1,
- allowance if last else 1,
- context, level)
- if not last:
- stream.write(delimnl)
+ self._format_namespace_items(items, stream, indent, allowance, context, level)
stream.write(')')
_dispatch[_types.SimpleNamespace.__repr__] = _pprint_simplenamespace
@@ -386,6 +390,25 @@
if not last:
write(delimnl)
+ def _format_namespace_items(self, items, stream, indent, allowance, context, level):
+ write = stream.write
+ delimnl = ',\n' + ' ' * indent
+ last_index = len(items) - 1
+ for i, (key, ent) in enumerate(items):
+ last = i == last_index
+ write(key)
+ write('=')
+ if id(ent) in context:
+ # Special-case representation of recursion to match standard
+ # recursive dataclass repr.
+ write("...")
+ else:
+ self._format(ent, stream, indent + len(key) + 1,
+ allowance if last else 1,
+ context, level)
+ if not last:
+ write(delimnl)
+
def _format_items(self, items, stream, indent, allowance, context, level):
write = stream.write
indent += self._indent_per_level
@@ -441,7 +464,7 @@
and flags indicating whether the representation is 'readable'
and whether the object represents a recursive construct.
"""
- return _safe_repr(object, context, maxlevels, level, self._sort_dicts)
+ return self._safe_repr(object, context, maxlevels, level)
def _pprint_default_dict(self, object, stream, indent, allowance, context, level):
if not len(object):
@@ -524,79 +547,88 @@
_dispatch[_collections.UserString.__repr__] = _pprint_user_string
-# Return triple (repr_string, isreadable, isrecursive).
+ def _safe_repr(self, object, context, maxlevels, level):
+ # Return triple (repr_string, isreadable, isrecursive).
+ typ = type(object)
+ if typ in _builtin_scalars:
+ return repr(object), True, False
-def _safe_repr(object, context, maxlevels, level, sort_dicts):
- typ = type(object)
- if typ in _builtin_scalars:
- return repr(object), True, False
+ r = getattr(typ, "__repr__", None)
- r = getattr(typ, "__repr__", None)
- if issubclass(typ, dict) and r is dict.__repr__:
- if not object:
- return "{}", True, False
- objid = id(object)
- if maxlevels and level >= maxlevels:
- return "{...}", False, objid in context
- if objid in context:
- return _recursion(object), False, True
- context[objid] = 1
- readable = True
- recursive = False
- components = []
- append = components.append
- level += 1
- if sort_dicts:
- items = sorted(object.items(), key=_safe_tuple)
- else:
- items = object.items()
- for k, v in items:
- krepr, kreadable, krecur = _safe_repr(k, context, maxlevels, level, sort_dicts)
- vrepr, vreadable, vrecur = _safe_repr(v, context, maxlevels, level, sort_dicts)
- append("%s: %s" % (krepr, vrepr))
- readable = readable and kreadable and vreadable
- if krecur or vrecur:
- recursive = True
- del context[objid]
- return "{%s}" % ", ".join(components), readable, recursive
+ if issubclass(typ, int) and r is int.__repr__:
+ if self._underscore_numbers:
+ return f"{object:_d}", True, False
+ else:
+ return repr(object), True, False
- if (issubclass(typ, list) and r is list.__repr__) or \
- (issubclass(typ, tuple) and r is tuple.__repr__):
- if issubclass(typ, list):
+ if issubclass(typ, dict) and r is dict.__repr__:
if not object:
- return "[]", True, False
- format = "[%s]"
- elif len(object) == 1:
- format = "(%s,)"
- else:
- if not object:
- return "()", True, False
- format = "(%s)"
- objid = id(object)
- if maxlevels and level >= maxlevels:
- return format % "...", False, objid in context
- if objid in context:
- return _recursion(object), False, True
- context[objid] = 1
- readable = True
- recursive = False
- components = []
- append = components.append
- level += 1
- for o in object:
- orepr, oreadable, orecur = _safe_repr(o, context, maxlevels, level, sort_dicts)
- append(orepr)
- if not oreadable:
- readable = False
- if orecur:
- recursive = True
- del context[objid]
- return format % ", ".join(components), readable, recursive
+ return "{}", True, False
+ objid = id(object)
+ if maxlevels and level >= maxlevels:
+ return "{...}", False, objid in context
+ if objid in context:
+ return _recursion(object), False, True
+ context[objid] = 1
+ readable = True
+ recursive = False
+ components = []
+ append = components.append
+ level += 1
+ if self._sort_dicts:
+ items = sorted(object.items(), key=_safe_tuple)
+ else:
+ items = object.items()
+ for k, v in items:
+ krepr, kreadable, krecur = self.format(
+ k, context, maxlevels, level)
+ vrepr, vreadable, vrecur = self.format(
+ v, context, maxlevels, level)
+ append("%s: %s" % (krepr, vrepr))
+ readable = readable and kreadable and vreadable
+ if krecur or vrecur:
+ recursive = True
+ del context[objid]
+ return "{%s}" % ", ".join(components), readable, recursive
- rep = repr(object)
- return rep, (rep and not rep.startswith('<')), False
+ if (issubclass(typ, list) and r is list.__repr__) or \
+ (issubclass(typ, tuple) and r is tuple.__repr__):
+ if issubclass(typ, list):
+ if not object:
+ return "[]", True, False
+ format = "[%s]"
+ elif len(object) == 1:
+ format = "(%s,)"
+ else:
+ if not object:
+ return "()", True, False
+ format = "(%s)"
+ objid = id(object)
+ if maxlevels and level >= maxlevels:
+ return format % "...", False, objid in context
+ if objid in context:
+ return _recursion(object), False, True
+ context[objid] = 1
+ readable = True
+ recursive = False
+ components = []
+ append = components.append
+ level += 1
+ for o in object:
+ orepr, oreadable, orecur = self.format(
+ o, context, maxlevels, level)
+ append(orepr)
+ if not oreadable:
+ readable = False
+ if orecur:
+ recursive = True
+ del context[objid]
+ return format % ", ".join(components), readable, recursive
-_builtin_scalars = frozenset({str, bytes, bytearray, int, float, complex,
+ rep = repr(object)
+ return rep, (rep and not rep.startswith('<')), False
+
+_builtin_scalars = frozenset({str, bytes, bytearray, float, complex,
bool, type(None)})
def _recursion(object):
@@ -610,7 +642,7 @@
object = [("string", (1, 2), [3, 4], {5: 6, 7: 8})] * 100000
p = PrettyPrinter()
t1 = time.perf_counter()
- _safe_repr(object, {}, None, 0, True)
+ p._safe_repr(object, {}, None, 0, True)
t2 = time.perf_counter()
p.pformat(object)
t3 = time.perf_counter()
diff --git a/common/py3-stdlib/profile.py b/common/py3-stdlib/profile.py
index 5cb017e..d8599fb 100755
--- a/common/py3-stdlib/profile.py
+++ b/common/py3-stdlib/profile.py
@@ -595,7 +595,12 @@
'__package__': None,
'__cached__': None,
}
- runctx(code, globs, None, options.outfile, options.sort)
+ try:
+ runctx(code, globs, None, options.outfile, options.sort)
+ except BrokenPipeError as exc:
+ # Prevent "Exception ignored" during interpreter shutdown.
+ sys.stdout = None
+ sys.exit(exc.errno)
else:
parser.print_usage()
return parser
diff --git a/common/py3-stdlib/pty.py b/common/py3-stdlib/pty.py
index a324320..8d8ce40 100644
--- a/common/py3-stdlib/pty.py
+++ b/common/py3-stdlib/pty.py
@@ -1,7 +1,7 @@
"""Pseudo terminal utilities."""
# Bugs: No signal handling. Doesn't set slave termios and window size.
-# Only tested on Linux.
+# Only tested on Linux, FreeBSD, and macOS.
# See: W. Richard Stevens. 1992. Advanced Programming in the
# UNIX Environment. Chapter 19.
# Author: Steen Lumholt -- with additions by Guido.
@@ -11,7 +11,11 @@
import sys
import tty
-__all__ = ["openpty","fork","spawn"]
+# names imported directly for test mocking purposes
+from os import close, waitpid
+from tty import setraw, tcgetattr, tcsetattr
+
+__all__ = ["openpty", "fork", "spawn"]
STDIN_FILENO = 0
STDOUT_FILENO = 1
@@ -105,8 +109,8 @@
os.dup2(slave_fd, STDIN_FILENO)
os.dup2(slave_fd, STDOUT_FILENO)
os.dup2(slave_fd, STDERR_FILENO)
- if (slave_fd > STDERR_FILENO):
- os.close (slave_fd)
+ if slave_fd > STDERR_FILENO:
+ os.close(slave_fd)
# Explicitly open the tty to make it become a controlling tty.
tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR)
@@ -133,14 +137,22 @@
pty master -> standard output (master_read)
standard input -> pty master (stdin_read)"""
fds = [master_fd, STDIN_FILENO]
- while True:
- rfds, wfds, xfds = select(fds, [], [])
+ while fds:
+ rfds, _wfds, _xfds = select(fds, [], [])
+
if master_fd in rfds:
- data = master_read(master_fd)
+ # Some OSes signal EOF by returning an empty byte string,
+ # some throw OSErrors.
+ try:
+ data = master_read(master_fd)
+ except OSError:
+ data = b""
if not data: # Reached EOF.
- fds.remove(master_fd)
+ return # Assume the child process has exited and is
+ # unreachable, so we clean up.
else:
os.write(STDOUT_FILENO, data)
+
if STDIN_FILENO in rfds:
data = stdin_read(STDIN_FILENO)
if not data:
@@ -153,20 +165,23 @@
if type(argv) == type(''):
argv = (argv,)
sys.audit('pty.spawn', argv)
+
pid, master_fd = fork()
if pid == CHILD:
os.execlp(argv[0], *argv)
+
try:
- mode = tty.tcgetattr(STDIN_FILENO)
- tty.setraw(STDIN_FILENO)
- restore = 1
+ mode = tcgetattr(STDIN_FILENO)
+ setraw(STDIN_FILENO)
+ restore = True
except tty.error: # This is the same as termios.error
- restore = 0
+ restore = False
+
try:
_copy(master_fd, master_read, stdin_read)
- except OSError:
+ finally:
if restore:
- tty.tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode)
+ tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode)
- os.close(master_fd)
- return os.waitpid(pid, 0)[1]
+ close(master_fd)
+ return waitpid(pid, 0)[1]
diff --git a/common/py3-stdlib/py_compile.py b/common/py3-stdlib/py_compile.py
index a81f493..388614e 100644
--- a/common/py3-stdlib/py_compile.py
+++ b/common/py3-stdlib/py_compile.py
@@ -173,43 +173,40 @@
return cfile
-def main(args=None):
- """Compile several source files.
+def main():
+ import argparse
- The files named in 'args' (or on the command line, if 'args' is
- not specified) are compiled and the resulting bytecode is cached
- in the normal manner. This function does not search a directory
- structure to locate source files; it only compiles files named
- explicitly. If '-' is the only parameter in args, the list of
- files is taken from standard input.
-
- """
- if args is None:
- args = sys.argv[1:]
- rv = 0
- if args == ['-']:
- while True:
- filename = sys.stdin.readline()
- if not filename:
- break
- filename = filename.rstrip('\n')
- try:
- compile(filename, doraise=True)
- except PyCompileError as error:
- rv = 1
- sys.stderr.write("%s\n" % error.msg)
- except OSError as error:
- rv = 1
- sys.stderr.write("%s\n" % error)
+ description = 'A simple command-line interface for py_compile module.'
+ parser = argparse.ArgumentParser(description=description)
+ parser.add_argument(
+ '-q', '--quiet',
+ action='store_true',
+ help='Suppress error output',
+ )
+ parser.add_argument(
+ 'filenames',
+ nargs='+',
+ help='Files to compile',
+ )
+ args = parser.parse_args()
+ if args.filenames == ['-']:
+ filenames = [filename.rstrip('\n') for filename in sys.stdin.readlines()]
else:
- for filename in args:
- try:
- compile(filename, doraise=True)
- except PyCompileError as error:
- # return value to indicate at least one failure
- rv = 1
- sys.stderr.write("%s\n" % error.msg)
- return rv
+ filenames = args.filenames
+ for filename in filenames:
+ try:
+ compile(filename, doraise=True)
+ except PyCompileError as error:
+ if args.quiet:
+ parser.exit(1)
+ else:
+ parser.exit(1, error.msg)
+ except OSError as error:
+ if args.quiet:
+ parser.exit(1)
+ else:
+ parser.exit(1, str(error))
+
if __name__ == "__main__":
- sys.exit(main())
+ main()
diff --git a/common/py3-stdlib/pyclbr.py b/common/py3-stdlib/pyclbr.py
index 99a1734..37f8699 100644
--- a/common/py3-stdlib/pyclbr.py
+++ b/common/py3-stdlib/pyclbr.py
@@ -21,11 +21,14 @@
name -- name of the object;
file -- file in which the object is defined;
lineno -- line in the file where the object's definition starts;
+ end_lineno -- line in the file where the object's definition ends;
parent -- parent of this object, if any;
children -- nested objects contained in this object.
The 'children' attribute is a dictionary mapping names to objects.
-Instances of Function describe functions with the attributes from _Object.
+Instances of Function describe functions with the attributes from _Object,
+plus the following:
+ is_async -- if a function is defined with an 'async' prefix
Instances of Class describe classes with the attributes from _Object,
plus the following:
@@ -38,11 +41,9 @@
shouldn't happen often.
"""
-import io
+import ast
import sys
import importlib.util
-import tokenize
-from token import NAME, DEDENT, OP
__all__ = ["readmodule", "readmodule_ex", "Class", "Function"]
@@ -51,48 +52,50 @@
class _Object:
"Information about Python class or function."
- def __init__(self, module, name, file, lineno, parent):
+ def __init__(self, module, name, file, lineno, end_lineno, parent):
self.module = module
self.name = name
self.file = file
self.lineno = lineno
+ self.end_lineno = end_lineno
self.parent = parent
self.children = {}
-
- def _addchild(self, name, obj):
- self.children[name] = obj
+ if parent is not None:
+ parent.children[name] = self
+# Odd Function and Class signatures are for back-compatibility.
class Function(_Object):
"Information about a Python function, including methods."
- def __init__(self, module, name, file, lineno, parent=None):
- _Object.__init__(self, module, name, file, lineno, parent)
+ def __init__(self, module, name, file, lineno,
+ parent=None, is_async=False, *, end_lineno=None):
+ super().__init__(module, name, file, lineno, end_lineno, parent)
+ self.is_async = is_async
+ if isinstance(parent, Class):
+ parent.methods[name] = lineno
class Class(_Object):
"Information about a Python class."
- def __init__(self, module, name, super, file, lineno, parent=None):
- _Object.__init__(self, module, name, file, lineno, parent)
- self.super = [] if super is None else super
+ def __init__(self, module, name, super_, file, lineno,
+ parent=None, *, end_lineno=None):
+ super().__init__(module, name, file, lineno, end_lineno, parent)
+ self.super = super_ or []
self.methods = {}
- def _addmethod(self, name, lineno):
- self.methods[name] = lineno
-
-def _nest_function(ob, func_name, lineno):
+# These 2 functions are used in these tests
+# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py
+def _nest_function(ob, func_name, lineno, end_lineno, is_async=False):
"Return a Function after nesting within ob."
- newfunc = Function(ob.module, func_name, ob.file, lineno, ob)
- ob._addchild(func_name, newfunc)
- if isinstance(ob, Class):
- ob._addmethod(func_name, lineno)
- return newfunc
+ return Function(ob.module, func_name, ob.file, lineno,
+ parent=ob, is_async=is_async, end_lineno=end_lineno)
-def _nest_class(ob, class_name, lineno, super=None):
+def _nest_class(ob, class_name, lineno, end_lineno, super=None):
"Return a Class after nesting within ob."
- newclass = Class(ob.module, class_name, super, ob.file, lineno, ob)
- ob._addchild(class_name, newclass)
- return newclass
+ return Class(ob.module, class_name, super, ob.file, lineno,
+ parent=ob, end_lineno=end_lineno)
+
def readmodule(module, path=None):
"""Return Class objects for the top-level classes in module.
@@ -115,6 +118,7 @@
"""
return _readmodule(module, path or [])
+
def _readmodule(module, path, inpackage=None):
"""Do the hard work for readmodule[_ex].
@@ -179,187 +183,93 @@
return _create_tree(fullmodule, path, fname, source, tree, inpackage)
-def _create_tree(fullmodule, path, fname, source, tree, inpackage):
- """Return the tree for a particular module.
+class _ModuleBrowser(ast.NodeVisitor):
+ def __init__(self, module, path, file, tree, inpackage):
+ self.path = path
+ self.tree = tree
+ self.file = file
+ self.module = module
+ self.inpackage = inpackage
+ self.stack = []
- fullmodule (full module name), inpackage+module, becomes o.module.
- path is passed to recursive calls of _readmodule.
- fname becomes o.file.
- source is tokenized. Imports cause recursive calls to _readmodule.
- tree is {} or {'__path__': <submodule search locations>}.
- inpackage, None or string, is passed to recursive calls of _readmodule.
+ def visit_ClassDef(self, node):
+ bases = []
+ for base in node.bases:
+ name = ast.unparse(base)
+ if name in self.tree:
+ # We know this super class.
+ bases.append(self.tree[name])
+ elif len(names := name.split(".")) > 1:
+ # Super class form is module.class:
+ # look in module for class.
+ *_, module, class_ = names
+ if module in _modules:
+ bases.append(_modules[module].get(class_, name))
+ else:
+ bases.append(name)
- The effect of recursive calls is mutation of global _modules.
- """
- f = io.StringIO(source)
+ parent = self.stack[-1] if self.stack else None
+ class_ = Class(self.module, node.name, bases, self.file, node.lineno,
+ parent=parent, end_lineno=node.end_lineno)
+ if parent is None:
+ self.tree[node.name] = class_
+ self.stack.append(class_)
+ self.generic_visit(node)
+ self.stack.pop()
- stack = [] # Initialize stack of (class, indent) pairs.
+ def visit_FunctionDef(self, node, *, is_async=False):
+ parent = self.stack[-1] if self.stack else None
+ function = Function(self.module, node.name, self.file, node.lineno,
+ parent, is_async, end_lineno=node.end_lineno)
+ if parent is None:
+ self.tree[node.name] = function
+ self.stack.append(function)
+ self.generic_visit(node)
+ self.stack.pop()
- g = tokenize.generate_tokens(f.readline)
- try:
- for tokentype, token, start, _end, _line in g:
- if tokentype == DEDENT:
- lineno, thisindent = start
- # Close previous nested classes and defs.
- while stack and stack[-1][1] >= thisindent:
- del stack[-1]
- elif token == 'def':
- lineno, thisindent = start
- # Close previous nested classes and defs.
- while stack and stack[-1][1] >= thisindent:
- del stack[-1]
- tokentype, func_name, start = next(g)[0:3]
- if tokentype != NAME:
- continue # Skip def with syntax error.
- cur_func = None
- if stack:
- cur_obj = stack[-1][0]
- cur_func = _nest_function(cur_obj, func_name, lineno)
- else:
- # It is just a function.
- cur_func = Function(fullmodule, func_name, fname, lineno)
- tree[func_name] = cur_func
- stack.append((cur_func, thisindent))
- elif token == 'class':
- lineno, thisindent = start
- # Close previous nested classes and defs.
- while stack and stack[-1][1] >= thisindent:
- del stack[-1]
- tokentype, class_name, start = next(g)[0:3]
- if tokentype != NAME:
- continue # Skip class with syntax error.
- # Parse what follows the class name.
- tokentype, token, start = next(g)[0:3]
- inherit = None
- if token == '(':
- names = [] # Initialize list of superclasses.
- level = 1
- super = [] # Tokens making up current superclass.
- while True:
- tokentype, token, start = next(g)[0:3]
- if token in (')', ',') and level == 1:
- n = "".join(super)
- if n in tree:
- # We know this super class.
- n = tree[n]
- else:
- c = n.split('.')
- if len(c) > 1:
- # Super class form is module.class:
- # look in module for class.
- m = c[-2]
- c = c[-1]
- if m in _modules:
- d = _modules[m]
- if c in d:
- n = d[c]
- names.append(n)
- super = []
- if token == '(':
- level += 1
- elif token == ')':
- level -= 1
- if level == 0:
- break
- elif token == ',' and level == 1:
- pass
- # Only use NAME and OP (== dot) tokens for type name.
- elif tokentype in (NAME, OP) and level == 1:
- super.append(token)
- # Expressions in the base list are not supported.
- inherit = names
- if stack:
- cur_obj = stack[-1][0]
- cur_class = _nest_class(
- cur_obj, class_name, lineno, inherit)
- else:
- cur_class = Class(fullmodule, class_name, inherit,
- fname, lineno)
- tree[class_name] = cur_class
- stack.append((cur_class, thisindent))
- elif token == 'import' and start[1] == 0:
- modules = _getnamelist(g)
- for mod, _mod2 in modules:
- try:
- # Recursively read the imported module.
- if inpackage is None:
- _readmodule(mod, path)
- else:
- try:
- _readmodule(mod, path, inpackage)
- except ImportError:
- _readmodule(mod, [])
- except:
- # If we can't find or parse the imported module,
- # too bad -- don't die here.
- pass
- elif token == 'from' and start[1] == 0:
- mod, token = _getname(g)
- if not mod or token != "import":
- continue
- names = _getnamelist(g)
+ def visit_AsyncFunctionDef(self, node):
+ self.visit_FunctionDef(node, is_async=True)
+
+ def visit_Import(self, node):
+ if node.col_offset != 0:
+ return
+
+ for module in node.names:
+ try:
try:
- # Recursively read the imported module.
- d = _readmodule(mod, path, inpackage)
- except:
- # If we can't find or parse the imported module,
- # too bad -- don't die here.
- continue
- # Add any classes that were defined in the imported module
- # to our name space if they were mentioned in the list.
- for n, n2 in names:
- if n in d:
- tree[n2 or n] = d[n]
- elif n == '*':
- # Don't add names that start with _.
- for n in d:
- if n[0] != '_':
- tree[n] = d[n]
- except StopIteration:
- pass
+ _readmodule(module.name, self.path, self.inpackage)
+ except ImportError:
+ _readmodule(module.name, [])
+ except (ImportError, SyntaxError):
+ # If we can't find or parse the imported module,
+ # too bad -- don't die here.
+ continue
- f.close()
- return tree
+ def visit_ImportFrom(self, node):
+ if node.col_offset != 0:
+ return
+ try:
+ module = "." * node.level
+ if node.module:
+ module += node.module
+ module = _readmodule(module, self.path, self.inpackage)
+ except (ImportError, SyntaxError):
+ return
+
+ for name in node.names:
+ if name.name in module:
+ self.tree[name.asname or name.name] = module[name.name]
+ elif name.name == "*":
+ for import_name, import_value in module.items():
+ if import_name.startswith("_"):
+ continue
+ self.tree[import_name] = import_value
-def _getnamelist(g):
- """Return list of (dotted-name, as-name or None) tuples for token source g.
-
- An as-name is the name that follows 'as' in an as clause.
- """
- names = []
- while True:
- name, token = _getname(g)
- if not name:
- break
- if token == 'as':
- name2, token = _getname(g)
- else:
- name2 = None
- names.append((name, name2))
- while token != "," and "\n" not in token:
- token = next(g)[1]
- if token != ",":
- break
- return names
-
-
-def _getname(g):
- "Return (dotted-name or None, next-token) tuple for token source g."
- parts = []
- tokentype, token = next(g)[0:2]
- if tokentype != NAME and token != '*':
- return (None, token)
- parts.append(token)
- while True:
- tokentype, token = next(g)[0:2]
- if token != '.':
- break
- tokentype, token = next(g)[0:2]
- if tokentype != NAME:
- break
- parts.append(token)
- return (".".join(parts), token)
+def _create_tree(fullmodule, path, fname, source, tree, inpackage):
+ mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage)
+ mbrowser.visit(ast.parse(source))
+ return mbrowser.tree
def _main():
diff --git a/common/py3-stdlib/pydoc.py b/common/py3-stdlib/pydoc.py
index 35ef3eb..4a8c10a 100755
--- a/common/py3-stdlib/pydoc.py
+++ b/common/py3-stdlib/pydoc.py
@@ -23,7 +23,7 @@
local machine. Port number 0 can be used to get an arbitrary unused port.
Run "pydoc -b" to start an HTTP server on an arbitrary unused port and
-open a Web browser to interactively browse documentation. Combine with
+open a web browser to interactively browse documentation. Combine with
the -n and -p options to control the hostname and port used.
Run "pydoc -w <name>" to write out the HTML documentation for a module
@@ -504,7 +504,7 @@
not file.startswith(os.path.join(basedir, 'site-packages')))) and
object.__name__ not in ('xml.etree', 'test.pydoc_mod')):
if docloc.startswith(("http://", "https://")):
- docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower())
+ docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower())
else:
docloc = os.path.join(docloc, object.__name__.lower() + ".html")
else:
@@ -694,7 +694,7 @@
url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif pep:
- url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep)
+ url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif selfdot:
# Create a link for methods like 'self.method(...)'
@@ -1594,9 +1594,10 @@
def pipepager(text, cmd):
"""Page through text by feeding it to another program."""
import subprocess
- proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
+ proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+ errors='backslashreplace')
try:
- with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe:
+ with proc.stdin as pipe:
try:
pipe.write(text)
except KeyboardInterrupt:
@@ -1617,13 +1618,14 @@
def tempfilepager(text, cmd):
"""Page through text by invoking a program on a temporary file."""
import tempfile
- filename = tempfile.mktemp()
- with open(filename, 'w', errors='backslashreplace') as file:
- file.write(text)
- try:
+ with tempfile.TemporaryDirectory() as tempdir:
+ filename = os.path.join(tempdir, 'pydoc.out')
+ with open(filename, 'w', errors='backslashreplace',
+ encoding=os.device_encoding(0) if
+ sys.platform == 'win32' else None
+ ) as file:
+ file.write(text)
os.system(cmd + ' "' + filename + '"')
- finally:
- os.unlink(filename)
def _escape_stdout(text):
# Escape non-encodable characters to avoid encoding errors later
@@ -1817,7 +1819,6 @@
'False': '',
'None': '',
'True': '',
- '__peg_parser__': '',
'and': 'BOOLEAN',
'as': 'with',
'assert': ('assert', ''),
@@ -2065,7 +2066,7 @@
Welcome to Python {0}'s help utility!
If this is your first time using Python, you should definitely check out
-the tutorial on the Internet at https://docs.python.org/{0}/tutorial/.
+the tutorial on the internet at https://docs.python.org/{0}/tutorial/.
Enter the name of any module, keyword, or topic to get help on writing
Python programs and using Python modules. To quit this help utility and
@@ -2279,13 +2280,13 @@
warnings.filterwarnings('ignore') # ignore problems during import
ModuleScanner().run(callback, key, onerror=onerror)
-# --------------------------------------- enhanced Web browser interface
+# --------------------------------------- enhanced web browser interface
def _start_server(urlhandler, hostname, port):
"""Start an HTTP server thread on a specific port.
Start an HTML/text server thread, so HTML or text documents can be
- browsed dynamically and interactively with a Web browser. Example use:
+ browsed dynamically and interactively with a web browser. Example use:
>>> import time
>>> import pydoc
@@ -2457,9 +2458,6 @@
%s</head><body bgcolor="#f0f0f8">%s<div style="clear:both;padding-top:.5em;">%s</div>
</body></html>''' % (title, css_link, html_navbar(), contents)
- def filelink(self, url, path):
- return '<a href="getfile?key=%s">%s</a>' % (url, path)
-
html = _HTMLDoc()
@@ -2545,19 +2543,6 @@
'key = %s' % key, '#ffffff', '#ee77aa', '<br>'.join(results))
return 'Search Results', contents
- def html_getfile(path):
- """Get and display a source file listing safely."""
- path = urllib.parse.unquote(path)
- with tokenize.open(path) as fp:
- lines = html.escape(fp.read())
- body = '<pre>%s</pre>' % lines
- heading = html.heading(
- '<big><big><strong>File Listing</strong></big></big>',
- '#ffffff', '#7799ee')
- contents = heading + html.bigsection(
- 'File: %s' % path, '#ffffff', '#ee77aa', body)
- return 'getfile %s' % path, contents
-
def html_topics():
"""Index of topic texts available."""
@@ -2649,8 +2634,6 @@
op, _, url = url.partition('=')
if op == "search?key":
title, content = html_search(url)
- elif op == "getfile?key":
- title, content = html_getfile(url)
elif op == "topic?key":
# try topics first, then objects.
try:
@@ -2689,7 +2672,7 @@
def browse(port=0, *, open_browser=True, hostname='localhost'):
- """Start the enhanced pydoc Web server and open a Web browser.
+ """Start the enhanced pydoc web server and open a web browser.
Use port '0' to start the server on an arbitrary port.
Set open_browser to False to suppress opening a browser.
@@ -2841,7 +2824,7 @@
number 0 can be used to get an arbitrary unused port.
{cmd} -b
- Start an HTTP server on an arbitrary unused port and open a Web browser
+ Start an HTTP server on an arbitrary unused port and open a web browser
to interactively browse documentation. This option can be used in
combination with -n and/or -p.
diff --git a/common/py3-stdlib/pydoc_data/topics.py b/common/py3-stdlib/pydoc_data/topics.py
index d8dd8c5..ac7d16c 100644
--- a/common/py3-stdlib/pydoc_data/topics.py
+++ b/common/py3-stdlib/pydoc_data/topics.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Autogenerated by Sphinx on Mon Dec 7 15:00:07 2020
+# Autogenerated by Sphinx on Wed Mar 16 11:26:55 2022
topics = {'assert': 'The "assert" statement\n'
'**********************\n'
'\n'
@@ -433,11 +433,9 @@
'\n'
'Execution of Python coroutines can be suspended and resumed at '
'many\n'
- 'points (see *coroutine*). Inside the body of a coroutine '
- 'function,\n'
- '"await" and "async" identifiers become reserved keywords; "await"\n'
- 'expressions, "async for" and "async with" can only be used in\n'
- 'coroutine function bodies.\n'
+ 'points (see *coroutine*). "await" expressions, "async for" and '
+ '"async\n'
+ 'with" can only be used in the body of a coroutine function.\n'
'\n'
'Functions defined with "async def" syntax are always coroutine\n'
'functions, even if they do not contain "await" or "async" '
@@ -453,19 +451,22 @@
' do_stuff()\n'
' await some_coroutine()\n'
'\n'
+ 'Changed in version 3.7: "await" and "async" are now keywords;\n'
+ 'previously they were only treated as such inside the body of a\n'
+ 'coroutine function.\n'
+ '\n'
'\n'
'The "async for" statement\n'
'=========================\n'
'\n'
' async_for_stmt ::= "async" for_stmt\n'
'\n'
- 'An *asynchronous iterable* is able to call asynchronous code in '
- 'its\n'
- '*iter* implementation, and *asynchronous iterator* can call\n'
- 'asynchronous code in its *next* method.\n'
+ 'An *asynchronous iterable* provides an "__aiter__" method that\n'
+ 'directly returns an *asynchronous iterator*, which can call\n'
+ 'asynchronous code in its "__anext__" method.\n'
'\n'
'The "async for" statement allows convenient iteration over\n'
- 'asynchronous iterators.\n'
+ 'asynchronous iterables.\n'
'\n'
'The following code:\n'
'\n'
@@ -550,13 +551,65 @@
'exception.\n'
' That new exception causes the old one to be lost.\n'
'\n'
- '[2] A string literal appearing as the first statement in the '
+ '[2] In pattern matching, a sequence is defined as one of the\n'
+ ' following:\n'
+ '\n'
+ ' * a class that inherits from "collections.abc.Sequence"\n'
+ '\n'
+ ' * a Python class that has been registered as\n'
+ ' "collections.abc.Sequence"\n'
+ '\n'
+ ' * a builtin class that has its (CPython) '
+ '"Py_TPFLAGS_SEQUENCE"\n'
+ ' bit set\n'
+ '\n'
+ ' * a class that inherits from any of the above\n'
+ '\n'
+ ' The following standard library classes are sequences:\n'
+ '\n'
+ ' * "array.array"\n'
+ '\n'
+ ' * "collections.deque"\n'
+ '\n'
+ ' * "list"\n'
+ '\n'
+ ' * "memoryview"\n'
+ '\n'
+ ' * "range"\n'
+ '\n'
+ ' * "tuple"\n'
+ '\n'
+ ' Note:\n'
+ '\n'
+ ' Subject values of type "str", "bytes", and "bytearray" do '
+ 'not\n'
+ ' match sequence patterns.\n'
+ '\n'
+ '[3] In pattern matching, a mapping is defined as one of the '
+ 'following:\n'
+ '\n'
+ ' * a class that inherits from "collections.abc.Mapping"\n'
+ '\n'
+ ' * a Python class that has been registered as\n'
+ ' "collections.abc.Mapping"\n'
+ '\n'
+ ' * a builtin class that has its (CPython) '
+ '"Py_TPFLAGS_MAPPING"\n'
+ ' bit set\n'
+ '\n'
+ ' * a class that inherits from any of the above\n'
+ '\n'
+ ' The standard library classes "dict" and '
+ '"types.MappingProxyType"\n'
+ ' are mappings.\n'
+ '\n'
+ '[4] A string literal appearing as the first statement in the '
'function\n'
' body is transformed into the function’s "__doc__" attribute '
'and\n'
' therefore the function’s *docstring*.\n'
'\n'
- '[3] A string literal appearing as the first statement in the class\n'
+ '[5] A string literal appearing as the first statement in the class\n'
' body is transformed into the namespace’s "__doc__" item and\n'
' therefore the class’s *docstring*.\n',
'atom-identifiers': 'Identifiers (Names)\n'
@@ -883,32 +936,6 @@
'*instance* of the\n'
' owner class.\n'
'\n'
- 'object.__set_name__(self, owner, name)\n'
- '\n'
- ' Called at the time the owning class *owner* is '
- 'created. The\n'
- ' descriptor has been assigned to *name*.\n'
- '\n'
- ' Note:\n'
- '\n'
- ' "__set_name__()" is only called implicitly as part '
- 'of the "type"\n'
- ' constructor, so it will need to be called '
- 'explicitly with the\n'
- ' appropriate parameters when a descriptor is added '
- 'to a class\n'
- ' after initial creation:\n'
- '\n'
- ' class A:\n'
- ' pass\n'
- ' descr = custom_descriptor()\n'
- ' A.attr = descr\n'
- " descr.__set_name__(A, 'attr')\n"
- '\n'
- ' See Creating the class object for more details.\n'
- '\n'
- ' New in version 3.6.\n'
- '\n'
'The attribute "__objclass__" is interpreted by the '
'"inspect" module as\n'
'specifying the class where this object was defined '
@@ -980,16 +1007,16 @@
'"super(B,\n'
' obj).m()" searches "obj.__class__.__mro__" for the '
'base class "A"\n'
- ' immediately preceding "B" and then invokes the '
+ ' immediately following "B" and then invokes the '
'descriptor with the\n'
' call: "A.__dict__[\'m\'].__get__(obj, '
'obj.__class__)".\n'
'\n'
'For instance bindings, the precedence of descriptor '
'invocation depends\n'
- 'on the which descriptor methods are defined. A '
- 'descriptor can define\n'
- 'any combination of "__get__()", "__set__()" and '
+ 'on which descriptor methods are defined. A descriptor '
+ 'can define any\n'
+ 'combination of "__get__()", "__set__()" and '
'"__delete__()". If it\n'
'does not define "__get__()", then accessing the '
'attribute will return\n'
@@ -1011,14 +1038,15 @@
'can be\n'
'overridden by instances.\n'
'\n'
- 'Python methods (including "staticmethod()" and '
- '"classmethod()") are\n'
- 'implemented as non-data descriptors. Accordingly, '
- 'instances can\n'
- 'redefine and override methods. This allows individual '
- 'instances to\n'
- 'acquire behaviors that differ from other instances of '
- 'the same class.\n'
+ 'Python methods (including those decorated with '
+ '"@staticmethod" and\n'
+ '"@classmethod") are implemented as non-data '
+ 'descriptors. Accordingly,\n'
+ 'instances can redefine and override methods. This '
+ 'allows individual\n'
+ 'instances to acquire behaviors that differ from other '
+ 'instances of the\n'
+ 'same class.\n'
'\n'
'The "property()" function is implemented as a data '
'descriptor.\n'
@@ -1031,12 +1059,12 @@
'\n'
'*__slots__* allow us to explicitly declare data members '
'(like\n'
- 'properties) and deny the creation of *__dict__* and '
+ 'properties) and deny the creation of "__dict__" and '
'*__weakref__*\n'
'(unless explicitly declared in *__slots__* or available '
'in a parent.)\n'
'\n'
- 'The space saved over using *__dict__* can be '
+ 'The space saved over using "__dict__" can be '
'significant. Attribute\n'
'lookup speed can be significantly improved as well.\n'
'\n'
@@ -1048,7 +1076,7 @@
'*__slots__*\n'
' reserves space for the declared variables and '
'prevents the\n'
- ' automatic creation of *__dict__* and *__weakref__* '
+ ' automatic creation of "__dict__" and *__weakref__* '
'for each\n'
' instance.\n'
'\n'
@@ -1057,11 +1085,11 @@
'--------------------------\n'
'\n'
'* When inheriting from a class without *__slots__*, the '
- '*__dict__* and\n'
+ '"__dict__" and\n'
' *__weakref__* attribute of the instances will always '
'be accessible.\n'
'\n'
- '* Without a *__dict__* variable, instances cannot be '
+ '* Without a "__dict__" variable, instances cannot be '
'assigned new\n'
' variables not listed in the *__slots__* definition. '
'Attempts to\n'
@@ -1075,28 +1103,28 @@
'\n'
'* Without a *__weakref__* variable for each instance, '
'classes defining\n'
- ' *__slots__* do not support weak references to its '
- 'instances. If weak\n'
- ' reference support is needed, then add '
+ ' *__slots__* do not support "weak references" to its '
+ 'instances. If\n'
+ ' weak reference support is needed, then add '
'"\'__weakref__\'" to the\n'
' sequence of strings in the *__slots__* declaration.\n'
'\n'
'* *__slots__* are implemented at the class level by '
'creating\n'
- ' descriptors (Implementing Descriptors) for each '
- 'variable name. As a\n'
- ' result, class attributes cannot be used to set default '
- 'values for\n'
- ' instance variables defined by *__slots__*; otherwise, '
- 'the class\n'
- ' attribute would overwrite the descriptor assignment.\n'
+ ' descriptors for each variable name. As a result, '
+ 'class attributes\n'
+ ' cannot be used to set default values for instance '
+ 'variables defined\n'
+ ' by *__slots__*; otherwise, the class attribute would '
+ 'overwrite the\n'
+ ' descriptor assignment.\n'
'\n'
'* The action of a *__slots__* declaration is not limited '
'to the class\n'
' where it is defined. *__slots__* declared in parents '
'are available\n'
' in child classes. However, child subclasses will get a '
- '*__dict__*\n'
+ '"__dict__"\n'
' and *__weakref__* unless they also define *__slots__* '
'(which should\n'
' only contain names of any *additional* slots).\n'
@@ -1116,13 +1144,19 @@
' “variable-length” built-in types such as "int", '
'"bytes" and "tuple".\n'
'\n'
- '* Any non-string iterable may be assigned to '
- '*__slots__*. Mappings may\n'
- ' also be used; however, in the future, special meaning '
- 'may be\n'
- ' assigned to the values corresponding to each key.\n'
+ '* Any non-string *iterable* may be assigned to '
+ '*__slots__*.\n'
'\n'
- '* *__class__* assignment works only if both classes have '
+ '* If a "dictionary" is used to assign *__slots__*, the '
+ 'dictionary keys\n'
+ ' will be used as the slot names. The values of the '
+ 'dictionary can be\n'
+ ' used to provide per-attribute docstrings that will be '
+ 'recognised by\n'
+ ' "inspect.getdoc()" and displayed in the output of '
+ '"help()".\n'
+ '\n'
+ '* "__class__" assignment works only if both classes have '
'the same\n'
' *__slots__*.\n'
'\n'
@@ -1134,10 +1168,10 @@
'violations\n'
' raise "TypeError".\n'
'\n'
- '* If an iterator is used for *__slots__* then a '
- 'descriptor is created\n'
- ' for each of the iterator’s values. However, the '
- '*__slots__*\n'
+ '* If an *iterator* is used for *__slots__* then a '
+ '*descriptor* is\n'
+ ' created for each of the iterator’s values. However, '
+ 'the *__slots__*\n'
' attribute will be an empty iterator.\n',
'attribute-references': 'Attribute references\n'
'********************\n'
@@ -1260,6 +1294,10 @@
'In the latter case, sequence repetition is performed; a negative\n'
'repetition factor yields an empty sequence.\n'
'\n'
+ 'This operation can be customized using the special "__mul__()" '
+ 'and\n'
+ '"__rmul__()" methods.\n'
+ '\n'
'The "@" (at) operator is intended to be used for matrix\n'
'multiplication. No builtin Python types implement this operator.\n'
'\n'
@@ -1275,6 +1313,10 @@
'result. Division by zero raises the "ZeroDivisionError" '
'exception.\n'
'\n'
+ 'This operation can be customized using the special "__truediv__()" '
+ 'and\n'
+ '"__floordiv__()" methods.\n'
+ '\n'
'The "%" (modulo) operator yields the remainder from the division '
'of\n'
'the first argument by the second. The numeric arguments are '
@@ -1306,6 +1348,10 @@
'string formatting is described in the Python Library Reference,\n'
'section printf-style String Formatting.\n'
'\n'
+ 'The *modulo* operation can be customized using the special '
+ '"__mod__()"\n'
+ 'method.\n'
+ '\n'
'The floor division operator, the modulo operator, and the '
'"divmod()"\n'
'function are not defined for complex numbers. Instead, convert to '
@@ -1320,9 +1366,16 @@
'and then added together. In the latter case, the sequences are\n'
'concatenated.\n'
'\n'
+ 'This operation can be customized using the special "__add__()" '
+ 'and\n'
+ '"__radd__()" methods.\n'
+ '\n'
'The "-" (subtraction) operator yields the difference of its '
'arguments.\n'
- 'The numeric arguments are first converted to a common type.\n',
+ 'The numeric arguments are first converted to a common type.\n'
+ '\n'
+ 'This operation can be customized using the special "__sub__()" '
+ 'method.\n',
'bitwise': 'Binary bitwise operations\n'
'*************************\n'
'\n'
@@ -1335,14 +1388,18 @@
'\n'
'The "&" operator yields the bitwise AND of its arguments, which '
'must\n'
- 'be integers.\n'
+ 'be integers or one of them must be a custom object overriding\n'
+ '"__and__()" or "__rand__()" special methods.\n'
'\n'
'The "^" operator yields the bitwise XOR (exclusive OR) of its\n'
- 'arguments, which must be integers.\n'
+ 'arguments, which must be integers or one of them must be a '
+ 'custom\n'
+ 'object overriding "__xor__()" or "__rxor__()" special methods.\n'
'\n'
'The "|" operator yields the bitwise (inclusive) OR of its '
'arguments,\n'
- 'which must be integers.\n',
+ 'which must be integers or one of them must be a custom object\n'
+ 'overriding "__or__()" or "__ror__()" special methods.\n',
'bltin-code-objects': 'Code Objects\n'
'************\n'
'\n'
@@ -1359,6 +1416,10 @@
'through their "__code__" attribute. See also the '
'"code" module.\n'
'\n'
+ 'Accessing "__code__" raises an auditing event '
+ '"object.__getattr__"\n'
+ 'with arguments "obj" and ""__code__"".\n'
+ '\n'
'A code object can be executed or evaluated by passing '
'it (instead of a\n'
'source string) to the "exec()" or "eval()" built-in '
@@ -1703,7 +1764,7 @@
'original global namespace. (Usually, the suite contains mostly\n'
'function definitions.) When the class’s suite finishes execution, '
'its\n'
- 'execution frame is discarded but its local namespace is saved. [3] '
+ 'execution frame is discarded but its local namespace is saved. [5] '
'A\n'
'class object is then created using the inheritance list for the '
'base\n'
@@ -1784,7 +1845,11 @@
' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n'
' | "is" ["not"] | ["not"] "in"\n'
'\n'
- 'Comparisons yield boolean values: "True" or "False".\n'
+ 'Comparisons yield boolean values: "True" or "False". Custom '
+ '*rich\n'
+ 'comparison methods* may return non-boolean values. In this '
+ 'case Python\n'
+ 'will call "bool()" on such value in boolean contexts.\n'
'\n'
'Comparisons can be chained arbitrarily, e.g., "x < y <= z" '
'is\n'
@@ -2182,6 +2247,7 @@
' | for_stmt\n'
' | try_stmt\n'
' | with_stmt\n'
+ ' | match_stmt\n'
' | funcdef\n'
' | classdef\n'
' | async_with_stmt\n'
@@ -2319,33 +2385,6 @@
':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, '
'2]".\n'
'\n'
- 'Note:\n'
- '\n'
- ' There is a subtlety when the sequence is being modified by the '
- 'loop\n'
- ' (this can only occur for mutable sequences, e.g. lists). An\n'
- ' internal counter is used to keep track of which item is used '
- 'next,\n'
- ' and this is incremented on each iteration. When this counter '
- 'has\n'
- ' reached the length of the sequence the loop terminates. This '
- 'means\n'
- ' that if the suite deletes the current (or a previous) item '
- 'from the\n'
- ' sequence, the next item will be skipped (since it gets the '
- 'index of\n'
- ' the current item which has already been treated). Likewise, '
- 'if the\n'
- ' suite inserts an item in the sequence before the current item, '
- 'the\n'
- ' current item will be treated again the next time through the '
- 'loop.\n'
- ' This can lead to nasty bugs that can be avoided by making a\n'
- ' temporary copy using a slice of the whole sequence, e.g.,\n'
- '\n'
- ' for x in a[:]:\n'
- ' if x < 0: a.remove(x)\n'
- '\n'
'\n'
'The "try" statement\n'
'===================\n'
@@ -2381,8 +2420,9 @@
'compatible\n'
'with an exception if it is the class or a base class of the '
'exception\n'
- 'object or a tuple containing an item compatible with the '
- 'exception.\n'
+ 'object, or a tuple containing an item that is the class or a '
+ 'base\n'
+ 'class of the exception object.\n'
'\n'
'If no except clause matches the exception, the search for an '
'exception\n'
@@ -2449,11 +2489,32 @@
'(see\n'
'section The standard type hierarchy) identifying the point in '
'the\n'
- 'program where the exception occurred. "sys.exc_info()" values '
- 'are\n'
- 'restored to their previous values (before the call) when '
- 'returning\n'
- 'from a function that handled an exception.\n'
+ 'program where the exception occurred. The details about the '
+ 'exception\n'
+ 'accessed via "sys.exc_info()" are restored to their previous '
+ 'values\n'
+ 'when leaving an exception handler:\n'
+ '\n'
+ ' >>> print(sys.exc_info())\n'
+ ' (None, None, None)\n'
+ ' >>> try:\n'
+ ' ... raise TypeError\n'
+ ' ... except:\n'
+ ' ... print(sys.exc_info())\n'
+ ' ... try:\n'
+ ' ... raise ValueError\n'
+ ' ... except:\n'
+ ' ... print(sys.exc_info())\n'
+ ' ... print(sys.exc_info())\n'
+ ' ...\n'
+ " (<class 'TypeError'>, TypeError(), <traceback object at "
+ '0x10efad080>)\n'
+ " (<class 'ValueError'>, ValueError(), <traceback object at "
+ '0x10efad040>)\n'
+ " (<class 'TypeError'>, TypeError(), <traceback object at "
+ '0x10efad080>)\n'
+ ' >>> print(sys.exc_info())\n'
+ ' (None, None, None)\n'
'\n'
'The optional "else" clause is executed if the control flow '
'leaves the\n'
@@ -2538,8 +2599,10 @@
'usage\n'
'patterns to be encapsulated for convenient reuse.\n'
'\n'
- ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n'
- ' with_item ::= expression ["as" target]\n'
+ ' with_stmt ::= "with" ( "(" with_stmt_contents ","? '
+ '")" | with_stmt_contents ) ":" suite\n'
+ ' with_stmt_contents ::= with_item ("," with_item)*\n'
+ ' with_item ::= expression ["as" target]\n'
'\n'
'The execution of the "with" statement with one “item” proceeds '
'as\n'
@@ -2631,9 +2694,23 @@
' with B() as b:\n'
' SUITE\n'
'\n'
+ 'You can also write multi-item context managers in multiple lines '
+ 'if\n'
+ 'the items are surrounded by parentheses. For example:\n'
+ '\n'
+ ' with (\n'
+ ' A() as a,\n'
+ ' B() as b,\n'
+ ' ):\n'
+ ' SUITE\n'
+ '\n'
'Changed in version 3.1: Support for multiple context '
'expressions.\n'
'\n'
+ 'Changed in version 3.10: Support for using grouping parentheses '
+ 'to\n'
+ 'break the statement in multiple lines.\n'
+ '\n'
'See also:\n'
'\n'
' **PEP 343** - The “with” statement\n'
@@ -2642,6 +2719,746 @@
' statement.\n'
'\n'
'\n'
+ 'The "match" statement\n'
+ '=====================\n'
+ '\n'
+ 'New in version 3.10.\n'
+ '\n'
+ 'The match statement is used for pattern matching. Syntax:\n'
+ '\n'
+ ' match_stmt ::= \'match\' subject_expr ":" NEWLINE INDENT '
+ 'case_block+ DEDENT\n'
+ ' subject_expr ::= star_named_expression "," '
+ 'star_named_expressions?\n'
+ ' | named_expression\n'
+ ' case_block ::= \'case\' patterns [guard] ":" block\n'
+ '\n'
+ 'Note:\n'
+ '\n'
+ ' This section uses single quotes to denote soft keywords.\n'
+ '\n'
+ 'Pattern matching takes a pattern as input (following "case") and '
+ 'a\n'
+ 'subject value (following "match"). The pattern (which may '
+ 'contain\n'
+ 'subpatterns) is matched against the subject value. The outcomes '
+ 'are:\n'
+ '\n'
+ '* A match success or failure (also termed a pattern success or\n'
+ ' failure).\n'
+ '\n'
+ '* Possible binding of matched values to a name. The '
+ 'prerequisites for\n'
+ ' this are further discussed below.\n'
+ '\n'
+ 'The "match" and "case" keywords are soft keywords.\n'
+ '\n'
+ 'See also:\n'
+ '\n'
+ ' * **PEP 634** – Structural Pattern Matching: Specification\n'
+ '\n'
+ ' * **PEP 636** – Structural Pattern Matching: Tutorial\n'
+ '\n'
+ '\n'
+ 'Overview\n'
+ '--------\n'
+ '\n'
+ 'Here’s an overview of the logical flow of a match statement:\n'
+ '\n'
+ '1. The subject expression "subject_expr" is evaluated and a '
+ 'resulting\n'
+ ' subject value obtained. If the subject expression contains a '
+ 'comma,\n'
+ ' a tuple is constructed using the standard rules.\n'
+ '\n'
+ '2. Each pattern in a "case_block" is attempted to match with '
+ 'the\n'
+ ' subject value. The specific rules for success or failure are\n'
+ ' described below. The match attempt can also bind some or all '
+ 'of the\n'
+ ' standalone names within the pattern. The precise pattern '
+ 'binding\n'
+ ' rules vary per pattern type and are specified below. **Name\n'
+ ' bindings made during a successful pattern match outlive the\n'
+ ' executed block and can be used after the match statement**.\n'
+ '\n'
+ ' Note:\n'
+ '\n'
+ ' During failed pattern matches, some subpatterns may '
+ 'succeed.\n'
+ ' Do not rely on bindings being made for a failed match.\n'
+ ' Conversely, do not rely on variables remaining unchanged '
+ 'after\n'
+ ' a failed match. The exact behavior is dependent on\n'
+ ' implementation and may vary. This is an intentional '
+ 'decision\n'
+ ' made to allow different implementations to add '
+ 'optimizations.\n'
+ '\n'
+ '3. If the pattern succeeds, the corresponding guard (if present) '
+ 'is\n'
+ ' evaluated. In this case all name bindings are guaranteed to '
+ 'have\n'
+ ' happened.\n'
+ '\n'
+ ' * If the guard evaluates as true or is missing, the "block" '
+ 'inside\n'
+ ' "case_block" is executed.\n'
+ '\n'
+ ' * Otherwise, the next "case_block" is attempted as described '
+ 'above.\n'
+ '\n'
+ ' * If there are no further case blocks, the match statement '
+ 'is\n'
+ ' completed.\n'
+ '\n'
+ 'Note:\n'
+ '\n'
+ ' Users should generally never rely on a pattern being '
+ 'evaluated.\n'
+ ' Depending on implementation, the interpreter may cache values '
+ 'or use\n'
+ ' other optimizations which skip repeated evaluations.\n'
+ '\n'
+ 'A sample match statement:\n'
+ '\n'
+ ' >>> flag = False\n'
+ ' >>> match (100, 200):\n'
+ ' ... case (100, 300): # Mismatch: 200 != 300\n'
+ " ... print('Case 1')\n"
+ ' ... case (100, 200) if flag: # Successful match, but '
+ 'guard fails\n'
+ " ... print('Case 2')\n"
+ ' ... case (100, y): # Matches and binds y to 200\n'
+ " ... print(f'Case 3, y: {y}')\n"
+ ' ... case _: # Pattern not attempted\n'
+ " ... print('Case 4, I match anything!')\n"
+ ' ...\n'
+ ' Case 3, y: 200\n'
+ '\n'
+ 'In this case, "if flag" is a guard. Read more about that in the '
+ 'next\n'
+ 'section.\n'
+ '\n'
+ '\n'
+ 'Guards\n'
+ '------\n'
+ '\n'
+ ' guard ::= "if" named_expression\n'
+ '\n'
+ 'A "guard" (which is part of the "case") must succeed for code '
+ 'inside\n'
+ 'the "case" block to execute. It takes the form: "if" followed '
+ 'by an\n'
+ 'expression.\n'
+ '\n'
+ 'The logical flow of a "case" block with a "guard" follows:\n'
+ '\n'
+ '1. Check that the pattern in the "case" block succeeded. If '
+ 'the\n'
+ ' pattern failed, the "guard" is not evaluated and the next '
+ '"case"\n'
+ ' block is checked.\n'
+ '\n'
+ '2. If the pattern succeeded, evaluate the "guard".\n'
+ '\n'
+ ' * If the "guard" condition evaluates as true, the case block '
+ 'is\n'
+ ' selected.\n'
+ '\n'
+ ' * If the "guard" condition evaluates as false, the case block '
+ 'is\n'
+ ' not selected.\n'
+ '\n'
+ ' * If the "guard" raises an exception during evaluation, the\n'
+ ' exception bubbles up.\n'
+ '\n'
+ 'Guards are allowed to have side effects as they are '
+ 'expressions.\n'
+ 'Guard evaluation must proceed from the first to the last case '
+ 'block,\n'
+ 'one at a time, skipping case blocks whose pattern(s) don’t all\n'
+ 'succeed. (I.e., guard evaluation must happen in order.) Guard\n'
+ 'evaluation must stop once a case block is selected.\n'
+ '\n'
+ '\n'
+ 'Irrefutable Case Blocks\n'
+ '-----------------------\n'
+ '\n'
+ 'An irrefutable case block is a match-all case block. A match\n'
+ 'statement may have at most one irrefutable case block, and it '
+ 'must be\n'
+ 'last.\n'
+ '\n'
+ 'A case block is considered irrefutable if it has no guard and '
+ 'its\n'
+ 'pattern is irrefutable. A pattern is considered irrefutable if '
+ 'we can\n'
+ 'prove from its syntax alone that it will always succeed. Only '
+ 'the\n'
+ 'following patterns are irrefutable:\n'
+ '\n'
+ '* AS Patterns whose left-hand side is irrefutable\n'
+ '\n'
+ '* OR Patterns containing at least one irrefutable pattern\n'
+ '\n'
+ '* Capture Patterns\n'
+ '\n'
+ '* Wildcard Patterns\n'
+ '\n'
+ '* parenthesized irrefutable patterns\n'
+ '\n'
+ '\n'
+ 'Patterns\n'
+ '--------\n'
+ '\n'
+ 'Note:\n'
+ '\n'
+ ' This section uses grammar notations beyond standard EBNF:\n'
+ '\n'
+ ' * the notation "SEP.RULE+" is shorthand for "RULE (SEP '
+ 'RULE)*"\n'
+ '\n'
+ ' * the notation "!RULE" is shorthand for a negative lookahead\n'
+ ' assertion\n'
+ '\n'
+ 'The top-level syntax for "patterns" is:\n'
+ '\n'
+ ' patterns ::= open_sequence_pattern | pattern\n'
+ ' pattern ::= as_pattern | or_pattern\n'
+ ' closed_pattern ::= | literal_pattern\n'
+ ' | capture_pattern\n'
+ ' | wildcard_pattern\n'
+ ' | value_pattern\n'
+ ' | group_pattern\n'
+ ' | sequence_pattern\n'
+ ' | mapping_pattern\n'
+ ' | class_pattern\n'
+ '\n'
+ 'The descriptions below will include a description “in simple '
+ 'terms” of\n'
+ 'what a pattern does for illustration purposes (credits to '
+ 'Raymond\n'
+ 'Hettinger for a document that inspired most of the '
+ 'descriptions). Note\n'
+ 'that these descriptions are purely for illustration purposes and '
+ '**may\n'
+ 'not** reflect the underlying implementation. Furthermore, they '
+ 'do not\n'
+ 'cover all valid forms.\n'
+ '\n'
+ '\n'
+ 'OR Patterns\n'
+ '~~~~~~~~~~~\n'
+ '\n'
+ 'An OR pattern is two or more patterns separated by vertical bars '
+ '"|".\n'
+ 'Syntax:\n'
+ '\n'
+ ' or_pattern ::= "|".closed_pattern+\n'
+ '\n'
+ 'Only the final subpattern may be irrefutable, and each '
+ 'subpattern must\n'
+ 'bind the same set of names to avoid ambiguity.\n'
+ '\n'
+ 'An OR pattern matches each of its subpatterns in turn to the '
+ 'subject\n'
+ 'value, until one succeeds. The OR pattern is then considered\n'
+ 'successful. Otherwise, if none of the subpatterns succeed, the '
+ 'OR\n'
+ 'pattern fails.\n'
+ '\n'
+ 'In simple terms, "P1 | P2 | ..." will try to match "P1", if it '
+ 'fails\n'
+ 'it will try to match "P2", succeeding immediately if any '
+ 'succeeds,\n'
+ 'failing otherwise.\n'
+ '\n'
+ '\n'
+ 'AS Patterns\n'
+ '~~~~~~~~~~~\n'
+ '\n'
+ 'An AS pattern matches an OR pattern on the left of the "as" '
+ 'keyword\n'
+ 'against a subject. Syntax:\n'
+ '\n'
+ ' as_pattern ::= or_pattern "as" capture_pattern\n'
+ '\n'
+ 'If the OR pattern fails, the AS pattern fails. Otherwise, the '
+ 'AS\n'
+ 'pattern binds the subject to the name on the right of the as '
+ 'keyword\n'
+ 'and succeeds. "capture_pattern" cannot be a a "_".\n'
+ '\n'
+ 'In simple terms "P as NAME" will match with "P", and on success '
+ 'it\n'
+ 'will set "NAME = <subject>".\n'
+ '\n'
+ '\n'
+ 'Literal Patterns\n'
+ '~~~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A literal pattern corresponds to most literals in Python. '
+ 'Syntax:\n'
+ '\n'
+ ' literal_pattern ::= signed_number\n'
+ ' | signed_number "+" NUMBER\n'
+ ' | signed_number "-" NUMBER\n'
+ ' | strings\n'
+ ' | "None"\n'
+ ' | "True"\n'
+ ' | "False"\n'
+ ' | signed_number: NUMBER | "-" NUMBER\n'
+ '\n'
+ 'The rule "strings" and the token "NUMBER" are defined in the '
+ 'standard\n'
+ 'Python grammar. Triple-quoted strings are supported. Raw '
+ 'strings and\n'
+ 'byte strings are supported. Formatted string literals are not\n'
+ 'supported.\n'
+ '\n'
+ 'The forms "signed_number \'+\' NUMBER" and "signed_number \'-\' '
+ 'NUMBER"\n'
+ 'are for expressing complex numbers; they require a real number '
+ 'on the\n'
+ 'left and an imaginary number on the right. E.g. "3 + 4j".\n'
+ '\n'
+ 'In simple terms, "LITERAL" will succeed only if "<subject> ==\n'
+ 'LITERAL". For the singletons "None", "True" and "False", the '
+ '"is"\n'
+ 'operator is used.\n'
+ '\n'
+ '\n'
+ 'Capture Patterns\n'
+ '~~~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A capture pattern binds the subject value to a name. Syntax:\n'
+ '\n'
+ " capture_pattern ::= !'_' NAME\n"
+ '\n'
+ 'A single underscore "_" is not a capture pattern (this is what '
+ '"!\'_\'"\n'
+ 'expresses). It is instead treated as a "wildcard_pattern".\n'
+ '\n'
+ 'In a given pattern, a given name can only be bound once. E.g. '
+ '"case\n'
+ 'x, x: ..." is invalid while "case [x] | x: ..." is allowed.\n'
+ '\n'
+ 'Capture patterns always succeed. The binding follows scoping '
+ 'rules\n'
+ 'established by the assignment expression operator in **PEP '
+ '572**; the\n'
+ 'name becomes a local variable in the closest containing function '
+ 'scope\n'
+ 'unless there’s an applicable "global" or "nonlocal" statement.\n'
+ '\n'
+ 'In simple terms "NAME" will always succeed and it will set "NAME '
+ '=\n'
+ '<subject>".\n'
+ '\n'
+ '\n'
+ 'Wildcard Patterns\n'
+ '~~~~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A wildcard pattern always succeeds (matches anything) and binds '
+ 'no\n'
+ 'name. Syntax:\n'
+ '\n'
+ " wildcard_pattern ::= '_'\n"
+ '\n'
+ '"_" is a soft keyword within any pattern, but only within '
+ 'patterns.\n'
+ 'It is an identifier, as usual, even within "match" subject\n'
+ 'expressions, "guard"s, and "case" blocks.\n'
+ '\n'
+ 'In simple terms, "_" will always succeed.\n'
+ '\n'
+ '\n'
+ 'Value Patterns\n'
+ '~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A value pattern represents a named value in Python. Syntax:\n'
+ '\n'
+ ' value_pattern ::= attr\n'
+ ' attr ::= name_or_attr "." NAME\n'
+ ' name_or_attr ::= attr | NAME\n'
+ '\n'
+ 'The dotted name in the pattern is looked up using standard '
+ 'Python name\n'
+ 'resolution rules. The pattern succeeds if the value found '
+ 'compares\n'
+ 'equal to the subject value (using the "==" equality operator).\n'
+ '\n'
+ 'In simple terms "NAME1.NAME2" will succeed only if "<subject> '
+ '==\n'
+ 'NAME1.NAME2"\n'
+ '\n'
+ 'Note:\n'
+ '\n'
+ ' If the same value occurs multiple times in the same match '
+ 'statement,\n'
+ ' the interpreter may cache the first value found and reuse it '
+ 'rather\n'
+ ' than repeat the same lookup. This cache is strictly tied to a '
+ 'given\n'
+ ' execution of a given match statement.\n'
+ '\n'
+ '\n'
+ 'Group Patterns\n'
+ '~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A group pattern allows users to add parentheses around patterns '
+ 'to\n'
+ 'emphasize the intended grouping. Otherwise, it has no '
+ 'additional\n'
+ 'syntax. Syntax:\n'
+ '\n'
+ ' group_pattern ::= "(" pattern ")"\n'
+ '\n'
+ 'In simple terms "(P)" has the same effect as "P".\n'
+ '\n'
+ '\n'
+ 'Sequence Patterns\n'
+ '~~~~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A sequence pattern contains several subpatterns to be matched '
+ 'against\n'
+ 'sequence elements. The syntax is similar to the unpacking of a '
+ 'list or\n'
+ 'tuple.\n'
+ '\n'
+ ' sequence_pattern ::= "[" [maybe_sequence_pattern] "]"\n'
+ ' | "(" [open_sequence_pattern] ")"\n'
+ ' open_sequence_pattern ::= maybe_star_pattern "," '
+ '[maybe_sequence_pattern]\n'
+ ' maybe_sequence_pattern ::= ",".maybe_star_pattern+ ","?\n'
+ ' maybe_star_pattern ::= star_pattern | pattern\n'
+ ' star_pattern ::= "*" (capture_pattern | '
+ 'wildcard_pattern)\n'
+ '\n'
+ 'There is no difference if parentheses or square brackets are '
+ 'used for\n'
+ 'sequence patterns (i.e. "(...)" vs "[...]" ).\n'
+ '\n'
+ 'Note:\n'
+ '\n'
+ ' A single pattern enclosed in parentheses without a trailing '
+ 'comma\n'
+ ' (e.g. "(3 | 4)") is a group pattern. While a single pattern '
+ 'enclosed\n'
+ ' in square brackets (e.g. "[3 | 4]") is still a sequence '
+ 'pattern.\n'
+ '\n'
+ 'At most one star subpattern may be in a sequence pattern. The '
+ 'star\n'
+ 'subpattern may occur in any position. If no star subpattern is\n'
+ 'present, the sequence pattern is a fixed-length sequence '
+ 'pattern;\n'
+ 'otherwise it is a variable-length sequence pattern.\n'
+ '\n'
+ 'The following is the logical flow for matching a sequence '
+ 'pattern\n'
+ 'against a subject value:\n'
+ '\n'
+ '1. If the subject value is not a sequence [2], the sequence '
+ 'pattern\n'
+ ' fails.\n'
+ '\n'
+ '2. If the subject value is an instance of "str", "bytes" or\n'
+ ' "bytearray" the sequence pattern fails.\n'
+ '\n'
+ '3. The subsequent steps depend on whether the sequence pattern '
+ 'is\n'
+ ' fixed or variable-length.\n'
+ '\n'
+ ' If the sequence pattern is fixed-length:\n'
+ '\n'
+ ' 1. If the length of the subject sequence is not equal to the '
+ 'number\n'
+ ' of subpatterns, the sequence pattern fails\n'
+ '\n'
+ ' 2. Subpatterns in the sequence pattern are matched to their\n'
+ ' corresponding items in the subject sequence from left to '
+ 'right.\n'
+ ' Matching stops as soon as a subpattern fails. If all\n'
+ ' subpatterns succeed in matching their corresponding item, '
+ 'the\n'
+ ' sequence pattern succeeds.\n'
+ '\n'
+ ' Otherwise, if the sequence pattern is variable-length:\n'
+ '\n'
+ ' 1. If the length of the subject sequence is less than the '
+ 'number of\n'
+ ' non-star subpatterns, the sequence pattern fails.\n'
+ '\n'
+ ' 2. The leading non-star subpatterns are matched to their\n'
+ ' corresponding items as for fixed-length sequences.\n'
+ '\n'
+ ' 3. If the previous step succeeds, the star subpattern matches '
+ 'a\n'
+ ' list formed of the remaining subject items, excluding the\n'
+ ' remaining items corresponding to non-star subpatterns '
+ 'following\n'
+ ' the star subpattern.\n'
+ '\n'
+ ' 4. Remaining non-star subpatterns are matched to their\n'
+ ' corresponding subject items, as for a fixed-length '
+ 'sequence.\n'
+ '\n'
+ ' Note:\n'
+ '\n'
+ ' The length of the subject sequence is obtained via "len()" '
+ '(i.e.\n'
+ ' via the "__len__()" protocol). This length may be cached '
+ 'by the\n'
+ ' interpreter in a similar manner as value patterns.\n'
+ '\n'
+ 'In simple terms "[P1, P2, P3," … ", P<N>]" matches only if all '
+ 'the\n'
+ 'following happens:\n'
+ '\n'
+ '* check "<subject>" is a sequence\n'
+ '\n'
+ '* "len(subject) == <N>"\n'
+ '\n'
+ '* "P1" matches "<subject>[0]" (note that this match can also '
+ 'bind\n'
+ ' names)\n'
+ '\n'
+ '* "P2" matches "<subject>[1]" (note that this match can also '
+ 'bind\n'
+ ' names)\n'
+ '\n'
+ '* … and so on for the corresponding pattern/element.\n'
+ '\n'
+ '\n'
+ 'Mapping Patterns\n'
+ '~~~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A mapping pattern contains one or more key-value patterns. The '
+ 'syntax\n'
+ 'is similar to the construction of a dictionary. Syntax:\n'
+ '\n'
+ ' mapping_pattern ::= "{" [items_pattern] "}"\n'
+ ' items_pattern ::= ",".key_value_pattern+ ","?\n'
+ ' key_value_pattern ::= (literal_pattern | value_pattern) ":" '
+ 'pattern\n'
+ ' | double_star_pattern\n'
+ ' double_star_pattern ::= "**" capture_pattern\n'
+ '\n'
+ 'At most one double star pattern may be in a mapping pattern. '
+ 'The\n'
+ 'double star pattern must be the last subpattern in the mapping\n'
+ 'pattern.\n'
+ '\n'
+ 'Duplicate keys in mapping patterns are disallowed. Duplicate '
+ 'literal\n'
+ 'keys will raise a "SyntaxError". Two keys that otherwise have '
+ 'the same\n'
+ 'value will raise a "ValueError" at runtime.\n'
+ '\n'
+ 'The following is the logical flow for matching a mapping '
+ 'pattern\n'
+ 'against a subject value:\n'
+ '\n'
+ '1. If the subject value is not a mapping [3],the mapping '
+ 'pattern\n'
+ ' fails.\n'
+ '\n'
+ '2. If every key given in the mapping pattern is present in the '
+ 'subject\n'
+ ' mapping, and the pattern for each key matches the '
+ 'corresponding\n'
+ ' item of the subject mapping, the mapping pattern succeeds.\n'
+ '\n'
+ '3. If duplicate keys are detected in the mapping pattern, the '
+ 'pattern\n'
+ ' is considered invalid. A "SyntaxError" is raised for '
+ 'duplicate\n'
+ ' literal values; or a "ValueError" for named keys of the same '
+ 'value.\n'
+ '\n'
+ 'Note:\n'
+ '\n'
+ ' Key-value pairs are matched using the two-argument form of '
+ 'the\n'
+ ' mapping subject’s "get()" method. Matched key-value pairs '
+ 'must\n'
+ ' already be present in the mapping, and not created on-the-fly '
+ 'via\n'
+ ' "__missing__()" or "__getitem__()".\n'
+ '\n'
+ 'In simple terms "{KEY1: P1, KEY2: P2, ... }" matches only if all '
+ 'the\n'
+ 'following happens:\n'
+ '\n'
+ '* check "<subject>" is a mapping\n'
+ '\n'
+ '* "KEY1 in <subject>"\n'
+ '\n'
+ '* "P1" matches "<subject>[KEY1]"\n'
+ '\n'
+ '* … and so on for the corresponding KEY/pattern pair.\n'
+ '\n'
+ '\n'
+ 'Class Patterns\n'
+ '~~~~~~~~~~~~~~\n'
+ '\n'
+ 'A class pattern represents a class and its positional and '
+ 'keyword\n'
+ 'arguments (if any). Syntax:\n'
+ '\n'
+ ' class_pattern ::= name_or_attr "(" [pattern_arguments '
+ '","?] ")"\n'
+ ' pattern_arguments ::= positional_patterns ["," '
+ 'keyword_patterns]\n'
+ ' | keyword_patterns\n'
+ ' positional_patterns ::= ",".pattern+\n'
+ ' keyword_patterns ::= ",".keyword_pattern+\n'
+ ' keyword_pattern ::= NAME "=" pattern\n'
+ '\n'
+ 'The same keyword should not be repeated in class patterns.\n'
+ '\n'
+ 'The following is the logical flow for matching a class pattern '
+ 'against\n'
+ 'a subject value:\n'
+ '\n'
+ '1. If "name_or_attr" is not an instance of the builtin "type" , '
+ 'raise\n'
+ ' "TypeError".\n'
+ '\n'
+ '2. If the subject value is not an instance of "name_or_attr" '
+ '(tested\n'
+ ' via "isinstance()"), the class pattern fails.\n'
+ '\n'
+ '3. If no pattern arguments are present, the pattern succeeds.\n'
+ ' Otherwise, the subsequent steps depend on whether keyword or\n'
+ ' positional argument patterns are present.\n'
+ '\n'
+ ' For a number of built-in types (specified below), a single\n'
+ ' positional subpattern is accepted which will match the '
+ 'entire\n'
+ ' subject; for these types keyword patterns also work as for '
+ 'other\n'
+ ' types.\n'
+ '\n'
+ ' If only keyword patterns are present, they are processed as\n'
+ ' follows, one by one:\n'
+ '\n'
+ ' I. The keyword is looked up as an attribute on the subject.\n'
+ '\n'
+ ' * If this raises an exception other than "AttributeError", '
+ 'the\n'
+ ' exception bubbles up.\n'
+ '\n'
+ ' * If this raises "AttributeError", the class pattern has '
+ 'failed.\n'
+ '\n'
+ ' * Else, the subpattern associated with the keyword pattern '
+ 'is\n'
+ ' matched against the subject’s attribute value. If this '
+ 'fails,\n'
+ ' the class pattern fails; if this succeeds, the match '
+ 'proceeds\n'
+ ' to the next keyword.\n'
+ '\n'
+ ' II. If all keyword patterns succeed, the class pattern '
+ 'succeeds.\n'
+ '\n'
+ ' If any positional patterns are present, they are converted '
+ 'to\n'
+ ' keyword patterns using the "__match_args__" attribute on the '
+ 'class\n'
+ ' "name_or_attr" before matching:\n'
+ '\n'
+ ' I. The equivalent of "getattr(cls, "__match_args__", ())" is\n'
+ ' called.\n'
+ '\n'
+ ' * If this raises an exception, the exception bubbles up.\n'
+ '\n'
+ ' * If the returned value is not a tuple, the conversion '
+ 'fails and\n'
+ ' "TypeError" is raised.\n'
+ '\n'
+ ' * If there are more positional patterns than\n'
+ ' "len(cls.__match_args__)", "TypeError" is raised.\n'
+ '\n'
+ ' * Otherwise, positional pattern "i" is converted to a '
+ 'keyword\n'
+ ' pattern using "__match_args__[i]" as the keyword.\n'
+ ' "__match_args__[i]" must be a string; if not "TypeError" '
+ 'is\n'
+ ' raised.\n'
+ '\n'
+ ' * If there are duplicate keywords, "TypeError" is raised.\n'
+ '\n'
+ ' See also:\n'
+ '\n'
+ ' Customizing positional arguments in class pattern '
+ 'matching\n'
+ '\n'
+ ' II. Once all positional patterns have been converted to '
+ 'keyword\n'
+ ' patterns,\n'
+ ' the match proceeds as if there were only keyword '
+ 'patterns.\n'
+ '\n'
+ ' For the following built-in types the handling of positional\n'
+ ' subpatterns is different:\n'
+ '\n'
+ ' * "bool"\n'
+ '\n'
+ ' * "bytearray"\n'
+ '\n'
+ ' * "bytes"\n'
+ '\n'
+ ' * "dict"\n'
+ '\n'
+ ' * "float"\n'
+ '\n'
+ ' * "frozenset"\n'
+ '\n'
+ ' * "int"\n'
+ '\n'
+ ' * "list"\n'
+ '\n'
+ ' * "set"\n'
+ '\n'
+ ' * "str"\n'
+ '\n'
+ ' * "tuple"\n'
+ '\n'
+ ' These classes accept a single positional argument, and the '
+ 'pattern\n'
+ ' there is matched against the whole object rather than an '
+ 'attribute.\n'
+ ' For example "int(0|1)" matches the value "0", but not the '
+ 'values\n'
+ ' "0.0" or "False".\n'
+ '\n'
+ 'In simple terms "CLS(P1, attr=P2)" matches only if the '
+ 'following\n'
+ 'happens:\n'
+ '\n'
+ '* "isinstance(<subject>, CLS)"\n'
+ '\n'
+ '* convert "P1" to a keyword pattern using "CLS.__match_args__"\n'
+ '\n'
+ '* For each keyword argument "attr=P2":\n'
+ ' * "hasattr(<subject>, "attr")"\n'
+ '\n'
+ ' * "P2" matches "<subject>.attr"\n'
+ '\n'
+ '* … and so on for the corresponding keyword argument/pattern '
+ 'pair.\n'
+ '\n'
+ 'See also:\n'
+ '\n'
+ ' * **PEP 634** – Structural Pattern Matching: Specification\n'
+ '\n'
+ ' * **PEP 636** – Structural Pattern Matching: Tutorial\n'
+ '\n'
+ '\n'
'Function definitions\n'
'====================\n'
'\n'
@@ -2655,7 +3472,6 @@
' decorators ::= decorator+\n'
' decorator ::= "@" assignment_expression '
'NEWLINE\n'
- ' dotted_name ::= identifier ("." identifier)*\n'
' parameter_list ::= defparameter ("," '
'defparameter)* "," "/" ["," [parameter_list_no_posonly]]\n'
' | parameter_list_no_posonly\n'
@@ -2680,7 +3496,7 @@
'\n'
'The function definition does not execute the function body; this '
'gets\n'
- 'executed only when the function is called. [2]\n'
+ 'executed only when the function is called. [4]\n'
'\n'
'A function definition may be wrapped by one or more *decorator*\n'
'expressions. Decorator expressions are evaluated when the '
@@ -2733,17 +3549,17 @@
'“pre-\n'
'computed” value is used for each call. This is especially '
'important\n'
- 'to understand when a default parameter is a mutable object, such '
- 'as a\n'
- 'list or a dictionary: if the function modifies the object (e.g. '
- 'by\n'
- 'appending an item to a list), the default value is in effect '
- 'modified.\n'
- 'This is generally not what was intended. A way around this is '
- 'to use\n'
- '"None" as the default, and explicitly test for it in the body of '
- 'the\n'
- 'function, e.g.:\n'
+ 'to understand when a default parameter value is a mutable '
+ 'object, such\n'
+ 'as a list or a dictionary: if the function modifies the object '
+ '(e.g.\n'
+ 'by appending an item to a list), the default parameter value is '
+ 'in\n'
+ 'effect modified. This is generally not what was intended. A '
+ 'way\n'
+ 'around this is to use "None" as the default, and explicitly test '
+ 'for\n'
+ 'it in the body of the function, e.g.:\n'
'\n'
' def whats_on_the_telly(penguin=None):\n'
' if penguin is None:\n'
@@ -2755,7 +3571,7 @@
'Calls.\n'
'A function call always assigns values to all parameters '
'mentioned in\n'
- 'the parameter list, either from position arguments, from '
+ 'the parameter list, either from positional arguments, from '
'keyword\n'
'arguments, or from default values. If the form “"*identifier"” '
'is\n'
@@ -2767,8 +3583,14 @@
'new\n'
'empty mapping of the same type. Parameters after “"*"” or\n'
'“"*identifier"” are keyword-only parameters and may only be '
- 'passed\n'
- 'used keyword arguments.\n'
+ 'passed by\n'
+ 'keyword arguments. Parameters before “"/"” are positional-only\n'
+ 'parameters and may only be passed by positional arguments.\n'
+ '\n'
+ 'Changed in version 3.8: The "/" function parameter syntax may be '
+ 'used\n'
+ 'to indicate positional-only parameters. See **PEP 570** for '
+ 'details.\n'
'\n'
'Parameters may have an *annotation* of the form “": '
'expression"”\n'
@@ -2879,7 +3701,7 @@
'function definitions.) When the class’s suite finishes '
'execution, its\n'
'execution frame is discarded but its local namespace is saved. '
- '[3] A\n'
+ '[5] A\n'
'class object is then created using the inheritance list for the '
'base\n'
'classes and the saved local namespace for the attribute '
@@ -2964,12 +3786,9 @@
'\n'
'Execution of Python coroutines can be suspended and resumed at '
'many\n'
- 'points (see *coroutine*). Inside the body of a coroutine '
- 'function,\n'
- '"await" and "async" identifiers become reserved keywords; '
- '"await"\n'
- 'expressions, "async for" and "async with" can only be used in\n'
- 'coroutine function bodies.\n'
+ 'points (see *coroutine*). "await" expressions, "async for" and '
+ '"async\n'
+ 'with" can only be used in the body of a coroutine function.\n'
'\n'
'Functions defined with "async def" syntax are always coroutine\n'
'functions, even if they do not contain "await" or "async" '
@@ -2985,19 +3804,22 @@
' do_stuff()\n'
' await some_coroutine()\n'
'\n'
+ 'Changed in version 3.7: "await" and "async" are now keywords;\n'
+ 'previously they were only treated as such inside the body of a\n'
+ 'coroutine function.\n'
+ '\n'
'\n'
'The "async for" statement\n'
'-------------------------\n'
'\n'
' async_for_stmt ::= "async" for_stmt\n'
'\n'
- 'An *asynchronous iterable* is able to call asynchronous code in '
- 'its\n'
- '*iter* implementation, and *asynchronous iterator* can call\n'
- 'asynchronous code in its *next* method.\n'
+ 'An *asynchronous iterable* provides an "__aiter__" method that\n'
+ 'directly returns an *asynchronous iterator*, which can call\n'
+ 'asynchronous code in its "__anext__" method.\n'
'\n'
'The "async for" statement allows convenient iteration over\n'
- 'asynchronous iterators.\n'
+ 'asynchronous iterables.\n'
'\n'
'The following code:\n'
'\n'
@@ -3083,13 +3905,65 @@
'exception.\n'
' That new exception causes the old one to be lost.\n'
'\n'
- '[2] A string literal appearing as the first statement in the '
+ '[2] In pattern matching, a sequence is defined as one of the\n'
+ ' following:\n'
+ '\n'
+ ' * a class that inherits from "collections.abc.Sequence"\n'
+ '\n'
+ ' * a Python class that has been registered as\n'
+ ' "collections.abc.Sequence"\n'
+ '\n'
+ ' * a builtin class that has its (CPython) '
+ '"Py_TPFLAGS_SEQUENCE"\n'
+ ' bit set\n'
+ '\n'
+ ' * a class that inherits from any of the above\n'
+ '\n'
+ ' The following standard library classes are sequences:\n'
+ '\n'
+ ' * "array.array"\n'
+ '\n'
+ ' * "collections.deque"\n'
+ '\n'
+ ' * "list"\n'
+ '\n'
+ ' * "memoryview"\n'
+ '\n'
+ ' * "range"\n'
+ '\n'
+ ' * "tuple"\n'
+ '\n'
+ ' Note:\n'
+ '\n'
+ ' Subject values of type "str", "bytes", and "bytearray" do '
+ 'not\n'
+ ' match sequence patterns.\n'
+ '\n'
+ '[3] In pattern matching, a mapping is defined as one of the '
+ 'following:\n'
+ '\n'
+ ' * a class that inherits from "collections.abc.Mapping"\n'
+ '\n'
+ ' * a Python class that has been registered as\n'
+ ' "collections.abc.Mapping"\n'
+ '\n'
+ ' * a builtin class that has its (CPython) '
+ '"Py_TPFLAGS_MAPPING"\n'
+ ' bit set\n'
+ '\n'
+ ' * a class that inherits from any of the above\n'
+ '\n'
+ ' The standard library classes "dict" and '
+ '"types.MappingProxyType"\n'
+ ' are mappings.\n'
+ '\n'
+ '[4] A string literal appearing as the first statement in the '
'function\n'
' body is transformed into the function’s "__doc__" attribute '
'and\n'
' therefore the function’s *docstring*.\n'
'\n'
- '[3] A string literal appearing as the first statement in the '
+ '[5] A string literal appearing as the first statement in the '
'class\n'
' body is transformed into the namespace’s "__doc__" item and\n'
' therefore the class’s *docstring*.\n',
@@ -3227,13 +4101,13 @@
'\n'
' If "__new__()" is invoked during object construction and '
'it returns\n'
- ' an instance or subclass of *cls*, then the new '
- 'instance’s\n'
- ' "__init__()" method will be invoked like '
- '"__init__(self[, ...])",\n'
- ' where *self* is the new instance and the remaining '
- 'arguments are\n'
- ' the same as were passed to the object constructor.\n'
+ ' an instance of *cls*, then the new instance’s '
+ '"__init__()" method\n'
+ ' will be invoked like "__init__(self[, ...])", where '
+ '*self* is the\n'
+ ' new instance and the remaining arguments are the same as '
+ 'were\n'
+ ' passed to the object constructor.\n'
'\n'
' If "__new__()" does not return an instance of *cls*, '
'then the new\n'
@@ -3728,17 +4602,16 @@
'debugger will pause execution just before the first line of the\n'
'module.\n'
'\n'
- 'The typical usage to break into the debugger from a running '
- 'program is\n'
- 'to insert\n'
+ 'The typical usage to break into the debugger is to insert:\n'
'\n'
' import pdb; pdb.set_trace()\n'
'\n'
- 'at the location you want to break into the debugger. You can '
- 'then\n'
- 'step through the code following this statement, and continue '
- 'running\n'
- 'without the debugger using the "continue" command.\n'
+ 'at the location you want to break into the debugger, and then '
+ 'run the\n'
+ 'program. You can then step through the code following this '
+ 'statement,\n'
+ 'and continue running without the debugger using the "continue"\n'
+ 'command.\n'
'\n'
'New in version 3.7: The built-in "breakpoint()", when called '
'with\n'
@@ -4594,20 +5467,32 @@
'binding\n'
'operations.\n'
'\n'
- 'The following constructs bind names: formal parameters to '
- 'functions,\n'
- '"import" statements, class and function definitions (these bind '
- 'the\n'
- 'class or function name in the defining block), and targets that '
- 'are\n'
- 'identifiers if occurring in an assignment, "for" loop header, '
- 'or after\n'
- '"as" in a "with" statement or "except" clause. The "import" '
- 'statement\n'
- 'of the form "from ... import *" binds all names defined in the\n'
- 'imported module, except those beginning with an underscore. '
- 'This form\n'
- 'may only be used at the module level.\n'
+ 'The following constructs bind names:\n'
+ '\n'
+ '* formal parameters to functions,\n'
+ '\n'
+ '* class definitions,\n'
+ '\n'
+ '* function definitions,\n'
+ '\n'
+ '* assignment expressions,\n'
+ '\n'
+ '* targets that are identifiers if occurring in an assignment:\n'
+ '\n'
+ ' * "for" loop header,\n'
+ '\n'
+ ' * after "as" in a "with" statement, "except" clause or in the '
+ 'as-\n'
+ ' pattern in structural pattern matching,\n'
+ '\n'
+ ' * in a capture pattern in structural pattern matching\n'
+ '\n'
+ '* "import" statements.\n'
+ '\n'
+ 'The "import" statement of the form "from ... import *" binds '
+ 'all names\n'
+ 'defined in the imported module, except those beginning with an\n'
+ 'underscore. This form may only be used at the module level.\n'
'\n'
'A target occurring in a "del" statement is also considered '
'bound for\n'
@@ -4680,9 +5565,9 @@
'operations.\n'
'\n'
'If the "global" statement occurs within a block, all uses of '
- 'the name\n'
- 'specified in the statement refer to the binding of that name in '
- 'the\n'
+ 'the names\n'
+ 'specified in the statement refer to the bindings of those names '
+ 'in the\n'
'top-level namespace. Names are resolved in the top-level '
'namespace by\n'
'searching the global namespace, i.e. the namespace of the '
@@ -4691,9 +5576,10 @@
'namespace\n'
'of the module "builtins". The global namespace is searched '
'first. If\n'
- 'the name is not found there, the builtins namespace is '
- 'searched. The\n'
- '"global" statement must precede all uses of the name.\n'
+ 'the names are not found there, the builtins namespace is '
+ 'searched.\n'
+ 'The "global" statement must precede all uses of the listed '
+ 'names.\n'
'\n'
'The "global" statement has the same scope as a name binding '
'operation\n'
@@ -4987,30 +5873,7 @@
'all by the loop. Hint: the built-in function "range()" returns an\n'
'iterator of integers suitable to emulate the effect of Pascal’s "for '
'i\n'
- ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n'
- '\n'
- 'Note:\n'
- '\n'
- ' There is a subtlety when the sequence is being modified by the '
- 'loop\n'
- ' (this can only occur for mutable sequences, e.g. lists). An\n'
- ' internal counter is used to keep track of which item is used next,\n'
- ' and this is incremented on each iteration. When this counter has\n'
- ' reached the length of the sequence the loop terminates. This '
- 'means\n'
- ' that if the suite deletes the current (or a previous) item from '
- 'the\n'
- ' sequence, the next item will be skipped (since it gets the index '
- 'of\n'
- ' the current item which has already been treated). Likewise, if '
- 'the\n'
- ' suite inserts an item in the sequence before the current item, the\n'
- ' current item will be treated again the next time through the loop.\n'
- ' This can lead to nasty bugs that can be avoided by making a\n'
- ' temporary copy using a slice of the whole sequence, e.g.,\n'
- '\n'
- ' for x in a[:]:\n'
- ' if x < 0: a.remove(x)\n',
+ ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n',
'formatstrings': 'Format String Syntax\n'
'********************\n'
'\n'
@@ -5020,9 +5883,11 @@
'"Formatter",\n'
'subclasses can define their own format string syntax). The '
'syntax is\n'
- 'related to that of formatted string literals, but there '
- 'are\n'
- 'differences.\n'
+ 'related to that of formatted string literals, but it is '
+ 'less\n'
+ 'sophisticated and, in particular, does not support '
+ 'arbitrary\n'
+ 'expressions.\n'
'\n'
'Format strings contain “replacement fields” surrounded by '
'curly braces\n'
@@ -5216,7 +6081,7 @@
'character that can be any character and defaults to a space '
'if\n'
'omitted. It is not possible to use a literal curly brace '
- '(“"{"” or\n'
+ '(”"{"” or\n'
'“"}"”) as the *fill* character in a formatted string '
'literal or when\n'
'using the "str.format()" method. However, it is possible '
@@ -5254,9 +6119,9 @@
' | | in the form ‘+000000120’. This alignment '
'option is only |\n'
' | | valid for numeric types. It becomes the '
- 'default when ‘0’ |\n'
- ' | | immediately precedes the field '
- 'width. |\n'
+ 'default for |\n'
+ ' | | numbers when ‘0’ immediately precedes the '
+ 'field width. |\n'
' '
'+-----------+------------------------------------------------------------+\n'
' | "\'^\'" | Forces the field to be centered within '
@@ -5311,19 +6176,19 @@
'complex\n'
'types. For integers, when binary, octal, or hexadecimal '
'output is\n'
- 'used, this option adds the prefix respective "\'0b\'", '
- '"\'0o\'", or "\'0x\'"\n'
- 'to the output value. For float and complex the alternate '
- 'form causes\n'
- 'the result of the conversion to always contain a '
- 'decimal-point\n'
- 'character, even if no digits follow it. Normally, a '
- 'decimal-point\n'
- 'character appears in the result of these conversions only '
- 'if a digit\n'
- 'follows it. In addition, for "\'g\'" and "\'G\'" '
- 'conversions, trailing\n'
- 'zeros are not removed from the result.\n'
+ 'used, this option adds the respective prefix "\'0b\'", '
+ '"\'0o\'", "\'0x\'",\n'
+ 'or "\'0X\'" to the output value. For float and complex the '
+ 'alternate\n'
+ 'form causes the result of the conversion to always contain '
+ 'a decimal-\n'
+ 'point character, even if no digits follow it. Normally, a '
+ 'decimal-\n'
+ 'point character appears in the result of these conversions '
+ 'only if a\n'
+ 'digit follows it. In addition, for "\'g\'" and "\'G\'" '
+ 'conversions,\n'
+ 'trailing zeros are not removed from the result.\n'
'\n'
'The "\',\'" option signals the use of a comma for a '
'thousands separator.\n'
@@ -5364,19 +6229,23 @@
'with an\n'
'*alignment* type of "\'=\'".\n'
'\n'
- 'The *precision* is a decimal number indicating how many '
+ 'Changed in version 3.10: Preceding the *width* field by '
+ '"\'0\'" no\n'
+ 'longer affects the default alignment for strings.\n'
+ '\n'
+ 'The *precision* is a decimal integer indicating how many '
'digits should\n'
- 'be displayed after the decimal point for a floating point '
- 'value\n'
- 'formatted with "\'f\'" and "\'F\'", or before and after the '
- 'decimal point\n'
- 'for a floating point value formatted with "\'g\'" or '
- '"\'G\'". For non-\n'
- 'number types the field indicates the maximum field size - '
- 'in other\n'
- 'words, how many characters will be used from the field '
- 'content. The\n'
- '*precision* is not allowed for integer values.\n'
+ 'be displayed after the decimal point for presentation types '
+ '"\'f\'" and\n'
+ '"\'F\'", or before and after the decimal point for '
+ 'presentation types\n'
+ '"\'g\'" or "\'G\'". For string presentation types the '
+ 'field indicates the\n'
+ 'maximum field size - in other words, how many characters '
+ 'will be used\n'
+ 'from the field content. The *precision* is not allowed for '
+ 'integer\n'
+ 'presentation types.\n'
'\n'
'Finally, the *type* determines how the data should be '
'presented.\n'
@@ -5436,8 +6305,12 @@
'+-----------+------------------------------------------------------------+\n'
' | "\'X\'" | Hex format. Outputs the number in base '
'16, using upper- |\n'
- ' | | case letters for the digits above '
- '9. |\n'
+ ' | | case letters for the digits above 9. In '
+ 'case "\'#\'" is |\n'
+ ' | | specified, the prefix "\'0x\'" will be '
+ 'upper-cased to "\'0X\'" |\n'
+ ' | | as '
+ 'well. |\n'
' '
'+-----------+------------------------------------------------------------+\n'
' | "\'n\'" | Number. This is the same as "\'d\'", '
@@ -5530,44 +6403,51 @@
' | | formats the result in either fixed-point '
'format or in |\n'
' | | scientific notation, depending on its '
- 'magnitude. The |\n'
- ' | | precise rules are as follows: suppose that '
- 'the result |\n'
+ 'magnitude. A |\n'
+ ' | | precision of "0" is treated as equivalent '
+ 'to a precision |\n'
+ ' | | of "1". The precise rules are as follows: '
+ 'suppose that |\n'
+ ' | | the result formatted with presentation '
+ 'type "\'e\'" and |\n'
+ ' | | precision "p-1" would have exponent '
+ '"exp". Then, if "m <= |\n'
+ ' | | exp < p", where "m" is -4 for floats and '
+ '-6 for |\n'
+ ' | | "Decimals", the number is formatted with '
+ 'presentation type |\n'
+ ' | | "\'f\'" and precision "p-1-exp". '
+ 'Otherwise, the number is |\n'
' | | formatted with presentation type "\'e\'" '
- 'and precision "p-1" |\n'
- ' | | would have exponent "exp". Then, if "m <= '
- 'exp < p", where |\n'
- ' | | "m" is -4 for floats and -6 for '
- '"Decimals", the number is |\n'
- ' | | formatted with presentation type "\'f\'" '
'and precision |\n'
- ' | | "p-1-exp". Otherwise, the number is '
- 'formatted with |\n'
- ' | | presentation type "\'e\'" and precision '
- '"p-1". In both cases |\n'
- ' | | insignificant trailing zeros are removed '
- 'from the |\n'
- ' | | significand, and the decimal point is also '
- 'removed if |\n'
- ' | | there are no remaining digits following '
- 'it, unless the |\n'
- ' | | "\'#\'" option is used. Positive and '
- 'negative infinity, |\n'
- ' | | positive and negative zero, and nans, are '
- 'formatted as |\n'
- ' | | "inf", "-inf", "0", "-0" and "nan" '
- 'respectively, |\n'
- ' | | regardless of the precision. A precision '
- 'of "0" is |\n'
- ' | | treated as equivalent to a precision of '
- '"1". With no |\n'
- ' | | precision given, uses a precision of "6" '
- 'significant |\n'
- ' | | digits for "float", and shows all '
- 'coefficient digits for |\n'
- ' | | '
- '"Decimal". '
- '|\n'
+ ' | | "p-1". In both cases insignificant '
+ 'trailing zeros are |\n'
+ ' | | removed from the significand, and the '
+ 'decimal point is |\n'
+ ' | | also removed if there are no remaining '
+ 'digits following |\n'
+ ' | | it, unless the "\'#\'" option is used. '
+ 'With no precision |\n'
+ ' | | given, uses a precision of "6" significant '
+ 'digits for |\n'
+ ' | | "float". For "Decimal", the coefficient of '
+ 'the result is |\n'
+ ' | | formed from the coefficient digits of the '
+ 'value; |\n'
+ ' | | scientific notation is used for values '
+ 'smaller than "1e-6" |\n'
+ ' | | in absolute value and values where the '
+ 'place value of the |\n'
+ ' | | least significant digit is larger than 1, '
+ 'and fixed-point |\n'
+ ' | | notation is used otherwise. Positive and '
+ 'negative |\n'
+ ' | | infinity, positive and negative zero, and '
+ 'nans, are |\n'
+ ' | | formatted as "inf", "-inf", "0", "-0" and '
+ '"nan" |\n'
+ ' | | respectively, regardless of the '
+ 'precision. |\n'
' '
'+-----------+------------------------------------------------------------+\n'
' | "\'G\'" | General format. Same as "\'g\'" except '
@@ -5592,19 +6472,24 @@
'percent sign. |\n'
' '
'+-----------+------------------------------------------------------------+\n'
- ' | None | Similar to "\'g\'", except that '
- 'fixed-point notation, when |\n'
- ' | | used, has at least one digit past the '
- 'decimal point. The |\n'
- ' | | default precision is as high as needed to '
- 'represent the |\n'
- ' | | particular value. The overall effect is to '
- 'match the |\n'
- ' | | output of "str()" as altered by the other '
- 'format |\n'
- ' | | '
- 'modifiers. '
- '|\n'
+ ' | None | For "float" this is the same as "\'g\'", '
+ 'except that when |\n'
+ ' | | fixed-point notation is used to format the '
+ 'result, it |\n'
+ ' | | always includes at least one digit past '
+ 'the decimal point. |\n'
+ ' | | The precision used is as large as needed '
+ 'to represent the |\n'
+ ' | | given value faithfully. For "Decimal", '
+ 'this is the same |\n'
+ ' | | as either "\'g\'" or "\'G\'" depending on '
+ 'the value of |\n'
+ ' | | "context.capitals" for the current decimal '
+ 'context. The |\n'
+ ' | | overall effect is to match the output of '
+ '"str()" as |\n'
+ ' | | altered by the other format '
+ 'modifiers. |\n'
' '
'+-----------+------------------------------------------------------------+\n'
'\n'
@@ -5782,7 +6667,6 @@
' decorators ::= decorator+\n'
' decorator ::= "@" assignment_expression '
'NEWLINE\n'
- ' dotted_name ::= identifier ("." identifier)*\n'
' parameter_list ::= defparameter ("," '
'defparameter)* "," "/" ["," [parameter_list_no_posonly]]\n'
' | parameter_list_no_posonly\n'
@@ -5807,7 +6691,7 @@
'\n'
'The function definition does not execute the function body; this '
'gets\n'
- 'executed only when the function is called. [2]\n'
+ 'executed only when the function is called. [4]\n'
'\n'
'A function definition may be wrapped by one or more *decorator*\n'
'expressions. Decorator expressions are evaluated when the '
@@ -5860,17 +6744,17 @@
'“pre-\n'
'computed” value is used for each call. This is especially '
'important\n'
- 'to understand when a default parameter is a mutable object, such '
- 'as a\n'
- 'list or a dictionary: if the function modifies the object (e.g. '
- 'by\n'
- 'appending an item to a list), the default value is in effect '
- 'modified.\n'
- 'This is generally not what was intended. A way around this is '
- 'to use\n'
- '"None" as the default, and explicitly test for it in the body of '
- 'the\n'
- 'function, e.g.:\n'
+ 'to understand when a default parameter value is a mutable '
+ 'object, such\n'
+ 'as a list or a dictionary: if the function modifies the object '
+ '(e.g.\n'
+ 'by appending an item to a list), the default parameter value is '
+ 'in\n'
+ 'effect modified. This is generally not what was intended. A '
+ 'way\n'
+ 'around this is to use "None" as the default, and explicitly test '
+ 'for\n'
+ 'it in the body of the function, e.g.:\n'
'\n'
' def whats_on_the_telly(penguin=None):\n'
' if penguin is None:\n'
@@ -5882,7 +6766,7 @@
'Calls.\n'
'A function call always assigns values to all parameters '
'mentioned in\n'
- 'the parameter list, either from position arguments, from '
+ 'the parameter list, either from positional arguments, from '
'keyword\n'
'arguments, or from default values. If the form “"*identifier"” '
'is\n'
@@ -5894,8 +6778,14 @@
'new\n'
'empty mapping of the same type. Parameters after “"*"” or\n'
'“"*identifier"” are keyword-only parameters and may only be '
- 'passed\n'
- 'used keyword arguments.\n'
+ 'passed by\n'
+ 'keyword arguments. Parameters before “"/"” are positional-only\n'
+ 'parameters and may only be passed by positional arguments.\n'
+ '\n'
+ 'Changed in version 3.8: The "/" function parameter syntax may be '
+ 'used\n'
+ 'to indicate positional-only parameters. See **PEP 570** for '
+ 'details.\n'
'\n'
'Parameters may have an *annotation* of the form “": '
'expression"”\n'
@@ -5987,8 +6877,10 @@
'\n'
'Names listed in a "global" statement must not be defined as '
'formal\n'
- 'parameters or in a "for" loop control target, "class" definition,\n'
- 'function definition, "import" statement, or variable annotation.\n'
+ 'parameters, or as targets in "with" statements or "except" '
+ 'clauses, or\n'
+ 'in a "for" target list, "class" definition, function definition,\n'
+ '"import" statement, or variable annotation.\n'
'\n'
'**CPython implementation detail:** The current implementation does '
'not\n'
@@ -6020,22 +6912,31 @@
'trailing underscore characters:\n'
'\n'
'"_*"\n'
- ' Not imported by "from module import *". The special '
- 'identifier "_"\n'
- ' is used in the interactive interpreter to store the result '
- 'of the\n'
- ' last evaluation; it is stored in the "builtins" module. '
- 'When not\n'
- ' in interactive mode, "_" has no special meaning and is not '
- 'defined.\n'
- ' See section The import statement.\n'
+ ' Not imported by "from module import *".\n'
+ '\n'
+ '"_"\n'
+ ' In a "case" pattern within a "match" statement, "_" is a '
+ 'soft\n'
+ ' keyword that denotes a wildcard.\n'
+ '\n'
+ ' Separately, the interactive interpreter makes the result of '
+ 'the\n'
+ ' last evaluation available in the variable "_". (It is '
+ 'stored in the\n'
+ ' "builtins" module, alongside built-in functions like '
+ '"print".)\n'
+ '\n'
+ ' Elsewhere, "_" is a regular identifier. It is often used to '
+ 'name\n'
+ ' “special” items, but it is not special to Python itself.\n'
'\n'
' Note:\n'
'\n'
' The name "_" is often used in conjunction with\n'
' internationalization; refer to the documentation for the\n'
' "gettext" module for more information on this '
- 'convention.\n'
+ 'convention.It is\n'
+ ' also commonly used for unused variables.\n'
'\n'
'"__*__"\n'
' System-defined names, informally known as “dunder” names. '
@@ -6158,6 +7059,28 @@
' async elif if or yield\n'
'\n'
'\n'
+ 'Soft Keywords\n'
+ '=============\n'
+ '\n'
+ 'New in version 3.10.\n'
+ '\n'
+ 'Some identifiers are only reserved under specific contexts. '
+ 'These are\n'
+ 'known as *soft keywords*. The identifiers "match", "case" '
+ 'and "_" can\n'
+ 'syntactically act as keywords in contexts related to the '
+ 'pattern\n'
+ 'matching statement, but this distinction is done at the '
+ 'parser level,\n'
+ 'not when tokenizing.\n'
+ '\n'
+ 'As soft keywords, their use with pattern matching is possible '
+ 'while\n'
+ 'still preserving compatibility with existing code that uses '
+ '"match",\n'
+ '"case" and "_" as identifier names.\n'
+ '\n'
+ '\n'
'Reserved classes of identifiers\n'
'===============================\n'
'\n'
@@ -6168,15 +7091,23 @@
'trailing underscore characters:\n'
'\n'
'"_*"\n'
- ' Not imported by "from module import *". The special '
- 'identifier "_"\n'
- ' is used in the interactive interpreter to store the result '
+ ' Not imported by "from module import *".\n'
+ '\n'
+ '"_"\n'
+ ' In a "case" pattern within a "match" statement, "_" is a '
+ 'soft\n'
+ ' keyword that denotes a wildcard.\n'
+ '\n'
+ ' Separately, the interactive interpreter makes the result '
'of the\n'
- ' last evaluation; it is stored in the "builtins" module. '
- 'When not\n'
- ' in interactive mode, "_" has no special meaning and is not '
- 'defined.\n'
- ' See section The import statement.\n'
+ ' last evaluation available in the variable "_". (It is '
+ 'stored in the\n'
+ ' "builtins" module, alongside built-in functions like '
+ '"print".)\n'
+ '\n'
+ ' Elsewhere, "_" is a regular identifier. It is often used '
+ 'to name\n'
+ ' “special” items, but it is not special to Python itself.\n'
'\n'
' Note:\n'
'\n'
@@ -6184,7 +7115,8 @@
' internationalization; refer to the documentation for '
'the\n'
' "gettext" module for more information on this '
- 'convention.\n'
+ 'convention.It is\n'
+ ' also commonly used for unused variables.\n'
'\n'
'"__*__"\n'
' System-defined names, informally known as “dunder” names. '
@@ -6256,7 +7188,7 @@
' | "from" relative_module "import" "(" '
'identifier ["as" identifier]\n'
' ("," identifier ["as" identifier])* [","] ")"\n'
- ' | "from" module "import" "*"\n'
+ ' | "from" relative_module "import" "*"\n'
' module ::= (identifier ".")* identifier\n'
' relative_module ::= "."* module | "."+\n'
'\n'
@@ -6600,10 +7532,7 @@
'lambda': 'Lambdas\n'
'*******\n'
'\n'
- ' lambda_expr ::= "lambda" [parameter_list] ":" '
- 'expression\n'
- ' lambda_expr_nocond ::= "lambda" [parameter_list] ":" '
- 'expression_nocond\n'
+ ' lambda_expr ::= "lambda" [parameter_list] ":" expression\n'
'\n'
'Lambda expressions (sometimes called lambda forms) are used to '
'create\n'
@@ -6648,20 +7577,32 @@
'*Names* refer to objects. Names are introduced by name binding\n'
'operations.\n'
'\n'
- 'The following constructs bind names: formal parameters to '
- 'functions,\n'
- '"import" statements, class and function definitions (these bind '
- 'the\n'
- 'class or function name in the defining block), and targets that '
- 'are\n'
- 'identifiers if occurring in an assignment, "for" loop header, or '
- 'after\n'
- '"as" in a "with" statement or "except" clause. The "import" '
- 'statement\n'
- 'of the form "from ... import *" binds all names defined in the\n'
- 'imported module, except those beginning with an underscore. This '
- 'form\n'
- 'may only be used at the module level.\n'
+ 'The following constructs bind names:\n'
+ '\n'
+ '* formal parameters to functions,\n'
+ '\n'
+ '* class definitions,\n'
+ '\n'
+ '* function definitions,\n'
+ '\n'
+ '* assignment expressions,\n'
+ '\n'
+ '* targets that are identifiers if occurring in an assignment:\n'
+ '\n'
+ ' * "for" loop header,\n'
+ '\n'
+ ' * after "as" in a "with" statement, "except" clause or in the '
+ 'as-\n'
+ ' pattern in structural pattern matching,\n'
+ '\n'
+ ' * in a capture pattern in structural pattern matching\n'
+ '\n'
+ '* "import" statements.\n'
+ '\n'
+ 'The "import" statement of the form "from ... import *" binds all '
+ 'names\n'
+ 'defined in the imported module, except those beginning with an\n'
+ 'underscore. This form may only be used at the module level.\n'
'\n'
'A target occurring in a "del" statement is also considered bound '
'for\n'
@@ -6731,8 +7672,8 @@
'operations.\n'
'\n'
'If the "global" statement occurs within a block, all uses of the '
- 'name\n'
- 'specified in the statement refer to the binding of that name in '
+ 'names\n'
+ 'specified in the statement refer to the bindings of those names in '
'the\n'
'top-level namespace. Names are resolved in the top-level '
'namespace by\n'
@@ -6741,9 +7682,9 @@
'namespace\n'
'of the module "builtins". The global namespace is searched '
'first. If\n'
- 'the name is not found there, the builtins namespace is searched. '
- 'The\n'
- '"global" statement must precede all uses of the name.\n'
+ 'the names are not found there, the builtins namespace is '
+ 'searched.\n'
+ 'The "global" statement must precede all uses of the listed names.\n'
'\n'
'The "global" statement has the same scope as a name binding '
'operation\n'
@@ -6883,7 +7824,7 @@
'\n'
'Note that numeric literals do not include a sign; a phrase like '
'"-1"\n'
- 'is actually an expression composed of the unary operator ‘"-"‘ '
+ 'is actually an expression composed of the unary operator ‘"-"’ '
'and the\n'
'literal "1".\n',
'numeric-types': 'Emulating numeric types\n'
@@ -7028,16 +7969,6 @@
'the data\n'
' model.\n'
'\n'
- ' Note:\n'
- '\n'
- ' Due to a bug in the dispatching mechanism for "**=", a '
- 'class that\n'
- ' defines "__ipow__()" but returns "NotImplemented" '
- 'would fail to\n'
- ' fall back to "x.__pow__(y)" and "y.__rpow__(x)". This '
- 'bug is\n'
- ' fixed in Python 3.10.\n'
- '\n'
'object.__neg__(self)\n'
'object.__pos__(self)\n'
'object.__abs__(self)\n'
@@ -7088,9 +8019,9 @@
' of the object truncated to an "Integral" (typically an '
'"int").\n'
'\n'
- ' If "__int__()" is not defined then the built-in function '
- '"int()"\n'
- ' falls back to "__trunc__()".\n',
+ ' The built-in function "int()" falls back to '
+ '"__trunc__()" if\n'
+ ' neither "__int__()" nor "__index__()" is defined.\n',
'objects': 'Objects, values and types\n'
'*************************\n'
'\n'
@@ -7224,8 +8155,8 @@
'\n'
'The following table summarizes the operator precedence '
'in Python, from\n'
- 'lowest precedence (least binding) to highest precedence '
- '(most\n'
+ 'highest precedence (most binding) to lowest precedence '
+ '(least\n'
'binding). Operators in the same box have the same '
'precedence. Unless\n'
'the syntax is explicitly given, operators are binary. '
@@ -7244,65 +8175,6 @@
'| Operator | '
'Description |\n'
'|=================================================|=======================================|\n'
- '| ":=" | '
- 'Assignment expression |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "lambda" | '
- 'Lambda expression |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "if" – "else" | '
- 'Conditional expression |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "or" | '
- 'Boolean OR |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "and" | '
- 'Boolean AND |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "not" "x" | '
- 'Boolean NOT |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "in", "not in", "is", "is not", "<", "<=", ">", | '
- 'Comparisons, including membership |\n'
- '| ">=", "!=", "==" | '
- 'tests and identity tests |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "|" | '
- 'Bitwise OR |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "^" | '
- 'Bitwise XOR |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "&" | '
- 'Bitwise AND |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "<<", ">>" | '
- 'Shifts |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "+", "-" | '
- 'Addition and subtraction |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "*", "@", "/", "//", "%" | '
- 'Multiplication, matrix |\n'
- '| | '
- 'multiplication, division, floor |\n'
- '| | '
- 'division, remainder [5] |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "+x", "-x", "~x" | '
- 'Positive, negative, bitwise NOT |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "**" | '
- 'Exponentiation [6] |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "await" "x" | '
- 'Await expression |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
- '| "x[index]", "x[index:index]", | '
- 'Subscription, slicing, call, |\n'
- '| "x(arguments...)", "x.attribute" | '
- 'attribute reference |\n'
- '+-------------------------------------------------+---------------------------------------+\n'
'| "(expressions...)", "[expressions...]", "{key: | '
'Binding or parenthesized expression, |\n'
'| value...}", "{expressions...}" | list '
@@ -7310,6 +8182,65 @@
'| | '
'display |\n'
'+-------------------------------------------------+---------------------------------------+\n'
+ '| "x[index]", "x[index:index]", | '
+ 'Subscription, slicing, call, |\n'
+ '| "x(arguments...)", "x.attribute" | '
+ 'attribute reference |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "await" "x" | '
+ 'Await expression |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "**" | '
+ 'Exponentiation [5] |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "+x", "-x", "~x" | '
+ 'Positive, negative, bitwise NOT |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "*", "@", "/", "//", "%" | '
+ 'Multiplication, matrix |\n'
+ '| | '
+ 'multiplication, division, floor |\n'
+ '| | '
+ 'division, remainder [6] |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "+", "-" | '
+ 'Addition and subtraction |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "<<", ">>" | '
+ 'Shifts |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "&" | '
+ 'Bitwise AND |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "^" | '
+ 'Bitwise XOR |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "|" | '
+ 'Bitwise OR |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "in", "not in", "is", "is not", "<", "<=", ">", | '
+ 'Comparisons, including membership |\n'
+ '| ">=", "!=", "==" | '
+ 'tests and identity tests |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "not" "x" | '
+ 'Boolean NOT |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "and" | '
+ 'Boolean AND |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "or" | '
+ 'Boolean OR |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "if" – "else" | '
+ 'Conditional expression |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| "lambda" | '
+ 'Lambda expression |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
+ '| ":=" | '
+ 'Assignment expression |\n'
+ '+-------------------------------------------------+---------------------------------------+\n'
'\n'
'-[ Footnotes ]-\n'
'\n'
@@ -7389,14 +8320,14 @@
'Check their\n'
' documentation for more info.\n'
'\n'
- '[5] The "%" operator is also used for string formatting; '
- 'the same\n'
- ' precedence applies.\n'
- '\n'
- '[6] The power operator "**" binds less tightly than an '
+ '[5] The power operator "**" binds less tightly than an '
'arithmetic or\n'
' bitwise unary operator on its right, that is, '
- '"2**-1" is "0.5".\n',
+ '"2**-1" is "0.5".\n'
+ '\n'
+ '[6] The "%" operator is also used for string formatting; '
+ 'the same\n'
+ ' precedence applies.\n',
'pass': 'The "pass" statement\n'
'********************\n'
'\n'
@@ -7444,18 +8375,21 @@
'"ZeroDivisionError".\n'
'Raising a negative number to a fractional power results in a '
'"complex"\n'
- 'number. (In earlier versions it raised a "ValueError".)\n',
+ 'number. (In earlier versions it raised a "ValueError".)\n'
+ '\n'
+ 'This operation can be customized using the special "__pow__()" '
+ 'method.\n',
'raise': 'The "raise" statement\n'
'*********************\n'
'\n'
' raise_stmt ::= "raise" [expression ["from" expression]]\n'
'\n'
- 'If no expressions are present, "raise" re-raises the last '
- 'exception\n'
- 'that was active in the current scope. If no exception is active '
- 'in\n'
- 'the current scope, a "RuntimeError" exception is raised indicating\n'
- 'that this is an error.\n'
+ 'If no expressions are present, "raise" re-raises the exception that '
+ 'is\n'
+ 'currently being handled, which is also known as the *active\n'
+ 'exception*. If there isn’t currently an active exception, a\n'
+ '"RuntimeError" exception is raised indicating that this is an '
+ 'error.\n'
'\n'
'Otherwise, "raise" evaluates the first expression as the exception\n'
'object. It must be either a subclass or an instance of\n'
@@ -7481,12 +8415,18 @@
'\n'
'The "from" clause is used for exception chaining: if given, the '
'second\n'
- '*expression* must be another exception class or instance, which '
- 'will\n'
- 'then be attached to the raised exception as the "__cause__" '
- 'attribute\n'
- '(which is writable). If the raised exception is not handled, both\n'
- 'exceptions will be printed:\n'
+ '*expression* must be another exception class or instance. If the\n'
+ 'second expression is an exception instance, it will be attached to '
+ 'the\n'
+ 'raised exception as the "__cause__" attribute (which is writable). '
+ 'If\n'
+ 'the expression is an exception class, the class will be '
+ 'instantiated\n'
+ 'and the resulting exception instance will be attached to the '
+ 'raised\n'
+ 'exception as the "__cause__" attribute. If the raised exception is '
+ 'not\n'
+ 'handled, both exceptions will be printed:\n'
'\n'
' >>> try:\n'
' ... print(1 / 0)\n'
@@ -7504,11 +8444,14 @@
' File "<stdin>", line 4, in <module>\n'
' RuntimeError: Something bad happened\n'
'\n'
- 'A similar mechanism works implicitly if an exception is raised '
- 'inside\n'
- 'an exception handler or a "finally" clause: the previous exception '
- 'is\n'
- 'then attached as the new exception’s "__context__" attribute:\n'
+ 'A similar mechanism works implicitly if a new exception is raised '
+ 'when\n'
+ 'an exception is already being handled. An exception may be '
+ 'handled\n'
+ 'when an "except" or "finally" clause, or a "with" statement, is '
+ 'used.\n'
+ 'The previous exception is then attached as the new exception’s\n'
+ '"__context__" attribute:\n'
'\n'
' >>> try:\n'
' ... print(1 / 0)\n'
@@ -7590,61 +8533,62 @@
'\n'
'The following methods can be defined to implement '
'container objects.\n'
- 'Containers usually are sequences (such as lists or tuples) '
- 'or mappings\n'
- '(like dictionaries), but can represent other containers as '
- 'well. The\n'
- 'first set of methods is used either to emulate a sequence '
- 'or to\n'
- 'emulate a mapping; the difference is that for a sequence, '
- 'the\n'
- 'allowable keys should be the integers *k* for which "0 <= '
- 'k < N" where\n'
- '*N* is the length of the sequence, or slice objects, which '
- 'define a\n'
- 'range of items. It is also recommended that mappings '
- 'provide the\n'
- 'methods "keys()", "values()", "items()", "get()", '
- '"clear()",\n'
- '"setdefault()", "pop()", "popitem()", "copy()", and '
- '"update()"\n'
- 'behaving similar to those for Python’s standard dictionary '
+ 'Containers usually are *sequences* (such as "lists" or '
+ '"tuples") or\n'
+ '*mappings* (like "dictionaries"), but can represent other '
+ 'containers\n'
+ 'as well. The first set of methods is used either to '
+ 'emulate a\n'
+ 'sequence or to emulate a mapping; the difference is that '
+ 'for a\n'
+ 'sequence, the allowable keys should be the integers *k* '
+ 'for which "0\n'
+ '<= k < N" where *N* is the length of the sequence, or '
+ '"slice" objects,\n'
+ 'which define a range of items. It is also recommended '
+ 'that mappings\n'
+ 'provide the methods "keys()", "values()", "items()", '
+ '"get()",\n'
+ '"clear()", "setdefault()", "pop()", "popitem()", "copy()", '
+ 'and\n'
+ '"update()" behaving similar to those for Python’s '
+ 'standard\n'
+ '"dictionary" objects. The "collections.abc" module '
+ 'provides a\n'
+ '"MutableMapping" *abstract base class* to help create '
+ 'those methods\n'
+ 'from a base set of "__getitem__()", "__setitem__()", '
+ '"__delitem__()",\n'
+ 'and "keys()". Mutable sequences should provide methods '
+ '"append()",\n'
+ '"count()", "index()", "extend()", "insert()", "pop()", '
+ '"remove()",\n'
+ '"reverse()" and "sort()", like Python standard "list" '
'objects.\n'
- 'The "collections.abc" module provides a "MutableMapping" '
- 'abstract base\n'
- 'class to help create those methods from a base set of '
- '"__getitem__()",\n'
- '"__setitem__()", "__delitem__()", and "keys()". Mutable '
- 'sequences\n'
- 'should provide methods "append()", "count()", "index()", '
- '"extend()",\n'
- '"insert()", "pop()", "remove()", "reverse()" and "sort()", '
- 'like Python\n'
- 'standard list objects. Finally, sequence types should '
- 'implement\n'
- 'addition (meaning concatenation) and multiplication '
+ 'Finally, sequence types should implement addition '
'(meaning\n'
- 'repetition) by defining the methods "__add__()", '
- '"__radd__()",\n'
- '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" '
- 'described\n'
- 'below; they should not define other numerical operators. '
+ 'concatenation) and multiplication (meaning repetition) by '
+ 'defining the\n'
+ 'methods "__add__()", "__radd__()", "__iadd__()", '
+ '"__mul__()",\n'
+ '"__rmul__()" and "__imul__()" described below; they should '
+ 'not define\n'
+ 'other numerical operators. It is recommended that both '
+ 'mappings and\n'
+ 'sequences implement the "__contains__()" method to allow '
+ 'efficient use\n'
+ 'of the "in" operator; for mappings, "in" should search the '
+ 'mapping’s\n'
+ 'keys; for sequences, it should search through the values. '
'It is\n'
- 'recommended that both mappings and sequences implement '
+ 'further recommended that both mappings and sequences '
+ 'implement the\n'
+ '"__iter__()" method to allow efficient iteration through '
'the\n'
- '"__contains__()" method to allow efficient use of the "in" '
- 'operator;\n'
- 'for mappings, "in" should search the mapping’s keys; for '
- 'sequences, it\n'
- 'should search through the values. It is further '
- 'recommended that both\n'
- 'mappings and sequences implement the "__iter__()" method '
- 'to allow\n'
- 'efficient iteration through the container; for mappings, '
- '"__iter__()"\n'
- 'should iterate through the object’s keys; for sequences, '
- 'it should\n'
- 'iterate through the values.\n'
+ 'container; for mappings, "__iter__()" should iterate '
+ 'through the\n'
+ 'object’s keys; for sequences, it should iterate through '
+ 'the values.\n'
'\n'
'object.__len__(self)\n'
'\n'
@@ -7703,22 +8647,24 @@
'object.__getitem__(self, key)\n'
'\n'
' Called to implement evaluation of "self[key]". For '
- 'sequence types,\n'
- ' the accepted keys should be integers and slice '
- 'objects. Note that\n'
- ' the special interpretation of negative indexes (if the '
- 'class wishes\n'
- ' to emulate a sequence type) is up to the '
- '"__getitem__()" method. If\n'
- ' *key* is of an inappropriate type, "TypeError" may be '
- 'raised; if of\n'
- ' a value outside the set of indexes for the sequence '
- '(after any\n'
- ' special interpretation of negative values), '
- '"IndexError" should be\n'
- ' raised. For mapping types, if *key* is missing (not in '
+ '*sequence*\n'
+ ' types, the accepted keys should be integers and slice '
+ 'objects.\n'
+ ' Note that the special interpretation of negative '
+ 'indexes (if the\n'
+ ' class wishes to emulate a *sequence* type) is up to '
'the\n'
- ' container), "KeyError" should be raised.\n'
+ ' "__getitem__()" method. If *key* is of an inappropriate '
+ 'type,\n'
+ ' "TypeError" may be raised; if of a value outside the '
+ 'set of indexes\n'
+ ' for the sequence (after any special interpretation of '
+ 'negative\n'
+ ' values), "IndexError" should be raised. For *mapping* '
+ 'types, if\n'
+ ' *key* is missing (not in the container), "KeyError" '
+ 'should be\n'
+ ' raised.\n'
'\n'
' Note:\n'
'\n'
@@ -7728,6 +8674,15 @@
'of the\n'
' sequence.\n'
'\n'
+ ' Note:\n'
+ '\n'
+ ' When subscripting a *class*, the special class '
+ 'method\n'
+ ' "__class_getitem__()" may be called instead of '
+ '"__getitem__()".\n'
+ ' See __class_getitem__ versus __getitem__ for more '
+ 'details.\n'
+ '\n'
'object.__setitem__(self, key, value)\n'
'\n'
' Called to implement assignment to "self[key]". Same '
@@ -7763,19 +8718,13 @@
'\n'
'object.__iter__(self)\n'
'\n'
- ' This method is called when an iterator is required for '
- 'a container.\n'
- ' This method should return a new iterator object that '
- 'can iterate\n'
- ' over all the objects in the container. For mappings, '
- 'it should\n'
- ' iterate over the keys of the container.\n'
- '\n'
- ' Iterator objects also need to implement this method; '
- 'they are\n'
- ' required to return themselves. For more information on '
- 'iterator\n'
- ' objects, see Iterator Types.\n'
+ ' This method is called when an *iterator* is required '
+ 'for a\n'
+ ' container. This method should return a new iterator '
+ 'object that can\n'
+ ' iterate over all the objects in the container. For '
+ 'mappings, it\n'
+ ' should iterate over the keys of the container.\n'
'\n'
'object.__reversed__(self)\n'
'\n'
@@ -7838,6 +8787,10 @@
'the\n'
'second argument.\n'
'\n'
+ 'This operation can be customized using the special '
+ '"__lshift__()" and\n'
+ '"__rshift__()" methods.\n'
+ '\n'
'A right shift by *n* bits is defined as floor division by '
'"pow(2,n)".\n'
'A left shift by *n* bits is defined as multiplication with '
@@ -7950,7 +8903,7 @@
'immediate\n'
' subclasses. This method returns a list of all those '
'references\n'
- ' still alive. Example:\n'
+ ' still alive. The list is in definition order. Example:\n'
'\n'
' >>> int.__subclasses__()\n'
" [<class 'bool'>]\n"
@@ -8052,13 +9005,13 @@
'\n'
' If "__new__()" is invoked during object construction and '
'it returns\n'
- ' an instance or subclass of *cls*, then the new '
- 'instance’s\n'
- ' "__init__()" method will be invoked like "__init__(self[, '
- '...])",\n'
- ' where *self* is the new instance and the remaining '
- 'arguments are\n'
- ' the same as were passed to the object constructor.\n'
+ ' an instance of *cls*, then the new instance’s '
+ '"__init__()" method\n'
+ ' will be invoked like "__init__(self[, ...])", where '
+ '*self* is the\n'
+ ' new instance and the remaining arguments are the same as '
+ 'were\n'
+ ' passed to the object constructor.\n'
'\n'
' If "__new__()" does not return an instance of *cls*, then '
'the new\n'
@@ -8726,32 +9679,6 @@
'of the\n'
' owner class.\n'
'\n'
- 'object.__set_name__(self, owner, name)\n'
- '\n'
- ' Called at the time the owning class *owner* is created. '
- 'The\n'
- ' descriptor has been assigned to *name*.\n'
- '\n'
- ' Note:\n'
- '\n'
- ' "__set_name__()" is only called implicitly as part of '
- 'the "type"\n'
- ' constructor, so it will need to be called explicitly '
- 'with the\n'
- ' appropriate parameters when a descriptor is added to a '
- 'class\n'
- ' after initial creation:\n'
- '\n'
- ' class A:\n'
- ' pass\n'
- ' descr = custom_descriptor()\n'
- ' A.attr = descr\n'
- " descr.__set_name__(A, 'attr')\n"
- '\n'
- ' See Creating the class object for more details.\n'
- '\n'
- ' New in version 3.6.\n'
- '\n'
'The attribute "__objclass__" is interpreted by the "inspect" '
'module as\n'
'specifying the class where this object was defined (setting '
@@ -8822,16 +9749,16 @@
'"super(B,\n'
' obj).m()" searches "obj.__class__.__mro__" for the base '
'class "A"\n'
- ' immediately preceding "B" and then invokes the descriptor '
+ ' immediately following "B" and then invokes the descriptor '
'with the\n'
' call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n'
'\n'
'For instance bindings, the precedence of descriptor '
'invocation depends\n'
- 'on the which descriptor methods are defined. A descriptor '
- 'can define\n'
- 'any combination of "__get__()", "__set__()" and '
- '"__delete__()". If it\n'
+ 'on which descriptor methods are defined. A descriptor can '
+ 'define any\n'
+ 'combination of "__get__()", "__set__()" and "__delete__()". '
+ 'If it\n'
'does not define "__get__()", then accessing the attribute '
'will return\n'
'the descriptor object itself unless there is a value in the '
@@ -8852,13 +9779,14 @@
'be\n'
'overridden by instances.\n'
'\n'
- 'Python methods (including "staticmethod()" and '
- '"classmethod()") are\n'
- 'implemented as non-data descriptors. Accordingly, instances '
- 'can\n'
- 'redefine and override methods. This allows individual '
- 'instances to\n'
- 'acquire behaviors that differ from other instances of the '
+ 'Python methods (including those decorated with '
+ '"@staticmethod" and\n'
+ '"@classmethod") are implemented as non-data descriptors. '
+ 'Accordingly,\n'
+ 'instances can redefine and override methods. This allows '
+ 'individual\n'
+ 'instances to acquire behaviors that differ from other '
+ 'instances of the\n'
'same class.\n'
'\n'
'The "property()" function is implemented as a data '
@@ -8872,12 +9800,12 @@
'\n'
'*__slots__* allow us to explicitly declare data members '
'(like\n'
- 'properties) and deny the creation of *__dict__* and '
+ 'properties) and deny the creation of "__dict__" and '
'*__weakref__*\n'
'(unless explicitly declared in *__slots__* or available in a '
'parent.)\n'
'\n'
- 'The space saved over using *__dict__* can be significant. '
+ 'The space saved over using "__dict__" can be significant. '
'Attribute\n'
'lookup speed can be significantly improved as well.\n'
'\n'
@@ -8889,7 +9817,7 @@
'*__slots__*\n'
' reserves space for the declared variables and prevents '
'the\n'
- ' automatic creation of *__dict__* and *__weakref__* for '
+ ' automatic creation of "__dict__" and *__weakref__* for '
'each\n'
' instance.\n'
'\n'
@@ -8898,11 +9826,11 @@
'~~~~~~~~~~~~~~~~~~~~~~~~~~\n'
'\n'
'* When inheriting from a class without *__slots__*, the '
- '*__dict__* and\n'
+ '"__dict__" and\n'
' *__weakref__* attribute of the instances will always be '
'accessible.\n'
'\n'
- '* Without a *__dict__* variable, instances cannot be '
+ '* Without a "__dict__" variable, instances cannot be '
'assigned new\n'
' variables not listed in the *__slots__* definition. '
'Attempts to\n'
@@ -8915,28 +9843,28 @@
'\n'
'* Without a *__weakref__* variable for each instance, '
'classes defining\n'
- ' *__slots__* do not support weak references to its '
- 'instances. If weak\n'
- ' reference support is needed, then add "\'__weakref__\'" to '
- 'the\n'
+ ' *__slots__* do not support "weak references" to its '
+ 'instances. If\n'
+ ' weak reference support is needed, then add '
+ '"\'__weakref__\'" to the\n'
' sequence of strings in the *__slots__* declaration.\n'
'\n'
'* *__slots__* are implemented at the class level by '
'creating\n'
- ' descriptors (Implementing Descriptors) for each variable '
- 'name. As a\n'
- ' result, class attributes cannot be used to set default '
- 'values for\n'
- ' instance variables defined by *__slots__*; otherwise, the '
- 'class\n'
- ' attribute would overwrite the descriptor assignment.\n'
+ ' descriptors for each variable name. As a result, class '
+ 'attributes\n'
+ ' cannot be used to set default values for instance '
+ 'variables defined\n'
+ ' by *__slots__*; otherwise, the class attribute would '
+ 'overwrite the\n'
+ ' descriptor assignment.\n'
'\n'
'* The action of a *__slots__* declaration is not limited to '
'the class\n'
' where it is defined. *__slots__* declared in parents are '
'available\n'
' in child classes. However, child subclasses will get a '
- '*__dict__*\n'
+ '"__dict__"\n'
' and *__weakref__* unless they also define *__slots__* '
'(which should\n'
' only contain names of any *additional* slots).\n'
@@ -8956,13 +9884,18 @@
' “variable-length” built-in types such as "int", "bytes" '
'and "tuple".\n'
'\n'
- '* Any non-string iterable may be assigned to *__slots__*. '
- 'Mappings may\n'
- ' also be used; however, in the future, special meaning may '
- 'be\n'
- ' assigned to the values corresponding to each key.\n'
+ '* Any non-string *iterable* may be assigned to *__slots__*.\n'
'\n'
- '* *__class__* assignment works only if both classes have the '
+ '* If a "dictionary" is used to assign *__slots__*, the '
+ 'dictionary keys\n'
+ ' will be used as the slot names. The values of the '
+ 'dictionary can be\n'
+ ' used to provide per-attribute docstrings that will be '
+ 'recognised by\n'
+ ' "inspect.getdoc()" and displayed in the output of '
+ '"help()".\n'
+ '\n'
+ '* "__class__" assignment works only if both classes have the '
'same\n'
' *__slots__*.\n'
'\n'
@@ -8974,9 +9907,9 @@
'violations\n'
' raise "TypeError".\n'
'\n'
- '* If an iterator is used for *__slots__* then a descriptor '
- 'is created\n'
- ' for each of the iterator’s values. However, the '
+ '* If an *iterator* is used for *__slots__* then a '
+ '*descriptor* is\n'
+ ' created for each of the iterator’s values. However, the '
'*__slots__*\n'
' attribute will be an empty iterator.\n'
'\n'
@@ -8985,15 +9918,15 @@
'==========================\n'
'\n'
'Whenever a class inherits from another class, '
- '*__init_subclass__* is\n'
- 'called on that class. This way, it is possible to write '
- 'classes which\n'
- 'change the behavior of subclasses. This is closely related '
- 'to class\n'
- 'decorators, but where class decorators only affect the '
- 'specific class\n'
- 'they’re applied to, "__init_subclass__" solely applies to '
- 'future\n'
+ '"__init_subclass__()" is\n'
+ 'called on the parent class. This way, it is possible to '
+ 'write classes\n'
+ 'which change the behavior of subclasses. This is closely '
+ 'related to\n'
+ 'class decorators, but where class decorators only affect the '
+ 'specific\n'
+ 'class they’re applied to, "__init_subclass__" solely applies '
+ 'to future\n'
'subclasses of the class defining the method.\n'
'\n'
'classmethod object.__init_subclass__(cls)\n'
@@ -9041,6 +9974,38 @@
'\n'
' New in version 3.6.\n'
'\n'
+ 'When a class is created, "type.__new__()" scans the class '
+ 'variables\n'
+ 'and makes callbacks to those with a "__set_name__()" hook.\n'
+ '\n'
+ 'object.__set_name__(self, owner, name)\n'
+ '\n'
+ ' Automatically called at the time the owning class *owner* '
+ 'is\n'
+ ' created. The object has been assigned to *name* in that '
+ 'class:\n'
+ '\n'
+ ' class A:\n'
+ ' x = C() # Automatically calls: x.__set_name__(A, '
+ "'x')\n"
+ '\n'
+ ' If the class variable is assigned after the class is '
+ 'created,\n'
+ ' "__set_name__()" will not be called automatically. If '
+ 'needed,\n'
+ ' "__set_name__()" can be called directly:\n'
+ '\n'
+ ' class A:\n'
+ ' pass\n'
+ '\n'
+ ' c = C()\n'
+ ' A.x = c # The hook is not called\n'
+ " c.__set_name__(A, 'x') # Manually invoke the hook\n"
+ '\n'
+ ' See Creating the class object for more details.\n'
+ '\n'
+ ' New in version 3.6.\n'
+ '\n'
'\n'
'Metaclasses\n'
'-----------\n'
@@ -9153,10 +10118,10 @@
'come from\n'
'the class definition). The "__prepare__" method should be '
'implemented\n'
- 'as a "classmethod()". The namespace returned by '
- '"__prepare__" is\n'
- 'passed in to "__new__", but when the final class object is '
- 'created the\n'
+ 'as a "classmethod". The namespace returned by "__prepare__" '
+ 'is passed\n'
+ 'in to "__new__", but when the final class object is created '
+ 'the\n'
'namespace is copied into a new "dict".\n'
'\n'
'If the metaclass has no "__prepare__" attribute, then the '
@@ -9236,22 +10201,21 @@
'When using the default metaclass "type", or any metaclass '
'that\n'
'ultimately calls "type.__new__", the following additional\n'
- 'customisation steps are invoked after creating the class '
+ 'customization steps are invoked after creating the class '
'object:\n'
'\n'
- '* first, "type.__new__" collects all of the descriptors in '
- 'the class\n'
- ' namespace that define a "__set_name__()" method;\n'
+ '1. The "type.__new__" method collects all of the attributes '
+ 'in the\n'
+ ' class namespace that define a "__set_name__()" method;\n'
'\n'
- '* second, all of these "__set_name__" methods are called '
- 'with the\n'
- ' class being defined and the assigned name of that '
- 'particular\n'
- ' descriptor;\n'
+ '2. Those "__set_name__" methods are called with the class '
+ 'being\n'
+ ' defined and the assigned name of that particular '
+ 'attribute;\n'
'\n'
- '* finally, the "__init_subclass__()" hook is called on the '
- 'immediate\n'
- ' parent of the new class in its method resolution order.\n'
+ '3. The "__init_subclass__()" hook is called on the immediate '
+ 'parent of\n'
+ ' the new class in its method resolution order.\n'
'\n'
'After the class object is created, it is passed to the '
'class\n'
@@ -9344,9 +10308,33 @@
'Emulating generic types\n'
'=======================\n'
'\n'
- 'One can implement the generic class syntax as specified by '
- '**PEP 484**\n'
- '(for example "List[int]") by defining a special method:\n'
+ 'When using *type annotations*, it is often useful to '
+ '*parameterize* a\n'
+ '*generic type* using Python’s square-brackets notation. For '
+ 'example,\n'
+ 'the annotation "list[int]" might be used to signify a "list" '
+ 'in which\n'
+ 'all the elements are of type "int".\n'
+ '\n'
+ 'See also:\n'
+ '\n'
+ ' **PEP 484** - Type Hints\n'
+ ' Introducing Python’s framework for type annotations\n'
+ '\n'
+ ' Generic Alias Types\n'
+ ' Documentation for objects representing parameterized '
+ 'generic\n'
+ ' classes\n'
+ '\n'
+ ' Generics, user-defined generics and "typing.Generic"\n'
+ ' Documentation on how to implement generic classes that '
+ 'can be\n'
+ ' parameterized at runtime and understood by static '
+ 'type-checkers.\n'
+ '\n'
+ 'A class can *generally* only be parameterized if it defines '
+ 'the\n'
+ 'special class method "__class_getitem__()".\n'
'\n'
'classmethod object.__class_getitem__(cls, key)\n'
'\n'
@@ -9354,18 +10342,144 @@
'generic class\n'
' by type arguments found in *key*.\n'
'\n'
- 'This method is looked up on the class object itself, and '
- 'when defined\n'
- 'in the class body, this method is implicitly a class '
- 'method. Note,\n'
- 'this mechanism is primarily reserved for use with static '
- 'type hints,\n'
- 'other usage is discouraged.\n'
+ ' When defined on a class, "__class_getitem__()" is '
+ 'automatically a\n'
+ ' class method. As such, there is no need for it to be '
+ 'decorated with\n'
+ ' "@classmethod" when it is defined.\n'
+ '\n'
+ '\n'
+ 'The purpose of *__class_getitem__*\n'
+ '----------------------------------\n'
+ '\n'
+ 'The purpose of "__class_getitem__()" is to allow runtime\n'
+ 'parameterization of standard-library generic classes in '
+ 'order to more\n'
+ 'easily apply *type hints* to these classes.\n'
+ '\n'
+ 'To implement custom generic classes that can be '
+ 'parameterized at\n'
+ 'runtime and understood by static type-checkers, users should '
+ 'either\n'
+ 'inherit from a standard library class that already '
+ 'implements\n'
+ '"__class_getitem__()", or inherit from "typing.Generic", '
+ 'which has its\n'
+ 'own implementation of "__class_getitem__()".\n'
+ '\n'
+ 'Custom implementations of "__class_getitem__()" on classes '
+ 'defined\n'
+ 'outside of the standard library may not be understood by '
+ 'third-party\n'
+ 'type-checkers such as mypy. Using "__class_getitem__()" on '
+ 'any class\n'
+ 'for purposes other than type hinting is discouraged.\n'
+ '\n'
+ '\n'
+ '*__class_getitem__* versus *__getitem__*\n'
+ '----------------------------------------\n'
+ '\n'
+ 'Usually, the subscription of an object using square brackets '
+ 'will call\n'
+ 'the "__getitem__()" instance method defined on the object’s '
+ 'class.\n'
+ 'However, if the object being subscribed is itself a class, '
+ 'the class\n'
+ 'method "__class_getitem__()" may be called instead.\n'
+ '"__class_getitem__()" should return a GenericAlias object if '
+ 'it is\n'
+ 'properly defined.\n'
+ '\n'
+ 'Presented with the *expression* "obj[x]", the Python '
+ 'interpreter\n'
+ 'follows something like the following process to decide '
+ 'whether\n'
+ '"__getitem__()" or "__class_getitem__()" should be called:\n'
+ '\n'
+ ' from inspect import isclass\n'
+ '\n'
+ ' def subscribe(obj, x):\n'
+ ' """Return the result of the expression `obj[x]`"""\n'
+ '\n'
+ ' class_of_obj = type(obj)\n'
+ '\n'
+ ' # If the class of obj defines __getitem__,\n'
+ ' # call class_of_obj.__getitem__(obj, x)\n'
+ " if hasattr(class_of_obj, '__getitem__'):\n"
+ ' return class_of_obj.__getitem__(obj, x)\n'
+ '\n'
+ ' # Else, if obj is a class and defines '
+ '__class_getitem__,\n'
+ ' # call obj.__class_getitem__(x)\n'
+ ' elif isclass(obj) and hasattr(obj, '
+ "'__class_getitem__'):\n"
+ ' return obj.__class_getitem__(x)\n'
+ '\n'
+ ' # Else, raise an exception\n'
+ ' else:\n'
+ ' raise TypeError(\n'
+ ' f"\'{class_of_obj.__name__}\' object is not '
+ 'subscriptable"\n'
+ ' )\n'
+ '\n'
+ 'In Python, all classes are themselves instances of other '
+ 'classes. The\n'
+ 'class of a class is known as that class’s *metaclass*, and '
+ 'most\n'
+ 'classes have the "type" class as their metaclass. "type" '
+ 'does not\n'
+ 'define "__getitem__()", meaning that expressions such as '
+ '"list[int]",\n'
+ '"dict[str, float]" and "tuple[str, bytes]" all result in\n'
+ '"__class_getitem__()" being called:\n'
+ '\n'
+ ' >>> # list has class "type" as its metaclass, like most '
+ 'classes:\n'
+ ' >>> type(list)\n'
+ " <class 'type'>\n"
+ ' >>> type(dict) == type(list) == type(tuple) == type(str) '
+ '== type(bytes)\n'
+ ' True\n'
+ ' >>> # "list[int]" calls "list.__class_getitem__(int)"\n'
+ ' >>> list[int]\n'
+ ' list[int]\n'
+ ' >>> # list.__class_getitem__ returns a GenericAlias '
+ 'object:\n'
+ ' >>> type(list[int])\n'
+ " <class 'types.GenericAlias'>\n"
+ '\n'
+ 'However, if a class has a custom metaclass that defines\n'
+ '"__getitem__()", subscribing the class may result in '
+ 'different\n'
+ 'behaviour. An example of this can be found in the "enum" '
+ 'module:\n'
+ '\n'
+ ' >>> from enum import Enum\n'
+ ' >>> class Menu(Enum):\n'
+ ' ... """A breakfast menu"""\n'
+ " ... SPAM = 'spam'\n"
+ " ... BACON = 'bacon'\n"
+ ' ...\n'
+ ' >>> # Enum classes have a custom metaclass:\n'
+ ' >>> type(Menu)\n'
+ " <class 'enum.EnumMeta'>\n"
+ ' >>> # EnumMeta defines __getitem__,\n'
+ ' >>> # so __class_getitem__ is not called,\n'
+ ' >>> # and the result is not a GenericAlias object:\n'
+ " >>> Menu['SPAM']\n"
+ " <Menu.SPAM: 'spam'>\n"
+ " >>> type(Menu['SPAM'])\n"
+ " <enum 'Menu'>\n"
'\n'
'See also:\n'
'\n'
- ' **PEP 560** - Core support for typing module and generic '
+ ' **PEP 560** - Core Support for typing module and generic '
'types\n'
+ ' Introducing "__class_getitem__()", and outlining when '
+ 'a\n'
+ ' subscription results in "__class_getitem__()" being '
+ 'called\n'
+ ' instead of "__getitem__()"\n'
'\n'
'\n'
'Emulating callable objects\n'
@@ -9384,60 +10498,60 @@
'\n'
'The following methods can be defined to implement container '
'objects.\n'
- 'Containers usually are sequences (such as lists or tuples) '
- 'or mappings\n'
- '(like dictionaries), but can represent other containers as '
- 'well. The\n'
- 'first set of methods is used either to emulate a sequence or '
- 'to\n'
- 'emulate a mapping; the difference is that for a sequence, '
- 'the\n'
- 'allowable keys should be the integers *k* for which "0 <= k '
- '< N" where\n'
- '*N* is the length of the sequence, or slice objects, which '
- 'define a\n'
- 'range of items. It is also recommended that mappings '
- 'provide the\n'
- 'methods "keys()", "values()", "items()", "get()", '
- '"clear()",\n'
- '"setdefault()", "pop()", "popitem()", "copy()", and '
- '"update()"\n'
- 'behaving similar to those for Python’s standard dictionary '
+ 'Containers usually are *sequences* (such as "lists" or '
+ '"tuples") or\n'
+ '*mappings* (like "dictionaries"), but can represent other '
+ 'containers\n'
+ 'as well. The first set of methods is used either to emulate '
+ 'a\n'
+ 'sequence or to emulate a mapping; the difference is that for '
+ 'a\n'
+ 'sequence, the allowable keys should be the integers *k* for '
+ 'which "0\n'
+ '<= k < N" where *N* is the length of the sequence, or '
+ '"slice" objects,\n'
+ 'which define a range of items. It is also recommended that '
+ 'mappings\n'
+ 'provide the methods "keys()", "values()", "items()", '
+ '"get()",\n'
+ '"clear()", "setdefault()", "pop()", "popitem()", "copy()", '
+ 'and\n'
+ '"update()" behaving similar to those for Python’s standard\n'
+ '"dictionary" objects. The "collections.abc" module provides '
+ 'a\n'
+ '"MutableMapping" *abstract base class* to help create those '
+ 'methods\n'
+ 'from a base set of "__getitem__()", "__setitem__()", '
+ '"__delitem__()",\n'
+ 'and "keys()". Mutable sequences should provide methods '
+ '"append()",\n'
+ '"count()", "index()", "extend()", "insert()", "pop()", '
+ '"remove()",\n'
+ '"reverse()" and "sort()", like Python standard "list" '
'objects.\n'
- 'The "collections.abc" module provides a "MutableMapping" '
- 'abstract base\n'
- 'class to help create those methods from a base set of '
- '"__getitem__()",\n'
- '"__setitem__()", "__delitem__()", and "keys()". Mutable '
- 'sequences\n'
- 'should provide methods "append()", "count()", "index()", '
- '"extend()",\n'
- '"insert()", "pop()", "remove()", "reverse()" and "sort()", '
- 'like Python\n'
- 'standard list objects. Finally, sequence types should '
- 'implement\n'
- 'addition (meaning concatenation) and multiplication '
- '(meaning\n'
- 'repetition) by defining the methods "__add__()", '
- '"__radd__()",\n'
- '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" '
- 'described\n'
- 'below; they should not define other numerical operators. It '
- 'is\n'
- 'recommended that both mappings and sequences implement the\n'
- '"__contains__()" method to allow efficient use of the "in" '
- 'operator;\n'
- 'for mappings, "in" should search the mapping’s keys; for '
- 'sequences, it\n'
- 'should search through the values. It is further recommended '
- 'that both\n'
- 'mappings and sequences implement the "__iter__()" method to '
- 'allow\n'
- 'efficient iteration through the container; for mappings, '
- '"__iter__()"\n'
- 'should iterate through the object’s keys; for sequences, it '
- 'should\n'
- 'iterate through the values.\n'
+ 'Finally, sequence types should implement addition (meaning\n'
+ 'concatenation) and multiplication (meaning repetition) by '
+ 'defining the\n'
+ 'methods "__add__()", "__radd__()", "__iadd__()", '
+ '"__mul__()",\n'
+ '"__rmul__()" and "__imul__()" described below; they should '
+ 'not define\n'
+ 'other numerical operators. It is recommended that both '
+ 'mappings and\n'
+ 'sequences implement the "__contains__()" method to allow '
+ 'efficient use\n'
+ 'of the "in" operator; for mappings, "in" should search the '
+ 'mapping’s\n'
+ 'keys; for sequences, it should search through the values. '
+ 'It is\n'
+ 'further recommended that both mappings and sequences '
+ 'implement the\n'
+ '"__iter__()" method to allow efficient iteration through '
+ 'the\n'
+ 'container; for mappings, "__iter__()" should iterate through '
+ 'the\n'
+ 'object’s keys; for sequences, it should iterate through the '
+ 'values.\n'
'\n'
'object.__len__(self)\n'
'\n'
@@ -9495,22 +10609,23 @@
'object.__getitem__(self, key)\n'
'\n'
' Called to implement evaluation of "self[key]". For '
- 'sequence types,\n'
- ' the accepted keys should be integers and slice objects. '
- 'Note that\n'
- ' the special interpretation of negative indexes (if the '
- 'class wishes\n'
- ' to emulate a sequence type) is up to the "__getitem__()" '
- 'method. If\n'
- ' *key* is of an inappropriate type, "TypeError" may be '
- 'raised; if of\n'
- ' a value outside the set of indexes for the sequence '
- '(after any\n'
- ' special interpretation of negative values), "IndexError" '
+ '*sequence*\n'
+ ' types, the accepted keys should be integers and slice '
+ 'objects.\n'
+ ' Note that the special interpretation of negative indexes '
+ '(if the\n'
+ ' class wishes to emulate a *sequence* type) is up to the\n'
+ ' "__getitem__()" method. If *key* is of an inappropriate '
+ 'type,\n'
+ ' "TypeError" may be raised; if of a value outside the set '
+ 'of indexes\n'
+ ' for the sequence (after any special interpretation of '
+ 'negative\n'
+ ' values), "IndexError" should be raised. For *mapping* '
+ 'types, if\n'
+ ' *key* is missing (not in the container), "KeyError" '
'should be\n'
- ' raised. For mapping types, if *key* is missing (not in '
- 'the\n'
- ' container), "KeyError" should be raised.\n'
+ ' raised.\n'
'\n'
' Note:\n'
'\n'
@@ -9520,6 +10635,14 @@
'the\n'
' sequence.\n'
'\n'
+ ' Note:\n'
+ '\n'
+ ' When subscripting a *class*, the special class method\n'
+ ' "__class_getitem__()" may be called instead of '
+ '"__getitem__()".\n'
+ ' See __class_getitem__ versus __getitem__ for more '
+ 'details.\n'
+ '\n'
'object.__setitem__(self, key, value)\n'
'\n'
' Called to implement assignment to "self[key]". Same note '
@@ -9555,19 +10678,13 @@
'\n'
'object.__iter__(self)\n'
'\n'
- ' This method is called when an iterator is required for a '
- 'container.\n'
- ' This method should return a new iterator object that can '
- 'iterate\n'
- ' over all the objects in the container. For mappings, it '
- 'should\n'
- ' iterate over the keys of the container.\n'
- '\n'
- ' Iterator objects also need to implement this method; they '
- 'are\n'
- ' required to return themselves. For more information on '
- 'iterator\n'
- ' objects, see Iterator Types.\n'
+ ' This method is called when an *iterator* is required for '
+ 'a\n'
+ ' container. This method should return a new iterator '
+ 'object that can\n'
+ ' iterate over all the objects in the container. For '
+ 'mappings, it\n'
+ ' should iterate over the keys of the container.\n'
'\n'
'object.__reversed__(self)\n'
'\n'
@@ -9760,16 +10877,6 @@
'the data\n'
' model.\n'
'\n'
- ' Note:\n'
- '\n'
- ' Due to a bug in the dispatching mechanism for "**=", a '
- 'class that\n'
- ' defines "__ipow__()" but returns "NotImplemented" would '
- 'fail to\n'
- ' fall back to "x.__pow__(y)" and "y.__rpow__(x)". This '
- 'bug is\n'
- ' fixed in Python 3.10.\n'
- '\n'
'object.__neg__(self)\n'
'object.__pos__(self)\n'
'object.__abs__(self)\n'
@@ -9820,9 +10927,9 @@
' of the object truncated to an "Integral" (typically an '
'"int").\n'
'\n'
- ' If "__int__()" is not defined then the built-in function '
- '"int()"\n'
- ' falls back to "__trunc__()".\n'
+ ' The built-in function "int()" falls back to "__trunc__()" '
+ 'if\n'
+ ' neither "__int__()" nor "__index__()" is defined.\n'
'\n'
'\n'
'With Statement Context Managers\n'
@@ -9888,6 +10995,51 @@
' statement.\n'
'\n'
'\n'
+ 'Customizing positional arguments in class pattern matching\n'
+ '==========================================================\n'
+ '\n'
+ 'When using a class name in a pattern, positional arguments '
+ 'in the\n'
+ 'pattern are not allowed by default, i.e. "case MyClass(x, '
+ 'y)" is\n'
+ 'typically invalid without special support in "MyClass". To '
+ 'be able to\n'
+ 'use that kind of patterns, the class needs to define a\n'
+ '*__match_args__* attribute.\n'
+ '\n'
+ 'object.__match_args__\n'
+ '\n'
+ ' This class variable can be assigned a tuple of strings. '
+ 'When this\n'
+ ' class is used in a class pattern with positional '
+ 'arguments, each\n'
+ ' positional argument will be converted into a keyword '
+ 'argument,\n'
+ ' using the corresponding value in *__match_args__* as the '
+ 'keyword.\n'
+ ' The absence of this attribute is equivalent to setting it '
+ 'to "()".\n'
+ '\n'
+ 'For example, if "MyClass.__match_args__" is "("left", '
+ '"center",\n'
+ '"right")" that means that "case MyClass(x, y)" is equivalent '
+ 'to "case\n'
+ 'MyClass(left=x, center=y)". Note that the number of '
+ 'arguments in the\n'
+ 'pattern must be smaller than or equal to the number of '
+ 'elements in\n'
+ '*__match_args__*; if it is larger, the pattern match attempt '
+ 'will\n'
+ 'raise a "TypeError".\n'
+ '\n'
+ 'New in version 3.10.\n'
+ '\n'
+ 'See also:\n'
+ '\n'
+ ' **PEP 634** - Structural Pattern Matching\n'
+ ' The specification for the Python "match" statement.\n'
+ '\n'
+ '\n'
'Special method lookup\n'
'=====================\n'
'\n'
@@ -10058,7 +11210,7 @@
'*start* and\n'
' *end* are interpreted as in slice notation.\n'
'\n'
- 'str.encode(encoding="utf-8", errors="strict")\n'
+ "str.encode(encoding='utf-8', errors='strict')\n"
'\n'
' Return an encoded version of the string as a bytes '
'object. Default\n'
@@ -10307,9 +11459,9 @@
' >>> from keyword import iskeyword\n'
'\n'
" >>> 'hello'.isidentifier(), iskeyword('hello')\n"
- ' True, False\n'
+ ' (True, False)\n'
" >>> 'def'.isidentifier(), iskeyword('def')\n"
- ' True, True\n'
+ ' (True, True)\n'
'\n'
'str.islower()\n'
'\n'
@@ -10564,7 +11716,7 @@
'followed by\n'
' the string itself.\n'
'\n'
- 'str.rsplit(sep=None, maxsplit=-1)\n'
+ 'str.rsplit(sep=None, maxsplit=- 1)\n'
'\n'
' Return a list of the words in the string, using *sep* '
'as the\n'
@@ -10605,7 +11757,7 @@
" >>> 'Monty Python'.removesuffix(' Python')\n"
" 'Monty'\n"
'\n'
- 'str.split(sep=None, maxsplit=-1)\n'
+ 'str.split(sep=None, maxsplit=- 1)\n'
'\n'
' Return a list of the words in the string, using *sep* '
'as the\n'
@@ -10660,7 +11812,7 @@
" >>> ' 1 2 3 '.split()\n"
" ['1', '2', '3']\n"
'\n'
- 'str.splitlines([keepends])\n'
+ 'str.splitlines(keepends=False)\n'
'\n'
' Return a list of the lines in the string, breaking at '
'line\n'
@@ -11141,67 +12293,86 @@
'subscriptions': 'Subscriptions\n'
'*************\n'
'\n'
- 'Subscription of a sequence (string, tuple or list) or '
- 'mapping\n'
- '(dictionary) object usually selects an item from the '
- 'collection:\n'
+ 'The subscription of an instance of a container class will '
+ 'generally\n'
+ 'select an element from the container. The subscription of a '
+ '*generic\n'
+ 'class* will generally return a GenericAlias object.\n'
'\n'
' subscription ::= primary "[" expression_list "]"\n'
'\n'
+ 'When an object is subscripted, the interpreter will '
+ 'evaluate the\n'
+ 'primary and the expression list.\n'
+ '\n'
'The primary must evaluate to an object that supports '
- 'subscription\n'
- '(lists or dictionaries for example). User-defined objects '
- 'can support\n'
- 'subscription by defining a "__getitem__()" method.\n'
+ 'subscription. An\n'
+ 'object may support subscription through defining one or '
+ 'both of\n'
+ '"__getitem__()" and "__class_getitem__()". When the primary '
+ 'is\n'
+ 'subscripted, the evaluated result of the expression list '
+ 'will be\n'
+ 'passed to one of these methods. For more details on when\n'
+ '"__class_getitem__" is called instead of "__getitem__", '
+ 'see\n'
+ '__class_getitem__ versus __getitem__.\n'
+ '\n'
+ 'If the expression list contains at least one comma, it will '
+ 'evaluate\n'
+ 'to a "tuple" containing the items of the expression list. '
+ 'Otherwise,\n'
+ 'the expression list will evaluate to the value of the '
+ 'list’s sole\n'
+ 'member.\n'
'\n'
'For built-in objects, there are two types of objects that '
'support\n'
- 'subscription:\n'
+ 'subscription via "__getitem__()":\n'
'\n'
- 'If the primary is a mapping, the expression list must '
- 'evaluate to an\n'
- 'object whose value is one of the keys of the mapping, and '
+ '1. Mappings. If the primary is a *mapping*, the expression '
+ 'list must\n'
+ ' evaluate to an object whose value is one of the keys of '
'the\n'
- 'subscription selects the value in the mapping that '
- 'corresponds to that\n'
- 'key. (The expression list is a tuple except if it has '
- 'exactly one\n'
- 'item.)\n'
+ ' mapping, and the subscription selects the value in the '
+ 'mapping that\n'
+ ' corresponds to that key. An example of a builtin mapping '
+ 'class is\n'
+ ' the "dict" class.\n'
'\n'
- 'If the primary is a sequence, the expression list must '
- 'evaluate to an\n'
- 'integer or a slice (as discussed in the following '
- 'section).\n'
+ '2. Sequences. If the primary is a *sequence*, the '
+ 'expression list must\n'
+ ' evaluate to an "int" or a "slice" (as discussed in the '
+ 'following\n'
+ ' section). Examples of builtin sequence classes include '
+ 'the "str",\n'
+ ' "list" and "tuple" classes.\n'
'\n'
'The formal syntax makes no special provision for negative '
'indices in\n'
- 'sequences; however, built-in sequences all provide a '
+ '*sequences*. However, built-in sequences all provide a '
'"__getitem__()"\n'
'method that interprets negative indices by adding the '
'length of the\n'
- 'sequence to the index (so that "x[-1]" selects the last '
- 'item of "x").\n'
- 'The resulting value must be a nonnegative integer less than '
- 'the number\n'
- 'of items in the sequence, and the subscription selects the '
- 'item whose\n'
- 'index is that value (counting from zero). Since the support '
- 'for\n'
- 'negative indices and slicing occurs in the object’s '
- '"__getitem__()"\n'
- 'method, subclasses overriding this method will need to '
- 'explicitly add\n'
- 'that support.\n'
+ 'sequence to the index so that, for example, "x[-1]" selects '
+ 'the last\n'
+ 'item of "x". The resulting value must be a nonnegative '
+ 'integer less\n'
+ 'than the number of items in the sequence, and the '
+ 'subscription selects\n'
+ 'the item whose index is that value (counting from zero). '
+ 'Since the\n'
+ 'support for negative indices and slicing occurs in the '
+ 'object’s\n'
+ '"__getitem__()" method, subclasses overriding this method '
+ 'will need to\n'
+ 'explicitly add that support.\n'
'\n'
- 'A string’s items are characters. A character is not a '
- 'separate data\n'
- 'type but a string of exactly one character.\n'
- '\n'
- 'Subscription of certain *classes* or *types* creates a '
- 'generic alias.\n'
- 'In this case, user-defined classes can support subscription '
- 'by\n'
- 'providing a "__class_getitem__()" classmethod.\n',
+ 'A "string" is a special kind of sequence whose items are '
+ '*characters*.\n'
+ 'A character is not a separate data type but a string of '
+ 'exactly one\n'
+ 'character.\n',
'truth': 'Truth Value Testing\n'
'*******************\n'
'\n'
@@ -11259,7 +12430,8 @@
'object is “compatible” with the exception. An object is compatible\n'
'with an exception if it is the class or a base class of the '
'exception\n'
- 'object or a tuple containing an item compatible with the exception.\n'
+ 'object, or a tuple containing an item that is the class or a base\n'
+ 'class of the exception object.\n'
'\n'
'If no except clause matches the exception, the search for an '
'exception\n'
@@ -11314,9 +12486,31 @@
'the\n'
'exception class, the exception instance and a traceback object (see\n'
'section The standard type hierarchy) identifying the point in the\n'
- 'program where the exception occurred. "sys.exc_info()" values are\n'
- 'restored to their previous values (before the call) when returning\n'
- 'from a function that handled an exception.\n'
+ 'program where the exception occurred. The details about the '
+ 'exception\n'
+ 'accessed via "sys.exc_info()" are restored to their previous values\n'
+ 'when leaving an exception handler:\n'
+ '\n'
+ ' >>> print(sys.exc_info())\n'
+ ' (None, None, None)\n'
+ ' >>> try:\n'
+ ' ... raise TypeError\n'
+ ' ... except:\n'
+ ' ... print(sys.exc_info())\n'
+ ' ... try:\n'
+ ' ... raise ValueError\n'
+ ' ... except:\n'
+ ' ... print(sys.exc_info())\n'
+ ' ... print(sys.exc_info())\n'
+ ' ...\n'
+ " (<class 'TypeError'>, TypeError(), <traceback object at "
+ '0x10efad080>)\n'
+ " (<class 'ValueError'>, ValueError(), <traceback object at "
+ '0x10efad040>)\n'
+ " (<class 'TypeError'>, TypeError(), <traceback object at "
+ '0x10efad080>)\n'
+ ' >>> print(sys.exc_info())\n'
+ ' (None, None, None)\n'
'\n'
'The optional "else" clause is executed if the control flow leaves '
'the\n'
@@ -11480,7 +12674,6 @@
' There are two types of integers:\n'
'\n'
' Integers ("int")\n'
- '\n'
' These represent numbers in an unlimited range, subject to\n'
' available (virtual) memory only. For the purpose of '
'shift\n'
@@ -11577,7 +12770,7 @@
' points. All the code points in the range "U+0000 - '
'U+10FFFF"\n'
' can be represented in a string. Python doesn’t have a '
- '"char"\n'
+ '*char*\n'
' type; instead, every code point in the string is '
'represented\n'
' as a string object with length "1". The built-in '
@@ -11837,7 +13030,13 @@
'| |\n'
' | | and "\'return\'" for the '
'return | |\n'
- ' | | annotation, if provided. '
+ ' | | annotation, if provided. For '
+ '| |\n'
+ ' | | more information on working '
+ '| |\n'
+ ' | | with this attribute, see '
+ '| |\n'
+ ' | | Annotations Best Practices. '
'| |\n'
' '
'+---------------------------+---------------------------------+-------------+\n'
@@ -11958,20 +13157,18 @@
' A function or method which uses the "yield" statement (see\n'
' section The yield statement) is called a *generator '
'function*.\n'
- ' Such a function, when called, always returns an iterator '
- 'object\n'
- ' which can be used to execute the body of the function: '
- 'calling\n'
- ' the iterator’s "iterator.__next__()" method will cause the\n'
- ' function to execute until it provides a value using the '
- '"yield"\n'
- ' statement. When the function executes a "return" statement '
- 'or\n'
- ' falls off the end, a "StopIteration" exception is raised and '
- 'the\n'
- ' iterator will have reached the end of the set of values to '
- 'be\n'
- ' returned.\n'
+ ' Such a function, when called, always returns an *iterator*\n'
+ ' object which can be used to execute the body of the '
+ 'function:\n'
+ ' calling the iterator’s "iterator.__next__()" method will '
+ 'cause\n'
+ ' the function to execute until it provides a value using the\n'
+ ' "yield" statement. When the function executes a "return"\n'
+ ' statement or falls off the end, a "StopIteration" exception '
+ 'is\n'
+ ' raised and the iterator will have reached the end of the set '
+ 'of\n'
+ ' values to be returned.\n'
'\n'
' Coroutine functions\n'
' A function or method which is defined using "async def" is\n'
@@ -11987,18 +13184,18 @@
' which uses the "yield" statement is called a *asynchronous\n'
' generator function*. Such a function, when called, returns '
'an\n'
- ' asynchronous iterator object which can be used in an "async '
- 'for"\n'
- ' statement to execute the body of the function.\n'
+ ' *asynchronous iterator* object which can be used in an '
+ '"async\n'
+ ' for" statement to execute the body of the function.\n'
'\n'
- ' Calling the asynchronous iterator’s "aiterator.__anext__()"\n'
- ' method will return an *awaitable* which when awaited will\n'
- ' execute until it provides a value using the "yield" '
- 'expression.\n'
- ' When the function executes an empty "return" statement or '
- 'falls\n'
- ' off the end, a "StopAsyncIteration" exception is raised and '
+ ' Calling the asynchronous iterator’s "aiterator.__anext__" '
+ 'method\n'
+ ' will return an *awaitable* which when awaited will execute '
+ 'until\n'
+ ' it provides a value using the "yield" expression. When the\n'
+ ' function executes an empty "return" statement or falls off '
'the\n'
+ ' end, a "StopAsyncIteration" exception is raised and the\n'
' asynchronous iterator will have reached the end of the set '
'of\n'
' values to be yielded.\n'
@@ -12062,20 +13259,34 @@
' Attribute assignment updates the module’s namespace dictionary,\n'
' e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n'
'\n'
- ' Predefined (writable) attributes: "__name__" is the module’s '
- 'name;\n'
- ' "__doc__" is the module’s documentation string, or "None" if\n'
- ' unavailable; "__annotations__" (optional) is a dictionary\n'
- ' containing *variable annotations* collected during module body\n'
- ' execution; "__file__" is the pathname of the file from which '
+ ' Predefined (writable) attributes:\n'
+ '\n'
+ ' "__name__"\n'
+ ' The module’s name.\n'
+ '\n'
+ ' "__doc__"\n'
+ ' The module’s documentation string, or "None" if '
+ 'unavailable.\n'
+ '\n'
+ ' "__file__"\n'
+ ' The pathname of the file from which the module was loaded, '
+ 'if\n'
+ ' it was loaded from a file. The "__file__" attribute may '
+ 'be\n'
+ ' missing for certain types of modules, such as C modules '
+ 'that\n'
+ ' are statically linked into the interpreter. For '
+ 'extension\n'
+ ' modules loaded dynamically from a shared library, it’s '
'the\n'
- ' module was loaded, if it was loaded from a file. The "__file__"\n'
- ' attribute may be missing for certain types of modules, such as '
- 'C\n'
- ' modules that are statically linked into the interpreter; for\n'
- ' extension modules loaded dynamically from a shared library, it '
- 'is\n'
- ' the pathname of the shared library file.\n'
+ ' pathname of the shared library file.\n'
+ '\n'
+ ' "__annotations__"\n'
+ ' A dictionary containing *variable annotations* collected\n'
+ ' during module body execution. For best practices on '
+ 'working\n'
+ ' with "__annotations__", please see Annotations Best\n'
+ ' Practices.\n'
'\n'
' Special read-only attribute: "__dict__" is the module’s '
'namespace\n'
@@ -12133,20 +13344,31 @@
'instance\n'
' (see below).\n'
'\n'
- ' Special attributes: "__name__" is the class name; "__module__" '
- 'is\n'
- ' the module name in which the class was defined; "__dict__" is '
- 'the\n'
- ' dictionary containing the class’s namespace; "__bases__" is a '
- 'tuple\n'
- ' containing the base classes, in the order of their occurrence '
- 'in\n'
- ' the base class list; "__doc__" is the class’s documentation '
- 'string,\n'
- ' or "None" if undefined; "__annotations__" (optional) is a\n'
- ' dictionary containing *variable annotations* collected during '
- 'class\n'
- ' body execution.\n'
+ ' Special attributes:\n'
+ '\n'
+ ' "__name__"\n'
+ ' The class name.\n'
+ '\n'
+ ' "__module__"\n'
+ ' The name of the module in which the class was defined.\n'
+ '\n'
+ ' "__dict__"\n'
+ ' The dictionary containing the class’s namespace.\n'
+ '\n'
+ ' "__bases__"\n'
+ ' A tuple containing the base classes, in the order of '
+ 'their\n'
+ ' occurrence in the base class list.\n'
+ '\n'
+ ' "__doc__"\n'
+ ' The class’s documentation string, or "None" if undefined.\n'
+ '\n'
+ ' "__annotations__"\n'
+ ' A dictionary containing *variable annotations* collected\n'
+ ' during class body execution. For best practices on '
+ 'working\n'
+ ' with "__annotations__", please see Annotations Best\n'
+ ' Practices.\n'
'\n'
'Class instances\n'
' A class instance is created by calling a class object (see '
@@ -12307,6 +13529,10 @@
' gives the precise instruction (this is an index into the\n'
' bytecode string of the code object).\n'
'\n'
+ ' Accessing "f_code" raises an auditing event '
+ '"object.__getattr__"\n'
+ ' with arguments "obj" and ""f_code"".\n'
+ '\n'
' Special writable attributes: "f_trace", if not "None", is a\n'
' function called for various events during code execution '
'(this\n'
@@ -12390,6 +13616,9 @@
' the exception occurred in a "try" statement with no matching\n'
' except clause or with a finally clause.\n'
'\n'
+ ' Accessing "tb_frame" raises an auditing event\n'
+ ' "object.__getattr__" with arguments "obj" and ""tb_frame"".\n'
+ '\n'
' Special writable attribute: "tb_next" is the next level in '
'the\n'
' stack trace (towards the frame where the exception occurred), '
@@ -12440,9 +13669,8 @@
' object actually returned is the wrapped object, which is not\n'
' subject to any further transformation. Static method objects '
'are\n'
- ' not themselves callable, although the objects they wrap '
- 'usually\n'
- ' are. Static method objects are created by the built-in\n'
+ ' also callable. Static method objects are created by the '
+ 'built-in\n'
' "staticmethod()" constructor.\n'
'\n'
' Class method objects\n'
@@ -12511,9 +13739,9 @@
'"dict"\n'
'constructor.\n'
'\n'
- 'class dict(**kwarg)\n'
- 'class dict(mapping, **kwarg)\n'
- 'class dict(iterable, **kwarg)\n'
+ 'class dict(**kwargs)\n'
+ 'class dict(mapping, **kwargs)\n'
+ 'class dict(iterable, **kwargs)\n'
'\n'
' Return a new dictionary initialized from an optional '
'positional\n'
@@ -12907,6 +14135,14 @@
' Changed in version 3.8: Dictionary views are now '
'reversible.\n'
'\n'
+ 'dictview.mapping\n'
+ '\n'
+ ' Return a "types.MappingProxyType" that wraps the '
+ 'original\n'
+ ' dictionary to which the view refers.\n'
+ '\n'
+ ' New in version 3.10.\n'
+ '\n'
'Keys views are set-like since their entries are unique and '
'hashable.\n'
'If all values are hashable, so that "(key, value)" pairs are '
@@ -12952,7 +14188,15 @@
" >>> keys & {'eggs', 'bacon', 'salad'}\n"
" {'bacon'}\n"
" >>> keys ^ {'sausage', 'juice'}\n"
- " {'juice', 'sausage', 'bacon', 'spam'}\n",
+ " {'juice', 'sausage', 'bacon', 'spam'}\n"
+ '\n'
+ ' >>> # get back a read-only proxy for the original '
+ 'dictionary\n'
+ ' >>> values.mapping\n'
+ " mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, "
+ "'spam': 500})\n"
+ " >>> values.mapping['spam']\n"
+ ' 500\n',
'typesmethods': 'Methods\n'
'*******\n'
'\n'
@@ -13147,6 +14391,14 @@
'Comparisons in\n'
'the language reference.)\n'
'\n'
+ 'Forward and reversed iterators over mutable sequences access '
+ 'values\n'
+ 'using an index. That index will continue to march forward (or\n'
+ 'backward) even if the underlying sequence is mutated. The '
+ 'iterator\n'
+ 'terminates only when an "IndexError" or a "StopIteration" is\n'
+ 'encountered (or when the index drops below zero).\n'
+ '\n'
'Notes:\n'
'\n'
'1. While the "in" and "not in" operations are used only for '
@@ -13378,7 +14630,7 @@
'| | "s[i:i] = '
'[x]") | |\n'
'+--------------------------------+----------------------------------+-----------------------+\n'
- '| "s.pop([i])" | retrieves the item at *i* '
+ '| "s.pop()" or "s.pop(i)" | retrieves the item at *i* '
'and | (2) |\n'
'| | also removes it from '
'*s* | |\n'
@@ -13618,7 +14870,8 @@
'\n'
' The arguments to the range constructor must be integers '
'(either\n'
- ' built-in "int" or any object that implements the "__index__"\n'
+ ' built-in "int" or any object that implements the '
+ '"__index__()"\n'
' special method). If the *step* argument is omitted, it '
'defaults to\n'
' "1". If the *start* argument is omitted, it defaults to "0". '
@@ -13841,7 +15094,7 @@
'| | "s[i:i] = '
'[x]") | |\n'
'+--------------------------------+----------------------------------+-----------------------+\n'
- '| "s.pop([i])" | retrieves the item at '
+ '| "s.pop()" or "s.pop(i)" | retrieves the item at '
'*i* and | (2) |\n'
'| | also removes it from '
'*s* | |\n'
@@ -13906,15 +15159,21 @@
' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n'
'\n'
'The unary "-" (minus) operator yields the negation of its numeric\n'
- 'argument.\n'
+ 'argument; the operation can be overridden with the "__neg__()" '
+ 'special\n'
+ 'method.\n'
'\n'
'The unary "+" (plus) operator yields its numeric argument '
- 'unchanged.\n'
+ 'unchanged;\n'
+ 'the operation can be overridden with the "__pos__()" special '
+ 'method.\n'
'\n'
'The unary "~" (invert) operator yields the bitwise inversion of '
'its\n'
'integer argument. The bitwise inversion of "x" is defined as\n'
- '"-(x+1)". It only applies to integral numbers.\n'
+ '"-(x+1)". It only applies to integral numbers or to custom '
+ 'objects\n'
+ 'that override the "__invert__()" special method.\n'
'\n'
'In all three cases, if the argument does not have the proper type, '
'a\n'
@@ -13952,8 +15211,10 @@
'usage\n'
'patterns to be encapsulated for convenient reuse.\n'
'\n'
- ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n'
- ' with_item ::= expression ["as" target]\n'
+ ' with_stmt ::= "with" ( "(" with_stmt_contents ","? ")" | '
+ 'with_stmt_contents ) ":" suite\n'
+ ' with_stmt_contents ::= with_item ("," with_item)*\n'
+ ' with_item ::= expression ["as" target]\n'
'\n'
'The execution of the "with" statement with one “item” proceeds as\n'
'follows:\n'
@@ -14039,8 +15300,20 @@
' with B() as b:\n'
' SUITE\n'
'\n'
+ 'You can also write multi-item context managers in multiple lines if\n'
+ 'the items are surrounded by parentheses. For example:\n'
+ '\n'
+ ' with (\n'
+ ' A() as a,\n'
+ ' B() as b,\n'
+ ' ):\n'
+ ' SUITE\n'
+ '\n'
'Changed in version 3.1: Support for multiple context expressions.\n'
'\n'
+ 'Changed in version 3.10: Support for using grouping parentheses to\n'
+ 'break the statement in multiple lines.\n'
+ '\n'
'See also:\n'
'\n'
' **PEP 343** - The “with” statement\n'
diff --git a/common/py3-stdlib/random.py b/common/py3-stdlib/random.py
index a6454f5..1310a2d 100644
--- a/common/py3-stdlib/random.py
+++ b/common/py3-stdlib/random.py
@@ -48,9 +48,10 @@
from warnings import warn as _warn
from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil
from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin
-from math import tau as TWOPI, floor as _floor
+from math import tau as TWOPI, floor as _floor, isfinite as _isfinite
from os import urandom as _urandom
from _collections_abc import Set as _Set, Sequence as _Sequence
+from operator import index as _index
from itertools import accumulate as _accumulate, repeat as _repeat
from bisect import bisect as _bisect
import os as _os
@@ -77,6 +78,7 @@
"lognormvariate",
"normalvariate",
"paretovariate",
+ "randbytes",
"randint",
"random",
"randrange",
@@ -95,6 +97,7 @@
SG_MAGICCONST = 1.0 + _log(4.5)
BPF = 53 # Number of bits in a float
RECIP_BPF = 2 ** -BPF
+_ONE = 1
class Random(_random.Random):
@@ -151,8 +154,7 @@
elif version == 2 and isinstance(a, (str, bytes, bytearray)):
if isinstance(a, str):
a = a.encode()
- a += _sha512(a).digest()
- a = int.from_bytes(a, 'big')
+ a = int.from_bytes(a + _sha512(a).digest(), 'big')
elif not isinstance(a, (type(None), int, float, str, bytes, bytearray)):
_warn('Seeding based on hashing is deprecated\n'
@@ -287,7 +289,7 @@
## -------------------- integer methods -------------------
- def randrange(self, start, stop=None, step=1):
+ def randrange(self, start, stop=None, step=_ONE):
"""Choose a random item from range(start, stop[, step]).
This fixes the problem with randint() which includes the
@@ -297,38 +299,68 @@
# This code is a bit messy to make it fast for the
# common case while still doing adequate error checking.
- istart = int(start)
- if istart != start:
- raise ValueError("non-integer arg 1 for randrange()")
+ try:
+ istart = _index(start)
+ except TypeError:
+ istart = int(start)
+ if istart != start:
+ _warn('randrange() will raise TypeError in the future',
+ DeprecationWarning, 2)
+ raise ValueError("non-integer arg 1 for randrange()")
+ _warn('non-integer arguments to randrange() have been deprecated '
+ 'since Python 3.10 and will be removed in a subsequent '
+ 'version',
+ DeprecationWarning, 2)
if stop is None:
+ # We don't check for "step != 1" because it hasn't been
+ # type checked and converted to an integer yet.
+ if step is not _ONE:
+ raise TypeError('Missing a non-None stop argument')
if istart > 0:
return self._randbelow(istart)
raise ValueError("empty range for randrange()")
# stop argument supplied.
- istop = int(stop)
- if istop != stop:
- raise ValueError("non-integer stop for randrange()")
+ try:
+ istop = _index(stop)
+ except TypeError:
+ istop = int(stop)
+ if istop != stop:
+ _warn('randrange() will raise TypeError in the future',
+ DeprecationWarning, 2)
+ raise ValueError("non-integer stop for randrange()")
+ _warn('non-integer arguments to randrange() have been deprecated '
+ 'since Python 3.10 and will be removed in a subsequent '
+ 'version',
+ DeprecationWarning, 2)
width = istop - istart
- if step == 1 and width > 0:
- return istart + self._randbelow(width)
- if step == 1:
+ try:
+ istep = _index(step)
+ except TypeError:
+ istep = int(step)
+ if istep != step:
+ _warn('randrange() will raise TypeError in the future',
+ DeprecationWarning, 2)
+ raise ValueError("non-integer step for randrange()")
+ _warn('non-integer arguments to randrange() have been deprecated '
+ 'since Python 3.10 and will be removed in a subsequent '
+ 'version',
+ DeprecationWarning, 2)
+ # Fast path.
+ if istep == 1:
+ if width > 0:
+ return istart + self._randbelow(width)
raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width))
# Non-unit step argument supplied.
- istep = int(step)
- if istep != step:
- raise ValueError("non-integer step for randrange()")
if istep > 0:
n = (width + istep - 1) // istep
elif istep < 0:
n = (width + istep + 1) // istep
else:
raise ValueError("zero step for randrange()")
-
if n <= 0:
raise ValueError("empty range for randrange()")
-
return istart + istep * self._randbelow(n)
def randint(self, a, b):
@@ -424,13 +456,14 @@
# too many calls to _randbelow(), making them slower and
# causing them to eat more entropy than necessary.
- if isinstance(population, _Set):
- _warn('Sampling from a set deprecated\n'
- 'since Python 3.9 and will be removed in a subsequent version.',
- DeprecationWarning, 2)
- population = tuple(population)
if not isinstance(population, _Sequence):
- raise TypeError("Population must be a sequence. For dicts or sets, use sorted(d).")
+ if isinstance(population, _Set):
+ _warn('Sampling from a set deprecated\n'
+ 'since Python 3.9 and will be removed in a subsequent version.',
+ DeprecationWarning, 2)
+ population = tuple(population)
+ else:
+ raise TypeError("Population must be a sequence. For dicts or sets, use sorted(d).")
n = len(population)
if counts is not None:
cum_counts = list(_accumulate(counts))
@@ -441,7 +474,7 @@
raise TypeError('Counts must be integers')
if total <= 0:
raise ValueError('Total of counts must be greater than zero')
- selections = sample(range(total), k=k)
+ selections = self.sample(range(total), k=k)
bisect = _bisect
return [population[bisect(cum_counts, s)] for s in selections]
randbelow = self._randbelow
@@ -484,7 +517,15 @@
floor = _floor
n += 0.0 # convert to float for a small speed improvement
return [population[floor(random() * n)] for i in _repeat(None, k)]
- cum_weights = list(_accumulate(weights))
+ try:
+ cum_weights = list(_accumulate(weights))
+ except TypeError:
+ if not isinstance(weights, int):
+ raise
+ k = weights
+ raise TypeError(
+ f'The number of choices must be a keyword argument: {k=}'
+ ) from None
elif weights is not None:
raise TypeError('Cannot specify both weights and cumulative weights')
if len(cum_weights) != n:
@@ -492,6 +533,8 @@
total = cum_weights[-1] + 0.0 # convert to float
if total <= 0.0:
raise ValueError('Total of weights must be greater than zero')
+ if not _isfinite(total):
+ raise ValueError('Total of weights must be finite')
bisect = _bisect
hi = n - 1
return [population[bisect(cum_weights, random() * total, 0, hi)]
@@ -682,7 +725,7 @@
bbb = alpha - LOG4
ccc = alpha + ainv
- while 1:
+ while True:
u1 = random()
if not 1e-7 < u1 < 0.9999999:
continue
@@ -749,7 +792,7 @@
# Jain, pg. 495
u = 1.0 - self.random()
- return 1.0 / u ** (1.0 / alpha)
+ return u ** (-1.0 / alpha)
def weibullvariate(self, alpha, beta):
"""Weibull distribution.
@@ -845,7 +888,7 @@
from time import perf_counter
t0 = perf_counter()
- data = [func(*args) for i in range(n)]
+ data = [func(*args) for i in _repeat(None, n)]
t1 = perf_counter()
xbar = mean(data)
diff --git a/common/py3-stdlib/re.py b/common/py3-stdlib/re.py
index bfb7b1c..1d82b50 100644
--- a/common/py3-stdlib/re.py
+++ b/common/py3-stdlib/re.py
@@ -176,7 +176,6 @@
res = f'~{res}'
return res
__str__ = object.__str__
-
globals().update(RegexFlag.__members__)
# sre exception
diff --git a/common/py3-stdlib/rlcompleter.py b/common/py3-stdlib/rlcompleter.py
index bca4a7b..98b7930 100644
--- a/common/py3-stdlib/rlcompleter.py
+++ b/common/py3-stdlib/rlcompleter.py
@@ -31,6 +31,7 @@
import atexit
import builtins
+import inspect
import __main__
__all__ = ["Completer"]
@@ -96,7 +97,13 @@
def _callable_postfix(self, val, word):
if callable(val):
- word = word + "("
+ word += "("
+ try:
+ if not inspect.signature(val).parameters:
+ word += ")"
+ except ValueError:
+ pass
+
return word
def global_matches(self, text):
@@ -169,13 +176,20 @@
if (word[:n] == attr and
not (noprefix and word[:n+1] == noprefix)):
match = "%s.%s" % (expr, word)
- try:
- val = getattr(thisobject, word)
- except Exception:
- pass # Include even if attribute not set
+ if isinstance(getattr(type(thisobject), word, None),
+ property):
+ # bpo-44752: thisobject.word is a method decorated by
+ # `@property`. What follows applies a postfix if
+ # thisobject.word is callable, but know we know that
+ # this is not callable (because it is a property).
+ # Also, getattr(thisobject, word) will evaluate the
+ # property method, which is not desirable.
+ matches.append(match)
+ continue
+ if (value := getattr(thisobject, word, None)) is not None:
+ matches.append(self._callable_postfix(value, match))
else:
- match = self._callable_postfix(val, match)
- matches.append(match)
+ matches.append(match)
if matches or not noprefix:
break
if noprefix == '_':
diff --git a/common/py3-stdlib/runpy.py b/common/py3-stdlib/runpy.py
index 7e1e1ac..caba121 100644
--- a/common/py3-stdlib/runpy.py
+++ b/common/py3-stdlib/runpy.py
@@ -16,7 +16,6 @@
import io
import types
import os
-from pkgutil import read_code, get_importer
__all__ = [
"run_module", "run_path",
@@ -233,6 +232,7 @@
def _get_code_from_file(run_name, fname):
# Check for a compiled file first
+ from pkgutil import read_code
decoded_path = os.path.abspath(os.fsdecode(fname))
with io.open_code(decoded_path) as f:
code = read_code(f)
@@ -255,6 +255,7 @@
if run_name is None:
run_name = "<run_path>"
pkg_name = run_name.rpartition(".")[0]
+ from pkgutil import get_importer
importer = get_importer(path_name)
# Trying to avoid importing imp so as to not consume the deprecation warning.
is_NullImporter = False
diff --git a/common/py3-stdlib/sched.py b/common/py3-stdlib/sched.py
index ff87874..14613cf 100644
--- a/common/py3-stdlib/sched.py
+++ b/common/py3-stdlib/sched.py
@@ -26,23 +26,19 @@
import time
import heapq
from collections import namedtuple
+from itertools import count
import threading
from time import monotonic as _time
__all__ = ["scheduler"]
-class Event(namedtuple('Event', 'time, priority, action, argument, kwargs')):
- __slots__ = []
- def __eq__(s, o): return (s.time, s.priority) == (o.time, o.priority)
- def __lt__(s, o): return (s.time, s.priority) < (o.time, o.priority)
- def __le__(s, o): return (s.time, s.priority) <= (o.time, o.priority)
- def __gt__(s, o): return (s.time, s.priority) > (o.time, o.priority)
- def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority)
-
+Event = namedtuple('Event', 'time, priority, sequence, action, argument, kwargs')
Event.time.__doc__ = ('''Numeric type compatible with the return value of the
timefunc function passed to the constructor.''')
Event.priority.__doc__ = ('''Events scheduled for the same time will be executed
in the order of their priority.''')
+Event.sequence.__doc__ = ('''A continually increasing sequence number that
+ separates events if time and priority are equal.''')
Event.action.__doc__ = ('''Executing the event means executing
action(*argument, **kwargs)''')
Event.argument.__doc__ = ('''argument is a sequence holding the positional
@@ -61,6 +57,7 @@
self._lock = threading.RLock()
self.timefunc = timefunc
self.delayfunc = delayfunc
+ self._sequence_generator = count()
def enterabs(self, time, priority, action, argument=(), kwargs=_sentinel):
"""Enter a new event in the queue at an absolute time.
@@ -71,8 +68,10 @@
"""
if kwargs is _sentinel:
kwargs = {}
- event = Event(time, priority, action, argument, kwargs)
+
with self._lock:
+ event = Event(time, priority, next(self._sequence_generator),
+ action, argument, kwargs)
heapq.heappush(self._queue, event)
return event # The ID
@@ -136,7 +135,8 @@
with lock:
if not q:
break
- time, priority, action, argument, kwargs = q[0]
+ (time, priority, sequence, action,
+ argument, kwargs) = q[0]
now = timefunc()
if time > now:
delay = True
diff --git a/common/py3-stdlib/shelve.py b/common/py3-stdlib/shelve.py
index 5d443a0..e053c39 100644
--- a/common/py3-stdlib/shelve.py
+++ b/common/py3-stdlib/shelve.py
@@ -56,7 +56,7 @@
the persistent dictionary on disk, if feasible).
"""
-from pickle import Pickler, Unpickler
+from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler
from io import BytesIO
import collections.abc
@@ -85,7 +85,7 @@
keyencoding="utf-8"):
self.dict = dict
if protocol is None:
- protocol = 3
+ protocol = DEFAULT_PROTOCOL
self._protocol = protocol
self.writeback = writeback
self.cache = {}
diff --git a/common/py3-stdlib/shutil.py b/common/py3-stdlib/shutil.py
index f0e833d..37bf98d 100644
--- a/common/py3-stdlib/shutil.py
+++ b/common/py3-stdlib/shutil.py
@@ -32,16 +32,6 @@
except ImportError:
_LZMA_SUPPORTED = False
-try:
- from pwd import getpwnam
-except ImportError:
- getpwnam = None
-
-try:
- from grp import getgrnam
-except ImportError:
- getgrnam = None
-
_WINDOWS = os.name == 'nt'
posix = nt = None
if os.name == 'posix':
@@ -261,28 +251,37 @@
if not follow_symlinks and _islink(src):
os.symlink(os.readlink(src), dst)
else:
- with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
- # macOS
- if _HAS_FCOPYFILE:
- try:
- _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA)
- return dst
- except _GiveupOnFastCopy:
- pass
- # Linux
- elif _USE_CP_SENDFILE:
- try:
- _fastcopy_sendfile(fsrc, fdst)
- return dst
- except _GiveupOnFastCopy:
- pass
- # Windows, see:
- # https://github.com/python/cpython/pull/7160#discussion_r195405230
- elif _WINDOWS and file_size > 0:
- _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE))
- return dst
+ with open(src, 'rb') as fsrc:
+ try:
+ with open(dst, 'wb') as fdst:
+ # macOS
+ if _HAS_FCOPYFILE:
+ try:
+ _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA)
+ return dst
+ except _GiveupOnFastCopy:
+ pass
+ # Linux
+ elif _USE_CP_SENDFILE:
+ try:
+ _fastcopy_sendfile(fsrc, fdst)
+ return dst
+ except _GiveupOnFastCopy:
+ pass
+ # Windows, see:
+ # https://github.com/python/cpython/pull/7160#discussion_r195405230
+ elif _WINDOWS and file_size > 0:
+ _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE))
+ return dst
- copyfileobj(fsrc, fdst)
+ copyfileobj(fsrc, fdst)
+
+ # Issue 43219, raise a less confusing exception
+ except IsADirectoryError as e:
+ if not os.path.exists(dst):
+ raise FileNotFoundError(f'Directory does not exist: {dst}') from e
+ else:
+ raise
return dst
@@ -647,6 +646,7 @@
if is_dir:
try:
dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd)
+ dirfd_closed = False
except OSError:
onerror(os.open, fullname, sys.exc_info())
else:
@@ -654,6 +654,8 @@
if os.path.samestat(orig_st, os.fstat(dirfd)):
_rmtree_safe_fd(dirfd, fullname, onerror)
try:
+ os.close(dirfd)
+ dirfd_closed = True
os.rmdir(entry.name, dir_fd=topfd)
except OSError:
onerror(os.rmdir, fullname, sys.exc_info())
@@ -667,7 +669,8 @@
except OSError:
onerror(os.path.islink, fullname, sys.exc_info())
finally:
- os.close(dirfd)
+ if not dirfd_closed:
+ os.close(dirfd)
else:
try:
os.unlink(entry.name, dir_fd=topfd)
@@ -710,6 +713,7 @@
return
try:
fd = os.open(path, os.O_RDONLY)
+ fd_closed = False
except Exception:
onerror(os.open, path, sys.exc_info())
return
@@ -717,6 +721,8 @@
if os.path.samestat(orig_st, os.fstat(fd)):
_rmtree_safe_fd(fd, path, onerror)
try:
+ os.close(fd)
+ fd_closed = True
os.rmdir(path)
except OSError:
onerror(os.rmdir, path, sys.exc_info())
@@ -727,7 +733,8 @@
except OSError:
onerror(os.path.islink, path, sys.exc_info())
finally:
- os.close(fd)
+ if not fd_closed:
+ os.close(fd)
else:
try:
if _rmtree_islink(path):
@@ -813,6 +820,12 @@
if _destinsrc(src, dst):
raise Error("Cannot move a directory '%s' into itself"
" '%s'." % (src, dst))
+ if (_is_immutable(src)
+ or (not os.access(src, os.W_OK) and os.listdir(src)
+ and sys.platform == 'darwin')):
+ raise PermissionError("Cannot move the non-empty directory "
+ "'%s': Lacking write permission to '%s'."
+ % (src, src))
copytree(src, real_dst, copy_function=copy_function,
symlinks=True)
rmtree(src)
@@ -830,10 +843,21 @@
dst += os.path.sep
return dst.startswith(src)
+def _is_immutable(src):
+ st = _stat(src)
+ immutable_states = [stat.UF_IMMUTABLE, stat.SF_IMMUTABLE]
+ return hasattr(st, 'st_flags') and st.st_flags in immutable_states
+
def _get_gid(name):
"""Returns a gid, given a group name."""
- if getgrnam is None or name is None:
+ if name is None:
return None
+
+ try:
+ from grp import getgrnam
+ except ImportError:
+ return None
+
try:
result = getgrnam(name)
except KeyError:
@@ -844,8 +868,14 @@
def _get_uid(name):
"""Returns an uid, given a user name."""
- if getpwnam is None or name is None:
+ if name is None:
return None
+
+ try:
+ from pwd import getpwnam
+ except ImportError:
+ return None
+
try:
result = getpwnam(name)
except KeyError:
@@ -1148,20 +1178,16 @@
if name.startswith('/') or '..' in name:
continue
- target = os.path.join(extract_dir, *name.split('/'))
- if not target:
+ targetpath = os.path.join(extract_dir, *name.split('/'))
+ if not targetpath:
continue
- _ensure_directory(target)
+ _ensure_directory(targetpath)
if not name.endswith('/'):
# file
- data = zip.read(info.filename)
- f = open(target, 'wb')
- try:
- f.write(data)
- finally:
- f.close()
- del data
+ with zip.open(name, 'r') as source, \
+ open(targetpath, 'wb') as target:
+ copyfileobj(source, target)
finally:
zip.close()
diff --git a/common/py3-stdlib/signal.py b/common/py3-stdlib/signal.py
index d4a6d6f..50b215b 100644
--- a/common/py3-stdlib/signal.py
+++ b/common/py3-stdlib/signal.py
@@ -1,6 +1,5 @@
import _signal
from _signal import *
-from functools import wraps as _wraps
from enum import IntEnum as _IntEnum
_globals = globals()
@@ -42,6 +41,16 @@
return value
+# Similar to functools.wraps(), but only assign __doc__.
+# __module__ should be preserved,
+# __name__ and __qualname__ are already fine,
+# __annotations__ is not set.
+def _wraps(wrapped):
+ def decorator(wrapper):
+ wrapper.__doc__ = wrapped.__doc__
+ return wrapper
+ return decorator
+
@_wraps(_signal.signal)
def signal(signalnum, handler):
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
@@ -59,7 +68,6 @@
def pthread_sigmask(how, mask):
sigs_set = _signal.pthread_sigmask(how, mask)
return set(_int_to_enum(x, Signals) for x in sigs_set)
- pthread_sigmask.__doc__ = _signal.pthread_sigmask.__doc__
if 'sigpending' in _globals:
@@ -73,7 +81,6 @@
def sigwait(sigset):
retsig = _signal.sigwait(sigset)
return _int_to_enum(retsig, Signals)
- sigwait.__doc__ = _signal.sigwait
if 'valid_signals' in _globals:
diff --git a/common/py3-stdlib/site.py b/common/py3-stdlib/site.py
index 9e617af..939893e 100644
--- a/common/py3-stdlib/site.py
+++ b/common/py3-stdlib/site.py
@@ -88,6 +88,11 @@
USER_BASE = None
+def _trace(message):
+ if sys.flags.verbose:
+ print(message, file=sys.stderr)
+
+
def makepath(*paths):
dir = os.path.join(*paths)
try:
@@ -100,8 +105,15 @@
def abs_paths():
"""Set all module __file__ and __cached__ attributes to an absolute path"""
for m in set(sys.modules.values()):
- if (getattr(getattr(m, '__loader__', None), '__module__', None) not in
- ('_frozen_importlib', '_frozen_importlib_external')):
+ loader_module = None
+ try:
+ loader_module = m.__loader__.__module__
+ except AttributeError:
+ try:
+ loader_module = m.__spec__.loader.__module__
+ except AttributeError:
+ pass
+ if loader_module not in {'_frozen_importlib', '_frozen_importlib_external'}:
continue # don't mess with a PEP 302-supplied __file__
try:
m.__file__ = os.path.abspath(m.__file__)
@@ -156,14 +168,19 @@
else:
reset = False
fullname = os.path.join(sitedir, name)
+ _trace(f"Processing .pth file: {fullname!r}")
try:
- f = io.TextIOWrapper(io.open_code(fullname))
+ # locale encoding is not ideal especially on Windows. But we have used
+ # it for a long time. setuptools uses the locale encoding too.
+ f = io.TextIOWrapper(io.open_code(fullname), encoding="locale")
except OSError:
return
with f:
for n, line in enumerate(f):
if line.startswith("#"):
continue
+ if line.strip() == "":
+ continue
try:
if line.startswith(("import ", "import\t")):
exec(line)
@@ -190,6 +207,7 @@
def addsitedir(sitedir, known_paths=None):
"""Add 'sitedir' argument to sys.path if missing and handle .pth files in
'sitedir'"""
+ _trace(f"Adding directory: {sitedir!r}")
if known_paths is None:
known_paths = _init_pathinfo()
reset = True
@@ -248,6 +266,10 @@
if env_base:
return env_base
+ # VxWorks has no home directories
+ if sys.platform == "vxworks":
+ return None
+
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
@@ -267,7 +289,8 @@
version = sys.version_info
if os.name == 'nt':
- return f'{userbase}\\Python{version[0]}{version[1]}\\site-packages'
+ ver_nodot = sys.winver.replace('.', '')
+ return f'{userbase}\\Python{ver_nodot}\\site-packages'
if sys.platform == 'darwin' and sys._framework:
return f'{userbase}/lib/python/site-packages'
@@ -294,11 +317,14 @@
If the global variable ``USER_SITE`` is not initialized yet, this
function will also set it.
"""
- global USER_SITE
+ global USER_SITE, ENABLE_USER_SITE
userbase = getuserbase() # this will also set USER_BASE
if USER_SITE is None:
- USER_SITE = _get_path(userbase)
+ if userbase is None:
+ ENABLE_USER_SITE = False # disable user site and return None
+ else:
+ USER_SITE = _get_path(userbase)
return USER_SITE
@@ -310,6 +336,7 @@
"""
# get the per user site-package path
# this call will also make sure USER_BASE and USER_SITE are set
+ _trace("Processing user site-packages")
user_site = getusersitepackages()
if ENABLE_USER_SITE and os.path.isdir(user_site):
@@ -354,6 +381,7 @@
def addsitepackages(known_paths, prefixes=None):
"""Add site-packages to sys.path"""
+ _trace("Processing global site-packages")
for sitedir in getsitepackages(prefixes):
if os.path.isdir(sitedir):
addsitedir(sitedir, known_paths)
@@ -611,11 +639,14 @@
for dir in sys.path:
print(" %r," % (dir,))
print("]")
- print("USER_BASE: %r (%s)" % (user_base,
- "exists" if os.path.isdir(user_base) else "doesn't exist"))
- print("USER_SITE: %r (%s)" % (user_site,
- "exists" if os.path.isdir(user_site) else "doesn't exist"))
- print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE)
+ def exists(path):
+ if path is not None and os.path.isdir(path):
+ return "exists"
+ else:
+ return "doesn't exist"
+ print(f"USER_BASE: {user_base!r} ({exists(user_base)})")
+ print(f"USER_SITE: {user_site!r} ({exists(user_site)})")
+ print(f"ENABLE_USER_SITE: {ENABLE_USER_SITE!r}")
sys.exit(0)
buffer = []
diff --git a/common/py3-stdlib/smtpd.py b/common/py3-stdlib/smtpd.py
index 8f1a22e..bc43331 100755
--- a/common/py3-stdlib/smtpd.py
+++ b/common/py3-stdlib/smtpd.py
@@ -83,8 +83,6 @@
import getopt
import time
import socket
-import asyncore
-import asynchat
import collections
from warnings import warn
from email._header_value_parser import get_addr_spec, get_angle_addr
@@ -94,6 +92,19 @@
"MailmanProxy",
]
+warn(
+ 'The smtpd module is deprecated and unmaintained. Please see aiosmtpd '
+ '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.',
+ DeprecationWarning,
+ stacklevel=2)
+
+
+# These are imported after the above warning so that users get the correct
+# deprecation warning.
+import asyncore
+import asynchat
+
+
program = sys.argv[0]
__version__ = 'Python SMTP proxy version 0.3'
@@ -163,7 +174,7 @@
# a race condition may occur if the other end is closing
# before we can get the peername
self.close()
- if err.args[0] != errno.ENOTCONN:
+ if err.errno != errno.ENOTCONN:
raise
return
print('Peer:', repr(self.peer), file=DEBUGSTREAM)
diff --git a/common/py3-stdlib/smtplib.py b/common/py3-stdlib/smtplib.py
index 7808ba0..324a1c1 100755
--- a/common/py3-stdlib/smtplib.py
+++ b/common/py3-stdlib/smtplib.py
@@ -64,6 +64,7 @@
CRLF = "\r\n"
bCRLF = b"\r\n"
_MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3
+_MAXCHALLENGE = 5 # Maximum number of AUTH challenges sent
OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I)
@@ -167,7 +168,7 @@
"""Quote data for email.
Double leading '.', and change Unix newline '\\n', or Mac '\\r' into
- Internet CRLF end-of-line.
+ internet CRLF end-of-line.
"""
return re.sub(r'(?m)^\.', '..',
re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data))
@@ -222,7 +223,7 @@
helo_resp = None
ehlo_msg = "ehlo"
ehlo_resp = None
- does_esmtp = 0
+ does_esmtp = False
default_port = SMTP_PORT
def __init__(self, host='', port=0, local_hostname=None,
@@ -230,8 +231,8 @@
source_address=None):
"""Initialize a new instance.
- If specified, `host' is the name of the remote host to which to
- connect. If specified, `port' specifies the port to which to connect.
+ If specified, `host` is the name of the remote host to which to
+ connect. If specified, `port` specifies the port to which to connect.
By default, smtplib.SMTP_PORT is used. If a host is specified the
connect method is called, and if it returns anything other than a
success code an SMTPConnectError is raised. If specified,
@@ -248,6 +249,7 @@
self.esmtp_features = {}
self.command_encoding = 'ascii'
self.source_address = source_address
+ self._auth_challenge_count = 0
if host:
(code, msg) = self.connect(host, port)
@@ -365,10 +367,15 @@
def putcmd(self, cmd, args=""):
"""Send a command to the server."""
if args == "":
- str = '%s%s' % (cmd, CRLF)
+ s = cmd
else:
- str = '%s %s%s' % (cmd, args, CRLF)
- self.send(str)
+ s = f'{cmd} {args}'
+ if '\r' in s or '\n' in s:
+ s = s.replace('\n', '\\n').replace('\r', '\\r')
+ raise ValueError(
+ f'command and arguments contain prohibited newline characters: {s}'
+ )
+ self.send(f'{s}{CRLF}')
def getreply(self):
"""Get a reply from the server.
@@ -452,7 +459,7 @@
self.ehlo_resp = msg
if code != 250:
return (code, msg)
- self.does_esmtp = 1
+ self.does_esmtp = True
#parse the ehlo response -ddm
assert isinstance(self.ehlo_resp, bytes), repr(self.ehlo_resp)
resp = self.ehlo_resp.decode("latin-1").split('\n')
@@ -633,14 +640,23 @@
if initial_response is not None:
response = encode_base64(initial_response.encode('ascii'), eol='')
(code, resp) = self.docmd("AUTH", mechanism + " " + response)
+ self._auth_challenge_count = 1
else:
(code, resp) = self.docmd("AUTH", mechanism)
+ self._auth_challenge_count = 0
# If server responds with a challenge, send the response.
- if code == 334:
+ while code == 334:
+ self._auth_challenge_count += 1
challenge = base64.decodebytes(resp)
response = encode_base64(
authobject(challenge).encode('ascii'), eol='')
(code, resp) = self.docmd(response)
+ # If server keeps sending challenges, something is wrong.
+ if self._auth_challenge_count > _MAXCHALLENGE:
+ raise SMTPException(
+ "Server AUTH mechanism infinite loop. Last response: "
+ + repr((code, resp))
+ )
if code in (235, 503):
return (code, resp)
raise SMTPAuthenticationError(code, resp)
@@ -662,7 +678,7 @@
def auth_login(self, challenge=None):
""" Authobject to use with LOGIN authentication. Requires self.user and
self.password to be set."""
- if challenge is None:
+ if challenge is None or self._auth_challenge_count < 2:
return self.user
else:
return self.password
@@ -781,7 +797,7 @@
self.helo_resp = None
self.ehlo_resp = None
self.esmtp_features = {}
- self.does_esmtp = 0
+ self.does_esmtp = False
else:
# RFC 3207:
# 501 Syntax error (no parameters allowed)
@@ -1082,7 +1098,8 @@
# Handle Unix-domain sockets.
try:
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- self.sock.settimeout(self.timeout)
+ if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+ self.sock.settimeout(self.timeout)
self.file = None
self.sock.connect(host)
except OSError:
diff --git a/common/py3-stdlib/socket.py b/common/py3-stdlib/socket.py
index cafa573..63ba0ac 100755
--- a/common/py3-stdlib/socket.py
+++ b/common/py3-stdlib/socket.py
@@ -337,6 +337,7 @@
buffer = io.BufferedWriter(raw, buffering)
if binary:
return buffer
+ encoding = io.text_encoding(encoding)
text = io.TextIOWrapper(buffer, encoding, errors, newline)
text.mode = mode
return text
@@ -377,7 +378,7 @@
try:
while True:
if timeout and not selector_select(timeout):
- raise _socket.timeout('timed out')
+ raise TimeoutError('timed out')
if count:
blocksize = count - total_sent
if blocksize <= 0:
@@ -706,7 +707,7 @@
self._timeout_occurred = True
raise
except error as e:
- if e.args[0] in _blocking_errnos:
+ if e.errno in _blocking_errnos:
return None
raise
@@ -722,7 +723,7 @@
return self._sock.send(b)
except error as e:
# XXX what about EINTR?
- if e.args[0] in _blocking_errnos:
+ if e.errno in _blocking_errnos:
return None
raise
@@ -781,8 +782,9 @@
An empty argument is interpreted as meaning the local host.
First the hostname returned by gethostbyaddr() is checked, then
- possibly existing aliases. In case no FQDN is available, hostname
- from gethostname() is returned.
+ possibly existing aliases. In case no FQDN is available and `name`
+ was given, it is returned unchanged. If `name` was empty or '0.0.0.0',
+ hostname from gethostname() is returned.
"""
name = name.strip()
if not name or name == '0.0.0.0':
diff --git a/common/py3-stdlib/socketserver.py b/common/py3-stdlib/socketserver.py
index 57c1ae6..0d9583d 100644
--- a/common/py3-stdlib/socketserver.py
+++ b/common/py3-stdlib/socketserver.py
@@ -628,6 +628,39 @@
self.collect_children(blocking=self.block_on_close)
+class _Threads(list):
+ """
+ Joinable list of all non-daemon threads.
+ """
+ def append(self, thread):
+ self.reap()
+ if thread.daemon:
+ return
+ super().append(thread)
+
+ def pop_all(self):
+ self[:], result = [], self[:]
+ return result
+
+ def join(self):
+ for thread in self.pop_all():
+ thread.join()
+
+ def reap(self):
+ self[:] = (thread for thread in self if thread.is_alive())
+
+
+class _NoThreads:
+ """
+ Degenerate version of _Threads.
+ """
+ def append(self, thread):
+ pass
+
+ def join(self):
+ pass
+
+
class ThreadingMixIn:
"""Mix-in class to handle each request in a new thread."""
@@ -636,9 +669,9 @@
daemon_threads = False
# If true, server_close() waits until all non-daemonic threads terminate.
block_on_close = True
- # For non-daemonic threads, list of threading.Threading objects
+ # Threads object
# used by server_close() to wait for all threads completion.
- _threads = None
+ _threads = _NoThreads()
def process_request_thread(self, request, client_address):
"""Same as in BaseServer but as a thread.
@@ -655,23 +688,17 @@
def process_request(self, request, client_address):
"""Start a new thread to process the request."""
+ if self.block_on_close:
+ vars(self).setdefault('_threads', _Threads())
t = threading.Thread(target = self.process_request_thread,
args = (request, client_address))
t.daemon = self.daemon_threads
- if not t.daemon and self.block_on_close:
- if self._threads is None:
- self._threads = []
- self._threads.append(t)
+ self._threads.append(t)
t.start()
def server_close(self):
super().server_close()
- if self.block_on_close:
- threads = self._threads
- self._threads = None
- if threads:
- for thread in threads:
- thread.join()
+ self._threads.join()
if hasattr(os, "fork"):
diff --git a/common/py3-stdlib/sqlite3/__init__.py b/common/py3-stdlib/sqlite3/__init__.py
index 6c91df2..0dedf18 100644
--- a/common/py3-stdlib/sqlite3/__init__.py
+++ b/common/py3-stdlib/sqlite3/__init__.py
@@ -20,4 +20,52 @@
# misrepresented as being the original software.
# 3. This notice may not be removed or altered from any source distribution.
+"""
+The sqlite3 extension module provides a DB-API 2.0 (PEP 249) compilant
+interface to the SQLite library, and requires SQLite 3.7.15 or newer.
+
+To use the module, start by creating a database Connection object:
+
+ import sqlite3
+ cx = sqlite3.connect("test.db") # test.db will be created or opened
+
+The special path name ":memory:" can be provided to connect to a transient
+in-memory database:
+
+ cx = sqlite3.connect(":memory:") # connect to a database in RAM
+
+Once a connection has been established, create a Cursor object and call
+its execute() method to perform SQL queries:
+
+ cu = cx.cursor()
+
+ # create a table
+ cu.execute("create table lang(name, first_appeared)")
+
+ # insert values into a table
+ cu.execute("insert into lang values (?, ?)", ("C", 1972))
+
+ # execute a query and iterate over the result
+ for row in cu.execute("select * from lang"):
+ print(row)
+
+ cx.close()
+
+The sqlite3 module is written by Gerhard Häring <gh@ghaering.de>.
+"""
+
from sqlite3.dbapi2 import *
+
+
+# bpo-42264: OptimizedUnicode was deprecated in Python 3.10. It's scheduled
+# for removal in Python 3.12.
+def __getattr__(name):
+ if name == "OptimizedUnicode":
+ import warnings
+ msg = ("""
+ OptimizedUnicode is deprecated and will be removed in Python 3.12.
+ Since Python 3.3 it has simply been an alias for 'str'.
+ """)
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
+ return str
+ raise AttributeError(f"module 'sqlite3' has no attribute '{name}'")
diff --git a/common/py3-stdlib/sqlite3/dbapi2.py b/common/py3-stdlib/sqlite3/dbapi2.py
index 991682c..cfe6225 100644
--- a/common/py3-stdlib/sqlite3/dbapi2.py
+++ b/common/py3-stdlib/sqlite3/dbapi2.py
@@ -84,6 +84,20 @@
register_adapters_and_converters()
+# bpo-24464: enable_shared_cache was deprecated in Python 3.10. It's
+# scheduled for removal in Python 3.12.
+def enable_shared_cache(enable):
+ from _sqlite3 import enable_shared_cache as _old_enable_shared_cache
+ import warnings
+ msg = (
+ "enable_shared_cache is deprecated and will be removed in Python 3.12. "
+ "Shared cache is strongly discouraged by the SQLite 3 documentation. "
+ "If shared cache must be used, open the database in URI mode using"
+ "the cache=shared query parameter."
+ )
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
+ return _old_enable_shared_cache(enable)
+
# Clean up namespace
del(register_adapters_and_converters)
diff --git a/common/py3-stdlib/sqlite3/test/backup.py b/common/py3-stdlib/sqlite3/test/backup.py
index ad1da97..4e30594 100644
--- a/common/py3-stdlib/sqlite3/test/backup.py
+++ b/common/py3-stdlib/sqlite3/test/backup.py
@@ -2,7 +2,6 @@
import unittest
-@unittest.skipIf(sqlite.sqlite_version_info < (3, 6, 11), "Backup API not supported")
class BackupTests(unittest.TestCase):
def setUp(self):
cx = self.cx = sqlite.connect(":memory:")
@@ -18,9 +17,11 @@
self.assertEqual(result[0][0], 3)
self.assertEqual(result[1][0], 4)
- def test_bad_target_none(self):
+ def test_bad_target(self):
with self.assertRaises(TypeError):
self.cx.backup(None)
+ with self.assertRaises(TypeError):
+ self.cx.backup()
def test_bad_target_filename(self):
with self.assertRaises(TypeError):
@@ -148,10 +149,7 @@
with self.assertRaises(sqlite.OperationalError) as cm:
with sqlite.connect(':memory:') as bck:
self.cx.backup(bck, name='non-existing')
- self.assertIn(
- str(cm.exception),
- ['SQL logic error', 'SQL logic error or missing database']
- )
+ self.assertIn("unknown database", str(cm.exception))
self.cx.execute("ATTACH DATABASE ':memory:' AS attached_db")
self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)')
@@ -163,7 +161,7 @@
def suite():
- return unittest.makeSuite(BackupTests)
+ return unittest.TestLoader().loadTestsFromTestCase(BackupTests)
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/sqlite3/test/dbapi.py b/common/py3-stdlib/sqlite3/test/dbapi.py
index ad9c9f0..e332184 100644
--- a/common/py3-stdlib/sqlite3/test/dbapi.py
+++ b/common/py3-stdlib/sqlite3/test/dbapi.py
@@ -1,7 +1,6 @@
-#-*- coding: iso-8859-1 -*-
# pysqlite2/test/dbapi.py: tests for DB-API compliance
#
-# Copyright (C) 2004-2010 Gerhard Häring <gh@ghaering.de>
+# Copyright (C) 2004-2010 Gerhard Häring <gh@ghaering.de>
#
# This file is part of pysqlite.
#
@@ -21,68 +20,85 @@
# misrepresented as being the original software.
# 3. This notice may not be removed or altered from any source distribution.
+import subprocess
import threading
import unittest
import sqlite3 as sqlite
+import sys
-from test.support import TESTFN, unlink
+from test.support import check_disallow_instantiation, SHORT_TIMEOUT
+from test.support.os_helper import TESTFN, unlink
class ModuleTests(unittest.TestCase):
- def CheckAPILevel(self):
+ def test_api_level(self):
self.assertEqual(sqlite.apilevel, "2.0",
"apilevel is %s, should be 2.0" % sqlite.apilevel)
- def CheckThreadSafety(self):
+ def test_thread_safety(self):
self.assertEqual(sqlite.threadsafety, 1,
"threadsafety is %d, should be 1" % sqlite.threadsafety)
- def CheckParamStyle(self):
+ def test_param_style(self):
self.assertEqual(sqlite.paramstyle, "qmark",
"paramstyle is '%s', should be 'qmark'" %
sqlite.paramstyle)
- def CheckWarning(self):
+ def test_warning(self):
self.assertTrue(issubclass(sqlite.Warning, Exception),
"Warning is not a subclass of Exception")
- def CheckError(self):
+ def test_error(self):
self.assertTrue(issubclass(sqlite.Error, Exception),
"Error is not a subclass of Exception")
- def CheckInterfaceError(self):
+ def test_interface_error(self):
self.assertTrue(issubclass(sqlite.InterfaceError, sqlite.Error),
"InterfaceError is not a subclass of Error")
- def CheckDatabaseError(self):
+ def test_database_error(self):
self.assertTrue(issubclass(sqlite.DatabaseError, sqlite.Error),
"DatabaseError is not a subclass of Error")
- def CheckDataError(self):
+ def test_data_error(self):
self.assertTrue(issubclass(sqlite.DataError, sqlite.DatabaseError),
"DataError is not a subclass of DatabaseError")
- def CheckOperationalError(self):
+ def test_operational_error(self):
self.assertTrue(issubclass(sqlite.OperationalError, sqlite.DatabaseError),
"OperationalError is not a subclass of DatabaseError")
- def CheckIntegrityError(self):
+ def test_integrity_error(self):
self.assertTrue(issubclass(sqlite.IntegrityError, sqlite.DatabaseError),
"IntegrityError is not a subclass of DatabaseError")
- def CheckInternalError(self):
+ def test_internal_error(self):
self.assertTrue(issubclass(sqlite.InternalError, sqlite.DatabaseError),
"InternalError is not a subclass of DatabaseError")
- def CheckProgrammingError(self):
+ def test_programming_error(self):
self.assertTrue(issubclass(sqlite.ProgrammingError, sqlite.DatabaseError),
"ProgrammingError is not a subclass of DatabaseError")
- def CheckNotSupportedError(self):
+ def test_not_supported_error(self):
self.assertTrue(issubclass(sqlite.NotSupportedError,
sqlite.DatabaseError),
"NotSupportedError is not a subclass of DatabaseError")
+ # sqlite3_enable_shared_cache() is deprecated on macOS and calling it may raise
+ # OperationalError on some buildbots.
+ @unittest.skipIf(sys.platform == "darwin", "shared cache is deprecated on macOS")
+ def test_shared_cache_deprecated(self):
+ for enable in (True, False):
+ with self.assertWarns(DeprecationWarning) as cm:
+ sqlite.enable_shared_cache(enable)
+ self.assertIn("dbapi.py", cm.filename)
+
+ def test_disallow_instantiation(self):
+ cx = sqlite.connect(":memory:")
+ check_disallow_instantiation(self, type(cx("select 1")))
+
+
class ConnectionTests(unittest.TestCase):
def setUp(self):
@@ -94,38 +110,38 @@
def tearDown(self):
self.cx.close()
- def CheckCommit(self):
+ def test_commit(self):
self.cx.commit()
- def CheckCommitAfterNoChanges(self):
+ def test_commit_after_no_changes(self):
"""
A commit should also work when no changes were made to the database.
"""
self.cx.commit()
self.cx.commit()
- def CheckRollback(self):
+ def test_rollback(self):
self.cx.rollback()
- def CheckRollbackAfterNoChanges(self):
+ def test_rollback_after_no_changes(self):
"""
A rollback should also work when no changes were made to the database.
"""
self.cx.rollback()
self.cx.rollback()
- def CheckCursor(self):
+ def test_cursor(self):
cu = self.cx.cursor()
- def CheckFailedOpen(self):
+ def test_failed_open(self):
YOU_CANNOT_OPEN_THIS = "/foo/bar/bla/23534/mydb.db"
with self.assertRaises(sqlite.OperationalError):
con = sqlite.connect(YOU_CANNOT_OPEN_THIS)
- def CheckClose(self):
+ def test_close(self):
self.cx.close()
- def CheckExceptions(self):
+ def test_exceptions(self):
# Optional DB-API extension.
self.assertEqual(self.cx.Warning, sqlite.Warning)
self.assertEqual(self.cx.Error, sqlite.Error)
@@ -138,7 +154,7 @@
self.assertEqual(self.cx.ProgrammingError, sqlite.ProgrammingError)
self.assertEqual(self.cx.NotSupportedError, sqlite.NotSupportedError)
- def CheckInTransaction(self):
+ def test_in_transaction(self):
# Can't use db from setUp because we want to test initial state.
cx = sqlite.connect(":memory:")
cu = cx.cursor()
@@ -156,11 +172,11 @@
row = cu.fetchone()
self.assertEqual(cx.in_transaction, False)
- def CheckInTransactionRO(self):
+ def test_in_transaction_ro(self):
with self.assertRaises(AttributeError):
self.cx.in_transaction = True
- def CheckOpenWithPathLikeObject(self):
+ def test_open_with_path_like_object(self):
""" Checks that we can successfully connect to a database using an object that
is PathLike, i.e. has __fspath__(). """
self.addCleanup(unlink, TESTFN)
@@ -171,11 +187,7 @@
with sqlite.connect(path) as cx:
cx.execute('create table test(id integer)')
- def CheckOpenUri(self):
- if sqlite.sqlite_version_info < (3, 7, 7):
- with self.assertRaises(sqlite.NotSupportedError):
- sqlite.connect(':memory:', uri=True)
- return
+ def test_open_uri(self):
self.addCleanup(unlink, TESTFN)
with sqlite.connect(TESTFN) as cx:
cx.execute('create table test(id integer)')
@@ -185,12 +197,26 @@
with self.assertRaises(sqlite.OperationalError):
cx.execute('insert into test(id) values(1)')
- @unittest.skipIf(sqlite.sqlite_version_info >= (3, 3, 1),
- 'needs sqlite versions older than 3.3.1')
- def CheckSameThreadErrorOnOldVersion(self):
- with self.assertRaises(sqlite.NotSupportedError) as cm:
- sqlite.connect(':memory:', check_same_thread=False)
- self.assertEqual(str(cm.exception), 'shared connections not available')
+
+class UninitialisedConnectionTests(unittest.TestCase):
+ def setUp(self):
+ self.cx = sqlite.Connection.__new__(sqlite.Connection)
+
+ def test_uninit_operations(self):
+ funcs = (
+ lambda: self.cx.isolation_level,
+ lambda: self.cx.total_changes,
+ lambda: self.cx.in_transaction,
+ lambda: self.cx.iterdump(),
+ lambda: self.cx.cursor(),
+ lambda: self.cx.close(),
+ )
+ for func in funcs:
+ with self.subTest(func=func):
+ self.assertRaisesRegex(sqlite.ProgrammingError,
+ "Base Connection.__init__ not called",
+ func)
+
class CursorTests(unittest.TestCase):
def setUp(self):
@@ -206,21 +232,21 @@
self.cu.close()
self.cx.close()
- def CheckExecuteNoArgs(self):
+ def test_execute_no_args(self):
self.cu.execute("delete from test")
- def CheckExecuteIllegalSql(self):
+ def test_execute_illegal_sql(self):
with self.assertRaises(sqlite.OperationalError):
self.cu.execute("select asdf")
- def CheckExecuteTooMuchSql(self):
+ def test_execute_too_much_sql(self):
with self.assertRaises(sqlite.Warning):
self.cu.execute("select 5+4; select 4+5")
- def CheckExecuteTooMuchSql2(self):
+ def test_execute_too_much_sql2(self):
self.cu.execute("select 5+4; -- foo bar")
- def CheckExecuteTooMuchSql3(self):
+ def test_execute_too_much_sql3(self):
self.cu.execute("""
select 5+4;
@@ -229,53 +255,53 @@
*/
""")
- def CheckExecuteWrongSqlArg(self):
+ def test_execute_wrong_sql_arg(self):
with self.assertRaises(TypeError):
self.cu.execute(42)
- def CheckExecuteArgInt(self):
+ def test_execute_arg_int(self):
self.cu.execute("insert into test(id) values (?)", (42,))
- def CheckExecuteArgFloat(self):
+ def test_execute_arg_float(self):
self.cu.execute("insert into test(income) values (?)", (2500.32,))
- def CheckExecuteArgString(self):
+ def test_execute_arg_string(self):
self.cu.execute("insert into test(name) values (?)", ("Hugo",))
- def CheckExecuteArgStringWithZeroByte(self):
+ def test_execute_arg_string_with_zero_byte(self):
self.cu.execute("insert into test(name) values (?)", ("Hu\x00go",))
self.cu.execute("select name from test where id=?", (self.cu.lastrowid,))
row = self.cu.fetchone()
self.assertEqual(row[0], "Hu\x00go")
- def CheckExecuteNonIterable(self):
+ def test_execute_non_iterable(self):
with self.assertRaises(ValueError) as cm:
self.cu.execute("insert into test(id) values (?)", 42)
self.assertEqual(str(cm.exception), 'parameters are of unsupported type')
- def CheckExecuteWrongNoOfArgs1(self):
+ def test_execute_wrong_no_of_args1(self):
# too many parameters
with self.assertRaises(sqlite.ProgrammingError):
self.cu.execute("insert into test(id) values (?)", (17, "Egon"))
- def CheckExecuteWrongNoOfArgs2(self):
+ def test_execute_wrong_no_of_args2(self):
# too little parameters
with self.assertRaises(sqlite.ProgrammingError):
self.cu.execute("insert into test(id) values (?)")
- def CheckExecuteWrongNoOfArgs3(self):
+ def test_execute_wrong_no_of_args3(self):
# no parameters, parameters are needed
with self.assertRaises(sqlite.ProgrammingError):
self.cu.execute("insert into test(id) values (?)")
- def CheckExecuteParamList(self):
+ def test_execute_param_list(self):
self.cu.execute("insert into test(name) values ('foo')")
self.cu.execute("select name from test where name=?", ["foo"])
row = self.cu.fetchone()
self.assertEqual(row[0], "foo")
- def CheckExecuteParamSequence(self):
+ def test_execute_param_sequence(self):
class L:
def __len__(self):
return 1
@@ -288,7 +314,7 @@
row = self.cu.fetchone()
self.assertEqual(row[0], "foo")
- def CheckExecuteParamSequenceBadLen(self):
+ def test_execute_param_sequence_bad_len(self):
# Issue41662: Error in __len__() was overridden with ProgrammingError.
class L:
def __len__(self):
@@ -300,13 +326,13 @@
with self.assertRaises(ZeroDivisionError):
self.cu.execute("select name from test where name=?", L())
- def CheckExecuteDictMapping(self):
+ def test_execute_dict_mapping(self):
self.cu.execute("insert into test(name) values ('foo')")
self.cu.execute("select name from test where name=:name", {"name": "foo"})
row = self.cu.fetchone()
self.assertEqual(row[0], "foo")
- def CheckExecuteDictMapping_Mapping(self):
+ def test_execute_dict_mapping_mapping(self):
class D(dict):
def __missing__(self, key):
return "foo"
@@ -316,32 +342,32 @@
row = self.cu.fetchone()
self.assertEqual(row[0], "foo")
- def CheckExecuteDictMappingTooLittleArgs(self):
+ def test_execute_dict_mapping_too_little_args(self):
self.cu.execute("insert into test(name) values ('foo')")
with self.assertRaises(sqlite.ProgrammingError):
self.cu.execute("select name from test where name=:name and id=:id", {"name": "foo"})
- def CheckExecuteDictMappingNoArgs(self):
+ def test_execute_dict_mapping_no_args(self):
self.cu.execute("insert into test(name) values ('foo')")
with self.assertRaises(sqlite.ProgrammingError):
self.cu.execute("select name from test where name=:name")
- def CheckExecuteDictMappingUnnamed(self):
+ def test_execute_dict_mapping_unnamed(self):
self.cu.execute("insert into test(name) values ('foo')")
with self.assertRaises(sqlite.ProgrammingError):
self.cu.execute("select name from test where name=?", {"name": "foo"})
- def CheckClose(self):
+ def test_close(self):
self.cu.close()
- def CheckRowcountExecute(self):
+ def test_rowcount_execute(self):
self.cu.execute("delete from test")
self.cu.execute("insert into test(name) values ('foo')")
self.cu.execute("insert into test(name) values ('foo')")
self.cu.execute("update test set name='bar'")
self.assertEqual(self.cu.rowcount, 2)
- def CheckRowcountSelect(self):
+ def test_rowcount_select(self):
"""
pysqlite does not know the rowcount of SELECT statements, because we
don't fetch all rows after executing the select statement. The rowcount
@@ -350,12 +376,12 @@
self.cu.execute("select 5 union select 6")
self.assertEqual(self.cu.rowcount, -1)
- def CheckRowcountExecutemany(self):
+ def test_rowcount_executemany(self):
self.cu.execute("delete from test")
self.cu.executemany("insert into test(name) values (?)", [(1,), (2,), (3,)])
self.assertEqual(self.cu.rowcount, 3)
- def CheckTotalChanges(self):
+ def test_total_changes(self):
self.cu.execute("insert into test(name) values ('foo')")
self.cu.execute("insert into test(name) values ('foo')")
self.assertLess(2, self.cx.total_changes, msg='total changes reported wrong value')
@@ -364,14 +390,17 @@
# Sequences are required by the DB-API, iterators
# enhancements in pysqlite.
- def CheckExecuteManySequence(self):
+ def test_execute_many_sequence(self):
self.cu.executemany("insert into test(income) values (?)", [(x,) for x in range(100, 110)])
- def CheckExecuteManyIterator(self):
+ def test_execute_many_iterator(self):
class MyIter:
def __init__(self):
self.value = 5
+ def __iter__(self):
+ return self
+
def __next__(self):
if self.value == 10:
raise StopIteration
@@ -381,26 +410,26 @@
self.cu.executemany("insert into test(income) values (?)", MyIter())
- def CheckExecuteManyGenerator(self):
+ def test_execute_many_generator(self):
def mygen():
for i in range(5):
yield (i,)
self.cu.executemany("insert into test(income) values (?)", mygen())
- def CheckExecuteManyWrongSqlArg(self):
+ def test_execute_many_wrong_sql_arg(self):
with self.assertRaises(TypeError):
self.cu.executemany(42, [(3,)])
- def CheckExecuteManySelect(self):
+ def test_execute_many_select(self):
with self.assertRaises(sqlite.ProgrammingError):
self.cu.executemany("select ?", [(3,)])
- def CheckExecuteManyNotIterable(self):
+ def test_execute_many_not_iterable(self):
with self.assertRaises(TypeError):
self.cu.executemany("insert into test(income) values (?)", 42)
- def CheckFetchIter(self):
+ def test_fetch_iter(self):
# Optional DB-API extension.
self.cu.execute("delete from test")
self.cu.execute("insert into test(id) values (?)", (5,))
@@ -412,20 +441,20 @@
self.assertEqual(lst[0], 5)
self.assertEqual(lst[1], 6)
- def CheckFetchone(self):
+ def test_fetchone(self):
self.cu.execute("select name from test")
row = self.cu.fetchone()
self.assertEqual(row[0], "foo")
row = self.cu.fetchone()
self.assertEqual(row, None)
- def CheckFetchoneNoStatement(self):
+ def test_fetchone_no_statement(self):
cur = self.cx.cursor()
row = cur.fetchone()
self.assertEqual(row, None)
- def CheckArraySize(self):
- # must default ot 1
+ def test_array_size(self):
+ # must default to 1
self.assertEqual(self.cu.arraysize, 1)
# now set to 2
@@ -441,51 +470,51 @@
self.assertEqual(len(res), 2)
- def CheckFetchmany(self):
+ def test_fetchmany(self):
self.cu.execute("select name from test")
res = self.cu.fetchmany(100)
self.assertEqual(len(res), 1)
res = self.cu.fetchmany(100)
self.assertEqual(res, [])
- def CheckFetchmanyKwArg(self):
+ def test_fetchmany_kw_arg(self):
"""Checks if fetchmany works with keyword arguments"""
self.cu.execute("select name from test")
res = self.cu.fetchmany(size=100)
self.assertEqual(len(res), 1)
- def CheckFetchall(self):
+ def test_fetchall(self):
self.cu.execute("select name from test")
res = self.cu.fetchall()
self.assertEqual(len(res), 1)
res = self.cu.fetchall()
self.assertEqual(res, [])
- def CheckSetinputsizes(self):
+ def test_setinputsizes(self):
self.cu.setinputsizes([3, 4, 5])
- def CheckSetoutputsize(self):
+ def test_setoutputsize(self):
self.cu.setoutputsize(5, 0)
- def CheckSetoutputsizeNoColumn(self):
+ def test_setoutputsize_no_column(self):
self.cu.setoutputsize(42)
- def CheckCursorConnection(self):
+ def test_cursor_connection(self):
# Optional DB-API extension.
self.assertEqual(self.cu.connection, self.cx)
- def CheckWrongCursorCallable(self):
+ def test_wrong_cursor_callable(self):
with self.assertRaises(TypeError):
def f(): pass
cur = self.cx.cursor(f)
- def CheckCursorWrongClass(self):
+ def test_cursor_wrong_class(self):
class Foo: pass
foo = Foo()
with self.assertRaises(TypeError):
cur = sqlite.Cursor(foo)
- def CheckLastRowIDOnReplace(self):
+ def test_last_row_id_on_replace(self):
"""
INSERT OR REPLACE and REPLACE INTO should produce the same behavior.
"""
@@ -495,7 +524,7 @@
self.cu.execute(sql.format(statement), (1, 'foo'))
self.assertEqual(self.cu.lastrowid, 1)
- def CheckLastRowIDOnIgnore(self):
+ def test_last_row_id_on_ignore(self):
self.cu.execute(
"insert or ignore into test(unique_test) values (?)",
('test',))
@@ -505,7 +534,7 @@
('test',))
self.assertEqual(self.cu.lastrowid, 2)
- def CheckLastRowIDInsertOR(self):
+ def test_last_row_id_insert_o_r(self):
results = []
for statement in ('FAIL', 'ABORT', 'ROLLBACK'):
sql = 'INSERT OR {} INTO test(unique_test) VALUES (?)'
@@ -533,7 +562,7 @@
self.cur.close()
self.con.close()
- def CheckConCursor(self):
+ def test_con_cursor(self):
def run(con, errors):
try:
cur = con.cursor()
@@ -551,7 +580,7 @@
if len(errors) > 0:
self.fail("\n".join(errors))
- def CheckConCommit(self):
+ def test_con_commit(self):
def run(con, errors):
try:
con.commit()
@@ -569,7 +598,7 @@
if len(errors) > 0:
self.fail("\n".join(errors))
- def CheckConRollback(self):
+ def test_con_rollback(self):
def run(con, errors):
try:
con.rollback()
@@ -587,7 +616,7 @@
if len(errors) > 0:
self.fail("\n".join(errors))
- def CheckConClose(self):
+ def test_con_close(self):
def run(con, errors):
try:
con.close()
@@ -605,7 +634,7 @@
if len(errors) > 0:
self.fail("\n".join(errors))
- def CheckCurImplicitBegin(self):
+ def test_cur_implicit_begin(self):
def run(cur, errors):
try:
cur.execute("insert into test(name) values ('a')")
@@ -623,7 +652,7 @@
if len(errors) > 0:
self.fail("\n".join(errors))
- def CheckCurClose(self):
+ def test_cur_close(self):
def run(cur, errors):
try:
cur.close()
@@ -641,7 +670,7 @@
if len(errors) > 0:
self.fail("\n".join(errors))
- def CheckCurExecute(self):
+ def test_cur_execute(self):
def run(cur, errors):
try:
cur.execute("select name from test")
@@ -660,7 +689,7 @@
if len(errors) > 0:
self.fail("\n".join(errors))
- def CheckCurIterNext(self):
+ def test_cur_iter_next(self):
def run(cur, errors):
try:
row = cur.fetchone()
@@ -681,29 +710,29 @@
self.fail("\n".join(errors))
class ConstructorTests(unittest.TestCase):
- def CheckDate(self):
+ def test_date(self):
d = sqlite.Date(2004, 10, 28)
- def CheckTime(self):
+ def test_time(self):
t = sqlite.Time(12, 39, 35)
- def CheckTimestamp(self):
+ def test_timestamp(self):
ts = sqlite.Timestamp(2004, 10, 28, 12, 39, 35)
- def CheckDateFromTicks(self):
+ def test_date_from_ticks(self):
d = sqlite.DateFromTicks(42)
- def CheckTimeFromTicks(self):
+ def test_time_from_ticks(self):
t = sqlite.TimeFromTicks(42)
- def CheckTimestampFromTicks(self):
+ def test_timestamp_from_ticks(self):
ts = sqlite.TimestampFromTicks(42)
- def CheckBinary(self):
+ def test_binary(self):
b = sqlite.Binary(b"\0'")
class ExtensionTests(unittest.TestCase):
- def CheckScriptStringSql(self):
+ def test_script_string_sql(self):
con = sqlite.connect(":memory:")
cur = con.cursor()
cur.executescript("""
@@ -716,31 +745,31 @@
res = cur.fetchone()[0]
self.assertEqual(res, 5)
- def CheckScriptSyntaxError(self):
+ def test_script_syntax_error(self):
con = sqlite.connect(":memory:")
cur = con.cursor()
with self.assertRaises(sqlite.OperationalError):
cur.executescript("create table test(x); asdf; create table test2(x)")
- def CheckScriptErrorNormal(self):
+ def test_script_error_normal(self):
con = sqlite.connect(":memory:")
cur = con.cursor()
with self.assertRaises(sqlite.OperationalError):
cur.executescript("create table test(sadfsadfdsa); select foo from hurz;")
- def CheckCursorExecutescriptAsBytes(self):
+ def test_cursor_executescript_as_bytes(self):
con = sqlite.connect(":memory:")
cur = con.cursor()
with self.assertRaises(ValueError) as cm:
cur.executescript(b"create table test(foo); insert into test(foo) values (5);")
self.assertEqual(str(cm.exception), 'script argument must be unicode.')
- def CheckConnectionExecute(self):
+ def test_connection_execute(self):
con = sqlite.connect(":memory:")
result = con.execute("select 5").fetchone()[0]
self.assertEqual(result, 5, "Basic test of Connection.execute")
- def CheckConnectionExecutemany(self):
+ def test_connection_executemany(self):
con = sqlite.connect(":memory:")
con.execute("create table test(foo)")
con.executemany("insert into test(foo) values (?)", [(3,), (4,)])
@@ -748,46 +777,46 @@
self.assertEqual(result[0][0], 3, "Basic test of Connection.executemany")
self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany")
- def CheckConnectionExecutescript(self):
+ def test_connection_executescript(self):
con = sqlite.connect(":memory:")
con.executescript("create table test(foo); insert into test(foo) values (5);")
result = con.execute("select foo from test").fetchone()[0]
self.assertEqual(result, 5, "Basic test of Connection.executescript")
class ClosedConTests(unittest.TestCase):
- def CheckClosedConCursor(self):
+ def test_closed_con_cursor(self):
con = sqlite.connect(":memory:")
con.close()
with self.assertRaises(sqlite.ProgrammingError):
cur = con.cursor()
- def CheckClosedConCommit(self):
+ def test_closed_con_commit(self):
con = sqlite.connect(":memory:")
con.close()
with self.assertRaises(sqlite.ProgrammingError):
con.commit()
- def CheckClosedConRollback(self):
+ def test_closed_con_rollback(self):
con = sqlite.connect(":memory:")
con.close()
with self.assertRaises(sqlite.ProgrammingError):
con.rollback()
- def CheckClosedCurExecute(self):
+ def test_closed_cur_execute(self):
con = sqlite.connect(":memory:")
cur = con.cursor()
con.close()
with self.assertRaises(sqlite.ProgrammingError):
cur.execute("select 4")
- def CheckClosedCreateFunction(self):
+ def test_closed_create_function(self):
con = sqlite.connect(":memory:")
con.close()
def f(x): return 17
with self.assertRaises(sqlite.ProgrammingError):
con.create_function("foo", 1, f)
- def CheckClosedCreateAggregate(self):
+ def test_closed_create_aggregate(self):
con = sqlite.connect(":memory:")
con.close()
class Agg:
@@ -800,7 +829,7 @@
with self.assertRaises(sqlite.ProgrammingError):
con.create_aggregate("foo", 1, Agg)
- def CheckClosedSetAuthorizer(self):
+ def test_closed_set_authorizer(self):
con = sqlite.connect(":memory:")
con.close()
def authorizer(*args):
@@ -808,21 +837,21 @@
with self.assertRaises(sqlite.ProgrammingError):
con.set_authorizer(authorizer)
- def CheckClosedSetProgressCallback(self):
+ def test_closed_set_progress_callback(self):
con = sqlite.connect(":memory:")
con.close()
def progress(): pass
with self.assertRaises(sqlite.ProgrammingError):
con.set_progress_handler(progress, 100)
- def CheckClosedCall(self):
+ def test_closed_call(self):
con = sqlite.connect(":memory:")
con.close()
with self.assertRaises(sqlite.ProgrammingError):
con()
class ClosedCurTests(unittest.TestCase):
- def CheckClosed(self):
+ def test_closed(self):
con = sqlite.connect(":memory:")
cur = con.cursor()
cur.close()
@@ -860,7 +889,7 @@
self.cu.close()
self.cx.close()
- def CheckOnConflictRollbackWithExplicitTransaction(self):
+ def test_on_conflict_rollback_with_explicit_transaction(self):
self.cx.isolation_level = None # autocommit mode
self.cu = self.cx.cursor()
# Start an explicit transaction.
@@ -875,7 +904,7 @@
# Transaction should have rolled back and nothing should be in table.
self.assertEqual(self.cu.fetchall(), [])
- def CheckOnConflictAbortRaisesWithExplicitTransactions(self):
+ def test_on_conflict_abort_raises_with_explicit_transactions(self):
# Abort cancels the current sql statement but doesn't change anything
# about the current transaction.
self.cx.isolation_level = None # autocommit mode
@@ -891,7 +920,7 @@
# Expect the first two inserts to work, third to do nothing.
self.assertEqual(self.cu.fetchall(), [('abort_test', None), (None, 'foo',)])
- def CheckOnConflictRollbackWithoutTransaction(self):
+ def test_on_conflict_rollback_without_transaction(self):
# Start of implicit transaction
self.cu.execute("INSERT INTO test(name) VALUES ('abort_test')")
self.cu.execute("INSERT OR ROLLBACK INTO test(unique_name) VALUES ('foo')")
@@ -901,7 +930,7 @@
# Implicit transaction is rolled back on error.
self.assertEqual(self.cu.fetchall(), [])
- def CheckOnConflictAbortRaisesWithoutTransactions(self):
+ def test_on_conflict_abort_raises_without_transactions(self):
# Abort cancels the current sql statement but doesn't change anything
# about the current transaction.
self.cu.execute("INSERT INTO test(name) VALUES ('abort_test')")
@@ -912,20 +941,20 @@
self.cu.execute("SELECT name, unique_name FROM test")
self.assertEqual(self.cu.fetchall(), [('abort_test', None), (None, 'foo',)])
- def CheckOnConflictFail(self):
+ def test_on_conflict_fail(self):
self.cu.execute("INSERT OR FAIL INTO test(unique_name) VALUES ('foo')")
with self.assertRaises(sqlite.IntegrityError):
self.cu.execute("INSERT OR FAIL INTO test(unique_name) VALUES ('foo')")
self.assertEqual(self.cu.fetchall(), [])
- def CheckOnConflictIgnore(self):
+ def test_on_conflict_ignore(self):
self.cu.execute("INSERT OR IGNORE INTO test(unique_name) VALUES ('foo')")
# Nothing should happen.
self.cu.execute("INSERT OR IGNORE INTO test(unique_name) VALUES ('foo')")
self.cu.execute("SELECT unique_name FROM test")
self.assertEqual(self.cu.fetchall(), [('foo',)])
- def CheckOnConflictReplace(self):
+ def test_on_conflict_replace(self):
self.cu.execute("INSERT OR REPLACE INTO test(name, unique_name) VALUES ('Data!', 'foo')")
# There shouldn't be an IntegrityError exception.
self.cu.execute("INSERT OR REPLACE INTO test(name, unique_name) VALUES ('Very different data!', 'foo')")
@@ -933,21 +962,94 @@
self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')])
+class MultiprocessTests(unittest.TestCase):
+ CONNECTION_TIMEOUT = SHORT_TIMEOUT / 1000. # Defaults to 30 ms
+
+ def tearDown(self):
+ unlink(TESTFN)
+
+ def test_ctx_mgr_rollback_if_commit_failed(self):
+ # bpo-27334: ctx manager does not rollback if commit fails
+ SCRIPT = f"""if 1:
+ import sqlite3
+ def wait():
+ print("started")
+ assert "database is locked" in input()
+
+ cx = sqlite3.connect("{TESTFN}", timeout={self.CONNECTION_TIMEOUT})
+ cx.create_function("wait", 0, wait)
+ with cx:
+ cx.execute("create table t(t)")
+ try:
+ # execute two transactions; both will try to lock the db
+ cx.executescript('''
+ -- start a transaction and wait for parent
+ begin transaction;
+ select * from t;
+ select wait();
+ rollback;
+
+ -- start a new transaction; would fail if parent holds lock
+ begin transaction;
+ select * from t;
+ rollback;
+ ''')
+ finally:
+ cx.close()
+ """
+
+ # spawn child process
+ proc = subprocess.Popen(
+ [sys.executable, "-c", SCRIPT],
+ encoding="utf-8",
+ bufsize=0,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ )
+ self.addCleanup(proc.communicate)
+
+ # wait for child process to start
+ self.assertEqual("started", proc.stdout.readline().strip())
+
+ cx = sqlite.connect(TESTFN, timeout=self.CONNECTION_TIMEOUT)
+ try: # context manager should correctly release the db lock
+ with cx:
+ cx.execute("insert into t values('test')")
+ except sqlite.OperationalError as exc:
+ proc.stdin.write(str(exc))
+ else:
+ proc.stdin.write("no error")
+ finally:
+ cx.close()
+
+ # terminate child process
+ self.assertIsNone(proc.returncode)
+ try:
+ proc.communicate(input="end", timeout=SHORT_TIMEOUT)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ proc.communicate()
+ raise
+ self.assertEqual(proc.returncode, 0)
+
+
def suite():
- module_suite = unittest.makeSuite(ModuleTests, "Check")
- connection_suite = unittest.makeSuite(ConnectionTests, "Check")
- cursor_suite = unittest.makeSuite(CursorTests, "Check")
- thread_suite = unittest.makeSuite(ThreadTests, "Check")
- constructor_suite = unittest.makeSuite(ConstructorTests, "Check")
- ext_suite = unittest.makeSuite(ExtensionTests, "Check")
- closed_con_suite = unittest.makeSuite(ClosedConTests, "Check")
- closed_cur_suite = unittest.makeSuite(ClosedCurTests, "Check")
- on_conflict_suite = unittest.makeSuite(SqliteOnConflictTests, "Check")
- return unittest.TestSuite((
- module_suite, connection_suite, cursor_suite, thread_suite,
- constructor_suite, ext_suite, closed_con_suite, closed_cur_suite,
- on_conflict_suite,
- ))
+ tests = [
+ ClosedConTests,
+ ClosedCurTests,
+ ConnectionTests,
+ ConstructorTests,
+ CursorTests,
+ ExtensionTests,
+ ModuleTests,
+ MultiprocessTests,
+ SqliteOnConflictTests,
+ ThreadTests,
+ UninitialisedConnectionTests,
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/sqlite3/test/dump.py b/common/py3-stdlib/sqlite3/test/dump.py
index a1f45a4..618a7fd 100644
--- a/common/py3-stdlib/sqlite3/test/dump.py
+++ b/common/py3-stdlib/sqlite3/test/dump.py
@@ -11,7 +11,7 @@
def tearDown(self):
self.cx.close()
- def CheckTableDump(self):
+ def test_table_dump(self):
expected_sqls = [
"""CREATE TABLE "index"("index" blob);"""
,
@@ -49,7 +49,7 @@
[self.assertEqual(expected_sqls[i], actual_sqls[i])
for i in range(len(expected_sqls))]
- def CheckUnorderableRow(self):
+ def test_unorderable_row(self):
# iterdump() should be able to cope with unorderable row types (issue #15545)
class UnorderableRow:
def __init__(self, cursor, row):
@@ -71,7 +71,12 @@
self.assertEqual(expected, got)
def suite():
- return unittest.TestSuite(unittest.makeSuite(DumpTests, "Check"))
+ tests = [
+ DumpTests,
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/sqlite3/test/factory.py b/common/py3-stdlib/sqlite3/test/factory.py
index 95dd24b..8764284 100644
--- a/common/py3-stdlib/sqlite3/test/factory.py
+++ b/common/py3-stdlib/sqlite3/test/factory.py
@@ -1,7 +1,6 @@
-#-*- coding: iso-8859-1 -*-
# pysqlite2/test/factory.py: tests for the various factories in pysqlite
#
-# Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
+# Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
#
# This file is part of pysqlite.
#
@@ -47,7 +46,7 @@
def tearDown(self):
self.con.close()
- def CheckIsInstance(self):
+ def test_is_instance(self):
self.assertIsInstance(self.con, MyConnection)
class CursorFactoryTests(unittest.TestCase):
@@ -57,7 +56,7 @@
def tearDown(self):
self.con.close()
- def CheckIsInstance(self):
+ def test_is_instance(self):
cur = self.con.cursor()
self.assertIsInstance(cur, sqlite.Cursor)
cur = self.con.cursor(MyCursor)
@@ -65,7 +64,7 @@
cur = self.con.cursor(factory=lambda con: MyCursor(con))
self.assertIsInstance(cur, MyCursor)
- def CheckInvalidFactory(self):
+ def test_invalid_factory(self):
# not a callable at all
self.assertRaises(TypeError, self.con.cursor, None)
# invalid callable with not exact one argument
@@ -77,7 +76,7 @@
def setUp(self):
self.con = sqlite.connect(":memory:")
- def CheckIsProducedByFactory(self):
+ def test_is_produced_by_factory(self):
cur = self.con.cursor(factory=MyCursor)
cur.execute("select 4+5 as foo")
row = cur.fetchone()
@@ -91,12 +90,12 @@
def setUp(self):
self.con = sqlite.connect(":memory:")
- def CheckCustomFactory(self):
+ def test_custom_factory(self):
self.con.row_factory = lambda cur, row: list(row)
row = self.con.execute("select 1, 2").fetchone()
self.assertIsInstance(row, list)
- def CheckSqliteRowIndex(self):
+ def test_sqlite_row_index(self):
self.con.row_factory = sqlite.Row
row = self.con.execute("select 1 as a_1, 2 as b").fetchone()
self.assertIsInstance(row, sqlite.Row)
@@ -125,7 +124,7 @@
with self.assertRaises(IndexError):
row[2**1000]
- def CheckSqliteRowIndexUnicode(self):
+ def test_sqlite_row_index_unicode(self):
self.con.row_factory = sqlite.Row
row = self.con.execute("select 1 as \xff").fetchone()
self.assertEqual(row["\xff"], 1)
@@ -134,7 +133,7 @@
with self.assertRaises(IndexError):
row['\xdf']
- def CheckSqliteRowSlice(self):
+ def test_sqlite_row_slice(self):
# A sqlite.Row can be sliced like a list.
self.con.row_factory = sqlite.Row
row = self.con.execute("select 1, 2, 3, 4").fetchone()
@@ -152,21 +151,21 @@
self.assertEqual(row[0:4:2], (1, 3))
self.assertEqual(row[3:0:-2], (4, 2))
- def CheckSqliteRowIter(self):
+ def test_sqlite_row_iter(self):
"""Checks if the row object is iterable"""
self.con.row_factory = sqlite.Row
row = self.con.execute("select 1 as a, 2 as b").fetchone()
for col in row:
pass
- def CheckSqliteRowAsTuple(self):
+ def test_sqlite_row_as_tuple(self):
"""Checks if the row object can be converted to a tuple"""
self.con.row_factory = sqlite.Row
row = self.con.execute("select 1 as a, 2 as b").fetchone()
t = tuple(row)
self.assertEqual(t, (row['a'], row['b']))
- def CheckSqliteRowAsDict(self):
+ def test_sqlite_row_as_dict(self):
"""Checks if the row object can be correctly converted to a dictionary"""
self.con.row_factory = sqlite.Row
row = self.con.execute("select 1 as a, 2 as b").fetchone()
@@ -174,7 +173,7 @@
self.assertEqual(d["a"], row["a"])
self.assertEqual(d["b"], row["b"])
- def CheckSqliteRowHashCmp(self):
+ def test_sqlite_row_hash_cmp(self):
"""Checks if the row object compares and hashes correctly"""
self.con.row_factory = sqlite.Row
row_1 = self.con.execute("select 1 as a, 2 as b").fetchone()
@@ -208,7 +207,7 @@
self.assertEqual(hash(row_1), hash(row_2))
- def CheckSqliteRowAsSequence(self):
+ def test_sqlite_row_as_sequence(self):
""" Checks if the row object can act like a sequence """
self.con.row_factory = sqlite.Row
row = self.con.execute("select 1 as a, 2 as b").fetchone()
@@ -217,7 +216,7 @@
self.assertEqual(list(reversed(row)), list(reversed(as_tuple)))
self.assertIsInstance(row, Sequence)
- def CheckFakeCursorClass(self):
+ def test_fake_cursor_class(self):
# Issue #24257: Incorrect use of PyObject_IsInstance() caused
# segmentation fault.
# Issue #27861: Also applies for cursor factory.
@@ -234,30 +233,31 @@
def setUp(self):
self.con = sqlite.connect(":memory:")
- def CheckUnicode(self):
- austria = "Österreich"
+ def test_unicode(self):
+ austria = "Österreich"
row = self.con.execute("select ?", (austria,)).fetchone()
self.assertEqual(type(row[0]), str, "type of row[0] must be unicode")
- def CheckString(self):
+ def test_string(self):
self.con.text_factory = bytes
- austria = "Österreich"
+ austria = "Österreich"
row = self.con.execute("select ?", (austria,)).fetchone()
self.assertEqual(type(row[0]), bytes, "type of row[0] must be bytes")
self.assertEqual(row[0], austria.encode("utf-8"), "column must equal original data in UTF-8")
- def CheckCustom(self):
+ def test_custom(self):
self.con.text_factory = lambda x: str(x, "utf-8", "ignore")
- austria = "Österreich"
+ austria = "Österreich"
row = self.con.execute("select ?", (austria,)).fetchone()
self.assertEqual(type(row[0]), str, "type of row[0] must be unicode")
self.assertTrue(row[0].endswith("reich"), "column must contain original data")
- def CheckOptimizedUnicode(self):
- # In py3k, str objects are always returned when text_factory
- # is OptimizedUnicode
- self.con.text_factory = sqlite.OptimizedUnicode
- austria = "Österreich"
+ def test_optimized_unicode(self):
+ # OptimizedUnicode is deprecated as of Python 3.10
+ with self.assertWarns(DeprecationWarning) as cm:
+ self.con.text_factory = sqlite.OptimizedUnicode
+ self.assertIn("factory.py", cm.filename)
+ austria = "Österreich"
germany = "Deutchland"
a_row = self.con.execute("select ?", (austria,)).fetchone()
d_row = self.con.execute("select ?", (germany,)).fetchone()
@@ -273,25 +273,25 @@
self.con.execute("create table test (value text)")
self.con.execute("insert into test (value) values (?)", ("a\x00b",))
- def CheckString(self):
+ def test_string(self):
# text_factory defaults to str
row = self.con.execute("select value from test").fetchone()
self.assertIs(type(row[0]), str)
self.assertEqual(row[0], "a\x00b")
- def CheckBytes(self):
+ def test_bytes(self):
self.con.text_factory = bytes
row = self.con.execute("select value from test").fetchone()
self.assertIs(type(row[0]), bytes)
self.assertEqual(row[0], b"a\x00b")
- def CheckBytearray(self):
+ def test_bytearray(self):
self.con.text_factory = bytearray
row = self.con.execute("select value from test").fetchone()
self.assertIs(type(row[0]), bytearray)
self.assertEqual(row[0], b"a\x00b")
- def CheckCustom(self):
+ def test_custom(self):
# A custom factory should receive a bytes argument
self.con.text_factory = lambda x: x
row = self.con.execute("select value from test").fetchone()
@@ -302,13 +302,17 @@
self.con.close()
def suite():
- connection_suite = unittest.makeSuite(ConnectionFactoryTests, "Check")
- cursor_suite = unittest.makeSuite(CursorFactoryTests, "Check")
- row_suite_compat = unittest.makeSuite(RowFactoryTestsBackwardsCompat, "Check")
- row_suite = unittest.makeSuite(RowFactoryTests, "Check")
- text_suite = unittest.makeSuite(TextFactoryTests, "Check")
- text_zero_bytes_suite = unittest.makeSuite(TextFactoryTestsWithEmbeddedZeroBytes, "Check")
- return unittest.TestSuite((connection_suite, cursor_suite, row_suite_compat, row_suite, text_suite, text_zero_bytes_suite))
+ tests = [
+ ConnectionFactoryTests,
+ CursorFactoryTests,
+ RowFactoryTests,
+ RowFactoryTestsBackwardsCompat,
+ TextFactoryTests,
+ TextFactoryTestsWithEmbeddedZeroBytes,
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/sqlite3/test/hooks.py b/common/py3-stdlib/sqlite3/test/hooks.py
index d74e74b..8c60bdc 100644
--- a/common/py3-stdlib/sqlite3/test/hooks.py
+++ b/common/py3-stdlib/sqlite3/test/hooks.py
@@ -1,7 +1,6 @@
-#-*- coding: iso-8859-1 -*-
# pysqlite2/test/hooks.py: tests for various SQLite-specific hooks
#
-# Copyright (C) 2006-2007 Gerhard Häring <gh@ghaering.de>
+# Copyright (C) 2006-2007 Gerhard Häring <gh@ghaering.de>
#
# This file is part of pysqlite.
#
@@ -24,26 +23,27 @@
import unittest
import sqlite3 as sqlite
-from test.support import TESTFN, unlink
+from test.support.os_helper import TESTFN, unlink
+
class CollationTests(unittest.TestCase):
- def CheckCreateCollationNotString(self):
+ def test_create_collation_not_string(self):
con = sqlite.connect(":memory:")
with self.assertRaises(TypeError):
con.create_collation(None, lambda x, y: (x > y) - (x < y))
- def CheckCreateCollationNotCallable(self):
+ def test_create_collation_not_callable(self):
con = sqlite.connect(":memory:")
with self.assertRaises(TypeError) as cm:
con.create_collation("X", 42)
self.assertEqual(str(cm.exception), 'parameter must be callable')
- def CheckCreateCollationNotAscii(self):
+ def test_create_collation_not_ascii(self):
con = sqlite.connect(":memory:")
with self.assertRaises(sqlite.ProgrammingError):
- con.create_collation("collä", lambda x, y: (x > y) - (x < y))
+ con.create_collation("collä", lambda x, y: (x > y) - (x < y))
- def CheckCreateCollationBadUpper(self):
+ def test_create_collation_bad_upper(self):
class BadUpperStr(str):
def upper(self):
return None
@@ -60,9 +60,7 @@
self.assertEqual(result[0][0], 'b')
self.assertEqual(result[1][0], 'a')
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 2, 1),
- 'old SQLite versions crash on this test')
- def CheckCollationIsUsed(self):
+ def test_collation_is_used(self):
def mycoll(x, y):
# reverse order
return -((x > y) - (x < y))
@@ -87,7 +85,7 @@
result = con.execute(sql).fetchall()
self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll')
- def CheckCollationReturnsLargeInteger(self):
+ def test_collation_returns_large_integer(self):
def mycoll(x, y):
# reverse order
return -((x > y) - (x < y)) * 2**32
@@ -106,7 +104,7 @@
self.assertEqual(result, [('c',), ('b',), ('a',)],
msg="the expected order was not returned")
- def CheckCollationRegisterTwice(self):
+ def test_collation_register_twice(self):
"""
Register two different collation functions under the same name.
Verify that the last one is actually used.
@@ -120,7 +118,7 @@
self.assertEqual(result[0][0], 'b')
self.assertEqual(result[1][0], 'a')
- def CheckDeregisterCollation(self):
+ def test_deregister_collation(self):
"""
Register a collation, then deregister it. Make sure an error is raised if we try
to use it.
@@ -133,7 +131,7 @@
self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll')
class ProgressTests(unittest.TestCase):
- def CheckProgressHandlerUsed(self):
+ def test_progress_handler_used(self):
"""
Test that the progress handler is invoked once it is set.
"""
@@ -149,7 +147,7 @@
self.assertTrue(progress_calls)
- def CheckOpcodeCount(self):
+ def test_opcode_count(self):
"""
Test that the opcode argument is respected.
"""
@@ -172,7 +170,7 @@
second_count = len(progress_calls)
self.assertGreaterEqual(first_count, second_count)
- def CheckCancelOperation(self):
+ def test_cancel_operation(self):
"""
Test that returning a non-zero value stops the operation in progress.
"""
@@ -186,7 +184,7 @@
curs.execute,
"create table bar (a, b)")
- def CheckClearHandler(self):
+ def test_clear_handler(self):
"""
Test that setting the progress handler to None clears the previously set handler.
"""
@@ -202,7 +200,7 @@
self.assertEqual(action, 0, "progress handler was not cleared")
class TraceCallbackTests(unittest.TestCase):
- def CheckTraceCallbackUsed(self):
+ def test_trace_callback_used(self):
"""
Test that the trace callback is invoked once it is set.
"""
@@ -215,7 +213,7 @@
self.assertTrue(traced_statements)
self.assertTrue(any("create table foo" in stmt for stmt in traced_statements))
- def CheckClearTraceCallback(self):
+ def test_clear_trace_callback(self):
"""
Test that setting the trace callback to None clears the previously set callback.
"""
@@ -228,7 +226,7 @@
con.execute("create table foo(a, b)")
self.assertFalse(traced_statements, "trace callback was not cleared")
- def CheckUnicodeContent(self):
+ def test_unicode_content(self):
"""
Test that the statement can contain unicode literals.
"""
@@ -239,17 +237,13 @@
traced_statements.append(statement)
con.set_trace_callback(trace)
con.execute("create table foo(x)")
- # Can't execute bound parameters as their values don't appear
- # in traced statements before SQLite 3.6.21
- # (cf. http://www.sqlite.org/draft/releaselog/3_6_21.html)
- con.execute('insert into foo(x) values ("%s")' % unicode_value)
+ con.execute("insert into foo(x) values ('%s')" % unicode_value)
con.commit()
self.assertTrue(any(unicode_value in stmt for stmt in traced_statements),
"Unicode data %s garbled in trace callback: %s"
% (ascii(unicode_value), ', '.join(map(ascii, traced_statements))))
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 3, 9), "sqlite3_prepare_v2 is not available")
- def CheckTraceCallbackContent(self):
+ def test_trace_callback_content(self):
# set_trace_callback() shouldn't produce duplicate content (bpo-26187)
traced_statements = []
def trace(statement):
@@ -269,10 +263,14 @@
def suite():
- collation_suite = unittest.makeSuite(CollationTests, "Check")
- progress_suite = unittest.makeSuite(ProgressTests, "Check")
- trace_suite = unittest.makeSuite(TraceCallbackTests, "Check")
- return unittest.TestSuite((collation_suite, progress_suite, trace_suite))
+ tests = [
+ CollationTests,
+ ProgressTests,
+ TraceCallbackTests,
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/sqlite3/test/regression.py b/common/py3-stdlib/sqlite3/test/regression.py
index 6aa86d5..70d0ff9 100644
--- a/common/py3-stdlib/sqlite3/test/regression.py
+++ b/common/py3-stdlib/sqlite3/test/regression.py
@@ -1,7 +1,6 @@
-#-*- coding: iso-8859-1 -*-
# pysqlite2/test/regression.py: pysqlite regression tests
#
-# Copyright (C) 2006-2010 Gerhard Häring <gh@ghaering.de>
+# Copyright (C) 2006-2010 Gerhard Häring <gh@ghaering.de>
#
# This file is part of pysqlite.
#
@@ -35,12 +34,12 @@
def tearDown(self):
self.con.close()
- def CheckPragmaUserVersion(self):
+ def test_pragma_user_version(self):
# This used to crash pysqlite because this pragma command returns NULL for the column name
cur = self.con.cursor()
cur.execute("pragma user_version")
- def CheckPragmaSchemaVersion(self):
+ def test_pragma_schema_version(self):
# This still crashed pysqlite <= 2.2.1
con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES)
try:
@@ -50,7 +49,7 @@
cur.close()
con.close()
- def CheckStatementReset(self):
+ def test_statement_reset(self):
# pysqlite 2.1.0 to 2.2.0 have the problem that not all statements are
# reset before a rollback, but only those that are still in the
# statement cache. The others are not accessible from the connection object.
@@ -65,7 +64,7 @@
con.rollback()
- def CheckColumnNameWithSpaces(self):
+ def test_column_name_with_spaces(self):
cur = self.con.cursor()
cur.execute('select 1 as "foo bar [datetime]"')
self.assertEqual(cur.description[0][0], "foo bar [datetime]")
@@ -73,7 +72,7 @@
cur.execute('select 1 as "foo baz"')
self.assertEqual(cur.description[0][0], "foo baz")
- def CheckStatementFinalizationOnCloseDb(self):
+ def test_statement_finalization_on_close_db(self):
# pysqlite versions <= 2.3.3 only finalized statements in the statement
# cache when closing the database. statements that were still
# referenced in cursors weren't closed and could provoke "
@@ -87,8 +86,7 @@
cur.execute("select 1 x union select " + str(i))
con.close()
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 2, 2), 'needs sqlite 3.2.2 or newer')
- def CheckOnConflictRollback(self):
+ def test_on_conflict_rollback(self):
con = sqlite.connect(":memory:")
con.execute("create table foo(x, unique(x) on conflict rollback)")
con.execute("insert into foo(x) values (1)")
@@ -102,7 +100,7 @@
except sqlite.OperationalError:
self.fail("pysqlite knew nothing about the implicit ROLLBACK")
- def CheckWorkaroundForBuggySqliteTransferBindings(self):
+ def test_workaround_for_buggy_sqlite_transfer_bindings(self):
"""
pysqlite would crash with older SQLite versions unless
a workaround is implemented.
@@ -111,14 +109,14 @@
self.con.execute("drop table foo")
self.con.execute("create table foo(bar)")
- def CheckEmptyStatement(self):
+ def test_empty_statement(self):
"""
pysqlite used to segfault with SQLite versions 3.5.x. These return NULL
for "no-operation" statements
"""
self.con.execute("")
- def CheckTypeMapUsage(self):
+ def test_type_map_usage(self):
"""
pysqlite until 2.4.1 did not rebuild the row_cast_map when recompiling
a statement. This test exhibits the problem.
@@ -127,13 +125,13 @@
con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES)
con.execute("create table foo(bar timestamp)")
con.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),))
- con.execute(SELECT)
+ con.execute(SELECT).close()
con.execute("drop table foo")
con.execute("create table foo(bar integer)")
con.execute("insert into foo(bar) values (5)")
- con.execute(SELECT)
+ con.execute(SELECT).close()
- def CheckBindMutatingList(self):
+ def test_bind_mutating_list(self):
# Issue41662: Crash when mutate a list of parameters during iteration.
class X:
def __conform__(self, protocol):
@@ -146,7 +144,7 @@
with self.assertRaises(IndexError):
con.execute("insert into foo(bar, baz) values (?, ?)", parameters)
- def CheckErrorMsgDecodeError(self):
+ def test_error_msg_decode_error(self):
# When porting the module to Python 3.0, the error message about
# decoding errors disappeared. This verifies they're back again.
with self.assertRaises(sqlite.OperationalError) as cm:
@@ -155,13 +153,13 @@
msg = "Could not decode to UTF-8 column 'colname' with text 'xxx"
self.assertIn(msg, str(cm.exception))
- def CheckRegisterAdapter(self):
+ def test_register_adapter(self):
"""
See issue 3312.
"""
self.assertRaises(TypeError, sqlite.register_adapter, {}, None)
- def CheckSetIsolationLevel(self):
+ def test_set_isolation_level(self):
# See issue 27881.
class CustomStr(str):
def upper(self):
@@ -191,7 +189,7 @@
con.isolation_level = value
self.assertEqual(con.isolation_level, "DEFERRED")
- def CheckCursorConstructorCallCheck(self):
+ def test_cursor_constructor_call_check(self):
"""
Verifies that cursor methods check whether base class __init__ was
called.
@@ -208,14 +206,14 @@
r'^Base Cursor\.__init__ not called\.$'):
cur.close()
- def CheckStrSubclass(self):
+ def test_str_subclass(self):
"""
The Python 3.0 port of the module didn't cope with values of subclasses of str.
"""
class MyStr(str): pass
self.con.execute("select ?", (MyStr("abc"),))
- def CheckConnectionConstructorCallCheck(self):
+ def test_connection_constructor_call_check(self):
"""
Verifies that connection methods check whether base class __init__ was
called.
@@ -228,7 +226,7 @@
with self.assertRaises(sqlite.ProgrammingError):
cur = con.cursor()
- def CheckCursorRegistration(self):
+ def test_cursor_registration(self):
"""
Verifies that subclassed cursor classes are correctly registered with
the connection object, too. (fetch-across-rollback problem)
@@ -250,7 +248,7 @@
with self.assertRaises(sqlite.InterfaceError):
cur.fetchall()
- def CheckAutoCommit(self):
+ def test_auto_commit(self):
"""
Verifies that creating a connection in autocommit mode works.
2.5.3 introduced a regression so that these could no longer
@@ -258,7 +256,7 @@
"""
con = sqlite.connect(":memory:", isolation_level=None)
- def CheckPragmaAutocommit(self):
+ def test_pragma_autocommit(self):
"""
Verifies that running a PRAGMA statement that does an autocommit does
work. This did not work in 2.5.3/2.5.4.
@@ -270,21 +268,21 @@
cur.execute("pragma page_size")
row = cur.fetchone()
- def CheckConnectionCall(self):
+ def test_connection_call(self):
"""
Call a connection with a non-string SQL request: check error handling
of the statement constructor.
"""
self.assertRaises(TypeError, self.con, 1)
- def CheckCollation(self):
+ def test_collation(self):
def collation_cb(a, b):
return 1
self.assertRaises(sqlite.ProgrammingError, self.con.create_collation,
# Lone surrogate cannot be encoded to the default encoding (utf8)
"\uDC80", collation_cb)
- def CheckRecursiveCursorUse(self):
+ def test_recursive_cursor_use(self):
"""
http://bugs.python.org/issue10811
@@ -305,7 +303,7 @@
cur.executemany("insert into b (baz) values (?)",
((i,) for i in foo()))
- def CheckConvertTimestampMicrosecondPadding(self):
+ def test_convert_timestamp_microsecond_padding(self):
"""
http://bugs.python.org/issue14720
@@ -331,13 +329,13 @@
datetime.datetime(2012, 4, 4, 15, 6, 0, 123456),
])
- def CheckInvalidIsolationLevelType(self):
+ def test_invalid_isolation_level_type(self):
# isolation level is a string, not an integer
self.assertRaises(TypeError,
sqlite.connect, ":memory:", isolation_level=123)
- def CheckNullCharacter(self):
+ def test_null_character(self):
# Issue #21147
con = sqlite.connect(":memory:")
self.assertRaises(ValueError, con, "\0select 1")
@@ -346,7 +344,7 @@
self.assertRaises(ValueError, cur.execute, " \0select 2")
self.assertRaises(ValueError, cur.execute, "select 2\0")
- def CheckCommitCursorReset(self):
+ def test_commit_cursor_reset(self):
"""
Connection.commit() did reset cursors, which made sqlite3
to return rows multiple times when fetched from cursors
@@ -377,7 +375,7 @@
counter += 1
self.assertEqual(counter, 3, "should have returned exactly three rows")
- def CheckBpo31770(self):
+ def test_bpo31770(self):
"""
The interpreter shouldn't crash in case Cursor.__init__() is called
more than once.
@@ -393,11 +391,11 @@
del ref
support.gc_collect()
- def CheckDelIsolation_levelSegfault(self):
+ def test_del_isolation_level_segfault(self):
with self.assertRaises(AttributeError):
del self.con.isolation_level
- def CheckBpo37347(self):
+ def test_bpo37347(self):
class Printer:
def log(self, *args):
return sqlite.SQLITE_OK
@@ -411,13 +409,19 @@
self.con.execute("select 1") # trigger seg fault
method(None)
+ def test_return_empty_bytestring(self):
+ cur = self.con.execute("select X''")
+ val = cur.fetchone()[0]
+ self.assertEqual(val, b'')
def suite():
- regression_suite = unittest.makeSuite(RegressionTests, "Check")
- return unittest.TestSuite((
- regression_suite,
- ))
+ tests = [
+ RegressionTests
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/sqlite3/test/transactions.py b/common/py3-stdlib/sqlite3/test/transactions.py
index b8a13de..8028490 100644
--- a/common/py3-stdlib/sqlite3/test/transactions.py
+++ b/common/py3-stdlib/sqlite3/test/transactions.py
@@ -1,7 +1,6 @@
-#-*- coding: iso-8859-1 -*-
# pysqlite2/test/transactions.py: tests transactions
#
-# Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
+# Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
#
# This file is part of pysqlite.
#
@@ -52,7 +51,7 @@
except OSError:
pass
- def CheckDMLDoesNotAutoCommitBefore(self):
+ def test_dml_does_not_auto_commit_before(self):
self.cur1.execute("create table test(i)")
self.cur1.execute("insert into test(i) values (5)")
self.cur1.execute("create table test2(j)")
@@ -60,14 +59,14 @@
res = self.cur2.fetchall()
self.assertEqual(len(res), 0)
- def CheckInsertStartsTransaction(self):
+ def test_insert_starts_transaction(self):
self.cur1.execute("create table test(i)")
self.cur1.execute("insert into test(i) values (5)")
self.cur2.execute("select i from test")
res = self.cur2.fetchall()
self.assertEqual(len(res), 0)
- def CheckUpdateStartsTransaction(self):
+ def test_update_starts_transaction(self):
self.cur1.execute("create table test(i)")
self.cur1.execute("insert into test(i) values (5)")
self.con1.commit()
@@ -76,7 +75,7 @@
res = self.cur2.fetchone()[0]
self.assertEqual(res, 5)
- def CheckDeleteStartsTransaction(self):
+ def test_delete_starts_transaction(self):
self.cur1.execute("create table test(i)")
self.cur1.execute("insert into test(i) values (5)")
self.con1.commit()
@@ -85,7 +84,7 @@
res = self.cur2.fetchall()
self.assertEqual(len(res), 1)
- def CheckReplaceStartsTransaction(self):
+ def test_replace_starts_transaction(self):
self.cur1.execute("create table test(i)")
self.cur1.execute("insert into test(i) values (5)")
self.con1.commit()
@@ -95,7 +94,7 @@
self.assertEqual(len(res), 1)
self.assertEqual(res[0][0], 5)
- def CheckToggleAutoCommit(self):
+ def test_toggle_auto_commit(self):
self.cur1.execute("create table test(i)")
self.cur1.execute("insert into test(i) values (5)")
self.con1.isolation_level = None
@@ -111,17 +110,13 @@
res = self.cur2.fetchall()
self.assertEqual(len(res), 1)
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 2, 2),
- 'test hangs on sqlite versions older than 3.2.2')
- def CheckRaiseTimeout(self):
+ def test_raise_timeout(self):
self.cur1.execute("create table test(i)")
self.cur1.execute("insert into test(i) values (5)")
with self.assertRaises(sqlite.OperationalError):
self.cur2.execute("insert into test(i) values (5)")
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 2, 2),
- 'test hangs on sqlite versions older than 3.2.2')
- def CheckLocking(self):
+ def test_locking(self):
"""
This tests the improved concurrency with pysqlite 2.3.4. You needed
to roll back con2 before you could commit con1.
@@ -133,7 +128,7 @@
# NO self.con2.rollback() HERE!!!
self.con1.commit()
- def CheckRollbackCursorConsistency(self):
+ def test_rollback_cursor_consistency(self):
"""
Checks if cursors on the connection are set into a "reset" state
when a rollback is done on the connection.
@@ -153,12 +148,12 @@
self.con = sqlite.connect(":memory:")
self.cur = self.con.cursor()
- def CheckDropTable(self):
+ def test_drop_table(self):
self.cur.execute("create table test(i)")
self.cur.execute("insert into test(i) values (5)")
self.cur.execute("drop table test")
- def CheckPragma(self):
+ def test_pragma(self):
self.cur.execute("create table test(i)")
self.cur.execute("insert into test(i) values (5)")
self.cur.execute("pragma count_changes=1")
@@ -171,7 +166,7 @@
def setUp(self):
self.con = sqlite.connect(":memory:")
- def CheckDdlDoesNotAutostartTransaction(self):
+ def test_ddl_does_not_autostart_transaction(self):
# For backwards compatibility reasons, DDL statements should not
# implicitly start a transaction.
self.con.execute("create table test(i)")
@@ -179,7 +174,7 @@
result = self.con.execute("select * from test").fetchall()
self.assertEqual(result, [])
- def CheckImmediateTransactionalDDL(self):
+ def test_immediate_transactional_ddl(self):
# You can achieve transactional DDL by issuing a BEGIN
# statement manually.
self.con.execute("begin immediate")
@@ -188,7 +183,7 @@
with self.assertRaises(sqlite.OperationalError):
self.con.execute("select * from test")
- def CheckTransactionalDDL(self):
+ def test_transactional_ddl(self):
# You can achieve transactional DDL by issuing a BEGIN
# statement manually.
self.con.execute("begin")
@@ -201,10 +196,14 @@
self.con.close()
def suite():
- default_suite = unittest.makeSuite(TransactionTests, "Check")
- special_command_suite = unittest.makeSuite(SpecialCommandTests, "Check")
- ddl_suite = unittest.makeSuite(TransactionalDDL, "Check")
- return unittest.TestSuite((default_suite, special_command_suite, ddl_suite))
+ tests = [
+ SpecialCommandTests,
+ TransactionTests,
+ TransactionalDDL,
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/sqlite3/test/types.py b/common/py3-stdlib/sqlite3/test/types.py
index d26a9cb..4bb1de8 100644
--- a/common/py3-stdlib/sqlite3/test/types.py
+++ b/common/py3-stdlib/sqlite3/test/types.py
@@ -1,7 +1,6 @@
-#-*- coding: iso-8859-1 -*-
# pysqlite2/test/types.py: tests for type conversion and detection
#
-# Copyright (C) 2005 Gerhard Häring <gh@ghaering.de>
+# Copyright (C) 2005 Gerhard Häring <gh@ghaering.de>
#
# This file is part of pysqlite.
#
@@ -40,33 +39,33 @@
self.cur.close()
self.con.close()
- def CheckString(self):
- self.cur.execute("insert into test(s) values (?)", ("Österreich",))
+ def test_string(self):
+ self.cur.execute("insert into test(s) values (?)", ("Österreich",))
self.cur.execute("select s from test")
row = self.cur.fetchone()
- self.assertEqual(row[0], "Österreich")
+ self.assertEqual(row[0], "Österreich")
- def CheckSmallInt(self):
+ def test_small_int(self):
self.cur.execute("insert into test(i) values (?)", (42,))
self.cur.execute("select i from test")
row = self.cur.fetchone()
self.assertEqual(row[0], 42)
- def CheckLargeInt(self):
+ def test_large_int(self):
num = 2**40
self.cur.execute("insert into test(i) values (?)", (num,))
self.cur.execute("select i from test")
row = self.cur.fetchone()
self.assertEqual(row[0], num)
- def CheckFloat(self):
+ def test_float(self):
val = 3.14
self.cur.execute("insert into test(f) values (?)", (val,))
self.cur.execute("select f from test")
row = self.cur.fetchone()
self.assertEqual(row[0], val)
- def CheckBlob(self):
+ def test_blob(self):
sample = b"Guglhupf"
val = memoryview(sample)
self.cur.execute("insert into test(b) values (?)", (val,))
@@ -74,10 +73,10 @@
row = self.cur.fetchone()
self.assertEqual(row[0], sample)
- def CheckUnicodeExecute(self):
- self.cur.execute("select 'Österreich'")
+ def test_unicode_execute(self):
+ self.cur.execute("select 'Österreich'")
row = self.cur.fetchone()
- self.assertEqual(row[0], "Österreich")
+ self.assertEqual(row[0], "Österreich")
class DeclTypesTests(unittest.TestCase):
class Foo:
@@ -111,7 +110,20 @@
def setUp(self):
self.con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES)
self.cur = self.con.cursor()
- self.cur.execute("create table test(i int, s str, f float, b bool, u unicode, foo foo, bin blob, n1 number, n2 number(5), bad bad)")
+ self.cur.execute("""
+ create table test(
+ i int,
+ s str,
+ f float,
+ b bool,
+ u unicode,
+ foo foo,
+ bin blob,
+ n1 number,
+ n2 number(5),
+ bad bad,
+ cbin cblob)
+ """)
# override float, make them always return the same number
sqlite.converters["FLOAT"] = lambda x: 47.2
@@ -122,6 +134,7 @@
sqlite.converters["BAD"] = DeclTypesTests.BadConform
sqlite.converters["WRONG"] = lambda x: "WRONG"
sqlite.converters["NUMBER"] = float
+ sqlite.converters["CBLOB"] = lambda x: b"blobish"
def tearDown(self):
del sqlite.converters["FLOAT"]
@@ -130,24 +143,25 @@
del sqlite.converters["BAD"]
del sqlite.converters["WRONG"]
del sqlite.converters["NUMBER"]
+ del sqlite.converters["CBLOB"]
self.cur.close()
self.con.close()
- def CheckString(self):
+ def test_string(self):
# default
self.cur.execute("insert into test(s) values (?)", ("foo",))
self.cur.execute('select s as "s [WRONG]" from test')
row = self.cur.fetchone()
self.assertEqual(row[0], "foo")
- def CheckSmallInt(self):
+ def test_small_int(self):
# default
self.cur.execute("insert into test(i) values (?)", (42,))
self.cur.execute("select i from test")
row = self.cur.fetchone()
self.assertEqual(row[0], 42)
- def CheckLargeInt(self):
+ def test_large_int(self):
# default
num = 2**40
self.cur.execute("insert into test(i) values (?)", (num,))
@@ -155,7 +169,7 @@
row = self.cur.fetchone()
self.assertEqual(row[0], num)
- def CheckFloat(self):
+ def test_float(self):
# custom
val = 3.14
self.cur.execute("insert into test(f) values (?)", (val,))
@@ -163,7 +177,7 @@
row = self.cur.fetchone()
self.assertEqual(row[0], 47.2)
- def CheckBool(self):
+ def test_bool(self):
# custom
self.cur.execute("insert into test(b) values (?)", (False,))
self.cur.execute("select b from test")
@@ -176,7 +190,7 @@
row = self.cur.fetchone()
self.assertIs(row[0], True)
- def CheckUnicode(self):
+ def test_unicode(self):
# default
val = "\xd6sterreich"
self.cur.execute("insert into test(u) values (?)", (val,))
@@ -184,14 +198,14 @@
row = self.cur.fetchone()
self.assertEqual(row[0], val)
- def CheckFoo(self):
+ def test_foo(self):
val = DeclTypesTests.Foo("bla")
self.cur.execute("insert into test(foo) values (?)", (val,))
self.cur.execute("select foo from test")
row = self.cur.fetchone()
self.assertEqual(row[0], val)
- def CheckErrorInConform(self):
+ def test_error_in_conform(self):
val = DeclTypesTests.BadConform(TypeError)
with self.assertRaises(sqlite.InterfaceError):
self.cur.execute("insert into test(bad) values (?)", (val,))
@@ -204,19 +218,19 @@
with self.assertRaises(KeyboardInterrupt):
self.cur.execute("insert into test(bad) values (:val)", {"val": val})
- def CheckUnsupportedSeq(self):
+ def test_unsupported_seq(self):
class Bar: pass
val = Bar()
with self.assertRaises(sqlite.InterfaceError):
self.cur.execute("insert into test(f) values (?)", (val,))
- def CheckUnsupportedDict(self):
+ def test_unsupported_dict(self):
class Bar: pass
val = Bar()
with self.assertRaises(sqlite.InterfaceError):
self.cur.execute("insert into test(f) values (:val)", {"val": val})
- def CheckBlob(self):
+ def test_blob(self):
# default
sample = b"Guglhupf"
val = memoryview(sample)
@@ -225,19 +239,27 @@
row = self.cur.fetchone()
self.assertEqual(row[0], sample)
- def CheckNumber1(self):
+ def test_number1(self):
self.cur.execute("insert into test(n1) values (5)")
value = self.cur.execute("select n1 from test").fetchone()[0]
# if the converter is not used, it's an int instead of a float
self.assertEqual(type(value), float)
- def CheckNumber2(self):
+ def test_number2(self):
"""Checks whether converter names are cut off at '(' characters"""
self.cur.execute("insert into test(n2) values (5)")
value = self.cur.execute("select n2 from test").fetchone()[0]
# if the converter is not used, it's an int instead of a float
self.assertEqual(type(value), float)
+ def test_convert_zero_sized_blob(self):
+ self.con.execute("insert into test(cbin) values (?)", (b"",))
+ cur = self.con.execute("select cbin from test")
+ # Zero-sized blobs with converters returns None. This differs from
+ # blobs without a converter, where b"" is returned.
+ self.assertIsNone(cur.fetchone()[0])
+
+
class ColNamesTests(unittest.TestCase):
def setUp(self):
self.con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES)
@@ -257,7 +279,7 @@
self.cur.close()
self.con.close()
- def CheckDeclTypeNotUsed(self):
+ def test_decl_type_not_used(self):
"""
Assures that the declared type is not used when PARSE_DECLTYPES
is not set.
@@ -267,13 +289,13 @@
val = self.cur.fetchone()[0]
self.assertEqual(val, "xxx")
- def CheckNone(self):
+ def test_none(self):
self.cur.execute("insert into test(x) values (?)", (None,))
self.cur.execute("select x from test")
val = self.cur.fetchone()[0]
self.assertEqual(val, None)
- def CheckColName(self):
+ def test_col_name(self):
self.cur.execute("insert into test(x) values (?)", ("xxx",))
self.cur.execute('select x as "x y [bar]" from test')
val = self.cur.fetchone()[0]
@@ -283,12 +305,12 @@
# '[' (and the preceeding space) should be stripped.
self.assertEqual(self.cur.description[0][0], "x y")
- def CheckCaseInConverterName(self):
+ def test_case_in_converter_name(self):
self.cur.execute("select 'other' as \"x [b1b1]\"")
val = self.cur.fetchone()[0]
self.assertEqual(val, "MARKER")
- def CheckCursorDescriptionNoRow(self):
+ def test_cursor_description_no_row(self):
"""
cursor.description should at least provide the column name(s), even if
no row returned.
@@ -296,7 +318,7 @@
self.cur.execute("select * from test where 0 = 1")
self.assertEqual(self.cur.description[0][0], "x")
- def CheckCursorDescriptionInsert(self):
+ def test_cursor_description_insert(self):
self.cur.execute("insert into test values (1)")
self.assertIsNone(self.cur.description)
@@ -313,19 +335,19 @@
self.cur.close()
self.con.close()
- def CheckCursorDescriptionCTESimple(self):
+ def test_cursor_description_cte_simple(self):
self.cur.execute("with one as (select 1) select * from one")
self.assertIsNotNone(self.cur.description)
self.assertEqual(self.cur.description[0][0], "1")
- def CheckCursorDescriptionCTESMultipleColumns(self):
+ def test_cursor_description_cte_multiple_columns(self):
self.cur.execute("insert into test values(1)")
self.cur.execute("insert into test values(2)")
self.cur.execute("with testCTE as (select * from test) select * from testCTE")
self.assertIsNotNone(self.cur.description)
self.assertEqual(self.cur.description[0][0], "x")
- def CheckCursorDescriptionCTE(self):
+ def test_cursor_description_cte(self):
self.cur.execute("insert into test values (1)")
self.cur.execute("with bar as (select * from test) select * from test where x = 1")
self.assertIsNotNone(self.cur.description)
@@ -354,7 +376,7 @@
self.cur.close()
self.con.close()
- def CheckCasterIsUsed(self):
+ def test_caster_is_used(self):
self.cur.execute("select ?", (4,))
val = self.cur.fetchone()[0]
self.assertEqual(type(val), float)
@@ -372,7 +394,7 @@
def tearDown(self):
self.con.close()
- def CheckBinaryInputForConverter(self):
+ def test_binary_input_for_converter(self):
testdata = b"abcdefg" * 10
result = self.con.execute('select ? as "x [bin]"', (memoryview(zlib.compress(testdata)),)).fetchone()[0]
self.assertEqual(testdata, result)
@@ -387,23 +409,21 @@
self.cur.close()
self.con.close()
- def CheckSqliteDate(self):
+ def test_sqlite_date(self):
d = sqlite.Date(2004, 2, 14)
self.cur.execute("insert into test(d) values (?)", (d,))
self.cur.execute("select d from test")
d2 = self.cur.fetchone()[0]
self.assertEqual(d, d2)
- def CheckSqliteTimestamp(self):
+ def test_sqlite_timestamp(self):
ts = sqlite.Timestamp(2004, 2, 14, 7, 15, 0)
self.cur.execute("insert into test(ts) values (?)", (ts,))
self.cur.execute("select ts from test")
ts2 = self.cur.fetchone()[0]
self.assertEqual(ts, ts2)
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 1),
- 'the date functions are available on 3.1 or later')
- def CheckSqlTimestamp(self):
+ def test_sql_timestamp(self):
now = datetime.datetime.utcnow()
self.cur.execute("insert into test(ts) values (current_timestamp)")
self.cur.execute("select ts from test")
@@ -411,14 +431,14 @@
self.assertEqual(type(ts), datetime.datetime)
self.assertEqual(ts.year, now.year)
- def CheckDateTimeSubSeconds(self):
+ def test_date_time_sub_seconds(self):
ts = sqlite.Timestamp(2004, 2, 14, 7, 15, 0, 500000)
self.cur.execute("insert into test(ts) values (?)", (ts,))
self.cur.execute("select ts from test")
ts2 = self.cur.fetchone()[0]
self.assertEqual(ts, ts2)
- def CheckDateTimeSubSecondsFloatingPoint(self):
+ def test_date_time_sub_seconds_floating_point(self):
ts = sqlite.Timestamp(2004, 2, 14, 7, 15, 0, 510241)
self.cur.execute("insert into test(ts) values (?)", (ts,))
self.cur.execute("select ts from test")
@@ -426,14 +446,18 @@
self.assertEqual(ts, ts2)
def suite():
- sqlite_type_suite = unittest.makeSuite(SqliteTypeTests, "Check")
- decltypes_type_suite = unittest.makeSuite(DeclTypesTests, "Check")
- colnames_type_suite = unittest.makeSuite(ColNamesTests, "Check")
- adaptation_suite = unittest.makeSuite(ObjectAdaptationTests, "Check")
- bin_suite = unittest.makeSuite(BinaryConverterTests, "Check")
- date_suite = unittest.makeSuite(DateTimeTests, "Check")
- cte_suite = unittest.makeSuite(CommonTableExpressionTests, "Check")
- return unittest.TestSuite((sqlite_type_suite, decltypes_type_suite, colnames_type_suite, adaptation_suite, bin_suite, date_suite, cte_suite))
+ tests = [
+ BinaryConverterTests,
+ ColNamesTests,
+ CommonTableExpressionTests,
+ DateTimeTests,
+ DeclTypesTests,
+ ObjectAdaptationTests,
+ SqliteTypeTests,
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/sqlite3/test/userfunctions.py b/common/py3-stdlib/sqlite3/test/userfunctions.py
index c11c82e..539fd4b 100644
--- a/common/py3-stdlib/sqlite3/test/userfunctions.py
+++ b/common/py3-stdlib/sqlite3/test/userfunctions.py
@@ -25,8 +25,13 @@
import unittest.mock
import sqlite3 as sqlite
+from test.support import gc_collect
+
+
def func_returntext():
return "foo"
+def func_returntextwithnull():
+ return "1\x002"
def func_returnunicode():
return "bar"
def func_returnint():
@@ -42,22 +47,6 @@
def func_raiseexception():
5/0
-def func_isstring(v):
- return type(v) is str
-def func_isint(v):
- return type(v) is int
-def func_isfloat(v):
- return type(v) is float
-def func_isnone(v):
- return type(v) is type(None)
-def func_isblob(v):
- return isinstance(v, (bytes, memoryview))
-def func_islonglong(v):
- return isinstance(v, int) and v >= 1<<31
-
-def func(*args):
- return len(args)
-
class AggrNoStep:
def __init__(self):
pass
@@ -137,36 +126,44 @@
def finalize(self):
return self.val
+class AggrText:
+ def __init__(self):
+ self.txt = ""
+ def step(self, txt):
+ self.txt = self.txt + txt
+ def finalize(self):
+ return self.txt
+
+
class FunctionTests(unittest.TestCase):
def setUp(self):
self.con = sqlite.connect(":memory:")
self.con.create_function("returntext", 0, func_returntext)
+ self.con.create_function("returntextwithnull", 0, func_returntextwithnull)
self.con.create_function("returnunicode", 0, func_returnunicode)
self.con.create_function("returnint", 0, func_returnint)
self.con.create_function("returnfloat", 0, func_returnfloat)
self.con.create_function("returnnull", 0, func_returnnull)
self.con.create_function("returnblob", 0, func_returnblob)
self.con.create_function("returnlonglong", 0, func_returnlonglong)
+ self.con.create_function("returnnan", 0, lambda: float("nan"))
+ self.con.create_function("returntoolargeint", 0, lambda: 1 << 65)
self.con.create_function("raiseexception", 0, func_raiseexception)
- self.con.create_function("isstring", 1, func_isstring)
- self.con.create_function("isint", 1, func_isint)
- self.con.create_function("isfloat", 1, func_isfloat)
- self.con.create_function("isnone", 1, func_isnone)
- self.con.create_function("isblob", 1, func_isblob)
- self.con.create_function("islonglong", 1, func_islonglong)
- self.con.create_function("spam", -1, func)
+ self.con.create_function("isblob", 1, lambda x: isinstance(x, bytes))
+ self.con.create_function("isnone", 1, lambda x: x is None)
+ self.con.create_function("spam", -1, lambda *x: len(x))
self.con.execute("create table test(t text)")
def tearDown(self):
self.con.close()
- def CheckFuncErrorOnCreate(self):
+ def test_func_error_on_create(self):
with self.assertRaises(sqlite.OperationalError):
self.con.create_function("bla", -100, lambda x: 2*x)
- def CheckFuncRefCount(self):
+ def test_func_ref_count(self):
def getfunc():
def f():
return 1
@@ -178,28 +175,34 @@
cur = self.con.cursor()
cur.execute("select reftest()")
- def CheckFuncReturnText(self):
+ def test_func_return_text(self):
cur = self.con.cursor()
cur.execute("select returntext()")
val = cur.fetchone()[0]
self.assertEqual(type(val), str)
self.assertEqual(val, "foo")
- def CheckFuncReturnUnicode(self):
+ def test_func_return_text_with_null_char(self):
+ cur = self.con.cursor()
+ res = cur.execute("select returntextwithnull()").fetchone()[0]
+ self.assertEqual(type(res), str)
+ self.assertEqual(res, "1\x002")
+
+ def test_func_return_unicode(self):
cur = self.con.cursor()
cur.execute("select returnunicode()")
val = cur.fetchone()[0]
self.assertEqual(type(val), str)
self.assertEqual(val, "bar")
- def CheckFuncReturnInt(self):
+ def test_func_return_int(self):
cur = self.con.cursor()
cur.execute("select returnint()")
val = cur.fetchone()[0]
self.assertEqual(type(val), int)
self.assertEqual(val, 42)
- def CheckFuncReturnFloat(self):
+ def test_func_return_float(self):
cur = self.con.cursor()
cur.execute("select returnfloat()")
val = cur.fetchone()[0]
@@ -207,75 +210,99 @@
if val < 3.139 or val > 3.141:
self.fail("wrong value")
- def CheckFuncReturnNull(self):
+ def test_func_return_null(self):
cur = self.con.cursor()
cur.execute("select returnnull()")
val = cur.fetchone()[0]
self.assertEqual(type(val), type(None))
self.assertEqual(val, None)
- def CheckFuncReturnBlob(self):
+ def test_func_return_blob(self):
cur = self.con.cursor()
cur.execute("select returnblob()")
val = cur.fetchone()[0]
self.assertEqual(type(val), bytes)
self.assertEqual(val, b"blob")
- def CheckFuncReturnLongLong(self):
+ def test_func_return_long_long(self):
cur = self.con.cursor()
cur.execute("select returnlonglong()")
val = cur.fetchone()[0]
self.assertEqual(val, 1<<31)
- def CheckFuncException(self):
+ def test_func_return_nan(self):
+ cur = self.con.cursor()
+ cur.execute("select returnnan()")
+ self.assertIsNone(cur.fetchone()[0])
+
+ def test_func_return_too_large_int(self):
+ cur = self.con.cursor()
+ with self.assertRaises(sqlite.OperationalError):
+ self.con.execute("select returntoolargeint()")
+
+ def test_func_exception(self):
cur = self.con.cursor()
with self.assertRaises(sqlite.OperationalError) as cm:
cur.execute("select raiseexception()")
cur.fetchone()
self.assertEqual(str(cm.exception), 'user-defined function raised exception')
- def CheckParamString(self):
- cur = self.con.cursor()
- cur.execute("select isstring(?)", ("foo",))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def CheckParamInt(self):
- cur = self.con.cursor()
- cur.execute("select isint(?)", (42,))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def CheckParamFloat(self):
- cur = self.con.cursor()
- cur.execute("select isfloat(?)", (3.14,))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def CheckParamNone(self):
- cur = self.con.cursor()
- cur.execute("select isnone(?)", (None,))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def CheckParamBlob(self):
- cur = self.con.cursor()
- cur.execute("select isblob(?)", (memoryview(b"blob"),))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def CheckParamLongLong(self):
- cur = self.con.cursor()
- cur.execute("select islonglong(?)", (1<<42,))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def CheckAnyArguments(self):
+ def test_any_arguments(self):
cur = self.con.cursor()
cur.execute("select spam(?, ?)", (1, 2))
val = cur.fetchone()[0]
self.assertEqual(val, 2)
+ def test_empty_blob(self):
+ cur = self.con.execute("select isblob(x'')")
+ self.assertTrue(cur.fetchone()[0])
+
+ def test_nan_float(self):
+ cur = self.con.execute("select isnone(?)", (float("nan"),))
+ # SQLite has no concept of nan; it is converted to NULL
+ self.assertTrue(cur.fetchone()[0])
+
+ def test_too_large_int(self):
+ err = "Python int too large to convert to SQLite INTEGER"
+ self.assertRaisesRegex(OverflowError, err, self.con.execute,
+ "select spam(?)", (1 << 65,))
+
+ def test_non_contiguous_blob(self):
+ self.assertRaisesRegex(ValueError, "could not convert BLOB to buffer",
+ self.con.execute, "select spam(?)",
+ (memoryview(b"blob")[::2],))
+
+ def test_param_surrogates(self):
+ self.assertRaisesRegex(UnicodeEncodeError, "surrogates not allowed",
+ self.con.execute, "select spam(?)",
+ ("\ud803\ude6d",))
+
+ def test_func_params(self):
+ results = []
+ def append_result(arg):
+ results.append((arg, type(arg)))
+ self.con.create_function("test_params", 1, append_result)
+
+ dataset = [
+ (42, int),
+ (-1, int),
+ (1234567890123456789, int),
+ (4611686018427387905, int), # 63-bit int with non-zero low bits
+ (3.14, float),
+ (float('inf'), float),
+ ("text", str),
+ ("1\x002", str),
+ ("\u02e2q\u02e1\u2071\u1d57\u1d49", str),
+ (b"blob", bytes),
+ (bytearray(range(2)), bytes),
+ (memoryview(b"blob"), bytes),
+ (None, type(None)),
+ ]
+ for val, _ in dataset:
+ cur = self.con.execute("select test_params(?)", (val,))
+ cur.fetchone()
+ self.assertEqual(dataset, results)
+
# Regarding deterministic functions:
#
# Between 3.8.3 and 3.15.0, deterministic functions were only used to
@@ -284,7 +311,7 @@
# deterministic functions were permitted in WHERE clauses of partial
# indices, which allows testing based on syntax, iso. the query optimizer.
@unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher")
- def CheckFuncNonDeterministic(self):
+ def test_func_non_deterministic(self):
mock = unittest.mock.Mock(return_value=None)
self.con.create_function("nondeterministic", 0, mock, deterministic=False)
if sqlite.sqlite_version_info < (3, 15, 0):
@@ -295,7 +322,7 @@
self.con.execute("create index t on test(t) where nondeterministic() is not null")
@unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher")
- def CheckFuncDeterministic(self):
+ def test_func_deterministic(self):
mock = unittest.mock.Mock(return_value=None)
self.con.create_function("deterministic", 0, mock, deterministic=True)
if sqlite.sqlite_version_info < (3, 15, 0):
@@ -308,14 +335,30 @@
self.fail("Unexpected failure while creating partial index")
@unittest.skipIf(sqlite.sqlite_version_info >= (3, 8, 3), "SQLite < 3.8.3 needed")
- def CheckFuncDeterministicNotSupported(self):
+ def test_func_deterministic_not_supported(self):
with self.assertRaises(sqlite.NotSupportedError):
self.con.create_function("deterministic", 0, int, deterministic=True)
- def CheckFuncDeterministicKeywordOnly(self):
+ def test_func_deterministic_keyword_only(self):
with self.assertRaises(TypeError):
self.con.create_function("deterministic", 0, int, True)
+ def test_function_destructor_via_gc(self):
+ # See bpo-44304: The destructor of the user function can
+ # crash if is called without the GIL from the gc functions
+ dest = sqlite.connect(':memory:')
+ def md5sum(t):
+ return
+
+ dest.create_function("md5", 1, md5sum)
+ x = dest("create table lang (name, first_appeared)")
+ del md5sum, dest
+
+ y = [x]
+ y.append(y)
+
+ del x,y
+ gc_collect()
class AggregateTests(unittest.TestCase):
def setUp(self):
@@ -341,87 +384,88 @@
self.con.create_aggregate("checkType", 2, AggrCheckType)
self.con.create_aggregate("checkTypes", -1, AggrCheckTypes)
self.con.create_aggregate("mysum", 1, AggrSum)
+ self.con.create_aggregate("aggtxt", 1, AggrText)
def tearDown(self):
#self.cur.close()
#self.con.close()
pass
- def CheckAggrErrorOnCreate(self):
+ def test_aggr_error_on_create(self):
with self.assertRaises(sqlite.OperationalError):
self.con.create_function("bla", -100, AggrSum)
- def CheckAggrNoStep(self):
+ def test_aggr_no_step(self):
cur = self.con.cursor()
with self.assertRaises(AttributeError) as cm:
cur.execute("select nostep(t) from test")
self.assertEqual(str(cm.exception), "'AggrNoStep' object has no attribute 'step'")
- def CheckAggrNoFinalize(self):
+ def test_aggr_no_finalize(self):
cur = self.con.cursor()
with self.assertRaises(sqlite.OperationalError) as cm:
cur.execute("select nofinalize(t) from test")
val = cur.fetchone()[0]
self.assertEqual(str(cm.exception), "user-defined aggregate's 'finalize' method raised error")
- def CheckAggrExceptionInInit(self):
+ def test_aggr_exception_in_init(self):
cur = self.con.cursor()
with self.assertRaises(sqlite.OperationalError) as cm:
cur.execute("select excInit(t) from test")
val = cur.fetchone()[0]
self.assertEqual(str(cm.exception), "user-defined aggregate's '__init__' method raised error")
- def CheckAggrExceptionInStep(self):
+ def test_aggr_exception_in_step(self):
cur = self.con.cursor()
with self.assertRaises(sqlite.OperationalError) as cm:
cur.execute("select excStep(t) from test")
val = cur.fetchone()[0]
self.assertEqual(str(cm.exception), "user-defined aggregate's 'step' method raised error")
- def CheckAggrExceptionInFinalize(self):
+ def test_aggr_exception_in_finalize(self):
cur = self.con.cursor()
with self.assertRaises(sqlite.OperationalError) as cm:
cur.execute("select excFinalize(t) from test")
val = cur.fetchone()[0]
self.assertEqual(str(cm.exception), "user-defined aggregate's 'finalize' method raised error")
- def CheckAggrCheckParamStr(self):
+ def test_aggr_check_param_str(self):
cur = self.con.cursor()
- cur.execute("select checkType('str', ?)", ("foo",))
+ cur.execute("select checkTypes('str', ?, ?)", ("foo", str()))
val = cur.fetchone()[0]
- self.assertEqual(val, 1)
+ self.assertEqual(val, 2)
- def CheckAggrCheckParamInt(self):
+ def test_aggr_check_param_int(self):
cur = self.con.cursor()
cur.execute("select checkType('int', ?)", (42,))
val = cur.fetchone()[0]
self.assertEqual(val, 1)
- def CheckAggrCheckParamsInt(self):
+ def test_aggr_check_params_int(self):
cur = self.con.cursor()
cur.execute("select checkTypes('int', ?, ?)", (42, 24))
val = cur.fetchone()[0]
self.assertEqual(val, 2)
- def CheckAggrCheckParamFloat(self):
+ def test_aggr_check_param_float(self):
cur = self.con.cursor()
cur.execute("select checkType('float', ?)", (3.14,))
val = cur.fetchone()[0]
self.assertEqual(val, 1)
- def CheckAggrCheckParamNone(self):
+ def test_aggr_check_param_none(self):
cur = self.con.cursor()
cur.execute("select checkType('None', ?)", (None,))
val = cur.fetchone()[0]
self.assertEqual(val, 1)
- def CheckAggrCheckParamBlob(self):
+ def test_aggr_check_param_blob(self):
cur = self.con.cursor()
cur.execute("select checkType('blob', ?)", (memoryview(b"blob"),))
val = cur.fetchone()[0]
self.assertEqual(val, 1)
- def CheckAggrCheckAggrSum(self):
+ def test_aggr_check_aggr_sum(self):
cur = self.con.cursor()
cur.execute("delete from test")
cur.executemany("insert into test(i) values (?)", [(10,), (20,), (30,)])
@@ -429,6 +473,20 @@
val = cur.fetchone()[0]
self.assertEqual(val, 60)
+ def test_aggr_no_match(self):
+ cur = self.con.execute("select mysum(i) from (select 1 as i) where i == 0")
+ val = cur.fetchone()[0]
+ self.assertIsNone(val)
+
+ def test_aggr_text(self):
+ cur = self.con.cursor()
+ for txt in ["foo", "1\x002"]:
+ with self.subTest(txt=txt):
+ cur.execute("select aggtxt(?) from test", (txt,))
+ val = cur.fetchone()[0]
+ self.assertEqual(val, txt)
+
+
class AuthorizerTests(unittest.TestCase):
@staticmethod
def authorizer_cb(action, arg1, arg2, dbname, source):
@@ -494,17 +552,17 @@
def suite():
- function_suite = unittest.makeSuite(FunctionTests, "Check")
- aggregate_suite = unittest.makeSuite(AggregateTests, "Check")
- authorizer_suite = unittest.makeSuite(AuthorizerTests)
- return unittest.TestSuite((
- function_suite,
- aggregate_suite,
- authorizer_suite,
- unittest.makeSuite(AuthorizerRaiseExceptionTests),
- unittest.makeSuite(AuthorizerIllegalTypeTests),
- unittest.makeSuite(AuthorizerLargeIntegerTests),
- ))
+ tests = [
+ AggregateTests,
+ AuthorizerIllegalTypeTests,
+ AuthorizerLargeIntegerTests,
+ AuthorizerRaiseExceptionTests,
+ AuthorizerTests,
+ FunctionTests,
+ ]
+ return unittest.TestSuite(
+ [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
+ )
def test():
runner = unittest.TextTestRunner()
diff --git a/common/py3-stdlib/ssl.py b/common/py3-stdlib/ssl.py
index 30f4e59..181065d 100644
--- a/common/py3-stdlib/ssl.py
+++ b/common/py3-stdlib/ssl.py
@@ -253,7 +253,7 @@
from _ssl import enum_certificates, enum_crls
from socket import socket, SOCK_STREAM, create_connection
-from socket import SOL_SOCKET, SO_TYPE
+from socket import SOL_SOCKET, SO_TYPE, _GLOBAL_DEFAULT_TIMEOUT
import socket as _socket
import base64 # for DER-to-PEM translation
import errno
@@ -381,6 +381,11 @@
CertificateError is raised on failure. On success, the function
returns nothing.
"""
+ warnings.warn(
+ "ssl.match_hostname() is deprecated",
+ category=DeprecationWarning,
+ stacklevel=2
+ )
if not cert:
raise ValueError("empty or no certificate, match_hostname needs a "
"SSL socket or SSL context with either "
@@ -479,7 +484,14 @@
sslsocket_class = None # SSLSocket is assigned later.
sslobject_class = None # SSLObject is assigned later.
- def __new__(cls, protocol=PROTOCOL_TLS, *args, **kwargs):
+ def __new__(cls, protocol=None, *args, **kwargs):
+ if protocol is None:
+ warnings.warn(
+ "ssl.SSLContext() without protocol argument is deprecated.",
+ category=DeprecationWarning,
+ stacklevel=2
+ )
+ protocol = PROTOCOL_TLS
self = _SSLContext.__new__(cls, protocol)
return self
@@ -518,6 +530,11 @@
)
def set_npn_protocols(self, npn_protocols):
+ warnings.warn(
+ "ssl NPN is deprecated, use ALPN instead",
+ DeprecationWarning,
+ stacklevel=2
+ )
protos = bytearray()
for protocol in npn_protocols:
b = bytes(protocol, 'ascii')
@@ -734,12 +751,15 @@
# SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION,
# OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE
# by default.
- context = SSLContext(PROTOCOL_TLS)
-
if purpose == Purpose.SERVER_AUTH:
# verify certs and host name in client mode
+ context = SSLContext(PROTOCOL_TLS_CLIENT)
context.verify_mode = CERT_REQUIRED
context.check_hostname = True
+ elif purpose == Purpose.CLIENT_AUTH:
+ context = SSLContext(PROTOCOL_TLS_SERVER)
+ else:
+ raise ValueError(purpose)
if cafile or capath or cadata:
context.load_verify_locations(cafile, capath, cadata)
@@ -755,7 +775,7 @@
context.keylog_filename = keylogfile
return context
-def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE,
+def _create_unverified_context(protocol=None, *, cert_reqs=CERT_NONE,
check_hostname=False, purpose=Purpose.SERVER_AUTH,
certfile=None, keyfile=None,
cafile=None, capath=None, cadata=None):
@@ -772,10 +792,18 @@
# SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION,
# OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE
# by default.
- context = SSLContext(protocol)
+ if purpose == Purpose.SERVER_AUTH:
+ # verify certs and host name in client mode
+ if protocol is None:
+ protocol = PROTOCOL_TLS_CLIENT
+ elif purpose == Purpose.CLIENT_AUTH:
+ if protocol is None:
+ protocol = PROTOCOL_TLS_SERVER
+ else:
+ raise ValueError(purpose)
- if not check_hostname:
- context.check_hostname = False
+ context = SSLContext(protocol)
+ context.check_hostname = check_hostname
if cert_reqs is not None:
context.verify_mode = cert_reqs
if check_hostname:
@@ -909,15 +937,17 @@
"""Return the currently selected NPN protocol as a string, or ``None``
if a next protocol was not negotiated or if NPN is not supported by one
of the peers."""
- if _ssl.HAS_NPN:
- return self._sslobj.selected_npn_protocol()
+ warnings.warn(
+ "ssl NPN is deprecated, use ALPN instead",
+ DeprecationWarning,
+ stacklevel=2
+ )
def selected_alpn_protocol(self):
"""Return the currently selected ALPN protocol as a string, or ``None``
if a next protocol was not negotiated or if ALPN is not supported by one
of the peers."""
- if _ssl.HAS_ALPN:
- return self._sslobj.selected_alpn_protocol()
+ return self._sslobj.selected_alpn_protocol()
def cipher(self):
"""Return the currently selected cipher as a 3-tuple ``(name,
@@ -1126,10 +1156,12 @@
@_sslcopydoc
def selected_npn_protocol(self):
self._checkClosed()
- if self._sslobj is None or not _ssl.HAS_NPN:
- return None
- else:
- return self._sslobj.selected_npn_protocol()
+ warnings.warn(
+ "ssl NPN is deprecated, use ALPN instead",
+ DeprecationWarning,
+ stacklevel=2
+ )
+ return None
@_sslcopydoc
def selected_alpn_protocol(self):
@@ -1388,7 +1420,11 @@
do_handshake_on_connect=True,
suppress_ragged_eofs=True,
ciphers=None):
-
+ warnings.warn(
+ "ssl.wrap_socket() is deprecated, use SSLContext.wrap_socket()",
+ category=DeprecationWarning,
+ stacklevel=2
+ )
if server_side and not certfile:
raise ValueError("certfile must be specified for server-side "
"operations")
@@ -1466,11 +1502,14 @@
d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)]
return base64.decodebytes(d.encode('ASCII', 'strict'))
-def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None):
+def get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT,
+ ca_certs=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
"""Retrieve the certificate from the server at the specified address,
and return it as a PEM-encoded string.
If 'ca_certs' is specified, validate the server cert against it.
- If 'ssl_version' is specified, use it in the connection attempt."""
+ If 'ssl_version' is specified, use it in the connection attempt.
+ If 'timeout' is specified, use it in the connection attempt.
+ """
host, port = addr
if ca_certs is not None:
@@ -1480,8 +1519,8 @@
context = _create_stdlib_context(ssl_version,
cert_reqs=cert_reqs,
cafile=ca_certs)
- with create_connection(addr) as sock:
- with context.wrap_socket(sock) as sslsock:
+ with create_connection(addr, timeout=timeout) as sock:
+ with context.wrap_socket(sock, server_hostname=host) as sslsock:
dercert = sslsock.getpeercert(True)
return DER_cert_to_PEM_cert(dercert)
diff --git a/common/py3-stdlib/statistics.py b/common/py3-stdlib/statistics.py
index f9d3802..f662453 100644
--- a/common/py3-stdlib/statistics.py
+++ b/common/py3-stdlib/statistics.py
@@ -73,6 +73,30 @@
2.5
+Statistics for relations between two inputs
+-------------------------------------------
+
+================== ====================================================
+Function Description
+================== ====================================================
+covariance Sample covariance for two variables.
+correlation Pearson's correlation coefficient for two variables.
+linear_regression Intercept and slope for simple linear regression.
+================== ====================================================
+
+Calculate covariance, Pearson's correlation, and simple linear regression
+for two inputs:
+
+>>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+>>> y = [1, 2, 3, 1, 2, 3, 1, 2, 3]
+>>> covariance(x, y)
+0.75
+>>> correlation(x, y) #doctest: +ELLIPSIS
+0.31622776601...
+>>> linear_regression(x, y) #doctest:
+LinearRegression(slope=0.1, intercept=1.5)
+
+
Exceptions
----------
@@ -83,9 +107,12 @@
__all__ = [
'NormalDist',
'StatisticsError',
+ 'correlation',
+ 'covariance',
'fmean',
'geometric_mean',
'harmonic_mean',
+ 'linear_regression',
'mean',
'median',
'median_grouped',
@@ -106,11 +133,11 @@
from fractions import Fraction
from decimal import Decimal
-from itertools import groupby
+from itertools import groupby, repeat
from bisect import bisect_left, bisect_right
from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum
from operator import itemgetter
-from collections import Counter
+from collections import Counter, namedtuple
# === Exceptions ===
@@ -120,21 +147,17 @@
# === Private utilities ===
-def _sum(data, start=0):
- """_sum(data [, start]) -> (type, sum, count)
+def _sum(data):
+ """_sum(data) -> (type, sum, count)
Return a high-precision sum of the given numeric data as a fraction,
together with the type to be converted to and the count of items.
- If optional argument ``start`` is given, it is added to the total.
- If ``data`` is empty, ``start`` (defaulting to 0) is returned.
-
-
Examples
--------
- >>> _sum([3, 2.25, 4.5, -0.5, 1.0], 0.75)
- (<class 'float'>, Fraction(11, 1), 5)
+ >>> _sum([3, 2.25, 4.5, -0.5, 0.25])
+ (<class 'float'>, Fraction(19, 2), 5)
Some sources of round-off error will be avoided:
@@ -157,10 +180,9 @@
allowed.
"""
count = 0
- n, d = _exact_ratio(start)
- partials = {d: n}
+ partials = {}
partials_get = partials.get
- T = _coerce(int, type(start))
+ T = int
for typ, values in groupby(data, type):
T = _coerce(T, typ) # or raise TypeError
for n, d in map(_exact_ratio, values):
@@ -173,8 +195,7 @@
assert not _isfinite(total)
else:
# Sum all the partial sums using builtin sum.
- # FIXME is this faster if we sum them in order of the denominator?
- total = sum(Fraction(n, d) for d, n in sorted(partials.items()))
+ total = sum(Fraction(n, d) for d, n in partials.items())
return (T, total, count)
@@ -225,27 +246,19 @@
x is expected to be an int, Fraction, Decimal or float.
"""
try:
- # Optimise the common case of floats. We expect that the most often
- # used numeric type will be builtin floats, so try to make this as
- # fast as possible.
- if type(x) is float or type(x) is Decimal:
- return x.as_integer_ratio()
- try:
- # x may be an int, Fraction, or Integral ABC.
- return (x.numerator, x.denominator)
- except AttributeError:
- try:
- # x may be a float or Decimal subclass.
- return x.as_integer_ratio()
- except AttributeError:
- # Just give up?
- pass
+ return x.as_integer_ratio()
+ except AttributeError:
+ pass
except (OverflowError, ValueError):
# float NAN or INF.
assert not _isfinite(x)
return (x, None)
- msg = "can't convert type '{}' to numerator/denominator"
- raise TypeError(msg.format(type(x).__name__))
+ try:
+ # x may be an Integral ABC.
+ return (x.numerator, x.denominator)
+ except AttributeError:
+ msg = f"can't convert type '{type(x).__name__}' to numerator/denominator"
+ raise TypeError(msg)
def _convert(value, T):
@@ -361,40 +374,39 @@
return exp(fmean(map(log, data)))
except ValueError:
raise StatisticsError('geometric mean requires a non-empty dataset '
- ' containing positive numbers') from None
+ 'containing positive numbers') from None
-def harmonic_mean(data):
+def harmonic_mean(data, weights=None):
"""Return the harmonic mean of data.
- The harmonic mean, sometimes called the subcontrary mean, is the
- reciprocal of the arithmetic mean of the reciprocals of the data,
- and is often appropriate when averaging quantities which are rates
- or ratios, for example speeds. Example:
+ The harmonic mean is the reciprocal of the arithmetic mean of the
+ reciprocals of the data. It can be used for averaging ratios or
+ rates, for example speeds.
- Suppose an investor purchases an equal value of shares in each of
- three companies, with P/E (price/earning) ratios of 2.5, 3 and 10.
- What is the average P/E ratio for the investor's portfolio?
+ Suppose a car travels 40 km/hr for 5 km and then speeds-up to
+ 60 km/hr for another 5 km. What is the average speed?
- >>> harmonic_mean([2.5, 3, 10]) # For an equal investment portfolio.
- 3.6
+ >>> harmonic_mean([40, 60])
+ 48.0
- Using the arithmetic mean would give an average of about 5.167, which
- is too high.
+ Suppose a car travels 40 km/hr for 5 km, and when traffic clears,
+ speeds-up to 60 km/hr for the remaining 30 km of the journey. What
+ is the average speed?
+
+ >>> harmonic_mean([40, 60], weights=[5, 30])
+ 56.0
If ``data`` is empty, or any element is less than zero,
``harmonic_mean`` will raise ``StatisticsError``.
"""
- # For a justification for using harmonic mean for P/E ratios, see
- # http://fixthepitch.pellucid.com/comps-analysis-the-missing-harmony-of-summary-statistics/
- # http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2621087
if iter(data) is data:
data = list(data)
errmsg = 'harmonic mean does not support negative values'
n = len(data)
if n < 1:
raise StatisticsError('harmonic_mean requires at least one data point')
- elif n == 1:
+ elif n == 1 and weights is None:
x = data[0]
if isinstance(x, (numbers.Real, Decimal)):
if x < 0:
@@ -402,13 +414,23 @@
return x
else:
raise TypeError('unsupported type')
+ if weights is None:
+ weights = repeat(1, n)
+ sum_weights = n
+ else:
+ if iter(weights) is weights:
+ weights = list(weights)
+ if len(weights) != n:
+ raise StatisticsError('Number of weights does not match data size')
+ _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg))
try:
- T, total, count = _sum(1 / x for x in _fail_neg(data, errmsg))
+ data = _fail_neg(data, errmsg)
+ T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data))
except ZeroDivisionError:
return 0
- assert count == n
- return _convert(n / total, T)
-
+ if total <= 0:
+ raise StatisticsError('Weighted sum must be positive')
+ return _convert(sum_weights / total, T)
# FIXME: investigate ways to calculate medians without sorting? Quickselect?
def median(data):
@@ -683,14 +705,20 @@
if c is not None:
T, total, count = _sum((x-c)**2 for x in data)
return (T, total)
- c = mean(data)
- T, total, count = _sum((x-c)**2 for x in data)
- # The following sum should mathematically equal zero, but due to rounding
- # error may not.
- U, total2, count2 = _sum((x - c) for x in data)
- assert T == U and count == count2
- total -= total2 ** 2 / len(data)
- assert not total < 0, 'negative sum of square deviations: %f' % total
+ T, total, count = _sum(data)
+ mean_n, mean_d = (total / count).as_integer_ratio()
+ partials = Counter()
+ for n, d in map(_exact_ratio, data):
+ diff_n = n * mean_d - d * mean_n
+ diff_d = d * mean_d
+ partials[diff_d * diff_d] += diff_n * diff_n
+ if None in partials:
+ # The sum will be a NAN or INF. We can ignore all the finite
+ # partials, and just look at this special one.
+ total = partials[None]
+ assert not _isfinite(total)
+ else:
+ total = sum(Fraction(n, d) for d, n in partials.items())
return (T, total)
@@ -794,6 +822,9 @@
1.0810874155219827
"""
+ # Fixme: Despite the exact sum of squared deviations, some inaccuracy
+ # remain because there are two rounding steps. The first occurs in
+ # the _convert() step for variance(), the second occurs in math.sqrt().
var = variance(data, xbar)
try:
return var.sqrt()
@@ -810,6 +841,9 @@
0.986893273527251
"""
+ # Fixme: Despite the exact sum of squared deviations, some inaccuracy
+ # remain because there are two rounding steps. The first occurs in
+ # the _convert() step for pvariance(), the second occurs in math.sqrt().
var = pvariance(data, mu)
try:
return var.sqrt()
@@ -817,6 +851,119 @@
return math.sqrt(var)
+# === Statistics for relations between two inputs ===
+
+# See https://en.wikipedia.org/wiki/Covariance
+# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient
+# https://en.wikipedia.org/wiki/Simple_linear_regression
+
+
+def covariance(x, y, /):
+ """Covariance
+
+ Return the sample covariance of two inputs *x* and *y*. Covariance
+ is a measure of the joint variability of two inputs.
+
+ >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+ >>> y = [1, 2, 3, 1, 2, 3, 1, 2, 3]
+ >>> covariance(x, y)
+ 0.75
+ >>> z = [9, 8, 7, 6, 5, 4, 3, 2, 1]
+ >>> covariance(x, z)
+ -7.5
+ >>> covariance(z, x)
+ -7.5
+
+ """
+ n = len(x)
+ if len(y) != n:
+ raise StatisticsError('covariance requires that both inputs have same number of data points')
+ if n < 2:
+ raise StatisticsError('covariance requires at least two data points')
+ xbar = fsum(x) / n
+ ybar = fsum(y) / n
+ sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))
+ return sxy / (n - 1)
+
+
+def correlation(x, y, /):
+ """Pearson's correlation coefficient
+
+ Return the Pearson's correlation coefficient for two inputs. Pearson's
+ correlation coefficient *r* takes values between -1 and +1. It measures the
+ strength and direction of the linear relationship, where +1 means very
+ strong, positive linear relationship, -1 very strong, negative linear
+ relationship, and 0 no linear relationship.
+
+ >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+ >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1]
+ >>> correlation(x, x)
+ 1.0
+ >>> correlation(x, y)
+ -1.0
+
+ """
+ n = len(x)
+ if len(y) != n:
+ raise StatisticsError('correlation requires that both inputs have same number of data points')
+ if n < 2:
+ raise StatisticsError('correlation requires at least two data points')
+ xbar = fsum(x) / n
+ ybar = fsum(y) / n
+ sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))
+ sxx = fsum((xi - xbar) ** 2.0 for xi in x)
+ syy = fsum((yi - ybar) ** 2.0 for yi in y)
+ try:
+ return sxy / sqrt(sxx * syy)
+ except ZeroDivisionError:
+ raise StatisticsError('at least one of the inputs is constant')
+
+
+LinearRegression = namedtuple('LinearRegression', ('slope', 'intercept'))
+
+
+def linear_regression(x, y, /):
+ """Slope and intercept for simple linear regression.
+
+ Return the slope and intercept of simple linear regression
+ parameters estimated using ordinary least squares. Simple linear
+ regression describes relationship between an independent variable
+ *x* and a dependent variable *y* in terms of linear function:
+
+ y = slope * x + intercept + noise
+
+ where *slope* and *intercept* are the regression parameters that are
+ estimated, and noise represents the variability of the data that was
+ not explained by the linear regression (it is equal to the
+ difference between predicted and actual values of the dependent
+ variable).
+
+ The parameters are returned as a named tuple.
+
+ >>> x = [1, 2, 3, 4, 5]
+ >>> noise = NormalDist().samples(5, seed=42)
+ >>> y = [3 * x[i] + 2 + noise[i] for i in range(5)]
+ >>> linear_regression(x, y) #doctest: +ELLIPSIS
+ LinearRegression(slope=3.09078914170..., intercept=1.75684970486...)
+
+ """
+ n = len(x)
+ if len(y) != n:
+ raise StatisticsError('linear regression requires that both inputs have same number of data points')
+ if n < 2:
+ raise StatisticsError('linear regression requires at least two data points')
+ xbar = fsum(x) / n
+ ybar = fsum(y) / n
+ sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))
+ sxx = fsum((xi - xbar) ** 2.0 for xi in x)
+ try:
+ slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x)
+ except ZeroDivisionError:
+ raise StatisticsError('x is constant')
+ intercept = ybar - slope * xbar
+ return LinearRegression(slope=slope, intercept=intercept)
+
+
## Normal Distribution #####################################################
diff --git a/common/py3-stdlib/subprocess.py b/common/py3-stdlib/subprocess.py
index f1d829a..ccb46a6 100644
--- a/common/py3-stdlib/subprocess.py
+++ b/common/py3-stdlib/subprocess.py
@@ -5,7 +5,6 @@
# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
#
# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/2.4/license for licensing details.
r"""Subprocesses with accessible I/O streams
@@ -55,13 +54,10 @@
import types
try:
- import pwd
+ import fcntl
except ImportError:
- pwd = None
-try:
- import grp
-except ImportError:
- grp = None
+ fcntl = None
+
__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
"getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
@@ -326,7 +322,7 @@
if dev_mode:
args.extend(('-X', 'dev'))
for opt in ('faulthandler', 'tracemalloc', 'importtime',
- 'showrefcount', 'utf8', 'oldparser'):
+ 'showrefcount', 'utf8'):
if opt in xoptions:
value = xoptions[opt]
if value is True:
@@ -415,7 +411,11 @@
if 'input' in kwargs and kwargs['input'] is None:
# Explicitly passing input=None was previously equivalent to passing an
# empty string. That is maintained here for backwards compatibility.
- kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
+ if kwargs.get('universal_newlines') or kwargs.get('text'):
+ empty = ''
+ else:
+ empty = b''
+ kwargs['input'] = empty
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
**kwargs).stdout
@@ -660,8 +660,9 @@
# os.posix_spawn() is not available
return False
- if sys.platform == 'darwin':
- # posix_spawn() is a syscall on macOS and properly reports errors
+ if sys.platform in ('darwin', 'sunos5'):
+ # posix_spawn() is a syscall on both macOS and Solaris,
+ # and properly reports errors
return True
# Check libc name and runtime libc version
@@ -693,7 +694,7 @@
_USE_POSIX_SPAWN = _use_posix_spawn()
-class Popen(object):
+class Popen:
""" Execute a child program in a new process.
For a complete description of the arguments see the Python documentation.
@@ -756,7 +757,7 @@
startupinfo=None, creationflags=0,
restore_signals=True, start_new_session=False,
pass_fds=(), *, user=None, group=None, extra_groups=None,
- encoding=None, errors=None, text=None, umask=-1):
+ encoding=None, errors=None, text=None, umask=-1, pipesize=-1):
"""Create new Popen instance."""
_cleanup()
# Held while anything is calling waitpid before returncode has been
@@ -773,6 +774,11 @@
if not isinstance(bufsize, int):
raise TypeError("bufsize must be an integer")
+ if pipesize is None:
+ pipesize = -1 # Restore default
+ if not isinstance(pipesize, int):
+ raise TypeError("pipesize must be an integer")
+
if _mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
@@ -797,6 +803,7 @@
self.returncode = None
self.encoding = encoding
self.errors = errors
+ self.pipesize = pipesize
# Validate the combinations of text and universal_newlines
if (text is not None and universal_newlines is not None
@@ -838,6 +845,13 @@
self.text_mode = encoding or errors or text or universal_newlines
+ # PEP 597: We suppress the EncodingWarning in subprocess module
+ # for now (at Python 3.10), because we focus on files for now.
+ # This will be changed to encoding = io.text_encoding(encoding)
+ # in the future.
+ if self.text_mode and encoding is None:
+ self.encoding = encoding = "locale"
+
# How long to resume waiting on a child after the first ^C.
# There is no right value for this. The purpose is to be polite
# yet remain good for interactive users trying to exit a tool.
@@ -861,7 +875,9 @@
"current platform")
elif isinstance(group, str):
- if grp is None:
+ try:
+ import grp
+ except ImportError:
raise ValueError("The group parameter cannot be a string "
"on systems without the grp module")
@@ -887,7 +903,9 @@
gids = []
for extra_group in extra_groups:
if isinstance(extra_group, str):
- if grp is None:
+ try:
+ import grp
+ except ImportError:
raise ValueError("Items in extra_groups cannot be "
"strings on systems without the "
"grp module")
@@ -913,10 +931,11 @@
"the current platform")
elif isinstance(user, str):
- if pwd is None:
+ try:
+ import pwd
+ except ImportError:
raise ValueError("The user parameter cannot be a string "
"on systems without the pwd module")
-
uid = pwd.getpwnam(user).pw_uid
elif isinstance(user, int):
uid = user
@@ -985,7 +1004,7 @@
def __repr__(self):
obj_repr = (
f"<{self.__class__.__name__}: "
- f"returncode: {self.returncode} args: {list(self.args)!r}>"
+ f"returncode: {self.returncode} args: {self.args!r}>"
)
if len(obj_repr) > 80:
obj_repr = obj_repr[:76] + "...>"
@@ -1521,10 +1540,8 @@
self.stderr.close()
# All data exchanged. Translate lists into strings.
- if stdout is not None:
- stdout = stdout[0]
- if stderr is not None:
- stderr = stderr[0]
+ stdout = stdout[0] if stdout else None
+ stderr = stderr[0] if stderr else None
return (stdout, stderr)
@@ -1575,6 +1592,8 @@
pass
elif stdin == PIPE:
p2cread, p2cwrite = os.pipe()
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
elif stdin == DEVNULL:
p2cread = self._get_devnull()
elif isinstance(stdin, int):
@@ -1587,6 +1606,8 @@
pass
elif stdout == PIPE:
c2pread, c2pwrite = os.pipe()
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
elif stdout == DEVNULL:
c2pwrite = self._get_devnull()
elif isinstance(stdout, int):
@@ -1599,6 +1620,8 @@
pass
elif stderr == PIPE:
errread, errwrite = os.pipe()
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
elif stderr == STDOUT:
if c2pwrite != -1:
errwrite = c2pwrite
diff --git a/common/py3-stdlib/symbol.py b/common/py3-stdlib/symbol.py
deleted file mode 100644
index aaac8c9..0000000
--- a/common/py3-stdlib/symbol.py
+++ /dev/null
@@ -1,122 +0,0 @@
-"""Non-terminal symbols of Python grammar (from "graminit.h")."""
-
-# This file is automatically generated; please don't muck it up!
-#
-# To update the symbols in this file, 'cd' to the top directory of
-# the python source tree after building the interpreter and run:
-#
-# python3 Tools/scripts/generate_symbol_py.py Include/graminit.h Lib/symbol.py
-#
-# or just
-#
-# make regen-symbol
-
-import warnings
-
-warnings.warn(
- "The symbol module is deprecated and will be removed "
- "in future versions of Python",
- DeprecationWarning,
- stacklevel=2,
-)
-
-#--start constants--
-single_input = 256
-file_input = 257
-eval_input = 258
-decorator = 259
-decorators = 260
-decorated = 261
-async_funcdef = 262
-funcdef = 263
-parameters = 264
-typedargslist = 265
-tfpdef = 266
-varargslist = 267
-vfpdef = 268
-stmt = 269
-simple_stmt = 270
-small_stmt = 271
-expr_stmt = 272
-annassign = 273
-testlist_star_expr = 274
-augassign = 275
-del_stmt = 276
-pass_stmt = 277
-flow_stmt = 278
-break_stmt = 279
-continue_stmt = 280
-return_stmt = 281
-yield_stmt = 282
-raise_stmt = 283
-import_stmt = 284
-import_name = 285
-import_from = 286
-import_as_name = 287
-dotted_as_name = 288
-import_as_names = 289
-dotted_as_names = 290
-dotted_name = 291
-global_stmt = 292
-nonlocal_stmt = 293
-assert_stmt = 294
-compound_stmt = 295
-async_stmt = 296
-if_stmt = 297
-while_stmt = 298
-for_stmt = 299
-try_stmt = 300
-with_stmt = 301
-with_item = 302
-except_clause = 303
-suite = 304
-namedexpr_test = 305
-test = 306
-test_nocond = 307
-lambdef = 308
-lambdef_nocond = 309
-or_test = 310
-and_test = 311
-not_test = 312
-comparison = 313
-comp_op = 314
-star_expr = 315
-expr = 316
-xor_expr = 317
-and_expr = 318
-shift_expr = 319
-arith_expr = 320
-term = 321
-factor = 322
-power = 323
-atom_expr = 324
-atom = 325
-testlist_comp = 326
-trailer = 327
-subscriptlist = 328
-subscript = 329
-sliceop = 330
-exprlist = 331
-testlist = 332
-dictorsetmaker = 333
-classdef = 334
-arglist = 335
-argument = 336
-comp_iter = 337
-sync_comp_for = 338
-comp_for = 339
-comp_if = 340
-encoding_decl = 341
-yield_expr = 342
-yield_arg = 343
-func_body_suite = 344
-func_type_input = 345
-func_type = 346
-typelist = 347
-#--end constants--
-
-sym_name = {}
-for _name, _value in list(globals().items()):
- if type(_value) is type(0):
- sym_name[_value] = _name
-del _name, _value
diff --git a/common/py3-stdlib/symtable.py b/common/py3-stdlib/symtable.py
index 521540f..98db1e2 100644
--- a/common/py3-stdlib/symtable.py
+++ b/common/py3-stdlib/symtable.py
@@ -10,6 +10,11 @@
__all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"]
def symtable(code, filename, compile_type):
+ """ Return the toplevel *SymbolTable* for the source code.
+
+ *filename* is the name of the file with the code
+ and *compile_type* is the *compile()* mode argument.
+ """
top = _symtable.symtable(code, filename, compile_type)
return _newSymbolTable(top, filename)
@@ -55,6 +60,11 @@
self._filename)
def get_type(self):
+ """Return the type of the symbol table.
+
+ The values retuned are 'class', 'module' and
+ 'function'.
+ """
if self._table.type == _symtable.TYPE_MODULE:
return "module"
if self._table.type == _symtable.TYPE_FUNCTION:
@@ -65,27 +75,51 @@
"unexpected type: {0}".format(self._table.type)
def get_id(self):
+ """Return an identifier for the table.
+ """
return self._table.id
def get_name(self):
+ """Return the table's name.
+
+ This corresponds to the name of the class, function
+ or 'top' if the table is for a class, function or
+ global respectively.
+ """
return self._table.name
def get_lineno(self):
+ """Return the number of the first line in the
+ block for the table.
+ """
return self._table.lineno
def is_optimized(self):
+ """Return *True* if the locals in the table
+ are optimizable.
+ """
return bool(self._table.type == _symtable.TYPE_FUNCTION)
def is_nested(self):
+ """Return *True* if the block is a nested class
+ or function."""
return bool(self._table.nested)
def has_children(self):
+ """Return *True* if the block has nested namespaces.
+ """
return bool(self._table.children)
def get_identifiers(self):
+ """Return a list of names of symbols in the table.
+ """
return self._table.symbols.keys()
def lookup(self, name):
+ """Lookup a *name* in the table.
+
+ Returns a *Symbol* instance.
+ """
sym = self._symbols.get(name)
if sym is None:
flags = self._table.symbols[name]
@@ -96,6 +130,9 @@
return sym
def get_symbols(self):
+ """Return a list of *Symbol* instances for
+ names in the table.
+ """
return [self.lookup(ident) for ident in self.get_identifiers()]
def __check_children(self, name):
@@ -104,6 +141,8 @@
if st.name == name]
def get_children(self):
+ """Return a list of the nested symbol tables.
+ """
return [_newSymbolTable(st, self._filename)
for st in self._table.children]
@@ -122,11 +161,15 @@
if test_func(self._table.symbols[ident]))
def get_parameters(self):
+ """Return a tuple of parameters to the function.
+ """
if self.__params is None:
self.__params = self.__idents_matching(lambda x:x & DEF_PARAM)
return self.__params
def get_locals(self):
+ """Return a tuple of locals in the function.
+ """
if self.__locals is None:
locs = (LOCAL, CELL)
test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs
@@ -134,6 +177,8 @@
return self.__locals
def get_globals(self):
+ """Return a tuple of globals in the function.
+ """
if self.__globals is None:
glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
@@ -141,11 +186,15 @@
return self.__globals
def get_nonlocals(self):
+ """Return a tuple of nonlocals in the function.
+ """
if self.__nonlocals is None:
self.__nonlocals = self.__idents_matching(lambda x:x & DEF_NONLOCAL)
return self.__nonlocals
def get_frees(self):
+ """Return a tuple of free variables in the function.
+ """
if self.__frees is None:
is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
self.__frees = self.__idents_matching(is_free)
@@ -157,6 +206,8 @@
__methods = None
def get_methods(self):
+ """Return a tuple of methods declared in the class.
+ """
if self.__methods is None:
d = {}
for st in self._table.children:
@@ -178,12 +229,19 @@
return "<symbol {0!r}>".format(self.__name)
def get_name(self):
+ """Return a name of a symbol.
+ """
return self.__name
def is_referenced(self):
+ """Return *True* if the symbol is used in
+ its block.
+ """
return bool(self.__flags & _symtable.USE)
def is_parameter(self):
+ """Return *True* if the symbol is a parameter.
+ """
return bool(self.__flags & DEF_PARAM)
def is_global(self):
@@ -193,9 +251,12 @@
or (self.__module_scope and self.__flags & DEF_BOUND))
def is_nonlocal(self):
+ """Return *True* if the symbol is nonlocal."""
return bool(self.__flags & DEF_NONLOCAL)
def is_declared_global(self):
+ """Return *True* if the symbol is declared global
+ with a global statement."""
return bool(self.__scope == GLOBAL_EXPLICIT)
def is_local(self):
@@ -205,19 +266,28 @@
or (self.__module_scope and self.__flags & DEF_BOUND))
def is_annotated(self):
+ """Return *True* if the symbol is annotated.
+ """
return bool(self.__flags & DEF_ANNOT)
def is_free(self):
+ """Return *True* if a referenced symbol is
+ not assigned to.
+ """
return bool(self.__scope == FREE)
def is_imported(self):
+ """Return *True* if the symbol is created from
+ an import statement.
+ """
return bool(self.__flags & DEF_IMPORT)
def is_assigned(self):
+ """Return *True* if a symbol is assigned to."""
return bool(self.__flags & DEF_LOCAL)
def is_namespace(self):
- """Returns true if name binding introduces new namespace.
+ """Returns *True* if name binding introduces new namespace.
If the name is used as the target of a function or class
statement, this will be true.
@@ -234,7 +304,7 @@
return self.__namespaces
def get_namespace(self):
- """Returns the single namespace bound to this name.
+ """Return the single namespace bound to this name.
Raises ValueError if the name is bound to multiple namespaces.
"""
diff --git a/common/py3-stdlib/sysconfig.py b/common/py3-stdlib/sysconfig.py
index bf04ac5..daf9f00 100644
--- a/common/py3-stdlib/sysconfig.py
+++ b/common/py3-stdlib/sysconfig.py
@@ -18,6 +18,11 @@
'parse_config_h',
]
+# Keys for get_config_var() that are never converted to Python integers.
+_ALWAYS_STR = {
+ 'MACOSX_DEPLOYMENT_TARGET',
+}
+
_INSTALL_SCHEMES = {
'posix_prefix': {
'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}',
@@ -51,42 +56,73 @@
'scripts': '{base}/Scripts',
'data': '{base}',
},
- # NOTE: When modifying "purelib" scheme, update site._get_path() too.
- 'nt_user': {
- 'stdlib': '{userbase}/Python{py_version_nodot}',
- 'platstdlib': '{userbase}/Python{py_version_nodot}',
- 'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
- 'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
- 'include': '{userbase}/Python{py_version_nodot}/Include',
- 'scripts': '{userbase}/Python{py_version_nodot}/Scripts',
- 'data': '{userbase}',
- },
- 'posix_user': {
- 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}',
- 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}',
- 'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
- 'platlib': '{userbase}/{platlibdir}/python{py_version_short}/site-packages',
- 'include': '{userbase}/include/python{py_version_short}',
- 'scripts': '{userbase}/bin',
- 'data': '{userbase}',
- },
- 'osx_framework_user': {
- 'stdlib': '{userbase}/lib/python',
- 'platstdlib': '{userbase}/lib/python',
- 'purelib': '{userbase}/lib/python/site-packages',
- 'platlib': '{userbase}/lib/python/site-packages',
- 'include': '{userbase}/include',
- 'scripts': '{userbase}/bin',
- 'data': '{userbase}',
- },
+ }
+
+
+# NOTE: site.py has copy of this function.
+# Sync it when modify this function.
+def _getuserbase():
+ env_base = os.environ.get("PYTHONUSERBASE", None)
+ if env_base:
+ return env_base
+
+ # VxWorks has no home directories
+ if sys.platform == "vxworks":
+ return None
+
+ def joinuser(*args):
+ return os.path.expanduser(os.path.join(*args))
+
+ if os.name == "nt":
+ base = os.environ.get("APPDATA") or "~"
+ return joinuser(base, "Python")
+
+ if sys.platform == "darwin" and sys._framework:
+ return joinuser("~", "Library", sys._framework,
+ f"{sys.version_info[0]}.{sys.version_info[1]}")
+
+ return joinuser("~", ".local")
+
+_HAS_USER_BASE = (_getuserbase() is not None)
+
+if _HAS_USER_BASE:
+ _INSTALL_SCHEMES |= {
+ # NOTE: When modifying "purelib" scheme, update site._get_path() too.
+ 'nt_user': {
+ 'stdlib': '{userbase}/Python{py_version_nodot_plat}',
+ 'platstdlib': '{userbase}/Python{py_version_nodot_plat}',
+ 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages',
+ 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages',
+ 'include': '{userbase}/Python{py_version_nodot_plat}/Include',
+ 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts',
+ 'data': '{userbase}',
+ },
+ 'posix_user': {
+ 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}',
+ 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}',
+ 'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
+ 'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
+ 'include': '{userbase}/include/python{py_version_short}',
+ 'scripts': '{userbase}/bin',
+ 'data': '{userbase}',
+ },
+ 'osx_framework_user': {
+ 'stdlib': '{userbase}/lib/python',
+ 'platstdlib': '{userbase}/lib/python',
+ 'purelib': '{userbase}/lib/python/site-packages',
+ 'platlib': '{userbase}/lib/python/site-packages',
+ 'include': '{userbase}/include/python{py_version_short}',
+ 'scripts': '{userbase}/bin',
+ 'data': '{userbase}',
+ },
}
_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include',
'scripts', 'data')
_PY_VERSION = sys.version.split()[0]
-_PY_VERSION_SHORT = '%d.%d' % sys.version_info[:2]
-_PY_VERSION_SHORT_NO_DOT = '%d%d' % sys.version_info[:2]
+_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}'
+_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}'
_PREFIX = os.path.normpath(sys.prefix)
_BASE_PREFIX = os.path.normpath(sys.base_prefix)
_EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
@@ -94,6 +130,12 @@
_CONFIG_VARS = None
_USER_BASE = None
+# Regexes needed for parsing Makefile (and similar syntaxes,
+# like old-style Setup files).
+_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)"
+_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)"
+_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}"
+
def _safe_realpath(path):
try:
@@ -142,18 +184,24 @@
if _PYTHON_BUILD:
for scheme in ('posix_prefix', 'posix_home'):
- _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include'
- _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.'
+ # On POSIX-y platforms, Python will:
+ # - Build from .h files in 'headers' (which is only added to the
+ # scheme when building CPython)
+ # - Install .h files to 'include'
+ scheme = _INSTALL_SCHEMES[scheme]
+ scheme['headers'] = scheme['include']
+ scheme['include'] = '{srcdir}/Include'
+ scheme['platinclude'] = '{projectbase}/.'
def _subst_vars(s, local_vars):
try:
return s.format(**local_vars)
- except KeyError:
+ except KeyError as var:
try:
return s.format(**os.environ)
- except KeyError as var:
- raise AttributeError('{%s}' % var) from None
+ except KeyError:
+ raise AttributeError(f'{var}') from None
def _extend_dict(target_dict, other_dict):
target_keys = target_dict.keys()
@@ -176,60 +224,62 @@
return res
-def _get_default_scheme():
- if os.name == 'posix':
- # the default scheme for posix is posix_prefix
- return 'posix_prefix'
- return os.name
+def _get_preferred_schemes():
+ if os.name == 'nt':
+ return {
+ 'prefix': 'nt',
+ 'home': 'posix_home',
+ 'user': 'nt_user',
+ }
+ if sys.platform == 'darwin' and sys._framework:
+ return {
+ 'prefix': 'posix_prefix',
+ 'home': 'posix_home',
+ 'user': 'osx_framework_user',
+ }
+ return {
+ 'prefix': 'posix_prefix',
+ 'home': 'posix_home',
+ 'user': 'posix_user',
+ }
-# NOTE: site.py has copy of this function.
-# Sync it when modify this function.
-def _getuserbase():
- env_base = os.environ.get("PYTHONUSERBASE", None)
- if env_base:
- return env_base
-
- def joinuser(*args):
- return os.path.expanduser(os.path.join(*args))
-
- if os.name == "nt":
- base = os.environ.get("APPDATA") or "~"
- return joinuser(base, "Python")
-
- if sys.platform == "darwin" and sys._framework:
- return joinuser("~", "Library", sys._framework,
- "%d.%d" % sys.version_info[:2])
-
- return joinuser("~", ".local")
+def get_preferred_scheme(key):
+ scheme = _get_preferred_schemes()[key]
+ if scheme not in _INSTALL_SCHEMES:
+ raise ValueError(
+ f"{key!r} returned {scheme!r}, which is not a valid scheme "
+ f"on this platform"
+ )
+ return scheme
-def _parse_makefile(filename, vars=None):
+def get_default_scheme():
+ return get_preferred_scheme('prefix')
+
+
+def _parse_makefile(filename, vars=None, keep_unresolved=True):
"""Parse a Makefile-style file.
A dictionary containing name/value pairs is returned. If an
optional dictionary is passed in as the second argument, it is
used instead of a new dictionary.
"""
- # Regexes needed for parsing Makefile (and similar syntaxes,
- # like old-style Setup files).
import re
- _variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
- _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
- _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
if vars is None:
vars = {}
done = {}
notdone = {}
- with open(filename, errors="surrogateescape") as f:
+ with open(filename, encoding=sys.getfilesystemencoding(),
+ errors="surrogateescape") as f:
lines = f.readlines()
for line in lines:
if line.startswith('#') or line.strip() == '':
continue
- m = _variable_rx.match(line)
+ m = re.match(_variable_rx, line)
if m:
n, v = m.group(1, 2)
v = v.strip()
@@ -240,6 +290,9 @@
notdone[n] = v
else:
try:
+ if n in _ALWAYS_STR:
+ raise ValueError
+
v = int(v)
except ValueError:
# insert literal `$'
@@ -259,8 +312,8 @@
while len(variables) > 0:
for name in tuple(variables):
value = notdone[name]
- m1 = _findvar1_rx.search(value)
- m2 = _findvar2_rx.search(value)
+ m1 = re.search(_findvar1_rx, value)
+ m2 = re.search(_findvar2_rx, value)
if m1 and m2:
m = m1 if m1.start() < m2.start() else m2
else:
@@ -298,6 +351,8 @@
notdone[name] = value
else:
try:
+ if name in _ALWAYS_STR:
+ raise ValueError
value = int(value)
except ValueError:
done[name] = value.strip()
@@ -313,9 +368,12 @@
done[name] = value
else:
+ # Adds unresolved variables to the done dict.
+ # This is disabled when called from distutils.sysconfig
+ if keep_unresolved:
+ done[name] = value
# bogus variable reference (e.g. "prefix=$/opt/python");
# just drop it since we can't deal
- done[name] = value
variables.remove(name)
# strip spurious spaces
@@ -333,21 +391,20 @@
if _PYTHON_BUILD:
return os.path.join(_sys_home or _PROJECT_BASE, "Makefile")
if hasattr(sys, 'abiflags'):
- config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags)
+ config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}'
else:
config_dir_name = 'config'
if hasattr(sys.implementation, '_multiarch'):
- config_dir_name += '-%s' % sys.implementation._multiarch
+ config_dir_name += f'-{sys.implementation._multiarch}'
return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile')
def _get_sysconfigdata_name():
- return os.environ.get('_PYTHON_SYSCONFIGDATA_NAME',
- '_sysconfigdata_{abi}_{platform}_{multiarch}'.format(
- abi=sys.abiflags,
- platform=sys.platform,
- multiarch=getattr(sys.implementation, '_multiarch', ''),
- ))
+ multiarch = getattr(sys.implementation, '_multiarch', '')
+ return os.environ.get(
+ '_PYTHON_SYSCONFIGDATA_NAME',
+ f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}',
+ )
def _generate_posix_vars():
@@ -359,19 +416,19 @@
try:
_parse_makefile(makefile, vars)
except OSError as e:
- msg = "invalid Python installation: unable to open %s" % makefile
+ msg = f"invalid Python installation: unable to open {makefile}"
if hasattr(e, "strerror"):
- msg = msg + " (%s)" % e.strerror
+ msg = f"{msg} ({e.strerror})"
raise OSError(msg)
# load the installed pyconfig.h:
config_h = get_config_h_filename()
try:
- with open(config_h) as f:
+ with open(config_h, encoding="utf-8") as f:
parse_config_h(f, vars)
except OSError as e:
- msg = "invalid Python installation: unable to open %s" % config_h
+ msg = f"invalid Python installation: unable to open {config_h}"
if hasattr(e, "strerror"):
- msg = msg + " (%s)" % e.strerror
+ msg = f"{msg} ({e.strerror})"
raise OSError(msg)
# On AIX, there are wrong paths to the linker scripts in the Makefile
# -- these paths are relative to the Python source, but when installed
@@ -397,7 +454,7 @@
module.build_time_vars = vars
sys.modules[name] = module
- pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT)
+ pybuilddir = f'build/lib.{get_platform()}-{_PY_VERSION_SHORT}'
if hasattr(sys, "gettotalrefcount"):
pybuilddir += '-pydebug'
os.makedirs(pybuilddir, exist_ok=True)
@@ -424,13 +481,15 @@
def _init_non_posix(vars):
"""Initialize the module as appropriate for NT"""
# set basic install directories
+ import _imp
vars['LIBDEST'] = get_path('stdlib')
vars['BINLIBDEST'] = get_path('platstdlib')
vars['INCLUDEPY'] = get_path('include')
- vars['EXT_SUFFIX'] = '.pyd'
+ vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0]
vars['EXE'] = '.exe'
vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable))
+ vars['TZPATH'] = ''
#
# public APIs
@@ -458,6 +517,8 @@
if m:
n, v = m.group(1, 2)
try:
+ if n in _ALWAYS_STR:
+ raise ValueError
v = int(v)
except ValueError:
pass
@@ -491,7 +552,7 @@
return _SCHEME_KEYS
-def get_paths(scheme=_get_default_scheme(), vars=None, expand=True):
+def get_paths(scheme=get_default_scheme(), vars=None, expand=True):
"""Return a mapping containing an install scheme.
``scheme`` is the install scheme name. If not provided, it will
@@ -503,7 +564,7 @@
return _INSTALL_SCHEMES[scheme]
-def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True):
+def get_path(name, scheme=get_default_scheme(), vars=None, expand=True):
"""Return a path corresponding to the scheme.
``scheme`` is the install scheme name.
@@ -543,20 +604,24 @@
except AttributeError:
# sys.abiflags may not be defined on all platforms.
_CONFIG_VARS['abiflags'] = ''
+ try:
+ _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '')
+ except AttributeError:
+ _CONFIG_VARS['py_version_nodot_plat'] = ''
if os.name == 'nt':
_init_non_posix(_CONFIG_VARS)
- _CONFIG_VARS['TZPATH'] = ''
if os.name == 'posix':
_init_posix(_CONFIG_VARS)
# For backward compatibility, see issue19555
SO = _CONFIG_VARS.get('EXT_SUFFIX')
if SO is not None:
_CONFIG_VARS['SO'] = SO
- # Setting 'userbase' is done below the call to the
- # init function to enable using 'get_config_var' in
- # the init-function.
- _CONFIG_VARS['userbase'] = _getuserbase()
+ if _HAS_USER_BASE:
+ # Setting 'userbase' is done below the call to the
+ # init function to enable using 'get_config_var' in
+ # the init-function.
+ _CONFIG_VARS['userbase'] = _getuserbase()
# Always convert srcdir to an absolute path
srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE)
@@ -653,16 +718,16 @@
# At least on Linux/Intel, 'machine' is the processor --
# i386, etc.
# XXX what about Alpha, SPARC, etc?
- return "%s-%s" % (osname, machine)
+ return f"{osname}-{machine}"
elif osname[:5] == "sunos":
if release[0] >= "5": # SunOS 5 == Solaris 2
osname = "solaris"
- release = "%d.%s" % (int(release[0]) - 3, release[2:])
+ release = f"{int(release[0]) - 3}.{release[2:]}"
# We can't use "platform.architecture()[0]" because a
# bootstrap problem. We use a dict to get an error
# if some suspicious happens.
bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
- machine += ".%s" % bitness[sys.maxsize]
+ machine += f".{bitness[sys.maxsize]}"
# fall through to standard osname-release-machine representation
elif osname[:3] == "aix":
from _aix_support import aix_platform
@@ -680,18 +745,44 @@
get_config_vars(),
osname, release, machine)
- return "%s-%s-%s" % (osname, release, machine)
+ return f"{osname}-{release}-{machine}"
def get_python_version():
return _PY_VERSION_SHORT
+def expand_makefile_vars(s, vars):
+ """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
+ 'string' according to 'vars' (a dictionary mapping variable names to
+ values). Variables not present in 'vars' are silently expanded to the
+ empty string. The variable values in 'vars' should not contain further
+ variable expansions; if 'vars' is the output of 'parse_makefile()',
+ you're fine. Returns a variable-expanded version of 's'.
+ """
+ import re
+
+ # This algorithm does multiple expansion, so if vars['foo'] contains
+ # "${bar}", it will expand ${foo} to ${bar}, and then expand
+ # ${bar}... and so forth. This is fine as long as 'vars' comes from
+ # 'parse_makefile()', which takes care of such expansions eagerly,
+ # according to make's variable expansion semantics.
+
+ while True:
+ m = re.search(_findvar1_rx, s) or re.search(_findvar2_rx, s)
+ if m:
+ (beg, end) = m.span()
+ s = s[0:beg] + vars.get(m.group(1)) + s[end:]
+ else:
+ break
+ return s
+
+
def _print_dict(title, data):
for index, (key, value) in enumerate(sorted(data.items())):
if index == 0:
- print('%s: ' % (title))
- print('\t%s = "%s"' % (key, value))
+ print(f'{title}: ')
+ print(f'\t{key} = "{value}"')
def _main():
@@ -699,9 +790,9 @@
if '--generate-posix-vars' in sys.argv:
_generate_posix_vars()
return
- print('Platform: "%s"' % get_platform())
- print('Python version: "%s"' % get_python_version())
- print('Current installation scheme: "%s"' % _get_default_scheme())
+ print(f'Platform: "{get_platform()}"')
+ print(f'Python version: "{get_python_version()}"')
+ print(f'Current installation scheme: "{get_default_scheme()}"')
print()
_print_dict('Paths', get_paths())
print()
diff --git a/common/py3-stdlib/tarfile.py b/common/py3-stdlib/tarfile.py
index 1d15612..6ada9a0 100755
--- a/common/py3-stdlib/tarfile.py
+++ b/common/py3-stdlib/tarfile.py
@@ -200,6 +200,7 @@
# base-256 representation. This allows values up to (256**(digits-1))-1.
# A 0o200 byte indicates a positive number, a 0o377 byte a negative
# number.
+ original_n = n
n = int(n)
if 0 <= n < 8 ** (digits - 1):
s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL
@@ -363,7 +364,7 @@
try:
import zlib
except ImportError:
- raise CompressionError("zlib module is not available")
+ raise CompressionError("zlib module is not available") from None
self.zlib = zlib
self.crc = zlib.crc32(b"")
if mode == "r":
@@ -376,7 +377,7 @@
try:
import bz2
except ImportError:
- raise CompressionError("bz2 module is not available")
+ raise CompressionError("bz2 module is not available") from None
if mode == "r":
self.dbuf = b""
self.cmp = bz2.BZ2Decompressor()
@@ -388,7 +389,7 @@
try:
import lzma
except ImportError:
- raise CompressionError("lzma module is not available")
+ raise CompressionError("lzma module is not available") from None
if mode == "r":
self.dbuf = b""
self.cmp = lzma.LZMADecompressor()
@@ -541,8 +542,8 @@
break
try:
buf = self.cmp.decompress(buf)
- except self.exception:
- raise ReadError("invalid compressed data")
+ except self.exception as e:
+ raise ReadError("invalid compressed data") from e
t.append(buf)
c += len(buf)
t = b"".join(t)
@@ -887,15 +888,24 @@
# Test number fields for values that exceed the field limit or values
# that like to be stored as float.
for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)):
- if name in pax_headers:
- # The pax header has priority. Avoid overflow.
- info[name] = 0
- continue
+ needs_pax = False
val = info[name]
- if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float):
- pax_headers[name] = str(val)
+ val_is_float = isinstance(val, float)
+ val_int = round(val) if val_is_float else val
+ if not 0 <= val_int < 8 ** (digits - 1):
+ # Avoid overflow.
info[name] = 0
+ needs_pax = True
+ elif val_is_float:
+ # Put rounded value in ustar header, and full
+ # precision value in pax header.
+ info[name] = val_int
+ needs_pax = True
+
+ # The existing pax header has priority.
+ if needs_pax and name not in pax_headers:
+ pax_headers[name] = str(val)
# Create a pax extended header if necessary.
if pax_headers:
@@ -1164,8 +1174,8 @@
# Fetch the next header and process it.
try:
next = self.fromtarfile(tarfile)
- except HeaderError:
- raise SubsequentHeaderError("missing or bad subsequent header")
+ except HeaderError as e:
+ raise SubsequentHeaderError(str(e)) from None
# Patch the TarInfo object from the next header with
# the longname information.
@@ -1277,8 +1287,8 @@
# Fetch the next header.
try:
next = self.fromtarfile(tarfile)
- except HeaderError:
- raise SubsequentHeaderError("missing or bad subsequent header")
+ except HeaderError as e:
+ raise SubsequentHeaderError(str(e)) from None
# Process GNU sparse information.
if "GNU.sparse.map" in pax_headers:
@@ -1533,7 +1543,7 @@
self.fileobj.seek(self.offset)
break
except HeaderError as e:
- raise ReadError(str(e))
+ raise ReadError(str(e)) from None
if self.mode in ("a", "w", "x"):
self._loaded = True
@@ -1603,17 +1613,20 @@
# Find out which *open() is appropriate for opening the file.
def not_compressed(comptype):
return cls.OPEN_METH[comptype] == 'taropen'
+ error_msgs = []
for comptype in sorted(cls.OPEN_METH, key=not_compressed):
func = getattr(cls, cls.OPEN_METH[comptype])
if fileobj is not None:
saved_pos = fileobj.tell()
try:
return func(name, "r", fileobj, **kwargs)
- except (ReadError, CompressionError):
+ except (ReadError, CompressionError) as e:
+ error_msgs.append(f'- method {comptype}: {e!r}')
if fileobj is not None:
fileobj.seek(saved_pos)
continue
- raise ReadError("file could not be opened successfully")
+ error_msgs_summary = '\n'.join(error_msgs)
+ raise ReadError(f"file could not be opened successfully:\n{error_msgs_summary}")
elif ":" in mode:
filemode, comptype = mode.split(":", 1)
@@ -1669,21 +1682,21 @@
try:
from gzip import GzipFile
except ImportError:
- raise CompressionError("gzip module is not available")
+ raise CompressionError("gzip module is not available") from None
try:
fileobj = GzipFile(name, mode + "b", compresslevel, fileobj)
- except OSError:
+ except OSError as e:
if fileobj is not None and mode == 'r':
- raise ReadError("not a gzip file")
+ raise ReadError("not a gzip file") from e
raise
try:
t = cls.taropen(name, mode, fileobj, **kwargs)
- except OSError:
+ except OSError as e:
fileobj.close()
if mode == 'r':
- raise ReadError("not a gzip file")
+ raise ReadError("not a gzip file") from e
raise
except:
fileobj.close()
@@ -1702,16 +1715,16 @@
try:
from bz2 import BZ2File
except ImportError:
- raise CompressionError("bz2 module is not available")
+ raise CompressionError("bz2 module is not available") from None
fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel)
try:
t = cls.taropen(name, mode, fileobj, **kwargs)
- except (OSError, EOFError):
+ except (OSError, EOFError) as e:
fileobj.close()
if mode == 'r':
- raise ReadError("not a bzip2 file")
+ raise ReadError("not a bzip2 file") from e
raise
except:
fileobj.close()
@@ -1730,16 +1743,16 @@
try:
from lzma import LZMAFile, LZMAError
except ImportError:
- raise CompressionError("lzma module is not available")
+ raise CompressionError("lzma module is not available") from None
fileobj = LZMAFile(fileobj or name, mode, preset=preset)
try:
t = cls.taropen(name, mode, fileobj, **kwargs)
- except (LZMAError, EOFError):
+ except (LZMAError, EOFError) as e:
fileobj.close()
if mode == 'r':
- raise ReadError("not an lzma file")
+ raise ReadError("not an lzma file") from e
raise
except:
fileobj.close()
@@ -1785,7 +1798,7 @@
than once in the archive, its last occurrence is assumed to be the
most up-to-date version.
"""
- tarinfo = self._getmember(name)
+ tarinfo = self._getmember(name.rstrip('/'))
if tarinfo is None:
raise KeyError("filename %r not found" % name)
return tarinfo
@@ -2253,7 +2266,7 @@
self._extract_member(self._find_link_target(tarinfo),
targetpath)
except KeyError:
- raise ExtractError("unable to resolve link inside archive")
+ raise ExtractError("unable to resolve link inside archive") from None
def chown(self, tarinfo, targetpath, numeric_owner):
"""Set owner of targetpath according to tarinfo. If numeric_owner
@@ -2281,16 +2294,16 @@
os.lchown(targetpath, u, g)
else:
os.chown(targetpath, u, g)
- except OSError:
- raise ExtractError("could not change owner")
+ except OSError as e:
+ raise ExtractError("could not change owner") from e
def chmod(self, tarinfo, targetpath):
"""Set file permissions of targetpath according to tarinfo.
"""
try:
os.chmod(targetpath, tarinfo.mode)
- except OSError:
- raise ExtractError("could not change mode")
+ except OSError as e:
+ raise ExtractError("could not change mode") from e
def utime(self, tarinfo, targetpath):
"""Set modification time of targetpath according to tarinfo.
@@ -2299,8 +2312,8 @@
return
try:
os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
- except OSError:
- raise ExtractError("could not change modification time")
+ except OSError as e:
+ raise ExtractError("could not change modification time") from e
#--------------------------------------------------------------------------
def next(self):
@@ -2336,15 +2349,24 @@
self.offset += BLOCKSIZE
continue
elif self.offset == 0:
- raise ReadError(str(e))
+ raise ReadError(str(e)) from None
except EmptyHeaderError:
if self.offset == 0:
- raise ReadError("empty file")
+ raise ReadError("empty file") from None
except TruncatedHeaderError as e:
if self.offset == 0:
- raise ReadError(str(e))
+ raise ReadError(str(e)) from None
except SubsequentHeaderError as e:
- raise ReadError(str(e))
+ raise ReadError(str(e)) from None
+ except Exception as e:
+ try:
+ import zlib
+ if isinstance(e, zlib.error):
+ raise ReadError(f'zlib error: {e}') from None
+ else:
+ raise e
+ except ImportError:
+ raise e
break
if tarinfo is not None:
diff --git a/common/py3-stdlib/tempfile.py b/common/py3-stdlib/tempfile.py
index 770f72c..7b68212 100644
--- a/common/py3-stdlib/tempfile.py
+++ b/common/py3-stdlib/tempfile.py
@@ -88,6 +88,10 @@
for arg in args:
if arg is None:
continue
+
+ if isinstance(arg, _os.PathLike):
+ arg = _os.fspath(arg)
+
if isinstance(arg, bytes):
if return_type is str:
raise TypeError("Can't mix bytes and non-bytes in "
@@ -99,7 +103,11 @@
"path components.")
return_type = str
if return_type is None:
- return str # tempfile APIs return a str by default.
+ if tempdir is None or isinstance(tempdir, str):
+ return str # tempfile APIs return a str by default.
+ else:
+ # we could check for bytes but it'll fail later on anyway
+ return bytes
return return_type
@@ -143,10 +151,7 @@
return self
def __next__(self):
- c = self.characters
- choose = self.rng.choice
- letters = [choose(c) for dummy in range(8)]
- return ''.join(letters)
+ return ''.join(self.rng.choices(self.characters, k=8))
def _candidate_tempdir_list():
"""Generate a list of candidate temporary directories which
@@ -268,17 +273,17 @@
# User visible interfaces.
def gettempprefix():
- """The default prefix for temporary directories."""
- return template
+ """The default prefix for temporary directories as string."""
+ return _os.fsdecode(template)
def gettempprefixb():
"""The default prefix for temporary directories as bytes."""
- return _os.fsencode(gettempprefix())
+ return _os.fsencode(template)
tempdir = None
-def gettempdir():
- """Accessor for tempfile.tempdir."""
+def _gettempdir():
+ """Private accessor for tempfile.tempdir."""
global tempdir
if tempdir is None:
_once_lock.acquire()
@@ -289,9 +294,13 @@
_once_lock.release()
return tempdir
+def gettempdir():
+ """Returns tempfile.tempdir as str."""
+ return _os.fsdecode(_gettempdir())
+
def gettempdirb():
- """A bytes version of tempfile.gettempdir()."""
- return _os.fsencode(gettempdir())
+ """Returns tempfile.tempdir as bytes."""
+ return _os.fsencode(_gettempdir())
def mkstemp(suffix=None, prefix=None, dir=None, text=False):
"""User-callable function to create and return a unique temporary
@@ -538,6 +547,9 @@
if _os.name == 'nt' and delete:
flags |= _os.O_TEMPORARY
+ if "b" not in mode:
+ encoding = _io.text_encoding(encoding)
+
(fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type)
try:
file = _io.open(fd, mode, buffering=buffering,
@@ -578,6 +590,9 @@
"""
global _O_TMPFILE_WORKS
+ if "b" not in mode:
+ encoding = _io.text_encoding(encoding)
+
prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir)
flags = _bin_openflags
@@ -633,6 +648,7 @@
if 'b' in mode:
self._file = _io.BytesIO()
else:
+ encoding = _io.text_encoding(encoding)
self._file = _io.TextIOWrapper(_io.BytesIO(),
encoding=encoding, errors=errors,
newline=newline)
@@ -763,7 +779,7 @@
return rv
-class TemporaryDirectory(object):
+class TemporaryDirectory:
"""Create and return a temporary directory. This has the same
behavior as mkdtemp but can be used as a context manager. For
example:
@@ -775,14 +791,17 @@
in it are removed.
"""
- def __init__(self, suffix=None, prefix=None, dir=None):
+ def __init__(self, suffix=None, prefix=None, dir=None,
+ ignore_cleanup_errors=False):
self.name = mkdtemp(suffix, prefix, dir)
+ self._ignore_cleanup_errors = ignore_cleanup_errors
self._finalizer = _weakref.finalize(
self, self._cleanup, self.name,
- warn_message="Implicitly cleaning up {!r}".format(self))
+ warn_message="Implicitly cleaning up {!r}".format(self),
+ ignore_errors=self._ignore_cleanup_errors)
@classmethod
- def _rmtree(cls, name):
+ def _rmtree(cls, name, ignore_errors=False):
def onerror(func, path, exc_info):
if issubclass(exc_info[0], PermissionError):
def resetperms(path):
@@ -801,19 +820,20 @@
_os.unlink(path)
# PermissionError is raised on FreeBSD for directories
except (IsADirectoryError, PermissionError):
- cls._rmtree(path)
+ cls._rmtree(path, ignore_errors=ignore_errors)
except FileNotFoundError:
pass
elif issubclass(exc_info[0], FileNotFoundError):
pass
else:
- raise
+ if not ignore_errors:
+ raise
_shutil.rmtree(name, onerror=onerror)
@classmethod
- def _cleanup(cls, name, warn_message):
- cls._rmtree(name)
+ def _cleanup(cls, name, warn_message, ignore_errors=False):
+ cls._rmtree(name, ignore_errors=ignore_errors)
_warnings.warn(warn_message, ResourceWarning)
def __repr__(self):
@@ -826,7 +846,7 @@
self.cleanup()
def cleanup(self):
- if self._finalizer.detach():
- self._rmtree(self.name)
+ if self._finalizer.detach() or _os.path.exists(self.name):
+ self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
__class_getitem__ = classmethod(_types.GenericAlias)
diff --git a/common/py3-stdlib/textwrap.py b/common/py3-stdlib/textwrap.py
index 30e693c..841de9b 100644
--- a/common/py3-stdlib/textwrap.py
+++ b/common/py3-stdlib/textwrap.py
@@ -215,8 +215,16 @@
# If we're allowed to break long words, then do so: put as much
# of the next chunk onto the current line as will fit.
if self.break_long_words:
- cur_line.append(reversed_chunks[-1][:space_left])
- reversed_chunks[-1] = reversed_chunks[-1][space_left:]
+ end = space_left
+ chunk = reversed_chunks[-1]
+ if self.break_on_hyphens and len(chunk) > space_left:
+ # break after last hyphen, but only if there are
+ # non-hyphens before it
+ hyphen = chunk.rfind('-', 0, space_left)
+ if hyphen > 0 and any(c != '-' for c in chunk[:hyphen]):
+ end = hyphen + 1
+ cur_line.append(chunk[:end])
+ reversed_chunks[-1] = chunk[end:]
# Otherwise, we have to preserve the long word intact. Only add
# it to the current line if there's nothing already there --
diff --git a/common/py3-stdlib/threading.py b/common/py3-stdlib/threading.py
index d96d99a..2d89742 100644
--- a/common/py3-stdlib/threading.py
+++ b/common/py3-stdlib/threading.py
@@ -28,7 +28,7 @@
'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread',
'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError',
'setprofile', 'settrace', 'local', 'stack_size',
- 'excepthook', 'ExceptHookArgs']
+ 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile']
# Rename some stuff so "from threading import *" is safe
_start_new_thread = _thread.start_new_thread
@@ -65,6 +65,10 @@
global _profile_hook
_profile_hook = func
+def getprofile():
+ """Get the profiler function as set by threading.setprofile()."""
+ return _profile_hook
+
def settrace(func):
"""Set a trace function for all threads started from the threading module.
@@ -75,6 +79,10 @@
global _trace_hook
_trace_hook = func
+def gettrace():
+ """Get the trace function as set by threading.settrace()."""
+ return _trace_hook
+
# Synchronization classes
Lock = _allocate_lock
@@ -380,7 +388,16 @@
"""
self.notify(len(self._waiters))
- notifyAll = notify_all
+ def notifyAll(self):
+ """Wake up all threads waiting on this condition.
+
+ This method is deprecated, use notify_all() instead.
+
+ """
+ import warnings
+ warnings.warn('notifyAll() is deprecated, use notify_all() instead',
+ DeprecationWarning, stacklevel=2)
+ self.notify_all()
class Semaphore:
@@ -530,7 +547,16 @@
"""Return true if and only if the internal flag is true."""
return self._flag
- isSet = is_set
+ def isSet(self):
+ """Return true if and only if the internal flag is true.
+
+ This method is deprecated, use notify_all() instead.
+
+ """
+ import warnings
+ warnings.warn('isSet() is deprecated, use is_set() instead',
+ DeprecationWarning, stacklevel=2)
+ return self.is_set()
def set(self):
"""Set the internal flag to true.
@@ -608,7 +634,7 @@
self._action = action
self._timeout = timeout
self._parties = parties
- self._state = 0 #0 filling, 1, draining, -1 resetting, -2 broken
+ self._state = 0 # 0 filling, 1 draining, -1 resetting, -2 broken
self._count = 0
def wait(self, timeout=None):
@@ -745,22 +771,39 @@
# Helper to generate new thread names
-_counter = _count().__next__
-_counter() # Consume 0 so first non-main thread has id 1.
-def _newname(template="Thread-%d"):
- return template % _counter()
+_counter = _count(1).__next__
+def _newname(name_template):
+ return name_template % _counter()
-# Active thread administration
-_active_limbo_lock = _allocate_lock()
+# Active thread administration.
+#
+# bpo-44422: Use a reentrant lock to allow reentrant calls to functions like
+# threading.enumerate().
+_active_limbo_lock = RLock()
_active = {} # maps thread id to Thread object
_limbo = {}
_dangling = WeakSet()
+
# Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown()
# to wait until all Python thread states get deleted:
# see Thread._set_tstate_lock().
_shutdown_locks_lock = _allocate_lock()
_shutdown_locks = set()
+def _maintain_shutdown_locks():
+ """
+ Drop any shutdown locks that don't correspond to running threads anymore.
+
+ Calling this from time to time avoids an ever-growing _shutdown_locks
+ set when Thread objects are not joined explicitly. See bpo-37788.
+
+ This must be called with _shutdown_locks_lock acquired.
+ """
+ # If a lock was released, the corresponding thread has exited
+ to_remove = [lock for lock in _shutdown_locks if not lock.locked()]
+ _shutdown_locks.difference_update(to_remove)
+
+
# Main class for threads
class Thread:
@@ -800,8 +843,19 @@
assert group is None, "group argument must be None for now"
if kwargs is None:
kwargs = {}
+ if name:
+ name = str(name)
+ else:
+ name = _newname("Thread-%d")
+ if target is not None:
+ try:
+ target_name = target.__name__
+ name += f" ({target_name})"
+ except AttributeError:
+ pass
+
self._target = target
- self._name = str(name or _newname())
+ self._name = name
self._args = args
self._kwargs = kwargs
if daemon is not None:
@@ -888,7 +942,7 @@
"""
try:
- if self._target:
+ if self._target is not None:
self._target(*self._args, **self._kwargs)
finally:
# Avoid a refcycle if the thread is running a function with
@@ -932,6 +986,7 @@
if not self.daemon:
with _shutdown_locks_lock:
+ _maintain_shutdown_locks()
_shutdown_locks.add(self._tstate_lock)
def _bootstrap_inner(self):
@@ -987,7 +1042,8 @@
self._tstate_lock = None
if not self.daemon:
with _shutdown_locks_lock:
- _shutdown_locks.discard(lock)
+ # Remove our lock and other released locks from _shutdown_locks
+ _maintain_shutdown_locks()
def _delete(self):
"Remove current thread from the dict of currently running threads."
@@ -1044,11 +1100,24 @@
# If the lock is acquired, the C code is done, and self._stop() is
# called. That sets ._is_stopped to True, and ._tstate_lock to None.
lock = self._tstate_lock
- if lock is None: # already determined that the C code is done
+ if lock is None:
+ # already determined that the C code is done
assert self._is_stopped
- elif lock.acquire(block, timeout):
- lock.release()
- self._stop()
+ return
+
+ try:
+ if lock.acquire(block, timeout):
+ lock.release()
+ self._stop()
+ except:
+ if lock.locked():
+ # bpo-45274: lock.acquire() acquired the lock, but the function
+ # was interrupted with an exception before reaching the
+ # lock.release(). It can happen if a signal handler raises an
+ # exception, like CTRL+C which raises KeyboardInterrupt.
+ lock.release()
+ self._stop()
+ raise
@property
def name(self):
@@ -1094,8 +1163,8 @@
"""Return whether the thread is alive.
This method returns True just before the run() method starts until just
- after the run() method terminates. The module function enumerate()
- returns a list of all alive threads.
+ after the run() method terminates. See also the module function
+ enumerate().
"""
assert self._initialized, "Thread.__init__() not called"
@@ -1128,15 +1197,47 @@
self._daemonic = daemonic
def isDaemon(self):
+ """Return whether this thread is a daemon.
+
+ This method is deprecated, use the daemon attribute instead.
+
+ """
+ import warnings
+ warnings.warn('isDaemon() is deprecated, get the daemon attribute instead',
+ DeprecationWarning, stacklevel=2)
return self.daemon
def setDaemon(self, daemonic):
+ """Set whether this thread is a daemon.
+
+ This method is deprecated, use the .daemon property instead.
+
+ """
+ import warnings
+ warnings.warn('setDaemon() is deprecated, set the daemon attribute instead',
+ DeprecationWarning, stacklevel=2)
self.daemon = daemonic
def getName(self):
+ """Return a string used for identification purposes only.
+
+ This method is deprecated, use the name attribute instead.
+
+ """
+ import warnings
+ warnings.warn('getName() is deprecated, get the name attribute instead',
+ DeprecationWarning, stacklevel=2)
return self.name
def setName(self, name):
+ """Set the name string for this thread.
+
+ This method is deprecated, use the name attribute instead.
+
+ """
+ import warnings
+ warnings.warn('setName() is deprecated, set the name attribute instead',
+ DeprecationWarning, stacklevel=2)
self.name = name
@@ -1186,6 +1287,10 @@
stderr.flush()
+# Original value of threading.excepthook
+__excepthook__ = excepthook
+
+
def _make_invoke_excepthook():
# Create a local namespace to ensure that variables remain alive
# when _invoke_excepthook() is called, even if it is called late during
@@ -1327,7 +1432,16 @@
except KeyError:
return _DummyThread()
-currentThread = current_thread
+def currentThread():
+ """Return the current Thread object, corresponding to the caller's thread of control.
+
+ This function is deprecated, use current_thread() instead.
+
+ """
+ import warnings
+ warnings.warn('currentThread() is deprecated, use current_thread() instead',
+ DeprecationWarning, stacklevel=2)
+ return current_thread()
def active_count():
"""Return the number of Thread objects currently alive.
@@ -1339,7 +1453,16 @@
with _active_limbo_lock:
return len(_active) + len(_limbo)
-activeCount = active_count
+def activeCount():
+ """Return the number of Thread objects currently alive.
+
+ This function is deprecated, use active_count() instead.
+
+ """
+ import warnings
+ warnings.warn('activeCount() is deprecated, use active_count() instead',
+ DeprecationWarning, stacklevel=2)
+ return active_count()
def _enumerate():
# Same as enumerate(), but without the lock. Internal use only.
@@ -1400,20 +1523,29 @@
global _SHUTTING_DOWN
_SHUTTING_DOWN = True
- # Main thread
- tlock = _main_thread._tstate_lock
- # The main thread isn't finished yet, so its thread state lock can't have
- # been released.
- assert tlock is not None
- assert tlock.locked()
- tlock.release()
- _main_thread._stop()
# Call registered threading atexit functions before threads are joined.
# Order is reversed, similar to atexit.
for atexit_call in reversed(_threading_atexits):
atexit_call()
+ # Main thread
+ if _main_thread.ident == get_ident():
+ tlock = _main_thread._tstate_lock
+ # The main thread isn't finished yet, so its thread state lock can't
+ # have been released.
+ assert tlock is not None
+ assert tlock.locked()
+ tlock.release()
+ _main_thread._stop()
+ else:
+ # bpo-1596321: _shutdown() must be called in the main thread.
+ # If the threading module was not imported by the main thread,
+ # _main_thread is the thread which imported the threading module.
+ # In this case, ignore _main_thread, similar behavior than for threads
+ # spawned by C libraries or using _thread.start_new_thread().
+ pass
+
# Join all non-deamon threads
while True:
with _shutdown_locks_lock:
@@ -1424,7 +1556,7 @@
break
for lock in locks:
- # mimick Thread.join()
+ # mimic Thread.join()
lock.acquire()
lock.release()
@@ -1457,7 +1589,7 @@
# by another (non-forked) thread. http://bugs.python.org/issue874900
global _active_limbo_lock, _main_thread
global _shutdown_locks_lock, _shutdown_locks
- _active_limbo_lock = _allocate_lock()
+ _active_limbo_lock = RLock()
# fork() only copied the current thread; clear references to others.
new_active = {}
diff --git a/common/py3-stdlib/timeit.py b/common/py3-stdlib/timeit.py
index 6c3ec01..9dfd454 100755
--- a/common/py3-stdlib/timeit.py
+++ b/common/py3-stdlib/timeit.py
@@ -72,6 +72,7 @@
_t0 = _timer()
for _i in _it:
{stmt}
+ pass
_t1 = _timer()
return _t1 - _t0
"""
diff --git a/common/py3-stdlib/token.py b/common/py3-stdlib/token.py
index 493bf04..9d0c0bf 100644
--- a/common/py3-stdlib/token.py
+++ b/common/py3-stdlib/token.py
@@ -62,12 +62,13 @@
ASYNC = 56
TYPE_IGNORE = 57
TYPE_COMMENT = 58
+SOFT_KEYWORD = 59
# These aren't used by the C tokenizer but are needed for tokenize.py
-ERRORTOKEN = 59
-COMMENT = 60
-NL = 61
-ENCODING = 62
-N_TOKENS = 63
+ERRORTOKEN = 60
+COMMENT = 61
+NL = 62
+ENCODING = 63
+N_TOKENS = 64
# Special definitions for cooperation with parser
NT_OFFSET = 256
diff --git a/common/py3-stdlib/tokenize.py b/common/py3-stdlib/tokenize.py
index 1aee21b..7d7736f 100644
--- a/common/py3-stdlib/tokenize.py
+++ b/common/py3-stdlib/tokenize.py
@@ -27,6 +27,7 @@
from builtins import open as _builtin_open
from codecs import lookup, BOM_UTF8
import collections
+import functools
from io import TextIOWrapper
import itertools as _itertools
import re
@@ -95,6 +96,7 @@
result.add(''.join(u))
return result
+@functools.lru_cache
def _compile(expr):
return re.compile(expr, re.UNICODE)
@@ -602,7 +604,7 @@
pos += 1
# Add an implicit NEWLINE if the input doesn't end in one
- if last_line and last_line[-1] not in '\r\n':
+ if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"):
yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
for indent in indents[1:]: # pop remaining indent levels
yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
diff --git a/common/py3-stdlib/trace.py b/common/py3-stdlib/trace.py
index c505d8b..2cf3643 100755
--- a/common/py3-stdlib/trace.py
+++ b/common/py3-stdlib/trace.py
@@ -116,7 +116,7 @@
return 0
def _modname(path):
- """Return a plausible module name for the patch."""
+ """Return a plausible module name for the path."""
base = os.path.basename(path)
filename, ext = os.path.splitext(base)
diff --git a/common/py3-stdlib/traceback.py b/common/py3-stdlib/traceback.py
index fb34de9..d6a010f 100644
--- a/common/py3-stdlib/traceback.py
+++ b/common/py3-stdlib/traceback.py
@@ -84,7 +84,25 @@
"another exception occurred:\n\n")
-def print_exception(etype, value, tb, limit=None, file=None, chain=True):
+class _Sentinel:
+ def __repr__(self):
+ return "<implicit>"
+
+_sentinel = _Sentinel()
+
+def _parse_value_tb(exc, value, tb):
+ if (value is _sentinel) != (tb is _sentinel):
+ raise ValueError("Both or neither of value and tb must be given")
+ if value is tb is _sentinel:
+ if exc is not None:
+ return exc, exc.__traceback__
+ else:
+ return None, None
+ return value, tb
+
+
+def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \
+ file=None, chain=True):
"""Print exception up to 'limit' stack trace entries from 'tb' to 'file'.
This differs from print_tb() in the following ways: (1) if
@@ -95,17 +113,16 @@
occurred with a caret on the next line indicating the approximate
position of the error.
"""
- # format_exception has ignored etype for some time, and code such as cgitb
- # passes in bogus values as a result. For compatibility with such code we
- # ignore it here (rather than in the new TracebackException API).
+ value, tb = _parse_value_tb(exc, value, tb)
if file is None:
file = sys.stderr
- for line in TracebackException(
- type(value), value, tb, limit=limit).format(chain=chain):
+ te = TracebackException(type(value), value, tb, limit=limit, compact=True)
+ for line in te.format(chain=chain):
print(line, file=file, end="")
-def format_exception(etype, value, tb, limit=None, chain=True):
+def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \
+ chain=True):
"""Format a stack trace and the exception information.
The arguments have the same meaning as the corresponding arguments
@@ -114,19 +131,15 @@
these lines are concatenated and printed, exactly the same text is
printed as does print_exception().
"""
- # format_exception has ignored etype for some time, and code such as cgitb
- # passes in bogus values as a result. For compatibility with such code we
- # ignore it here (rather than in the new TracebackException API).
- return list(TracebackException(
- type(value), value, tb, limit=limit).format(chain=chain))
+ value, tb = _parse_value_tb(exc, value, tb)
+ te = TracebackException(type(value), value, tb, limit=limit, compact=True)
+ return list(te.format(chain=chain))
-def format_exception_only(etype, value):
+def format_exception_only(exc, /, value=_sentinel):
"""Format the exception part of a traceback.
- The arguments are the exception type and value such as given by
- sys.last_type and sys.last_value. The return value is a list of
- strings, each ending in a newline.
+ The return value is a list of strings, each ending in a newline.
Normally, the list contains a single string; however, for
SyntaxError exceptions, it contains several lines that (when
@@ -137,7 +150,10 @@
string in the list.
"""
- return list(TracebackException(etype, value, None).format_exception_only())
+ if value is _sentinel:
+ value = exc
+ te = TracebackException(type(value), value, None, compact=True)
+ return list(te.format_exception_only())
# -- not official API but folk probably use these two functions.
@@ -285,9 +301,10 @@
@property
def line(self):
if self._line is None:
- self._line = linecache.getline(self.filename, self.lineno).strip()
- return self._line
-
+ if self.lineno is None:
+ return None
+ self._line = linecache.getline(self.filename, self.lineno)
+ return self._line.strip()
def walk_stack(f):
"""Walk a stack yielding the frame and line number for each frame.
@@ -458,52 +475,29 @@
occurred.
- :attr:`lineno` For syntax errors - the linenumber where the error
occurred.
+ - :attr:`end_lineno` For syntax errors - the end linenumber where the error
+ occurred. Can be `None` if not present.
- :attr:`text` For syntax errors - the text where the error
occurred.
- :attr:`offset` For syntax errors - the offset into the text where the
error occurred.
+ - :attr:`end_offset` For syntax errors - the offset into the text where the
+ error occurred. Can be `None` if not present.
- :attr:`msg` For syntax errors - the compiler error message.
"""
def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None,
- lookup_lines=True, capture_locals=False, _seen=None):
+ lookup_lines=True, capture_locals=False, compact=False,
+ _seen=None):
# NB: we need to accept exc_traceback, exc_value, exc_traceback to
# permit backwards compat with the existing API, otherwise we
# need stub thunk objects just to glue it together.
# Handle loops in __cause__ or __context__.
+ is_recursive_call = _seen is not None
if _seen is None:
_seen = set()
_seen.add(id(exc_value))
- # Gracefully handle (the way Python 2.4 and earlier did) the case of
- # being called with no type or value (None, None, None).
- if (exc_value and exc_value.__cause__ is not None
- and id(exc_value.__cause__) not in _seen):
- cause = TracebackException(
- type(exc_value.__cause__),
- exc_value.__cause__,
- exc_value.__cause__.__traceback__,
- limit=limit,
- lookup_lines=False,
- capture_locals=capture_locals,
- _seen=_seen)
- else:
- cause = None
- if (exc_value and exc_value.__context__ is not None
- and id(exc_value.__context__) not in _seen):
- context = TracebackException(
- type(exc_value.__context__),
- exc_value.__context__,
- exc_value.__context__.__traceback__,
- limit=limit,
- lookup_lines=False,
- capture_locals=capture_locals,
- _seen=_seen)
- else:
- context = None
- self.__cause__ = cause
- self.__context__ = context
- self.__suppress_context__ = \
- exc_value.__suppress_context__ if exc_value else False
+
# TODO: locals.
self.stack = StackSummary.extract(
walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines,
@@ -515,12 +509,62 @@
if exc_type and issubclass(exc_type, SyntaxError):
# Handle SyntaxError's specially
self.filename = exc_value.filename
- self.lineno = str(exc_value.lineno)
+ lno = exc_value.lineno
+ self.lineno = str(lno) if lno is not None else None
+ end_lno = exc_value.end_lineno
+ self.end_lineno = str(end_lno) if end_lno is not None else None
self.text = exc_value.text
self.offset = exc_value.offset
+ self.end_offset = exc_value.end_offset
self.msg = exc_value.msg
if lookup_lines:
self._load_lines()
+ self.__suppress_context__ = \
+ exc_value.__suppress_context__ if exc_value is not None else False
+
+ # Convert __cause__ and __context__ to `TracebackExceptions`s, use a
+ # queue to avoid recursion (only the top-level call gets _seen == None)
+ if not is_recursive_call:
+ queue = [(self, exc_value)]
+ while queue:
+ te, e = queue.pop()
+ if (e and e.__cause__ is not None
+ and id(e.__cause__) not in _seen):
+ cause = TracebackException(
+ type(e.__cause__),
+ e.__cause__,
+ e.__cause__.__traceback__,
+ limit=limit,
+ lookup_lines=lookup_lines,
+ capture_locals=capture_locals,
+ _seen=_seen)
+ else:
+ cause = None
+
+ if compact:
+ need_context = (cause is None and
+ e is not None and
+ not e.__suppress_context__)
+ else:
+ need_context = True
+ if (e and e.__context__ is not None
+ and need_context and id(e.__context__) not in _seen):
+ context = TracebackException(
+ type(e.__context__),
+ e.__context__,
+ e.__context__.__traceback__,
+ limit=limit,
+ lookup_lines=lookup_lines,
+ capture_locals=capture_locals,
+ _seen=_seen)
+ else:
+ context = None
+ te.__cause__ = cause
+ te.__context__ = context
+ if cause:
+ queue.append((te.__cause__, e.__cause__))
+ if context:
+ queue.append((te.__context__, e.__context__))
@classmethod
def from_exception(cls, exc, *args, **kwargs):
@@ -531,10 +575,6 @@
"""Private API. force all lines in the stack to be loaded."""
for frame in self.stack:
frame.line
- if self.__context__:
- self.__context__._load_lines()
- if self.__cause__:
- self.__cause__._load_lines()
def __eq__(self, other):
if isinstance(other, TracebackException):
@@ -564,6 +604,8 @@
stype = self.exc_type.__qualname__
smod = self.exc_type.__module__
if smod not in ("__main__", "builtins"):
+ if not isinstance(smod, str):
+ smod = "<unknown>"
stype = smod + '.' + stype
if not issubclass(self.exc_type, SyntaxError):
@@ -574,9 +616,12 @@
def _format_syntax_error(self, stype):
"""Format SyntaxError exceptions (internal helper)."""
# Show exactly where the problem was found.
- filename = self.filename or "<string>"
- lineno = str(self.lineno) or '?'
- yield ' File "{}", line {}\n'.format(filename, lineno)
+ filename_suffix = ''
+ if self.lineno is not None:
+ yield ' File "{}", line {}\n'.format(
+ self.filename or "<string>", self.lineno)
+ elif self.filename is not None:
+ filename_suffix = ' ({})'.format(self.filename)
text = self.text
if text is not None:
@@ -587,14 +632,22 @@
ltext = rtext.lstrip(' \n\f')
spaces = len(rtext) - len(ltext)
yield ' {}\n'.format(ltext)
- # Convert 1-based column offset to 0-based index into stripped text
- caret = (self.offset or 0) - 1 - spaces
- if caret >= 0:
- # non-space whitespace (likes tabs) must be kept for alignment
- caretspace = ((c if c.isspace() else ' ') for c in ltext[:caret])
- yield ' {}^\n'.format(''.join(caretspace))
+
+ if self.offset is not None:
+ offset = self.offset
+ end_offset = self.end_offset if self.end_offset not in {None, 0} else offset
+ if offset == end_offset or end_offset == -1:
+ end_offset = offset + 1
+
+ # Convert 1-based column offset to 0-based index into stripped text
+ colno = offset - 1 - spaces
+ end_colno = end_offset - 1 - spaces
+ if colno >= 0:
+ # non-space whitespace (likes tabs) must be kept for alignment
+ caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno])
+ yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n"))
msg = self.msg or "<no detail available>"
- yield "{}: {}\n".format(stype, msg)
+ yield "{}: {}{}\n".format(stype, msg, filename_suffix)
def format(self, *, chain=True):
"""Format the exception.
@@ -608,15 +661,32 @@
The message indicating which exception occurred is always the last
string in the output.
"""
- if chain:
- if self.__cause__ is not None:
- yield from self.__cause__.format(chain=chain)
- yield _cause_message
- elif (self.__context__ is not None and
- not self.__suppress_context__):
- yield from self.__context__.format(chain=chain)
- yield _context_message
- if self.stack:
- yield 'Traceback (most recent call last):\n'
- yield from self.stack.format()
- yield from self.format_exception_only()
+
+ output = []
+ exc = self
+ while exc:
+ if chain:
+ if exc.__cause__ is not None:
+ chained_msg = _cause_message
+ chained_exc = exc.__cause__
+ elif (exc.__context__ is not None and
+ not exc.__suppress_context__):
+ chained_msg = _context_message
+ chained_exc = exc.__context__
+ else:
+ chained_msg = None
+ chained_exc = None
+
+ output.append((chained_msg, exc))
+ exc = chained_exc
+ else:
+ output.append((None, exc))
+ exc = None
+
+ for msg, exc in reversed(output):
+ if msg is not None:
+ yield msg
+ if exc.stack:
+ yield 'Traceback (most recent call last):\n'
+ yield from exc.stack.format()
+ yield from exc.format_exception_only()
diff --git a/common/py3-stdlib/tracemalloc.py b/common/py3-stdlib/tracemalloc.py
index 69b4170..cec99c5 100644
--- a/common/py3-stdlib/tracemalloc.py
+++ b/common/py3-stdlib/tracemalloc.py
@@ -226,7 +226,7 @@
return str(self[0])
def __repr__(self):
- s = "<Traceback %r" % tuple(self)
+ s = f"<Traceback {tuple(self)}"
if self._total_nframe is None:
s += ">"
else:
diff --git a/common/py3-stdlib/turtle.py b/common/py3-stdlib/turtle.py
index ba8288d..f3b320b 100644
--- a/common/py3-stdlib/turtle.py
+++ b/common/py3-stdlib/turtle.py
@@ -38,7 +38,7 @@
----- turtle.py
This module is an extended reimplementation of turtle.py from the
-Python standard distribution up to Python 2.5. (See: http://www.python.org)
+Python standard distribution up to Python 2.5. (See: https://www.python.org)
It tries to keep the merits of turtle.py and to be (nearly) 100%
compatible with it. This means in the first place to enable the
@@ -264,12 +264,12 @@
def __neg__(self):
return Vec2D(-self[0], -self[1])
def __abs__(self):
- return (self[0]**2 + self[1]**2)**0.5
+ return math.hypot(*self)
def rotate(self, angle):
"""rotate self counterclockwise by angle
"""
perp = Vec2D(-self[1], self[0])
- angle = angle * math.pi / 180.0
+ angle = math.radians(angle)
c, s = math.cos(angle), math.sin(angle)
return Vec2D(self[0]*c+perp[0]*s, self[1]*c+perp[1]*s)
def __getnewargs__(self):
@@ -464,20 +464,18 @@
a corresponding TurtleScreenBase class has to be implemented.
"""
- @staticmethod
- def _blankimage():
+ def _blankimage(self):
"""return a blank image object
"""
- img = TK.PhotoImage(width=1, height=1)
+ img = TK.PhotoImage(width=1, height=1, master=self.cv)
img.blank()
return img
- @staticmethod
- def _image(filename):
+ def _image(self, filename):
"""return an image object containing the
imagedata from a gif-file named filename.
"""
- return TK.PhotoImage(file=filename)
+ return TK.PhotoImage(file=filename, master=self.cv)
def __init__(self, cv):
self.cv = cv
@@ -811,7 +809,7 @@
>>> screen.mainloop()
"""
- TK.mainloop()
+ self.cv.tk.mainloop()
def textinput(self, title, prompt):
"""Pop up a dialog window for input of a string.
@@ -826,7 +824,7 @@
>>> screen.textinput("NIM", "Name of first player:")
"""
- return simpledialog.askstring(title, prompt)
+ return simpledialog.askstring(title, prompt, parent=self.cv)
def numinput(self, title, prompt, default=None, minval=None, maxval=None):
"""Pop up a dialog window for input of a number.
@@ -847,7 +845,8 @@
"""
return simpledialog.askfloat(title, prompt, initialvalue=default,
- minvalue=minval, maxvalue=maxval)
+ minvalue=minval, maxvalue=maxval,
+ parent=self.cv)
##############################################################################
@@ -965,6 +964,8 @@
def __init__(self, cv, mode=_CFG["mode"],
colormode=_CFG["colormode"], delay=_CFG["delay"]):
+ TurtleScreenBase.__init__(self, cv)
+
self._shapes = {
"arrow" : Shape("polygon", ((-10,0), (10,0), (0,10))),
"turtle" : Shape("polygon", ((0,16), (-2,14), (-1,10), (-4,7),
@@ -988,7 +989,6 @@
self._bgpics = {"nopic" : ""}
- TurtleScreenBase.__init__(self, cv)
self._mode = mode
self._delayvalue = delay
self._colormode = _CFG["colormode"]
@@ -1598,7 +1598,7 @@
>>> turtle.heading()
1.5707963267948966
"""
- self._setDegreesPerAU(2*math.pi)
+ self._setDegreesPerAU(math.tau)
def _go(self, distance):
"""move turtle forward by specified distance"""
@@ -1645,7 +1645,7 @@
Argument:
distance -- a number
- Move the turtle backward by distance ,opposite to the direction the
+ Move the turtle backward by distance, opposite to the direction the
turtle is headed. Do not change the turtle's heading.
Example (for a Turtle instance named turtle):
@@ -1889,7 +1889,7 @@
elif isinstance(x, TNavigator):
pos = x._position
x, y = pos - self._position
- result = round(math.atan2(y, x)*180.0/math.pi, 10) % 360.0
+ result = round(math.degrees(math.atan2(y, x)), 10) % 360.0
result /= self._degreesPerAU
return (self._angleOffset + self._angleOrient*result) % self._fullcircle
@@ -1904,7 +1904,7 @@
67.0
"""
x, y = self._orient
- result = round(math.atan2(y, x)*180.0/math.pi, 10) % 360.0
+ result = round(math.degrees(math.atan2(y, x)), 10) % 360.0
result /= self._degreesPerAU
return (self._angleOffset + self._angleOrient*result) % self._fullcircle
@@ -1977,7 +1977,7 @@
steps = 1+int(min(11+abs(radius)/6.0, 59.0)*frac)
w = 1.0 * extent / steps
w2 = 0.5 * w
- l = 2.0 * radius * math.sin(w2*math.pi/180.0*self._degreesPerAU)
+ l = 2.0 * radius * math.sin(math.radians(w2)*self._degreesPerAU)
if radius < 0:
l, w, w2 = -l, -w, -w2
tr = self._tracer()
@@ -2862,7 +2862,7 @@
>>> turtle.fd(50)
"""
tilt = -angle * self._degreesPerAU * self._angleOrient
- tilt = (tilt * math.pi / 180.0) % (2*math.pi)
+ tilt = math.radians(tilt) % math.tau
self.pen(resizemode="user", tilt=tilt)
def tiltangle(self, angle=None):
@@ -2877,7 +2877,8 @@
between the orientation of the turtleshape and the heading of the
turtle (its direction of movement).
- Deprecated since Python 3.1
+ (Incorrectly marked as deprecated since Python 3.1, it is really
+ settiltangle that is deprecated.)
Examples (for a Turtle instance named turtle):
>>> turtle.shape("circle")
@@ -2886,7 +2887,7 @@
>>> turtle.tiltangle()
"""
if angle is None:
- tilt = -self._tilt * (180.0/math.pi) * self._angleOrient
+ tilt = -math.degrees(self._tilt) * self._angleOrient
return (tilt / self._degreesPerAU) % self._fullcircle
else:
self.settiltangle(angle)
@@ -2940,7 +2941,7 @@
if t11 * t22 - t12 * t21 == 0:
raise TurtleGraphicsError("Bad shape transform matrix: must not be singular")
self._shapetrafo = (m11, m12, m21, m22)
- alfa = math.atan2(-m21, m11) % (2 * math.pi)
+ alfa = math.atan2(-m21, m11) % math.tau
sa, ca = math.sin(alfa), math.cos(alfa)
a11, a12, a21, a22 = (ca*m11 - sa*m21, ca*m12 - sa*m22,
sa*m11 + ca*m21, sa*m12 + ca*m22)
diff --git a/common/py3-stdlib/types.py b/common/py3-stdlib/types.py
index ad2020e..62122a9 100644
--- a/common/py3-stdlib/types.py
+++ b/common/py3-stdlib/types.py
@@ -82,7 +82,7 @@
updated = False
shift = 0
for i, base in enumerate(bases):
- if isinstance(base, type):
+ if isinstance(base, type) and not isinstance(base, GenericAlias):
continue
if not hasattr(base, "__mro_entries__"):
continue
@@ -155,7 +155,12 @@
class's __getattr__ method; this is done by raising AttributeError.
This allows one to have properties active on an instance, and have virtual
- attributes on the class with the same name (see Enum for an example).
+ attributes on the class with the same name. (Enum used this between Python
+ versions 3.4 - 3.9 .)
+
+ Subclass from this to use a different method of accessing virtual atributes
+ and still be treated properly by the inspect module. (Enum uses this since
+ Python 3.10 .)
"""
def __init__(self, fget=None, fset=None, fdel=None, doc=None):
@@ -292,8 +297,11 @@
return wrapped
-
GenericAlias = type(list[int])
+UnionType = type(int | str)
+EllipsisType = type(Ellipsis)
+NoneType = type(None)
+NotImplementedType = type(NotImplemented)
__all__ = [n for n in globals() if n[:1] != '_']
diff --git a/common/py3-stdlib/typing.py b/common/py3-stdlib/typing.py
index f5316ab..086d0f3 100644
--- a/common/py3-stdlib/typing.py
+++ b/common/py3-stdlib/typing.py
@@ -4,8 +4,10 @@
At large scale, the structure of the module is following:
* Imports and exports, all public names should be explicitly added to __all__.
* Internal helper functions: these should never be used in code outside this module.
-* _SpecialForm and its instances (special forms): Any, NoReturn, ClassVar, Union, Optional
-* Two classes whose instances can be type arguments in addition to types: ForwardRef and TypeVar
+* _SpecialForm and its instances (special forms):
+ Any, NoReturn, ClassVar, Union, Optional, Concatenate
+* Classes whose instances can be type arguments in addition to types:
+ ForwardRef, TypeVar and ParamSpec
* The core of internal generics API: _GenericAlias and _VariadicGenericAlias, the latter is
currently only used by Tuple and Callable. All subscripted types like X[int], Union[int, str],
etc., are instances of either of these classes.
@@ -35,11 +37,13 @@
'Any',
'Callable',
'ClassVar',
+ 'Concatenate',
'Final',
'ForwardRef',
'Generic',
'Literal',
'Optional',
+ 'ParamSpec',
'Protocol',
'Tuple',
'Type',
@@ -96,6 +100,13 @@
'TypedDict', # Not really a type.
'Generator',
+ # Other concrete types.
+ 'BinaryIO',
+ 'IO',
+ 'Match',
+ 'Pattern',
+ 'TextIO',
+
# One-off things.
'AnyStr',
'cast',
@@ -103,14 +114,19 @@
'get_args',
'get_origin',
'get_type_hints',
+ 'is_typeddict',
'NewType',
'no_type_check',
'no_type_check_decorator',
'NoReturn',
'overload',
+ 'ParamSpecArgs',
+ 'ParamSpecKwargs',
'runtime_checkable',
'Text',
'TYPE_CHECKING',
+ 'TypeAlias',
+ 'TypeGuard',
]
# The pseudo-submodules 're' and 'io' are part of the public
@@ -118,7 +134,16 @@
# legitimate imports of those modules.
-def _type_check(arg, msg, is_argument=True):
+def _type_convert(arg, module=None, *, allow_special_forms=False):
+ """For converting None to type(None), and strings to ForwardRef."""
+ if arg is None:
+ return type(None)
+ if isinstance(arg, str):
+ return ForwardRef(arg, module=module, is_class=allow_special_forms)
+ return arg
+
+
+def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False):
"""Check that the argument is a type, and return it (internal helper).
As a special case, accept None and return type(None) instead. Also wrap strings
@@ -131,27 +156,32 @@
We append the repr() of the actual value (truncated to 100 chars).
"""
invalid_generic_forms = (Generic, Protocol)
- if is_argument:
- invalid_generic_forms = invalid_generic_forms + (ClassVar, Final)
+ if not allow_special_forms:
+ invalid_generic_forms += (ClassVar,)
+ if is_argument:
+ invalid_generic_forms += (Final,)
- if arg is None:
- return type(None)
- if isinstance(arg, str):
- return ForwardRef(arg)
+ arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms)
if (isinstance(arg, _GenericAlias) and
arg.__origin__ in invalid_generic_forms):
raise TypeError(f"{arg} is not valid as type argument")
- if arg in (Any, NoReturn):
+ if arg in (Any, NoReturn, Final, TypeAlias):
return arg
if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol):
raise TypeError(f"Plain {arg} is not valid as type argument")
- if isinstance(arg, (type, TypeVar, ForwardRef)):
+ if isinstance(arg, (type, TypeVar, ForwardRef, types.UnionType, ParamSpec,
+ ParamSpecArgs, ParamSpecKwargs)):
return arg
if not callable(arg):
raise TypeError(f"{msg} Got {arg!r:.100}.")
return arg
+def _is_param_expr(arg):
+ return arg is ... or isinstance(arg,
+ (tuple, list, ParamSpec, _ConcatenateGenericAlias))
+
+
def _type_repr(obj):
"""Return the repr() of an object, special-casing types (internal helper).
@@ -173,17 +203,19 @@
return repr(obj)
-def _collect_type_vars(types):
- """Collect all type variable contained in types in order of
- first appearance (lexicographic order). For example::
+def _collect_type_vars(types_, typevar_types=None):
+ """Collect all type variable contained
+ in types in order of first appearance (lexicographic order). For example::
_collect_type_vars((T, List[S, T])) == (T, S)
"""
+ if typevar_types is None:
+ typevar_types = TypeVar
tvars = []
- for t in types:
- if isinstance(t, TypeVar) and t not in tvars:
+ for t in types_:
+ if isinstance(t, typevar_types) and t not in tvars:
tvars.append(t)
- if isinstance(t, (_GenericAlias, GenericAlias)):
+ if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)):
tvars.extend([t for t in t.__parameters__ if t not in tvars])
return tuple(tvars)
@@ -196,9 +228,27 @@
raise TypeError(f"{cls} is not a generic class")
alen = len(parameters)
if alen != elen:
- raise TypeError(f"Too {'many' if alen > elen else 'few'} parameters for {cls};"
+ raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};"
f" actual {alen}, expected {elen}")
+def _prepare_paramspec_params(cls, params):
+ """Prepares the parameters for a Generic containing ParamSpec
+ variables (internal helper).
+ """
+ # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612.
+ if (len(cls.__parameters__) == 1
+ and params and not _is_param_expr(params[0])):
+ assert isinstance(cls.__parameters__[0], ParamSpec)
+ return (params,)
+ else:
+ _check_generic(cls, params, len(cls.__parameters__))
+ _params = []
+ # Convert lists to tuples to help other libraries cache the results.
+ for p, tvar in zip(params, cls.__parameters__):
+ if isinstance(tvar, ParamSpec) and isinstance(p, list):
+ p = tuple(p)
+ _params.append(p)
+ return tuple(_params)
def _deduplicate(params):
# Weed out strict duplicates, preserving the first of each occurrence.
@@ -221,7 +271,7 @@
# Flatten out Union[Union[...], ...].
params = []
for p in parameters:
- if isinstance(p, _UnionGenericAlias):
+ if isinstance(p, (_UnionGenericAlias, types.UnionType)):
params.extend(p.__args__)
elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union:
params.extend(p[1:])
@@ -270,17 +320,19 @@
def _eval_type(t, globalns, localns, recursive_guard=frozenset()):
"""Evaluate all forward references in the given type t.
For use of globalns and localns see the docstring for get_type_hints().
- recursive_guard is used to prevent prevent infinite recursion
- with recursive ForwardRef.
+ recursive_guard is used to prevent infinite recursion with a recursive
+ ForwardRef.
"""
if isinstance(t, ForwardRef):
return t._evaluate(globalns, localns, recursive_guard)
- if isinstance(t, (_GenericAlias, GenericAlias)):
+ if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)):
ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__)
if ev_args == t.__args__:
return t
if isinstance(t, GenericAlias):
return GenericAlias(t.__origin__, ev_args)
+ if isinstance(t, types.UnionType):
+ return functools.reduce(operator.or_, ev_args)
else:
return t.copy_with(ev_args)
return t
@@ -316,6 +368,12 @@
self._name = getitem.__name__
self.__doc__ = getitem.__doc__
+ def __getattr__(self, item):
+ if item in {'__name__', '__qualname__'}:
+ return self._name
+
+ raise AttributeError(item)
+
def __mro_entries__(self, bases):
raise TypeError(f"Cannot subclass {self!r}")
@@ -328,6 +386,12 @@
def __call__(self, *args, **kwds):
raise TypeError(f"Cannot instantiate {self!r}")
+ def __or__(self, other):
+ return Union[self, other]
+
+ def __ror__(self, other):
+ return Union[other, self]
+
def __instancecheck__(self, obj):
raise TypeError(f"{self} cannot be used with isinstance()")
@@ -340,9 +404,10 @@
class _LiteralSpecialForm(_SpecialForm, _root=True):
- @_tp_cache(typed=True)
def __getitem__(self, parameters):
- return self._getitem(self, parameters)
+ if not isinstance(parameters, tuple):
+ parameters = (parameters,)
+ return self._getitem(self, *parameters)
@_SpecialForm
@@ -451,6 +516,8 @@
parameters = _remove_dups_flatten(parameters)
if len(parameters) == 1:
return parameters[0]
+ if len(parameters) == 2 and type(None) in parameters:
+ return _UnionGenericAlias(self, parameters, name="Optional")
return _UnionGenericAlias(self, parameters)
@_SpecialForm
@@ -463,7 +530,8 @@
return Union[arg, type(None)]
@_LiteralSpecialForm
-def Literal(self, parameters):
+@_tp_cache(typed=True)
+def Literal(self, *parameters):
"""Special typing form to define literal types (a.k.a. value types).
This form can be used to indicate to type checkers that the corresponding
@@ -486,9 +554,6 @@
"""
# There is no '_type_check' call because arguments to Literal[...] are
# values, not types.
- if not isinstance(parameters, tuple):
- parameters = (parameters,)
-
parameters = _flatten_literal_params(parameters)
try:
@@ -499,14 +564,104 @@
return _LiteralGenericAlias(self, parameters)
+@_SpecialForm
+def TypeAlias(self, parameters):
+ """Special marker indicating that an assignment should
+ be recognized as a proper type alias definition by type
+ checkers.
+
+ For example::
+
+ Predicate: TypeAlias = Callable[..., bool]
+
+ It's invalid when used anywhere except as in the example above.
+ """
+ raise TypeError(f"{self} is not subscriptable")
+
+
+@_SpecialForm
+def Concatenate(self, parameters):
+ """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a
+ higher order function which adds, removes or transforms parameters of a
+ callable.
+
+ For example::
+
+ Callable[Concatenate[int, P], int]
+
+ See PEP 612 for detailed information.
+ """
+ if parameters == ():
+ raise TypeError("Cannot take a Concatenate of no types.")
+ if not isinstance(parameters, tuple):
+ parameters = (parameters,)
+ if not isinstance(parameters[-1], ParamSpec):
+ raise TypeError("The last parameter to Concatenate should be a "
+ "ParamSpec variable.")
+ msg = "Concatenate[arg, ...]: each arg must be a type."
+ parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1])
+ return _ConcatenateGenericAlias(self, parameters,
+ _typevar_types=(TypeVar, ParamSpec),
+ _paramspec_tvars=True)
+
+
+@_SpecialForm
+def TypeGuard(self, parameters):
+ """Special typing form used to annotate the return type of a user-defined
+ type guard function. ``TypeGuard`` only accepts a single type argument.
+ At runtime, functions marked this way should return a boolean.
+
+ ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static
+ type checkers to determine a more precise type of an expression within a
+ program's code flow. Usually type narrowing is done by analyzing
+ conditional code flow and applying the narrowing to a block of code. The
+ conditional expression here is sometimes referred to as a "type guard".
+
+ Sometimes it would be convenient to use a user-defined boolean function
+ as a type guard. Such a function should use ``TypeGuard[...]`` as its
+ return type to alert static type checkers to this intention.
+
+ Using ``-> TypeGuard`` tells the static type checker that for a given
+ function:
+
+ 1. The return value is a boolean.
+ 2. If the return value is ``True``, the type of its argument
+ is the type inside ``TypeGuard``.
+
+ For example::
+
+ def is_str(val: Union[str, float]):
+ # "isinstance" type guard
+ if isinstance(val, str):
+ # Type of ``val`` is narrowed to ``str``
+ ...
+ else:
+ # Else, type of ``val`` is narrowed to ``float``.
+ ...
+
+ Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower
+ form of ``TypeA`` (it can even be a wider form) and this may lead to
+ type-unsafe results. The main reason is to allow for things like
+ narrowing ``List[object]`` to ``List[str]`` even though the latter is not
+ a subtype of the former, since ``List`` is invariant. The responsibility of
+ writing type-safe type guards is left to the user.
+
+ ``TypeGuard`` also works with type variables. For more information, see
+ PEP 647 (User-Defined Type Guards).
+ """
+ item = _type_check(parameters, f'{self} accepts only single type.')
+ return _GenericAlias(self, (item,))
+
+
class ForwardRef(_Final, _root=True):
"""Internal wrapper to hold a forward reference."""
__slots__ = ('__forward_arg__', '__forward_code__',
'__forward_evaluated__', '__forward_value__',
- '__forward_is_argument__')
+ '__forward_is_argument__', '__forward_is_class__',
+ '__forward_module__')
- def __init__(self, arg, is_argument=True):
+ def __init__(self, arg, is_argument=True, module=None, *, is_class=False):
if not isinstance(arg, str):
raise TypeError(f"Forward reference must be a string -- got {arg!r}")
try:
@@ -518,6 +673,8 @@
self.__forward_evaluated__ = False
self.__forward_value__ = None
self.__forward_is_argument__ = is_argument
+ self.__forward_is_class__ = is_class
+ self.__forward_module__ = module
def _evaluate(self, globalns, localns, recursive_guard):
if self.__forward_arg__ in recursive_guard:
@@ -529,10 +686,15 @@
globalns = localns
elif localns is None:
localns = globalns
- type_ =_type_check(
+ if self.__forward_module__ is not None:
+ globalns = getattr(
+ sys.modules.get(self.__forward_module__, None), '__dict__', globalns
+ )
+ type_ = _type_check(
eval(self.__forward_code__, globalns, localns),
"Forward references must evaluate to types.",
is_argument=self.__forward_is_argument__,
+ allow_special_forms=self.__forward_is_class__,
)
self.__forward_value__ = _eval_type(
type_, globalns, localns, recursive_guard | {self.__forward_arg__}
@@ -546,16 +708,50 @@
if self.__forward_evaluated__ and other.__forward_evaluated__:
return (self.__forward_arg__ == other.__forward_arg__ and
self.__forward_value__ == other.__forward_value__)
- return self.__forward_arg__ == other.__forward_arg__
+ return (self.__forward_arg__ == other.__forward_arg__ and
+ self.__forward_module__ == other.__forward_module__)
def __hash__(self):
- return hash(self.__forward_arg__)
+ return hash((self.__forward_arg__, self.__forward_module__))
def __repr__(self):
return f'ForwardRef({self.__forward_arg__!r})'
+class _TypeVarLike:
+ """Mixin for TypeVar-like types (TypeVar and ParamSpec)."""
+ def __init__(self, bound, covariant, contravariant):
+ """Used to setup TypeVars and ParamSpec's bound, covariant and
+ contravariant attributes.
+ """
+ if covariant and contravariant:
+ raise ValueError("Bivariant types are not supported.")
+ self.__covariant__ = bool(covariant)
+ self.__contravariant__ = bool(contravariant)
+ if bound:
+ self.__bound__ = _type_check(bound, "Bound must be a type.")
+ else:
+ self.__bound__ = None
-class TypeVar(_Final, _Immutable, _root=True):
+ def __or__(self, right):
+ return Union[self, right]
+
+ def __ror__(self, left):
+ return Union[left, self]
+
+ def __repr__(self):
+ if self.__covariant__:
+ prefix = '+'
+ elif self.__contravariant__:
+ prefix = '-'
+ else:
+ prefix = '~'
+ return prefix + self.__name__
+
+ def __reduce__(self):
+ return self.__name__
+
+
+class TypeVar( _Final, _Immutable, _TypeVarLike, _root=True):
"""Type variable.
Usage::
@@ -605,20 +801,13 @@
def __init__(self, name, *constraints, bound=None,
covariant=False, contravariant=False):
self.__name__ = name
- if covariant and contravariant:
- raise ValueError("Bivariant types are not supported.")
- self.__covariant__ = bool(covariant)
- self.__contravariant__ = bool(contravariant)
+ super().__init__(bound, covariant, contravariant)
if constraints and bound is not None:
raise TypeError("Constraints cannot be combined with bound=...")
if constraints and len(constraints) == 1:
raise TypeError("A single constraint is not allowed")
msg = "TypeVar(name, constraint, ...): constraints must be types."
self.__constraints__ = tuple(_type_check(t, msg) for t in constraints)
- if bound:
- self.__bound__ = _type_check(bound, "Bound must be a type.")
- else:
- self.__bound__ = None
try:
def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling
except (AttributeError, ValueError):
@@ -626,17 +815,121 @@
if def_mod != 'typing':
self.__module__ = def_mod
- def __repr__(self):
- if self.__covariant__:
- prefix = '+'
- elif self.__contravariant__:
- prefix = '-'
- else:
- prefix = '~'
- return prefix + self.__name__
- def __reduce__(self):
- return self.__name__
+class ParamSpecArgs(_Final, _Immutable, _root=True):
+ """The args for a ParamSpec object.
+
+ Given a ParamSpec object P, P.args is an instance of ParamSpecArgs.
+
+ ParamSpecArgs objects have a reference back to their ParamSpec:
+
+ P.args.__origin__ is P
+
+ This type is meant for runtime introspection and has no special meaning to
+ static type checkers.
+ """
+ def __init__(self, origin):
+ self.__origin__ = origin
+
+ def __repr__(self):
+ return f"{self.__origin__.__name__}.args"
+
+ def __eq__(self, other):
+ if not isinstance(other, ParamSpecArgs):
+ return NotImplemented
+ return self.__origin__ == other.__origin__
+
+
+class ParamSpecKwargs(_Final, _Immutable, _root=True):
+ """The kwargs for a ParamSpec object.
+
+ Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs.
+
+ ParamSpecKwargs objects have a reference back to their ParamSpec:
+
+ P.kwargs.__origin__ is P
+
+ This type is meant for runtime introspection and has no special meaning to
+ static type checkers.
+ """
+ def __init__(self, origin):
+ self.__origin__ = origin
+
+ def __repr__(self):
+ return f"{self.__origin__.__name__}.kwargs"
+
+ def __eq__(self, other):
+ if not isinstance(other, ParamSpecKwargs):
+ return NotImplemented
+ return self.__origin__ == other.__origin__
+
+
+class ParamSpec(_Final, _Immutable, _TypeVarLike, _root=True):
+ """Parameter specification variable.
+
+ Usage::
+
+ P = ParamSpec('P')
+
+ Parameter specification variables exist primarily for the benefit of static
+ type checkers. They are used to forward the parameter types of one
+ callable to another callable, a pattern commonly found in higher order
+ functions and decorators. They are only valid when used in ``Concatenate``,
+ or as the first argument to ``Callable``, or as parameters for user-defined
+ Generics. See class Generic for more information on generic types. An
+ example for annotating a decorator::
+
+ T = TypeVar('T')
+ P = ParamSpec('P')
+
+ def add_logging(f: Callable[P, T]) -> Callable[P, T]:
+ '''A type-safe decorator to add logging to a function.'''
+ def inner(*args: P.args, **kwargs: P.kwargs) -> T:
+ logging.info(f'{f.__name__} was called')
+ return f(*args, **kwargs)
+ return inner
+
+ @add_logging
+ def add_two(x: float, y: float) -> float:
+ '''Add two numbers together.'''
+ return x + y
+
+ Parameter specification variables defined with covariant=True or
+ contravariant=True can be used to declare covariant or contravariant
+ generic types. These keyword arguments are valid, but their actual semantics
+ are yet to be decided. See PEP 612 for details.
+
+ Parameter specification variables can be introspected. e.g.:
+
+ P.__name__ == 'T'
+ P.__bound__ == None
+ P.__covariant__ == False
+ P.__contravariant__ == False
+
+ Note that only parameter specification variables defined in global scope can
+ be pickled.
+ """
+
+ __slots__ = ('__name__', '__bound__', '__covariant__', '__contravariant__',
+ '__dict__')
+
+ @property
+ def args(self):
+ return ParamSpecArgs(self)
+
+ @property
+ def kwargs(self):
+ return ParamSpecKwargs(self)
+
+ def __init__(self, name, *, bound=None, covariant=False, contravariant=False):
+ self.__name__ = name
+ super().__init__(bound, covariant, contravariant)
+ try:
+ def_mod = sys._getframe(1).f_globals.get('__name__', '__main__')
+ except (AttributeError, ValueError):
+ def_mod = None
+ if def_mod != 'typing':
+ self.__module__ = def_mod
def _is_dunder(attr):
@@ -681,14 +974,18 @@
return tuple(res)
def __getattr__(self, attr):
+ if attr in {'__name__', '__qualname__'}:
+ return self._name or self.__origin__.__name__
+
# We are careful for copy and pickle.
- # Also for simplicity we just don't relay all dunder names
+ # Also for simplicity we don't relay any dunder names
if '__origin__' in self.__dict__ and not _is_dunder(attr):
return getattr(self.__origin__, attr)
raise AttributeError(attr)
def __setattr__(self, attr, val):
- if _is_dunder(attr) or attr in ('_name', '_inst', '_nparams'):
+ if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams',
+ '_typevar_types', '_paramspec_tvars'}:
super().__setattr__(attr, val)
else:
setattr(self.__origin__, attr, val)
@@ -700,6 +997,9 @@
raise TypeError("Subscripted generics cannot be used with"
" class and instance checks")
+ def __dir__(self):
+ return list(set(super().__dir__()
+ + [attr for attr in dir(self.__origin__) if not _is_dunder(attr)]))
# Special typing constructs Union, Optional, Generic, Callable and Tuple
# use three special attributes for internal bookkeeping of generic types:
@@ -713,14 +1013,18 @@
class _GenericAlias(_BaseGenericAlias, _root=True):
- def __init__(self, origin, params, *, inst=True, name=None):
+ def __init__(self, origin, params, *, inst=True, name=None,
+ _typevar_types=TypeVar,
+ _paramspec_tvars=False):
super().__init__(origin, inst=inst, name=name)
if not isinstance(params, tuple):
params = (params,)
self.__args__ = tuple(... if a is _TypingEllipsis else
() if a is _TypingEmpty else
a for a in params)
- self.__parameters__ = _collect_type_vars(params)
+ self.__parameters__ = _collect_type_vars(params, typevar_types=_typevar_types)
+ self._typevar_types = _typevar_types
+ self._paramspec_tvars = _paramspec_tvars
if not name:
self.__module__ = origin.__module__
@@ -733,6 +1037,12 @@
def __hash__(self):
return hash((self.__origin__, self.__args__))
+ def __or__(self, right):
+ return Union[self, right]
+
+ def __ror__(self, left):
+ return Union[left, self]
+
@_tp_cache
def __getitem__(self, params):
if self.__origin__ in (Generic, Protocol):
@@ -740,25 +1050,40 @@
raise TypeError(f"Cannot subscript already-subscripted {self}")
if not isinstance(params, tuple):
params = (params,)
- msg = "Parameters to generic types must be types."
- params = tuple(_type_check(p, msg) for p in params)
- _check_generic(self, params, len(self.__parameters__))
+ params = tuple(_type_convert(p) for p in params)
+ if (self._paramspec_tvars
+ and any(isinstance(t, ParamSpec) for t in self.__parameters__)):
+ params = _prepare_paramspec_params(self, params)
+ else:
+ _check_generic(self, params, len(self.__parameters__))
subst = dict(zip(self.__parameters__, params))
new_args = []
for arg in self.__args__:
- if isinstance(arg, TypeVar):
- arg = subst[arg]
- elif isinstance(arg, (_GenericAlias, GenericAlias)):
+ if isinstance(arg, self._typevar_types):
+ if isinstance(arg, ParamSpec):
+ arg = subst[arg]
+ if not _is_param_expr(arg):
+ raise TypeError(f"Expected a list of types, an ellipsis, "
+ f"ParamSpec, or Concatenate. Got {arg}")
+ else:
+ arg = subst[arg]
+ elif isinstance(arg, (_GenericAlias, GenericAlias, types.UnionType)):
subparams = arg.__parameters__
if subparams:
subargs = tuple(subst[x] for x in subparams)
arg = arg[subargs]
- new_args.append(arg)
+ # Required to flatten out the args for CallableGenericAlias
+ if self.__origin__ == collections.abc.Callable and isinstance(arg, tuple):
+ new_args.extend(arg)
+ else:
+ new_args.append(arg)
return self.copy_with(tuple(new_args))
def copy_with(self, params):
- return self.__class__(self.__origin__, params, name=self._name, inst=self._inst)
+ return self.__class__(self.__origin__, params, name=self._name, inst=self._inst,
+ _typevar_types=self._typevar_types,
+ _paramspec_tvars=self._paramspec_tvars)
def __repr__(self):
if self._name:
@@ -779,6 +1104,9 @@
return operator.getitem, (origin, args)
def __mro_entries__(self, bases):
+ if isinstance(self.__origin__, _SpecialForm):
+ raise TypeError(f"Cannot subclass {self!r}")
+
if self._name: # generic version of an ABC or built-in class
return super().__mro_entries__(bases)
if self.__origin__ is Generic:
@@ -832,19 +1160,25 @@
def __reduce__(self):
return self._name
+ def __or__(self, right):
+ return Union[self, right]
+
+ def __ror__(self, left):
+ return Union[left, self]
class _CallableGenericAlias(_GenericAlias, _root=True):
def __repr__(self):
assert self._name == 'Callable'
- if len(self.__args__) == 2 and self.__args__[0] is Ellipsis:
+ args = self.__args__
+ if len(args) == 2 and _is_param_expr(args[0]):
return super().__repr__()
return (f'typing.Callable'
- f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
- f'{_type_repr(self.__args__[-1])}]')
+ f'[[{", ".join([_type_repr(a) for a in args[:-1]])}], '
+ f'{_type_repr(args[-1])}]')
def __reduce__(self):
args = self.__args__
- if not (len(args) == 2 and args[0] is ...):
+ if not (len(args) == 2 and _is_param_expr(args[0])):
args = list(args[:-1]), args[-1]
return operator.getitem, (Callable, args)
@@ -852,20 +1186,22 @@
class _CallableType(_SpecialGenericAlias, _root=True):
def copy_with(self, params):
return _CallableGenericAlias(self.__origin__, params,
- name=self._name, inst=self._inst)
+ name=self._name, inst=self._inst,
+ _typevar_types=(TypeVar, ParamSpec),
+ _paramspec_tvars=True)
def __getitem__(self, params):
if not isinstance(params, tuple) or len(params) != 2:
raise TypeError("Callable must be used as "
"Callable[[arg, ...], result].")
args, result = params
- if args is Ellipsis:
- params = (Ellipsis, result)
- else:
- if not isinstance(args, list):
- raise TypeError(f"Callable[args, result]: args must be a list."
- f" Got {args}")
+ # This relaxes what args can be on purpose to allow things like
+ # PEP 612 ParamSpec. Responsibility for whether a user is using
+ # Callable[...] properly is deferred to static type checkers.
+ if isinstance(args, list):
params = (tuple(args), result)
+ else:
+ params = (args, result)
return self.__getitem_inner__(params)
@_tp_cache
@@ -875,8 +1211,9 @@
result = _type_check(result, msg)
if args is Ellipsis:
return self.copy_with((_TypingEllipsis, result))
- msg = "Callable[[arg, ...], result]: each arg must be a type."
- args = tuple(_type_check(arg, msg) for arg in args)
+ if not isinstance(args, tuple):
+ args = (args,)
+ args = tuple(_type_convert(arg) for arg in args)
params = args + (result,)
return self.copy_with(params)
@@ -902,7 +1239,7 @@
return Union[params]
def __eq__(self, other):
- if not isinstance(other, _UnionGenericAlias):
+ if not isinstance(other, (_UnionGenericAlias, types.UnionType)):
return NotImplemented
return set(self.__args__) == set(other.__args__)
@@ -918,6 +1255,18 @@
return f'typing.Optional[{_type_repr(args[0])}]'
return super().__repr__()
+ def __instancecheck__(self, obj):
+ return self.__subclasscheck__(type(obj))
+
+ def __subclasscheck__(self, cls):
+ for arg in self.__args__:
+ if issubclass(cls, arg):
+ return True
+
+ def __reduce__(self):
+ func, (origin, args) = super().__reduce__()
+ return func, (Union, args)
+
def _value_and_type_iter(parameters):
return ((p, type(p)) for p in parameters)
@@ -935,6 +1284,18 @@
return hash(frozenset(_value_and_type_iter(self.__args__)))
+class _ConcatenateGenericAlias(_GenericAlias, _root=True):
+ def copy_with(self, params):
+ if isinstance(params[-1], (list, tuple)):
+ return (*params[:-1], *params[-1])
+ if isinstance(params[-1], _ConcatenateGenericAlias):
+ params = (*params[:-1], *params[-1].__args__)
+ elif not isinstance(params[-1], ParamSpec):
+ raise TypeError("The last parameter to Concatenate should be a "
+ "ParamSpec variable.")
+ return super().copy_with(params)
+
+
class Generic:
"""Abstract base class for generic types.
@@ -965,20 +1326,25 @@
if not params and cls is not Tuple:
raise TypeError(
f"Parameter list to {cls.__qualname__}[...] cannot be empty")
- msg = "Parameters to generic types must be types."
- params = tuple(_type_check(p, msg) for p in params)
+ params = tuple(_type_convert(p) for p in params)
if cls in (Generic, Protocol):
# Generic and Protocol can only be subscripted with unique type variables.
- if not all(isinstance(p, TypeVar) for p in params):
+ if not all(isinstance(p, (TypeVar, ParamSpec)) for p in params):
raise TypeError(
- f"Parameters to {cls.__name__}[...] must all be type variables")
+ f"Parameters to {cls.__name__}[...] must all be type variables "
+ f"or parameter specification variables.")
if len(set(params)) != len(params):
raise TypeError(
f"Parameters to {cls.__name__}[...] must all be unique")
else:
# Subscripting a regular Generic subclass.
- _check_generic(cls, params, len(cls.__parameters__))
- return _GenericAlias(cls, params)
+ if any(isinstance(t, ParamSpec) for t in cls.__parameters__):
+ params = _prepare_paramspec_params(cls, params)
+ else:
+ _check_generic(cls, params, len(cls.__parameters__))
+ return _GenericAlias(cls, params,
+ _typevar_types=(TypeVar, ParamSpec),
+ _paramspec_tvars=True)
def __init_subclass__(cls, *args, **kwargs):
super().__init_subclass__(*args, **kwargs)
@@ -990,7 +1356,7 @@
if error:
raise TypeError("Cannot inherit from plain Generic")
if '__orig_bases__' in cls.__dict__:
- tvars = _collect_type_vars(cls.__orig_bases__)
+ tvars = _collect_type_vars(cls.__orig_bases__, (TypeVar, ParamSpec))
# Look for Generic[T1, ..., Tn].
# If found, tvars must be a subset of it.
# If not found, tvars is it.
@@ -1060,24 +1426,55 @@
return all(callable(getattr(cls, attr, None)) for attr in _get_protocol_attrs(cls))
-def _no_init(self, *args, **kwargs):
- if type(self)._is_protocol:
+def _no_init_or_replace_init(self, *args, **kwargs):
+ cls = type(self)
+
+ if cls._is_protocol:
raise TypeError('Protocols cannot be instantiated')
+ # Already using a custom `__init__`. No need to calculate correct
+ # `__init__` to call. This can lead to RecursionError. See bpo-45121.
+ if cls.__init__ is not _no_init_or_replace_init:
+ return
-def _allow_reckless_class_cheks():
+ # Initially, `__init__` of a protocol subclass is set to `_no_init_or_replace_init`.
+ # The first instantiation of the subclass will call `_no_init_or_replace_init` which
+ # searches for a proper new `__init__` in the MRO. The new `__init__`
+ # replaces the subclass' old `__init__` (ie `_no_init_or_replace_init`). Subsequent
+ # instantiation of the protocol subclass will thus use the new
+ # `__init__` and no longer call `_no_init_or_replace_init`.
+ for base in cls.__mro__:
+ init = base.__dict__.get('__init__', _no_init_or_replace_init)
+ if init is not _no_init_or_replace_init:
+ cls.__init__ = init
+ break
+ else:
+ # should not happen
+ cls.__init__ = object.__init__
+
+ cls.__init__(self, *args, **kwargs)
+
+
+def _caller(depth=1, default='__main__'):
+ try:
+ return sys._getframe(depth + 1).f_globals.get('__name__', default)
+ except (AttributeError, ValueError): # For platforms without _getframe()
+ return None
+
+
+def _allow_reckless_class_checks(depth=3):
"""Allow instance and class checks for special stdlib modules.
The abc and functools modules indiscriminately call isinstance() and
issubclass() on the whole MRO of a user class, which may contain protocols.
"""
try:
- return sys._getframe(3).f_globals['__name__'] in ['abc', 'functools']
+ return sys._getframe(depth).f_globals['__name__'] in ['abc', 'functools']
except (AttributeError, ValueError): # For platforms without _getframe().
return True
-_PROTO_WHITELIST = {
+_PROTO_ALLOWLIST = {
'collections.abc': [
'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable',
'Hashable', 'Sized', 'Container', 'Collection', 'Reversible',
@@ -1092,6 +1489,14 @@
def __instancecheck__(cls, instance):
# We need this method for situations where attributes are
# assigned in __init__.
+ if (
+ getattr(cls, '_is_protocol', False) and
+ not getattr(cls, '_is_runtime_protocol', False) and
+ not _allow_reckless_class_checks(depth=2)
+ ):
+ raise TypeError("Instance and class checks can only be used with"
+ " @runtime_checkable protocols")
+
if ((not getattr(cls, '_is_protocol', False) or
_is_callable_members_only(cls)) and
issubclass(instance.__class__, cls)):
@@ -1154,12 +1559,12 @@
# First, perform various sanity checks.
if not getattr(cls, '_is_runtime_protocol', False):
- if _allow_reckless_class_cheks():
+ if _allow_reckless_class_checks():
return NotImplemented
raise TypeError("Instance and class checks can only be used with"
" @runtime_checkable protocols")
if not _is_callable_members_only(cls):
- if _allow_reckless_class_cheks():
+ if _allow_reckless_class_checks():
return NotImplemented
raise TypeError("Protocols with non-method members"
" don't support issubclass()")
@@ -1196,12 +1601,12 @@
# ... otherwise check consistency of bases, and prohibit instantiation.
for base in cls.__bases__:
if not (base in (object, Generic) or
- base.__module__ in _PROTO_WHITELIST and
- base.__name__ in _PROTO_WHITELIST[base.__module__] or
+ base.__module__ in _PROTO_ALLOWLIST and
+ base.__name__ in _PROTO_ALLOWLIST[base.__module__] or
issubclass(base, Generic) and base._is_protocol):
raise TypeError('Protocols can only inherit from other'
' protocols, got %r' % base)
- cls.__init__ = _no_init
+ cls.__init__ = _no_init_or_replace_init
class _AnnotatedAlias(_GenericAlias, _root=True):
@@ -1244,6 +1649,11 @@
def __hash__(self):
return hash((self.__origin__, self.__metadata__))
+ def __getattr__(self, attr):
+ if attr in {'__name__', '__qualname__'}:
+ return 'Annotated'
+ return super().__getattr__(attr)
+
class Annotated:
"""Add context specific metadata to a type.
@@ -1288,7 +1698,7 @@
"with at least two arguments (a type and an "
"annotation).")
msg = "Annotated[t, ...]: t must be a type."
- origin = _type_check(params[0], msg)
+ origin = _type_check(params[0], msg, allow_special_forms=True)
metadata = tuple(params[1:])
return _AnnotatedAlias(origin, metadata)
@@ -1382,7 +1792,8 @@
- If no dict arguments are passed, an attempt is made to use the
globals from obj (or the respective module's globals for classes),
and these are also used as the locals. If the object does not appear
- to have globals, an empty dictionary is used.
+ to have globals, an empty dictionary is used. For classes, the search
+ order is globals first then locals.
- If one dict argument is passed, it is used for both globals and
locals.
@@ -1398,16 +1809,27 @@
hints = {}
for base in reversed(obj.__mro__):
if globalns is None:
- base_globals = sys.modules[base.__module__].__dict__
+ base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {})
else:
base_globals = globalns
ann = base.__dict__.get('__annotations__', {})
+ if isinstance(ann, types.GetSetDescriptorType):
+ ann = {}
+ base_locals = dict(vars(base)) if localns is None else localns
+ if localns is None and globalns is None:
+ # This is surprising, but required. Before Python 3.10,
+ # get_type_hints only evaluated the globalns of
+ # a class. To maintain backwards compatibility, we reverse
+ # the globalns and localns order so that eval() looks into
+ # *base_globals* first rather than *base_locals*.
+ # This only affects ForwardRefs.
+ base_globals, base_locals = base_locals, base_globals
for name, value in ann.items():
if value is None:
value = type(None)
if isinstance(value, str):
- value = ForwardRef(value, is_argument=False)
- value = _eval_type(value, base_globals, localns)
+ value = ForwardRef(value, is_argument=False, is_class=True)
+ value = _eval_type(value, base_globals, base_locals)
hints[name] = value
return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()}
@@ -1438,7 +1860,13 @@
if value is None:
value = type(None)
if isinstance(value, str):
- value = ForwardRef(value)
+ # class-level forward refs were handled above, this must be either
+ # a module-level annotation or a function argument annotation
+ value = ForwardRef(
+ value,
+ is_argument=not isinstance(obj, types.ModuleType),
+ is_class=False,
+ )
value = _eval_type(value, globalns, localns)
if name in defaults and defaults[name] is None:
value = Optional[value]
@@ -1461,6 +1889,12 @@
if stripped_args == t.__args__:
return t
return GenericAlias(t.__origin__, stripped_args)
+ if isinstance(t, types.UnionType):
+ stripped_args = tuple(_strip_annotations(a) for a in t.__args__)
+ if stripped_args == t.__args__:
+ return t
+ return functools.reduce(operator.or_, stripped_args)
+
return t
@@ -1477,13 +1911,17 @@
get_origin(Generic[T]) is Generic
get_origin(Union[T, int]) is Union
get_origin(List[Tuple[T, T]][int]) == list
+ get_origin(P.args) is P
"""
if isinstance(tp, _AnnotatedAlias):
return Annotated
- if isinstance(tp, (_BaseGenericAlias, GenericAlias)):
+ if isinstance(tp, (_BaseGenericAlias, GenericAlias,
+ ParamSpecArgs, ParamSpecKwargs)):
return tp.__origin__
if tp is Generic:
return Generic
+ if isinstance(tp, types.UnionType):
+ return types.UnionType
return None
@@ -1500,16 +1938,31 @@
"""
if isinstance(tp, _AnnotatedAlias):
return (tp.__origin__,) + tp.__metadata__
- if isinstance(tp, _GenericAlias):
+ if isinstance(tp, (_GenericAlias, GenericAlias)):
res = tp.__args__
- if tp.__origin__ is collections.abc.Callable and res[0] is not Ellipsis:
+ if (tp.__origin__ is collections.abc.Callable
+ and not (len(res) == 2 and _is_param_expr(res[0]))):
res = (list(res[:-1]), res[-1])
return res
- if isinstance(tp, GenericAlias):
+ if isinstance(tp, types.UnionType):
return tp.__args__
return ()
+def is_typeddict(tp):
+ """Check if an annotation is a TypedDict class
+
+ For example::
+ class Film(TypedDict):
+ title: str
+ year: int
+
+ is_typeddict(Film) # => True
+ is_typeddict(Union[list, str]) # => False
+ """
+ return isinstance(tp, _TypedDictMeta)
+
+
def no_type_check(arg):
"""Decorator to indicate that annotations are not type hints.
@@ -1900,7 +2353,8 @@
own_annotation_keys = set(own_annotations.keys())
msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type"
own_annotations = {
- n: _type_check(tp, msg) for n, tp in own_annotations.items()
+ n: _type_check(tp, msg, module=tp_dict.__module__)
+ for n, tp in own_annotations.items()
}
required_keys = set()
optional_keys = set()
@@ -1980,24 +2434,24 @@
raise TypeError("TypedDict takes either a dict or keyword arguments,"
" but not both")
- ns = {'__annotations__': dict(fields), '__total__': total}
+ ns = {'__annotations__': dict(fields)}
try:
# Setting correct module is necessary to make typed dict classes pickleable.
ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__')
except (AttributeError, ValueError):
pass
- return _TypedDictMeta(typename, (), ns)
+ return _TypedDictMeta(typename, (), ns, total=total)
_TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {})
TypedDict.__mro_entries__ = lambda bases: (_TypedDict,)
-def NewType(name, tp):
+class NewType:
"""NewType creates simple unique types with almost zero
runtime overhead. NewType(name, tp) is considered a subtype of tp
by static type checkers. At runtime, NewType(name, tp) returns
- a dummy function that simply returns its argument. Usage::
+ a dummy callable that simply returns its argument. Usage::
UserId = NewType('UserId', int)
@@ -2012,12 +2466,30 @@
num = UserId(5) + 1 # type: int
"""
- def new_type(x):
+ def __init__(self, name, tp):
+ self.__qualname__ = name
+ if '.' in name:
+ name = name.rpartition('.')[-1]
+ self.__name__ = name
+ self.__supertype__ = tp
+ def_mod = _caller()
+ if def_mod != 'typing':
+ self.__module__ = def_mod
+
+ def __repr__(self):
+ return f'{self.__module__}.{self.__qualname__}'
+
+ def __call__(self, x):
return x
- new_type.__name__ = name
- new_type.__supertype__ = tp
- return new_type
+ def __reduce__(self):
+ return self.__qualname__
+
+ def __or__(self, other):
+ return Union[self, other]
+
+ def __ror__(self, other):
+ return Union[other, self]
# Python-version-specific alias (Python 2: unicode; Python 3: str)
diff --git a/common/py3-stdlib/unittest/_log.py b/common/py3-stdlib/unittest/_log.py
index 94e7e75..94868e5 100644
--- a/common/py3-stdlib/unittest/_log.py
+++ b/common/py3-stdlib/unittest/_log.py
@@ -26,11 +26,11 @@
class _AssertLogsContext(_BaseTestCaseContext):
- """A context manager used to implement TestCase.assertLogs()."""
+ """A context manager for assertLogs() and assertNoLogs() """
LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s"
- def __init__(self, test_case, logger_name, level):
+ def __init__(self, test_case, logger_name, level, no_logs):
_BaseTestCaseContext.__init__(self, test_case)
self.logger_name = logger_name
if level:
@@ -38,6 +38,7 @@
else:
self.level = logging.INFO
self.msg = None
+ self.no_logs = no_logs
def __enter__(self):
if isinstance(self.logger_name, logging.Logger):
@@ -46,6 +47,7 @@
logger = self.logger = logging.getLogger(self.logger_name)
formatter = logging.Formatter(self.LOGGING_FORMAT)
handler = _CapturingHandler()
+ handler.setLevel(self.level)
handler.setFormatter(formatter)
self.watcher = handler.watcher
self.old_handlers = logger.handlers[:]
@@ -54,16 +56,31 @@
logger.handlers = [handler]
logger.setLevel(self.level)
logger.propagate = False
+ if self.no_logs:
+ return
return handler.watcher
def __exit__(self, exc_type, exc_value, tb):
self.logger.handlers = self.old_handlers
self.logger.propagate = self.old_propagate
self.logger.setLevel(self.old_level)
+
if exc_type is not None:
# let unexpected exceptions pass through
return False
- if len(self.watcher.records) == 0:
- self._raiseFailure(
- "no logs of level {} or higher triggered on {}"
- .format(logging.getLevelName(self.level), self.logger.name))
+
+ if self.no_logs:
+ # assertNoLogs
+ if len(self.watcher.records) > 0:
+ self._raiseFailure(
+ "Unexpected logs found: {!r}".format(
+ self.watcher.output
+ )
+ )
+
+ else:
+ # assertLogs
+ if len(self.watcher.records) == 0:
+ self._raiseFailure(
+ "no logs of level {} or higher triggered on {}"
+ .format(logging.getLevelName(self.level), self.logger.name))
diff --git a/common/py3-stdlib/unittest/async_case.py b/common/py3-stdlib/unittest/async_case.py
index 1bc1312..2323119 100644
--- a/common/py3-stdlib/unittest/async_case.py
+++ b/common/py3-stdlib/unittest/async_case.py
@@ -4,7 +4,6 @@
from .case import TestCase
-
class IsolatedAsyncioTestCase(TestCase):
# Names intentionally have a long prefix
# to reduce a chance of clashing with user-defined attributes
@@ -52,7 +51,7 @@
# We intentionally don't add inspect.iscoroutinefunction() check
# for func argument because there is no way
# to check for async function reliably:
- # 1. It can be "async def func()" iself
+ # 1. It can be "async def func()" itself
# 2. Class can implement "async def __call__()" method
# 3. Regular "def func()" that returns awaitable object
self.addCleanup(*(func, *args), **kwargs)
@@ -72,15 +71,15 @@
self._callMaybeAsync(function, *args, **kwargs)
def _callAsync(self, func, /, *args, **kwargs):
- assert self._asyncioTestLoop is not None
+ assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized'
ret = func(*args, **kwargs)
- assert inspect.isawaitable(ret)
+ assert inspect.isawaitable(ret), f'{func!r} returned non-awaitable'
fut = self._asyncioTestLoop.create_future()
self._asyncioCallsQueue.put_nowait((fut, ret))
return self._asyncioTestLoop.run_until_complete(fut)
def _callMaybeAsync(self, func, /, *args, **kwargs):
- assert self._asyncioTestLoop is not None
+ assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized'
ret = func(*args, **kwargs)
if inspect.isawaitable(ret):
fut = self._asyncioTestLoop.create_future()
@@ -102,14 +101,14 @@
ret = await awaitable
if not fut.cancelled():
fut.set_result(ret)
- except asyncio.CancelledError:
+ except (SystemExit, KeyboardInterrupt):
raise
- except Exception as ex:
+ except (BaseException, asyncio.CancelledError) as ex:
if not fut.cancelled():
fut.set_exception(ex)
def _setupAsyncioLoop(self):
- assert self._asyncioTestLoop is None
+ assert self._asyncioTestLoop is None, 'asyncio test loop already initialized'
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.set_debug(True)
@@ -119,7 +118,7 @@
loop.run_until_complete(fut)
def _tearDownAsyncioLoop(self):
- assert self._asyncioTestLoop is not None
+ assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized'
loop = self._asyncioTestLoop
self._asyncioTestLoop = None
self._asyncioCallsQueue.put_nowait(None)
@@ -135,7 +134,7 @@
task.cancel()
loop.run_until_complete(
- asyncio.gather(*to_cancel, loop=loop, return_exceptions=True))
+ asyncio.gather(*to_cancel, return_exceptions=True))
for task in to_cancel:
if task.cancelled():
@@ -158,3 +157,12 @@
return super().run(result)
finally:
self._tearDownAsyncioLoop()
+
+ def debug(self):
+ self._setupAsyncioLoop()
+ super().debug()
+ self._tearDownAsyncioLoop()
+
+ def __del__(self):
+ if self._asyncioTestLoop is not None:
+ self._tearDownAsyncioLoop()
diff --git a/common/py3-stdlib/unittest/case.py b/common/py3-stdlib/unittest/case.py
index f8bc865..61003d0 100644
--- a/common/py3-stdlib/unittest/case.py
+++ b/common/py3-stdlib/unittest/case.py
@@ -252,7 +252,7 @@
def __enter__(self):
# The __warningregistry__'s need to be in a pristine state for tests
# to work properly.
- for v in sys.modules.values():
+ for v in list(sys.modules.values()):
if getattr(v, '__warningregistry__', None):
v.__warningregistry__ = {}
self.warnings_manager = warnings.catch_warnings(record=True)
@@ -295,7 +295,6 @@
self._raiseFailure("{} not triggered".format(exc_name))
-
class _OrderedChainMap(collections.ChainMap):
def __iter__(self):
seen = set()
@@ -556,73 +555,71 @@
function(*args, **kwargs)
def run(self, result=None):
- orig_result = result
if result is None:
result = self.defaultTestResult()
startTestRun = getattr(result, 'startTestRun', None)
+ stopTestRun = getattr(result, 'stopTestRun', None)
if startTestRun is not None:
startTestRun()
+ else:
+ stopTestRun = None
result.startTest(self)
-
- testMethod = getattr(self, self._testMethodName)
- if (getattr(self.__class__, "__unittest_skip__", False) or
- getattr(testMethod, "__unittest_skip__", False)):
- # If the class or method was skipped.
- try:
+ try:
+ testMethod = getattr(self, self._testMethodName)
+ if (getattr(self.__class__, "__unittest_skip__", False) or
+ getattr(testMethod, "__unittest_skip__", False)):
+ # If the class or method was skipped.
skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
or getattr(testMethod, '__unittest_skip_why__', ''))
self._addSkip(result, self, skip_why)
- finally:
- result.stopTest(self)
- return
- expecting_failure_method = getattr(testMethod,
- "__unittest_expecting_failure__", False)
- expecting_failure_class = getattr(self,
- "__unittest_expecting_failure__", False)
- expecting_failure = expecting_failure_class or expecting_failure_method
- outcome = _Outcome(result)
- try:
- self._outcome = outcome
+ return result
- with outcome.testPartExecutor(self):
- self._callSetUp()
- if outcome.success:
- outcome.expecting_failure = expecting_failure
- with outcome.testPartExecutor(self, isTest=True):
- self._callTestMethod(testMethod)
- outcome.expecting_failure = False
+ expecting_failure = (
+ getattr(self, "__unittest_expecting_failure__", False) or
+ getattr(testMethod, "__unittest_expecting_failure__", False)
+ )
+ outcome = _Outcome(result)
+ try:
+ self._outcome = outcome
+
with outcome.testPartExecutor(self):
- self._callTearDown()
+ self._callSetUp()
+ if outcome.success:
+ outcome.expecting_failure = expecting_failure
+ with outcome.testPartExecutor(self, isTest=True):
+ self._callTestMethod(testMethod)
+ outcome.expecting_failure = False
+ with outcome.testPartExecutor(self):
+ self._callTearDown()
- self.doCleanups()
- for test, reason in outcome.skipped:
- self._addSkip(result, test, reason)
- self._feedErrorsToResult(result, outcome.errors)
- if outcome.success:
- if expecting_failure:
- if outcome.expectedFailure:
- self._addExpectedFailure(result, outcome.expectedFailure)
+ self.doCleanups()
+ for test, reason in outcome.skipped:
+ self._addSkip(result, test, reason)
+ self._feedErrorsToResult(result, outcome.errors)
+ if outcome.success:
+ if expecting_failure:
+ if outcome.expectedFailure:
+ self._addExpectedFailure(result, outcome.expectedFailure)
+ else:
+ self._addUnexpectedSuccess(result)
else:
- self._addUnexpectedSuccess(result)
- else:
- result.addSuccess(self)
- return result
+ result.addSuccess(self)
+ return result
+ finally:
+ # explicitly break reference cycles:
+ # outcome.errors -> frame -> outcome -> outcome.errors
+ # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure
+ outcome.errors.clear()
+ outcome.expectedFailure = None
+
+ # clear the outcome, no more needed
+ self._outcome = None
+
finally:
result.stopTest(self)
- if orig_result is None:
- stopTestRun = getattr(result, 'stopTestRun', None)
- if stopTestRun is not None:
- stopTestRun()
-
- # explicitly break reference cycles:
- # outcome.errors -> frame -> outcome -> outcome.errors
- # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure
- outcome.errors.clear()
- outcome.expectedFailure = None
-
- # clear the outcome, no more needed
- self._outcome = None
+ if stopTestRun is not None:
+ stopTestRun()
def doCleanups(self):
"""Execute all cleanup functions. Normally called for you after
@@ -654,12 +651,20 @@
def debug(self):
"""Run the test without collecting errors in a TestResult"""
- self.setUp()
- getattr(self, self._testMethodName)()
- self.tearDown()
+ testMethod = getattr(self, self._testMethodName)
+ if (getattr(self.__class__, "__unittest_skip__", False) or
+ getattr(testMethod, "__unittest_skip__", False)):
+ # If the class or method was skipped.
+ skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
+ or getattr(testMethod, '__unittest_skip_why__', ''))
+ raise SkipTest(skip_why)
+
+ self._callSetUp()
+ self._callTestMethod(testMethod)
+ self._callTearDown()
while self._cleanups:
- function, args, kwargs = self._cleanups.pop(-1)
- function(*args, **kwargs)
+ function, args, kwargs = self._cleanups.pop()
+ self._callCleanup(function, *args, **kwargs)
def skipTest(self, reason):
"""Skip this test."""
@@ -788,7 +793,16 @@
"""
# Lazy import to avoid importing logging if it is not needed.
from ._log import _AssertLogsContext
- return _AssertLogsContext(self, logger, level)
+ return _AssertLogsContext(self, logger, level, no_logs=False)
+
+ def assertNoLogs(self, logger=None, level=None):
+ """ Fail unless no log messages of level *level* or higher are emitted
+ on *logger_name* or its children.
+
+ This method must be used as a context manager.
+ """
+ from ._log import _AssertLogsContext
+ return _AssertLogsContext(self, logger, level, no_logs=True)
def _getAssertEqualityFunc(self, first, second):
"""Get a detailed comparison function for the types of the two args.
@@ -1132,7 +1146,8 @@
def assertDictContainsSubset(self, subset, dictionary, msg=None):
"""Checks whether dictionary is a superset of subset."""
warnings.warn('assertDictContainsSubset is deprecated',
- DeprecationWarning)
+ DeprecationWarning,
+ stacklevel=2)
missing = []
mismatched = []
for key, value in subset.items():
diff --git a/common/py3-stdlib/unittest/mock.py b/common/py3-stdlib/unittest/mock.py
index b495a5f..7152f86 100644
--- a/common/py3-stdlib/unittest/mock.py
+++ b/common/py3-stdlib/unittest/mock.py
@@ -36,6 +36,10 @@
from functools import wraps, partial
+class InvalidSpecError(Exception):
+ """Indicates that an invalid value was used as a mock spec."""
+
+
_builtins = {name for name in dir(builtins) if not name.startswith('_')}
FILTER_DIR = True
@@ -406,7 +410,7 @@
# Check if spec is an async object or function
bound_args = _MOCK_SIG.bind_partial(cls, *args, **kw).arguments
spec_arg = bound_args.get('spec_set', bound_args.get('spec'))
- if spec_arg and _is_async_obj(spec_arg):
+ if spec_arg is not None and _is_async_obj(spec_arg):
bases = (AsyncMockMixin, cls)
new = type(cls.__name__, bases, {'__doc__': cls.__doc__})
instance = _safe_super(NonCallableMock, cls).__new__(new)
@@ -631,9 +635,10 @@
elif _is_magic(name):
raise AttributeError(name)
if not self._mock_unsafe:
- if name.startswith(('assert', 'assret')):
- raise AttributeError("Attributes cannot start with 'assert' "
- "or 'assret'")
+ if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')):
+ raise AttributeError(
+ f"{name!r} is not a valid assertion. Use a spec "
+ f"for the mock if {name!r} is meant to be an attribute.")
result = self._mock_children.get(name)
if result is _deleted:
@@ -652,10 +657,17 @@
self._mock_children[name] = result
elif isinstance(result, _SpecState):
- result = create_autospec(
- result.spec, result.spec_set, result.instance,
- result.parent, result.name
- )
+ try:
+ result = create_autospec(
+ result.spec, result.spec_set, result.instance,
+ result.parent, result.name
+ )
+ except InvalidSpecError:
+ target_name = self.__dict__['_mock_name'] or self
+ raise InvalidSpecError(
+ f'Cannot autospec attr {name!r} from target '
+ f'{target_name!r} as it has already been mocked out. '
+ f'[target={self!r}, attr={result.spec!r}]')
self._mock_children[name] = result
return result
@@ -992,6 +1004,11 @@
if _new_name in self.__dict__['_spec_asyncs']:
return AsyncMock(**kw)
+ if self._mock_sealed:
+ attribute = f".{kw['name']}" if "name" in kw else "()"
+ mock_name = self._extract_mock_name() + attribute
+ raise AttributeError(mock_name)
+
_type = type(self)
if issubclass(_type, MagicMock) and _new_name in _async_method_magics:
# Any asynchronous magic becomes an AsyncMock
@@ -1010,12 +1027,6 @@
klass = Mock
else:
klass = _type.__mro__[1]
-
- if self._mock_sealed:
- attribute = "." + kw["name"] if "name" in kw else "()"
- mock_name = self._extract_mock_name() + attribute
- raise AttributeError(mock_name)
-
return klass(**kw)
@@ -1241,6 +1252,17 @@
return thing
+# _check_spec_arg_typos takes kwargs from commands like patch and checks that
+# they don't contain common misspellings of arguments related to autospeccing.
+def _check_spec_arg_typos(kwargs_to_check):
+ typos = ("autospect", "auto_spec", "set_spec")
+ for typo in typos:
+ if typo in kwargs_to_check:
+ raise RuntimeError(
+ f"{typo!r} might be a typo; use unsafe=True if this is intended"
+ )
+
+
class _patch(object):
attribute_name = None
@@ -1248,7 +1270,7 @@
def __init__(
self, getter, attribute, new, spec, create,
- spec_set, autospec, new_callable, kwargs
+ spec_set, autospec, new_callable, kwargs, *, unsafe=False
):
if new_callable is not None:
if new is not DEFAULT:
@@ -1259,6 +1281,16 @@
raise ValueError(
"Cannot use 'autospec' and 'new_callable' together"
)
+ if not unsafe:
+ _check_spec_arg_typos(kwargs)
+ if _is_instance_mock(spec):
+ raise InvalidSpecError(
+ f'Cannot spec attr {attribute!r} as the spec '
+ f'has already been mocked out. [spec={spec!r}]')
+ if _is_instance_mock(spec_set):
+ raise InvalidSpecError(
+ f'Cannot spec attr {attribute!r} as the spec_set '
+ f'target has already been mocked out. [spec_set={spec_set!r}]')
self.getter = getter
self.attribute = attribute
@@ -1486,6 +1518,18 @@
if autospec is True:
autospec = original
+ if _is_instance_mock(self.target):
+ raise InvalidSpecError(
+ f'Cannot autospec attr {self.attribute!r} as the patch '
+ f'target has already been mocked out. '
+ f'[target={self.target!r}, attr={autospec!r}]')
+ if _is_instance_mock(autospec):
+ target_name = getattr(self.target, '__name__', self.target)
+ raise InvalidSpecError(
+ f'Cannot autospec attr {self.attribute!r} from target '
+ f'{target_name!r} as it has already been mocked out. '
+ f'[target={self.target!r}, attr={autospec!r}]')
+
new = create_autospec(autospec, spec_set=spec_set,
_name=self.attribute, **kwargs)
elif kwargs:
@@ -1558,9 +1602,9 @@
def _get_target(target):
try:
target, attribute = target.rsplit('.', 1)
- except (TypeError, ValueError):
- raise TypeError("Need a valid target to patch. You supplied: %r" %
- (target,))
+ except (TypeError, ValueError, AttributeError):
+ raise TypeError(
+ f"Need a valid target to patch. You supplied: {target!r}")
getter = lambda: _importer(target)
return getter, attribute
@@ -1568,7 +1612,7 @@
def _patch_object(
target, attribute, new=DEFAULT, spec=None,
create=False, spec_set=None, autospec=None,
- new_callable=None, **kwargs
+ new_callable=None, *, unsafe=False, **kwargs
):
"""
patch the named member (`attribute`) on an object (`target`) with a mock
@@ -1590,7 +1634,7 @@
getter = lambda: target
return _patch(
getter, attribute, new, spec, create,
- spec_set, autospec, new_callable, kwargs
+ spec_set, autospec, new_callable, kwargs, unsafe=unsafe
)
@@ -1645,7 +1689,7 @@
def patch(
target, new=DEFAULT, spec=None, create=False,
- spec_set=None, autospec=None, new_callable=None, **kwargs
+ spec_set=None, autospec=None, new_callable=None, *, unsafe=False, **kwargs
):
"""
`patch` acts as a function decorator, class decorator or a context
@@ -1707,6 +1751,10 @@
use "as" then the patched object will be bound to the name after the
"as"; very useful if `patch` is creating a mock object for you.
+ Patch will raise a `RuntimeError` if passed some common misspellings of
+ the arguments autospec and spec_set. Pass the argument `unsafe` with the
+ value True to disable that check.
+
`patch` takes arbitrary keyword arguments. These will be passed to
`AsyncMock` if the patched object is asynchronous, to `MagicMock`
otherwise or to `new_callable` if specified.
@@ -1717,7 +1765,7 @@
getter, attribute = _get_target(target)
return _patch(
getter, attribute, new, spec, create,
- spec_set, autospec, new_callable, kwargs
+ spec_set, autospec, new_callable, kwargs, unsafe=unsafe
)
@@ -2567,7 +2615,7 @@
def create_autospec(spec, spec_set=False, instance=False, _parent=None,
- _name=None, **kwargs):
+ _name=None, *, unsafe=False, **kwargs):
"""Create a mock object using another object as a spec. Attributes on the
mock will use the corresponding attribute on the `spec` object as their
spec.
@@ -2583,6 +2631,10 @@
spec for an instance object by passing `instance=True`. The returned mock
will only be callable if instances of the mock are callable.
+ `create_autospec` will raise a `RuntimeError` if passed some common
+ misspellings of the arguments autospec and spec_set. Pass the argument
+ `unsafe` with the value True to disable that check.
+
`create_autospec` also takes arbitrary keyword arguments that are passed to
the constructor of the created mock."""
if _is_list(spec):
@@ -2591,6 +2643,9 @@
spec = type(spec)
is_type = isinstance(spec, type)
+ if _is_instance_mock(spec):
+ raise InvalidSpecError(f'Cannot autospec a Mock object. '
+ f'[object={spec!r}]')
is_async_func = _is_async_func(spec)
_kwargs = {'spec': spec}
if spec_set:
@@ -2600,6 +2655,8 @@
_kwargs = {}
if _kwargs and instance:
_kwargs['_spec_as_instance'] = True
+ if not unsafe:
+ _check_spec_arg_typos(kwargs)
_kwargs.update(kwargs)
@@ -2869,6 +2926,8 @@
continue
if not isinstance(m, NonCallableMock):
continue
+ if isinstance(m._mock_children.get(attr), _SpecState):
+ continue
if m._mock_new_parent is mock:
seal(m)
diff --git a/common/py3-stdlib/unittest/result.py b/common/py3-stdlib/unittest/result.py
index 111317b..3da7005 100644
--- a/common/py3-stdlib/unittest/result.py
+++ b/common/py3-stdlib/unittest/result.py
@@ -173,17 +173,10 @@
def _exc_info_to_string(self, err, test):
"""Converts a sys.exc_info()-style tuple of values into a string."""
exctype, value, tb = err
- # Skip test runner traceback levels
- while tb and self._is_relevant_tb_level(tb):
- tb = tb.tb_next
-
- if exctype is test.failureException:
- # Skip assert*() traceback levels
- length = self._count_relevant_tb_levels(tb)
- else:
- length = None
+ tb = self._clean_tracebacks(exctype, value, tb, test)
tb_e = traceback.TracebackException(
- exctype, value, tb, limit=length, capture_locals=self.tb_locals)
+ exctype, value, tb,
+ capture_locals=self.tb_locals, compact=True)
msgLines = list(tb_e.format())
if self.buffer:
@@ -199,16 +192,49 @@
msgLines.append(STDERR_LINE % error)
return ''.join(msgLines)
+ def _clean_tracebacks(self, exctype, value, tb, test):
+ ret = None
+ first = True
+ excs = [(exctype, value, tb)]
+ while excs:
+ (exctype, value, tb) = excs.pop()
+ # Skip test runner traceback levels
+ while tb and self._is_relevant_tb_level(tb):
+ tb = tb.tb_next
+
+ # Skip assert*() traceback levels
+ if exctype is test.failureException:
+ self._remove_unittest_tb_frames(tb)
+
+ if first:
+ ret = tb
+ first = False
+ else:
+ value.__traceback__ = tb
+
+ if value is not None:
+ for c in (value.__cause__, value.__context__):
+ if c is not None:
+ excs.append((type(c), c, c.__traceback__))
+ return ret
def _is_relevant_tb_level(self, tb):
return '__unittest' in tb.tb_frame.f_globals
- def _count_relevant_tb_levels(self, tb):
- length = 0
+ def _remove_unittest_tb_frames(self, tb):
+ '''Truncates usercode tb at the first unittest frame.
+
+ If the first frame of the traceback is in user code,
+ the prefix up to the first unittest frame is returned.
+ If the first frame is already in the unittest module,
+ the traceback is not modified.
+ '''
+ prev = None
while tb and not self._is_relevant_tb_level(tb):
- length += 1
+ prev = tb
tb = tb.tb_next
- return length
+ if prev is not None:
+ prev.tb_next = None
def __repr__(self):
return ("<%s run=%i errors=%i failures=%i>" %
diff --git a/common/py3-stdlib/unittest/runner.py b/common/py3-stdlib/unittest/runner.py
index 45e7e4c..caf1590 100644
--- a/common/py3-stdlib/unittest/runner.py
+++ b/common/py3-stdlib/unittest/runner.py
@@ -59,6 +59,7 @@
super(TextTestResult, self).addSuccess(test)
if self.showAll:
self.stream.writeln("ok")
+ self.stream.flush()
elif self.dots:
self.stream.write('.')
self.stream.flush()
@@ -67,6 +68,7 @@
super(TextTestResult, self).addError(test, err)
if self.showAll:
self.stream.writeln("ERROR")
+ self.stream.flush()
elif self.dots:
self.stream.write('E')
self.stream.flush()
@@ -75,6 +77,7 @@
super(TextTestResult, self).addFailure(test, err)
if self.showAll:
self.stream.writeln("FAIL")
+ self.stream.flush()
elif self.dots:
self.stream.write('F')
self.stream.flush()
@@ -83,6 +86,7 @@
super(TextTestResult, self).addSkip(test, reason)
if self.showAll:
self.stream.writeln("skipped {0!r}".format(reason))
+ self.stream.flush()
elif self.dots:
self.stream.write("s")
self.stream.flush()
@@ -91,6 +95,7 @@
super(TextTestResult, self).addExpectedFailure(test, err)
if self.showAll:
self.stream.writeln("expected failure")
+ self.stream.flush()
elif self.dots:
self.stream.write("x")
self.stream.flush()
@@ -99,6 +104,7 @@
super(TextTestResult, self).addUnexpectedSuccess(test)
if self.showAll:
self.stream.writeln("unexpected success")
+ self.stream.flush()
elif self.dots:
self.stream.write("u")
self.stream.flush()
@@ -106,6 +112,7 @@
def printErrors(self):
if self.dots or self.showAll:
self.stream.writeln()
+ self.stream.flush()
self.printErrorList('ERROR', self.errors)
self.printErrorList('FAIL', self.failures)
@@ -115,6 +122,7 @@
self.stream.writeln("%s: %s" % (flavour,self.getDescription(test)))
self.stream.writeln(self.separator2)
self.stream.writeln("%s" % err)
+ self.stream.flush()
class TextTestRunner(object):
@@ -218,4 +226,5 @@
self.stream.writeln(" (%s)" % (", ".join(infos),))
else:
self.stream.write("\n")
+ self.stream.flush()
return result
diff --git a/common/py3-stdlib/unittest/suite.py b/common/py3-stdlib/unittest/suite.py
index 41993f9..6f45b6f 100644
--- a/common/py3-stdlib/unittest/suite.py
+++ b/common/py3-stdlib/unittest/suite.py
@@ -149,6 +149,7 @@
if getattr(currentClass, "__unittest_skip__", False):
return
+ failed = False
try:
currentClass._classSetupFailed = False
except TypeError:
@@ -157,27 +158,32 @@
pass
setUpClass = getattr(currentClass, 'setUpClass', None)
+ doClassCleanups = getattr(currentClass, 'doClassCleanups', None)
if setUpClass is not None:
_call_if_exists(result, '_setupStdout')
try:
- setUpClass()
- except Exception as e:
- if isinstance(result, _DebugResult):
- raise
- currentClass._classSetupFailed = True
- className = util.strclass(currentClass)
- self._createClassOrModuleLevelException(result, e,
- 'setUpClass',
- className)
+ try:
+ setUpClass()
+ except Exception as e:
+ if isinstance(result, _DebugResult):
+ raise
+ failed = True
+ try:
+ currentClass._classSetupFailed = True
+ except TypeError:
+ pass
+ className = util.strclass(currentClass)
+ self._createClassOrModuleLevelException(result, e,
+ 'setUpClass',
+ className)
+ if failed and doClassCleanups is not None:
+ doClassCleanups()
+ for exc_info in currentClass.tearDown_exceptions:
+ self._createClassOrModuleLevelException(
+ result, exc_info[1], 'setUpClass', className,
+ info=exc_info)
finally:
_call_if_exists(result, '_restoreStdout')
- if currentClass._classSetupFailed is True:
- currentClass.doClassCleanups()
- if len(currentClass.tearDown_exceptions) > 0:
- for exc in currentClass.tearDown_exceptions:
- self._createClassOrModuleLevelException(
- result, exc[1], 'setUpClass', className,
- info=exc)
def _get_previous_module(self, result):
previousModule = None
@@ -205,20 +211,22 @@
if setUpModule is not None:
_call_if_exists(result, '_setupStdout')
try:
- setUpModule()
- except Exception as e:
try:
- case.doModuleCleanups()
- except Exception as exc:
- self._createClassOrModuleLevelException(result, exc,
+ setUpModule()
+ except Exception as e:
+ if isinstance(result, _DebugResult):
+ raise
+ result._moduleSetUpFailed = True
+ self._createClassOrModuleLevelException(result, e,
'setUpModule',
currentModule)
- if isinstance(result, _DebugResult):
- raise
- result._moduleSetUpFailed = True
- self._createClassOrModuleLevelException(result, e,
- 'setUpModule',
- currentModule)
+ if result._moduleSetUpFailed:
+ try:
+ case.doModuleCleanups()
+ except Exception as e:
+ self._createClassOrModuleLevelException(result, e,
+ 'setUpModule',
+ currentModule)
finally:
_call_if_exists(result, '_restoreStdout')
@@ -251,30 +259,33 @@
except KeyError:
return
- tearDownModule = getattr(module, 'tearDownModule', None)
- if tearDownModule is not None:
- _call_if_exists(result, '_setupStdout')
+ _call_if_exists(result, '_setupStdout')
+ try:
+ tearDownModule = getattr(module, 'tearDownModule', None)
+ if tearDownModule is not None:
+ try:
+ tearDownModule()
+ except Exception as e:
+ if isinstance(result, _DebugResult):
+ raise
+ self._createClassOrModuleLevelException(result, e,
+ 'tearDownModule',
+ previousModule)
try:
- tearDownModule()
+ case.doModuleCleanups()
except Exception as e:
if isinstance(result, _DebugResult):
raise
self._createClassOrModuleLevelException(result, e,
'tearDownModule',
previousModule)
- finally:
- _call_if_exists(result, '_restoreStdout')
- try:
- case.doModuleCleanups()
- except Exception as e:
- self._createClassOrModuleLevelException(result, e,
- 'tearDownModule',
- previousModule)
+ finally:
+ _call_if_exists(result, '_restoreStdout')
def _tearDownPreviousClass(self, test, result):
previousClass = getattr(result, '_previousTestClass', None)
currentClass = test.__class__
- if currentClass == previousClass:
+ if currentClass == previousClass or previousClass is None:
return
if getattr(previousClass, '_classSetupFailed', False):
return
@@ -284,27 +295,34 @@
return
tearDownClass = getattr(previousClass, 'tearDownClass', None)
- if tearDownClass is not None:
- _call_if_exists(result, '_setupStdout')
- try:
- tearDownClass()
- except Exception as e:
- if isinstance(result, _DebugResult):
- raise
- className = util.strclass(previousClass)
- self._createClassOrModuleLevelException(result, e,
- 'tearDownClass',
- className)
- finally:
- _call_if_exists(result, '_restoreStdout')
- previousClass.doClassCleanups()
- if len(previousClass.tearDown_exceptions) > 0:
- for exc in previousClass.tearDown_exceptions:
- className = util.strclass(previousClass)
- self._createClassOrModuleLevelException(result, exc[1],
- 'tearDownClass',
- className,
- info=exc)
+ doClassCleanups = getattr(previousClass, 'doClassCleanups', None)
+ if tearDownClass is None and doClassCleanups is None:
+ return
+
+ _call_if_exists(result, '_setupStdout')
+ try:
+ if tearDownClass is not None:
+ try:
+ tearDownClass()
+ except Exception as e:
+ if isinstance(result, _DebugResult):
+ raise
+ className = util.strclass(previousClass)
+ self._createClassOrModuleLevelException(result, e,
+ 'tearDownClass',
+ className)
+ if doClassCleanups is not None:
+ doClassCleanups()
+ for exc_info in previousClass.tearDown_exceptions:
+ if isinstance(result, _DebugResult):
+ raise exc_info[1]
+ className = util.strclass(previousClass)
+ self._createClassOrModuleLevelException(result, exc_info[1],
+ 'tearDownClass',
+ className,
+ info=exc_info)
+ finally:
+ _call_if_exists(result, '_restoreStdout')
class _ErrorHolder(object):
diff --git a/common/py3-stdlib/unittest/test/test_assertions.py b/common/py3-stdlib/unittest/test/test_assertions.py
index f5e64d6..a0db342 100644
--- a/common/py3-stdlib/unittest/test/test_assertions.py
+++ b/common/py3-stdlib/unittest/test/test_assertions.py
@@ -2,6 +2,7 @@
import warnings
import weakref
import unittest
+from test.support import gc_collect
from itertools import product
@@ -124,8 +125,10 @@
self.foo()
Foo("test_functional").run()
+ gc_collect() # For PyPy or other GCs.
self.assertIsNone(wr())
Foo("test_with").run()
+ gc_collect() # For PyPy or other GCs.
self.assertIsNone(wr())
def testAssertNotRegex(self):
diff --git a/common/py3-stdlib/unittest/test/test_async_case.py b/common/py3-stdlib/unittest/test/test_async_case.py
index 2db441d..e46b99f 100644
--- a/common/py3-stdlib/unittest/test/test_async_case.py
+++ b/common/py3-stdlib/unittest/test/test_async_case.py
@@ -1,5 +1,10 @@
import asyncio
import unittest
+from test import support
+
+
+class MyException(Exception):
+ pass
def tearDownModule():
@@ -7,9 +12,14 @@
class TestAsyncCase(unittest.TestCase):
- def test_full_cycle(self):
- events = []
+ maxDiff = None
+ def tearDown(self):
+ # Ensure that IsolatedAsyncioTestCase instances are destroyed before
+ # starting a new event loop
+ support.gc_collect()
+
+ def test_full_cycle(self):
class Test(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.assertEqual(events, [])
@@ -18,12 +28,13 @@
async def asyncSetUp(self):
self.assertEqual(events, ['setUp'])
events.append('asyncSetUp')
+ self.addAsyncCleanup(self.on_cleanup1)
async def test_func(self):
self.assertEqual(events, ['setUp',
'asyncSetUp'])
events.append('test')
- self.addAsyncCleanup(self.on_cleanup)
+ self.addAsyncCleanup(self.on_cleanup2)
async def asyncTearDown(self):
self.assertEqual(events, ['setUp',
@@ -38,34 +49,48 @@
'asyncTearDown'])
events.append('tearDown')
- async def on_cleanup(self):
+ async def on_cleanup1(self):
+ self.assertEqual(events, ['setUp',
+ 'asyncSetUp',
+ 'test',
+ 'asyncTearDown',
+ 'tearDown',
+ 'cleanup2'])
+ events.append('cleanup1')
+
+ async def on_cleanup2(self):
self.assertEqual(events, ['setUp',
'asyncSetUp',
'test',
'asyncTearDown',
'tearDown'])
- events.append('cleanup')
+ events.append('cleanup2')
+ events = []
test = Test("test_func")
- test.run()
- self.assertEqual(events, ['setUp',
- 'asyncSetUp',
- 'test',
- 'asyncTearDown',
- 'tearDown',
- 'cleanup'])
+ result = test.run()
+ self.assertEqual(result.errors, [])
+ self.assertEqual(result.failures, [])
+ expected = ['setUp', 'asyncSetUp', 'test',
+ 'asyncTearDown', 'tearDown', 'cleanup2', 'cleanup1']
+ self.assertEqual(events, expected)
+
+ events = []
+ test = Test("test_func")
+ test.debug()
+ self.assertEqual(events, expected)
+ test.doCleanups()
+ self.assertEqual(events, expected)
def test_exception_in_setup(self):
- events = []
-
class Test(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
events.append('asyncSetUp')
- raise Exception()
+ self.addAsyncCleanup(self.on_cleanup)
+ raise MyException()
async def test_func(self):
events.append('test')
- self.addAsyncCleanup(self.on_cleanup)
async def asyncTearDown(self):
events.append('asyncTearDown')
@@ -74,35 +99,26 @@
events.append('cleanup')
+ events = []
test = Test("test_func")
- test.run()
+ result = test.run()
+ self.assertEqual(events, ['asyncSetUp', 'cleanup'])
+ self.assertIs(result.errors[0][0], test)
+ self.assertIn('MyException', result.errors[0][1])
+
+ events = []
+ test = Test("test_func")
+ try:
+ test.debug()
+ except MyException:
+ pass
+ else:
+ self.fail('Expected a MyException exception')
self.assertEqual(events, ['asyncSetUp'])
+ test.doCleanups()
+ self.assertEqual(events, ['asyncSetUp', 'cleanup'])
def test_exception_in_test(self):
- events = []
-
- class Test(unittest.IsolatedAsyncioTestCase):
- async def asyncSetUp(self):
- events.append('asyncSetUp')
-
- async def test_func(self):
- events.append('test')
- raise Exception()
- self.addAsyncCleanup(self.on_cleanup)
-
- async def asyncTearDown(self):
- events.append('asyncTearDown')
-
- async def on_cleanup(self):
- events.append('cleanup')
-
- test = Test("test_func")
- test.run()
- self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown'])
-
- def test_exception_in_test_after_adding_cleanup(self):
- events = []
-
class Test(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
events.append('asyncSetUp')
@@ -110,7 +126,7 @@
async def test_func(self):
events.append('test')
self.addAsyncCleanup(self.on_cleanup)
- raise Exception()
+ raise MyException()
async def asyncTearDown(self):
events.append('asyncTearDown')
@@ -118,13 +134,26 @@
async def on_cleanup(self):
events.append('cleanup')
+ events = []
test = Test("test_func")
- test.run()
+ result = test.run()
self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup'])
+ self.assertIs(result.errors[0][0], test)
+ self.assertIn('MyException', result.errors[0][1])
+
+ events = []
+ test = Test("test_func")
+ try:
+ test.debug()
+ except MyException:
+ pass
+ else:
+ self.fail('Expected a MyException exception')
+ self.assertEqual(events, ['asyncSetUp', 'test'])
+ test.doCleanups()
+ self.assertEqual(events, ['asyncSetUp', 'test', 'cleanup'])
def test_exception_in_tear_down(self):
- events = []
-
class Test(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
events.append('asyncSetUp')
@@ -135,37 +164,70 @@
async def asyncTearDown(self):
events.append('asyncTearDown')
- raise Exception()
+ raise MyException()
async def on_cleanup(self):
events.append('cleanup')
+ events = []
test = Test("test_func")
- test.run()
+ result = test.run()
self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup'])
+ self.assertIs(result.errors[0][0], test)
+ self.assertIn('MyException', result.errors[0][1])
+ events = []
+ test = Test("test_func")
+ try:
+ test.debug()
+ except MyException:
+ pass
+ else:
+ self.fail('Expected a MyException exception')
+ self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown'])
+ test.doCleanups()
+ self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup'])
def test_exception_in_tear_clean_up(self):
- events = []
-
class Test(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
events.append('asyncSetUp')
async def test_func(self):
events.append('test')
- self.addAsyncCleanup(self.on_cleanup)
+ self.addAsyncCleanup(self.on_cleanup1)
+ self.addAsyncCleanup(self.on_cleanup2)
async def asyncTearDown(self):
events.append('asyncTearDown')
- async def on_cleanup(self):
- events.append('cleanup')
- raise Exception()
+ async def on_cleanup1(self):
+ events.append('cleanup1')
+ raise MyException('some error')
+ async def on_cleanup2(self):
+ events.append('cleanup2')
+ raise MyException('other error')
+
+ events = []
test = Test("test_func")
- test.run()
- self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup'])
+ result = test.run()
+ self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2', 'cleanup1'])
+ self.assertIs(result.errors[0][0], test)
+ self.assertIn('MyException: other error', result.errors[0][1])
+ self.assertIn('MyException: some error', result.errors[1][1])
+
+ events = []
+ test = Test("test_func")
+ try:
+ test.debug()
+ except MyException:
+ pass
+ else:
+ self.fail('Expected a MyException exception')
+ self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2'])
+ test.doCleanups()
+ self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2', 'cleanup1'])
def test_cleanups_interleave_order(self):
events = []
@@ -190,6 +252,95 @@
'async_cleanup 2',
'sync_cleanup 1'])
+ def test_base_exception_from_async_method(self):
+ events = []
+ class Test(unittest.IsolatedAsyncioTestCase):
+ async def test_base(self):
+ events.append("test_base")
+ raise BaseException()
+ events.append("not it")
+
+ async def test_no_err(self):
+ events.append("test_no_err")
+
+ async def test_cancel(self):
+ raise asyncio.CancelledError()
+
+ test = Test("test_base")
+ output = test.run()
+ self.assertFalse(output.wasSuccessful())
+
+ test = Test("test_no_err")
+ test.run()
+ self.assertEqual(events, ['test_base', 'test_no_err'])
+
+ test = Test("test_cancel")
+ output = test.run()
+ self.assertFalse(output.wasSuccessful())
+
+ def test_cancellation_hanging_tasks(self):
+ cancelled = False
+ class Test(unittest.IsolatedAsyncioTestCase):
+ async def test_leaking_task(self):
+ async def coro():
+ nonlocal cancelled
+ try:
+ await asyncio.sleep(1)
+ except asyncio.CancelledError:
+ cancelled = True
+ raise
+
+ # Leave this running in the background
+ asyncio.create_task(coro())
+
+ test = Test("test_leaking_task")
+ output = test.run()
+ self.assertTrue(cancelled)
+
+ def test_debug_cleanup_same_loop(self):
+ class Test(unittest.IsolatedAsyncioTestCase):
+ async def asyncSetUp(self):
+ async def coro():
+ await asyncio.sleep(0)
+ fut = asyncio.ensure_future(coro())
+ self.addAsyncCleanup(self.cleanup, fut)
+ events.append('asyncSetUp')
+
+ async def test_func(self):
+ events.append('test')
+ raise MyException()
+
+ async def asyncTearDown(self):
+ events.append('asyncTearDown')
+
+ async def cleanup(self, fut):
+ try:
+ # Raises an exception if in different loop
+ await asyncio.wait([fut])
+ events.append('cleanup')
+ except:
+ import traceback
+ traceback.print_exc()
+ raise
+
+ events = []
+ test = Test("test_func")
+ result = test.run()
+ self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup'])
+ self.assertIn('MyException', result.errors[0][1])
+
+ events = []
+ test = Test("test_func")
+ try:
+ test.debug()
+ except MyException:
+ pass
+ else:
+ self.fail('Expected a MyException exception')
+ self.assertEqual(events, ['asyncSetUp', 'test'])
+ test.doCleanups()
+ self.assertEqual(events, ['asyncSetUp', 'test', 'cleanup'])
+
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/unittest/test/test_case.py b/common/py3-stdlib/unittest/test/test_case.py
index f855c4d..9b3a598 100644
--- a/common/py3-stdlib/unittest/test/test_case.py
+++ b/common/py3-stdlib/unittest/test/test_case.py
@@ -8,6 +8,7 @@
import warnings
import weakref
import inspect
+import types
from copy import deepcopy
from test import support
@@ -18,7 +19,7 @@
TestEquality, TestHashing, LoggingResult, LegacyLoggingResult,
ResultWithNoStartTestRunStopTestRun
)
-from test.support import captured_stderr
+from test.support import captured_stderr, gc_collect
log_foo = logging.getLogger('foo')
@@ -610,6 +611,8 @@
'Tests shortDescription() for a method with a longer '
'docstring.')
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
def testShortDescriptionWhitespaceTrimming(self):
"""
Tests shortDescription() whitespace is trimmed, so that the first
@@ -705,6 +708,10 @@
with self.assertRaises(self.failureException):
self.assertDictContainsSubset({'foo': one}, {'foo': '\uFFFD'})
+ with self.assertWarns(DeprecationWarning) as warninfo:
+ self.assertDictContainsSubset({}, {})
+ self.assertEqual(warninfo.warnings[0].filename, __file__)
+
def testAssertEqual(self):
equal_pairs = [
((), ()),
@@ -1350,6 +1357,20 @@
pass
self.assertRaises(TypeError, self.assertWarnsRegex, MyWarn, lambda: True)
+ def testAssertWarnsModifySysModules(self):
+ # bpo-29620: handle modified sys.modules during iteration
+ class Foo(types.ModuleType):
+ @property
+ def __warningregistry__(self):
+ sys.modules['@bar@'] = 'bar'
+
+ sys.modules['@foo@'] = Foo('foo')
+ try:
+ self.assertWarns(UserWarning, warnings.warn, 'expected')
+ finally:
+ del sys.modules['@foo@']
+ del sys.modules['@bar@']
+
def testAssertRaisesRegexMismatch(self):
def Stub():
raise Exception('Unexpected')
@@ -1658,6 +1679,18 @@
with self.assertLogs(level='WARNING'):
log_foo.info("1")
+ def testAssertLogsFailureLevelTooHigh_FilterInRootLogger(self):
+ # Failure due to level too high - message propagated to root
+ with self.assertNoStderr():
+ oldLevel = log_foo.level
+ log_foo.setLevel(logging.INFO)
+ try:
+ with self.assertRaises(self.failureException):
+ with self.assertLogs(level='WARNING'):
+ log_foo.info("1")
+ finally:
+ log_foo.setLevel(oldLevel)
+
def testAssertLogsFailureMismatchingLogger(self):
# Failure due to mismatching logger (and the logged message is
# passed through)
@@ -1666,6 +1699,81 @@
with self.assertLogs('foo'):
log_quux.error("1")
+ def testAssertLogsUnexpectedException(self):
+ # Check unexpected exception will go through.
+ with self.assertRaises(ZeroDivisionError):
+ with self.assertLogs():
+ raise ZeroDivisionError("Unexpected")
+
+ def testAssertNoLogsDefault(self):
+ with self.assertRaises(self.failureException) as cm:
+ with self.assertNoLogs():
+ log_foo.info("1")
+ log_foobar.debug("2")
+ self.assertEqual(
+ str(cm.exception),
+ "Unexpected logs found: ['INFO:foo:1']",
+ )
+
+ def testAssertNoLogsFailureFoundLogs(self):
+ with self.assertRaises(self.failureException) as cm:
+ with self.assertNoLogs():
+ log_quux.error("1")
+ log_foo.error("foo")
+
+ self.assertEqual(
+ str(cm.exception),
+ "Unexpected logs found: ['ERROR:quux:1', 'ERROR:foo:foo']",
+ )
+
+ def testAssertNoLogsPerLogger(self):
+ with self.assertNoStderr():
+ with self.assertLogs(log_quux):
+ with self.assertNoLogs(logger=log_foo):
+ log_quux.error("1")
+
+ def testAssertNoLogsFailurePerLogger(self):
+ # Failure due to unexpected logs for the given logger or its
+ # children.
+ with self.assertRaises(self.failureException) as cm:
+ with self.assertLogs(log_quux):
+ with self.assertNoLogs(logger=log_foo):
+ log_quux.error("1")
+ log_foobar.info("2")
+ self.assertEqual(
+ str(cm.exception),
+ "Unexpected logs found: ['INFO:foo.bar:2']",
+ )
+
+ def testAssertNoLogsPerLevel(self):
+ # Check per-level filtering
+ with self.assertNoStderr():
+ with self.assertNoLogs(level="ERROR"):
+ log_foo.info("foo")
+ log_quux.debug("1")
+
+ def testAssertNoLogsFailurePerLevel(self):
+ # Failure due to unexpected logs at the specified level.
+ with self.assertRaises(self.failureException) as cm:
+ with self.assertNoLogs(level="DEBUG"):
+ log_foo.debug("foo")
+ log_quux.debug("1")
+ self.assertEqual(
+ str(cm.exception),
+ "Unexpected logs found: ['DEBUG:foo:foo', 'DEBUG:quux:1']",
+ )
+
+ def testAssertNoLogsUnexpectedException(self):
+ # Check unexpected exception will go through.
+ with self.assertRaises(ZeroDivisionError):
+ with self.assertNoLogs():
+ raise ZeroDivisionError("Unexpected")
+
+ def testAssertNoLogsYieldsNone(self):
+ with self.assertNoLogs() as value:
+ pass
+ self.assertIsNone(value)
+
def testDeprecatedMethodNames(self):
"""
Test that the deprecated methods raise a DeprecationWarning. See #9424.
@@ -1845,6 +1953,7 @@
for method_name in ('test1', 'test2'):
testcase = TestCase(method_name)
testcase.run()
+ gc_collect() # For PyPy or other GCs.
self.assertEqual(MyException.ninstance, 0)
diff --git a/common/py3-stdlib/unittest/test/test_discovery.py b/common/py3-stdlib/unittest/test/test_discovery.py
index 16e081e..9d502c5 100644
--- a/common/py3-stdlib/unittest/test/test_discovery.py
+++ b/common/py3-stdlib/unittest/test/test_discovery.py
@@ -5,6 +5,7 @@
import types
import pickle
from test import support
+from test.support import import_helper
import test.test_importlib.util
import unittest
@@ -848,7 +849,7 @@
with unittest.mock.patch('builtins.__import__', _import):
# Since loader.discover() can modify sys.path, restore it when done.
- with support.DirsOnSysPath():
+ with import_helper.DirsOnSysPath():
# Make sure to remove 'package' from sys.modules when done.
with test.test_importlib.util.uncache('package'):
suite = loader.discover('package')
@@ -865,7 +866,7 @@
with unittest.mock.patch('builtins.__import__', _import):
# Since loader.discover() can modify sys.path, restore it when done.
- with support.DirsOnSysPath():
+ with import_helper.DirsOnSysPath():
# Make sure to remove 'package' from sys.modules when done.
with test.test_importlib.util.uncache('package'):
with self.assertRaises(TypeError) as cm:
diff --git a/common/py3-stdlib/unittest/test/test_program.py b/common/py3-stdlib/unittest/test/test_program.py
index eef82ff..b7fbbc1 100644
--- a/common/py3-stdlib/unittest/test/test_program.py
+++ b/common/py3-stdlib/unittest/test/test_program.py
@@ -6,6 +6,7 @@
from test import support
import unittest
import unittest.test
+from unittest.test.test_result import BufferedWriter
class Test_TestProgram(unittest.TestCase):
@@ -57,9 +58,9 @@
class FooBar(unittest.TestCase):
def testPass(self):
- assert True
+ pass
def testFail(self):
- assert False
+ raise AssertionError
class FooBarLoader(unittest.TestLoader):
"""Test loader that returns a suite containing FooBar."""
@@ -104,30 +105,39 @@
program.testNames)
def test_NonExit(self):
+ stream = BufferedWriter()
program = unittest.main(exit=False,
argv=["foobar"],
- testRunner=unittest.TextTestRunner(stream=io.StringIO()),
+ testRunner=unittest.TextTestRunner(stream=stream),
testLoader=self.FooBarLoader())
self.assertTrue(hasattr(program, 'result'))
+ self.assertIn('\nFAIL: testFail ', stream.getvalue())
+ self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
def test_Exit(self):
+ stream = BufferedWriter()
self.assertRaises(
SystemExit,
unittest.main,
argv=["foobar"],
- testRunner=unittest.TextTestRunner(stream=io.StringIO()),
+ testRunner=unittest.TextTestRunner(stream=stream),
exit=True,
testLoader=self.FooBarLoader())
+ self.assertIn('\nFAIL: testFail ', stream.getvalue())
+ self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
def test_ExitAsDefault(self):
+ stream = BufferedWriter()
self.assertRaises(
SystemExit,
unittest.main,
argv=["foobar"],
- testRunner=unittest.TextTestRunner(stream=io.StringIO()),
+ testRunner=unittest.TextTestRunner(stream=stream),
testLoader=self.FooBarLoader())
+ self.assertIn('\nFAIL: testFail ', stream.getvalue())
+ self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
class InitialisableProgram(unittest.TestProgram):
diff --git a/common/py3-stdlib/unittest/test/test_result.py b/common/py3-stdlib/unittest/test/test_result.py
index 0ffb87b..c5aaba0 100644
--- a/common/py3-stdlib/unittest/test/test_result.py
+++ b/common/py3-stdlib/unittest/test/test_result.py
@@ -2,10 +2,11 @@
import sys
import textwrap
-from test import support
+from test.support import warnings_helper, captured_stdout, captured_stderr
import traceback
import unittest
+from unittest.util import strclass
class MockTraceback(object):
@@ -22,6 +23,32 @@
unittest.result.traceback = traceback
+def bad_cleanup1():
+ print('do cleanup1')
+ raise TypeError('bad cleanup1')
+
+
+def bad_cleanup2():
+ print('do cleanup2')
+ raise ValueError('bad cleanup2')
+
+
+class BufferedWriter:
+ def __init__(self):
+ self.result = ''
+ self.buffer = ''
+
+ def write(self, arg):
+ self.buffer += arg
+
+ def flush(self):
+ self.result += self.buffer
+ self.buffer = ''
+
+ def getvalue(self):
+ return self.result
+
+
class Test_TestResult(unittest.TestCase):
# Note: there are not separate tests for TestResult.wasSuccessful(),
# TestResult.errors, TestResult.failures, TestResult.testsRun or
@@ -193,6 +220,61 @@
self.assertIs(test_case, test)
self.assertIsInstance(formatted_exc, str)
+ def test_addFailure_filter_traceback_frames(self):
+ class Foo(unittest.TestCase):
+ def test_1(self):
+ pass
+
+ test = Foo('test_1')
+ def get_exc_info():
+ try:
+ test.fail("foo")
+ except:
+ return sys.exc_info()
+
+ exc_info_tuple = get_exc_info()
+
+ full_exc = traceback.format_exception(*exc_info_tuple)
+
+ result = unittest.TestResult()
+ result.startTest(test)
+ result.addFailure(test, exc_info_tuple)
+ result.stopTest(test)
+
+ formatted_exc = result.failures[0][1]
+ dropped = [l for l in full_exc if l not in formatted_exc]
+ self.assertEqual(len(dropped), 1)
+ self.assertIn("raise self.failureException(msg)", dropped[0])
+
+ def test_addFailure_filter_traceback_frames_context(self):
+ class Foo(unittest.TestCase):
+ def test_1(self):
+ pass
+
+ test = Foo('test_1')
+ def get_exc_info():
+ try:
+ try:
+ test.fail("foo")
+ except:
+ raise ValueError(42)
+ except:
+ return sys.exc_info()
+
+ exc_info_tuple = get_exc_info()
+
+ full_exc = traceback.format_exception(*exc_info_tuple)
+
+ result = unittest.TestResult()
+ result.startTest(test)
+ result.addFailure(test, exc_info_tuple)
+ result.stopTest(test)
+
+ formatted_exc = result.failures[0][1]
+ dropped = [l for l in full_exc if l not in formatted_exc]
+ self.assertEqual(len(dropped), 1)
+ self.assertIn("raise self.failureException(msg)", dropped[0])
+
# "addError(test, err)"
# ...
# "Called when the test case test raises an unexpected exception err
@@ -433,10 +515,13 @@
self.assertTrue(result.shouldStop)
def testFailFastSetByRunner(self):
- runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
+ stream = BufferedWriter()
+ runner = unittest.TextTestRunner(stream=stream, failfast=True)
def test(result):
self.assertTrue(result.failfast)
result = runner.run(test)
+ stream.flush()
+ self.assertTrue(stream.getvalue().endswith('\n\nOK\n'))
classDict = dict(unittest.TestResult.__dict__)
@@ -458,8 +543,8 @@
class Test_OldTestResult(unittest.TestCase):
def assertOldResultWarning(self, test, failures):
- with support.check_warnings(("TestResult has no add.+ method,",
- RuntimeWarning)):
+ with warnings_helper.check_warnings(
+ ("TestResult has no add.+ method,", RuntimeWarning)):
result = OldResult()
test.run(result)
self.assertEqual(len(result.failures), failures)
@@ -633,36 +718,320 @@
self.assertEqual(result._original_stderr.getvalue(), expectedErrMessage)
self.assertMultiLineEqual(message, expectedFullMessage)
+ def testBufferSetUp(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ print('set up')
+ 1/0
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\nset up\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 1)
+ description = f'test_foo ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferTearDown(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def tearDown(self):
+ print('tear down')
+ 1/0
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\ntear down\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 1)
+ description = f'test_foo ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferDoCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ print('set up')
+ self.addCleanup(bad_cleanup1)
+ self.addCleanup(bad_cleanup2)
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 2)
+ description = f'test_foo ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('TypeError: bad cleanup1', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferSetUp_DoCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ print('set up')
+ self.addCleanup(bad_cleanup1)
+ self.addCleanup(bad_cleanup2)
+ 1/0
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 3)
+ description = f'test_foo ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[2]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('TypeError: bad cleanup1', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferTearDown_DoCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ print('set up')
+ self.addCleanup(bad_cleanup1)
+ self.addCleanup(bad_cleanup2)
+ def tearDown(self):
+ print('tear down')
+ 1/0
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\nset up\ntear down\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 3)
+ description = f'test_foo ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[2]
+ self.assertEqual(str(test_case), description)
+ self.assertIn('TypeError: bad cleanup1', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
def testBufferSetupClass(self):
- result = unittest.TestResult()
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@classmethod
def setUpClass(cls):
+ print('set up class')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
+ expected_out = '\nStdout:\nset up class\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
+ description = f'setUpClass ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
def testBufferTearDownClass(self):
- result = unittest.TestResult()
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@classmethod
def tearDownClass(cls):
+ print('tear down class')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
+ expected_out = '\nStdout:\ntear down class\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
+ description = f'tearDownClass ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferDoClassCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ print('set up class')
+ cls.addClassCleanup(bad_cleanup1)
+ cls.addClassCleanup(bad_cleanup2)
+ @classmethod
+ def tearDownClass(cls):
+ print('tear down class')
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 2)
+ description = f'tearDownClass ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('TypeError: bad cleanup1', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferSetupClass_DoClassCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ print('set up class')
+ cls.addClassCleanup(bad_cleanup1)
+ cls.addClassCleanup(bad_cleanup2)
+ 1/0
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\nset up class\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 3)
+ description = f'setUpClass ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn('\nStdout:\nset up class\n', formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[2]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('TypeError: bad cleanup1', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferTearDownClass_DoClassCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ print('set up class')
+ cls.addClassCleanup(bad_cleanup1)
+ cls.addClassCleanup(bad_cleanup2)
+ @classmethod
+ def tearDownClass(cls):
+ print('tear down class')
+ 1/0
+ def test_foo(self):
+ pass
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 3)
+ description = f'tearDownClass ({strclass(Foo)})'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn('\nStdout:\ntear down class\n', formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+ test_case, formatted_exc = result.errors[2]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('TypeError: bad cleanup1', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
def testBufferSetUpModule(self):
- result = unittest.TestResult()
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@@ -671,6 +1040,7 @@
class Module(object):
@staticmethod
def setUpModule():
+ print('set up module')
1/0
Foo.__module__ = 'Module'
@@ -678,10 +1048,18 @@
self.addCleanup(sys.modules.pop, 'Module')
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
+ expected_out = '\nStdout:\nset up module\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
+ description = 'setUpModule (Module)'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
def testBufferTearDownModule(self):
- result = unittest.TestResult()
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@@ -690,6 +1068,7 @@
class Module(object):
@staticmethod
def tearDownModule():
+ print('tear down module')
1/0
Foo.__module__ = 'Module'
@@ -697,7 +1076,124 @@
self.addCleanup(sys.modules.pop, 'Module')
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
+ expected_out = '\nStdout:\ntear down module\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
+ description = 'tearDownModule (Module)'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferDoModuleCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def test_foo(self):
+ pass
+ class Module(object):
+ @staticmethod
+ def setUpModule():
+ print('set up module')
+ unittest.addModuleCleanup(bad_cleanup1)
+ unittest.addModuleCleanup(bad_cleanup2)
+
+ Foo.__module__ = 'Module'
+ sys.modules['Module'] = Module
+ self.addCleanup(sys.modules.pop, 'Module')
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 1)
+ description = 'tearDownModule (Module)'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferSetUpModule_DoModuleCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def test_foo(self):
+ pass
+ class Module(object):
+ @staticmethod
+ def setUpModule():
+ print('set up module')
+ unittest.addModuleCleanup(bad_cleanup1)
+ unittest.addModuleCleanup(bad_cleanup2)
+ 1/0
+
+ Foo.__module__ = 'Module'
+ sys.modules['Module'] = Module
+ self.addCleanup(sys.modules.pop, 'Module')
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\nset up module\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 2)
+ description = 'setUpModule (Module)'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn('\nStdout:\nset up module\n', formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertIn(expected_out, formatted_exc)
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
+
+ def testBufferTearDownModule_DoModuleCleanups(self):
+ with captured_stdout() as stdout:
+ result = unittest.TestResult()
+ result.buffer = True
+
+ class Foo(unittest.TestCase):
+ def test_foo(self):
+ pass
+ class Module(object):
+ @staticmethod
+ def setUpModule():
+ print('set up module')
+ unittest.addModuleCleanup(bad_cleanup1)
+ unittest.addModuleCleanup(bad_cleanup2)
+ @staticmethod
+ def tearDownModule():
+ print('tear down module')
+ 1/0
+
+ Foo.__module__ = 'Module'
+ sys.modules['Module'] = Module
+ self.addCleanup(sys.modules.pop, 'Module')
+ suite = unittest.TestSuite([Foo('test_foo')])
+ suite(result)
+ expected_out = '\nStdout:\ntear down module\ndo cleanup2\ndo cleanup1\n'
+ self.assertEqual(stdout.getvalue(), expected_out)
+ self.assertEqual(len(result.errors), 2)
+ description = 'tearDownModule (Module)'
+ test_case, formatted_exc = result.errors[0]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
+ self.assertNotIn('ValueError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn('\nStdout:\ntear down module\n', formatted_exc)
+ test_case, formatted_exc = result.errors[1]
+ self.assertEqual(test_case.description, description)
+ self.assertIn('ValueError: bad cleanup2', formatted_exc)
+ self.assertNotIn('ZeroDivisionError', formatted_exc)
+ self.assertNotIn('TypeError', formatted_exc)
+ self.assertIn(expected_out, formatted_exc)
if __name__ == '__main__':
diff --git a/common/py3-stdlib/unittest/test/test_runner.py b/common/py3-stdlib/unittest/test/test_runner.py
index dd9a1b6..453e6c3 100644
--- a/common/py3-stdlib/unittest/test/test_runner.py
+++ b/common/py3-stdlib/unittest/test/test_runner.py
@@ -222,14 +222,42 @@
self.assertEqual(ordering,
['setUpClass', 'test', 'tearDownClass', 'cleanup_good'])
- def test_debug_executes_classCleanUp(self):
+ def test_run_class_cleanUp_without_tearDownClass(self):
ordering = []
+ blowUp = True
class TestableTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
ordering.append('setUpClass')
cls.addClassCleanup(cleanup, ordering)
+ if blowUp:
+ raise Exception()
+ def testNothing(self):
+ ordering.append('test')
+ @classmethod
+ @property
+ def tearDownClass(cls):
+ raise AttributeError
+
+ runTests(TestableTest)
+ self.assertEqual(ordering, ['setUpClass', 'cleanup_good'])
+
+ ordering = []
+ blowUp = False
+ runTests(TestableTest)
+ self.assertEqual(ordering,
+ ['setUpClass', 'test', 'cleanup_good'])
+
+ def test_debug_executes_classCleanUp(self):
+ ordering = []
+ blowUp = False
+
+ class TestableTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ ordering.append('setUpClass')
+ cls.addClassCleanup(cleanup, ordering, blowUp=blowUp)
def testNothing(self):
ordering.append('test')
@classmethod
@@ -241,6 +269,48 @@
self.assertEqual(ordering,
['setUpClass', 'test', 'tearDownClass', 'cleanup_good'])
+ ordering = []
+ blowUp = True
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
+ with self.assertRaises(Exception) as cm:
+ suite.debug()
+ self.assertEqual(str(cm.exception), 'CleanUpExc')
+ self.assertEqual(ordering,
+ ['setUpClass', 'test', 'tearDownClass', 'cleanup_exc'])
+
+ def test_debug_executes_classCleanUp_when_teardown_exception(self):
+ ordering = []
+ blowUp = False
+
+ class TestableTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ ordering.append('setUpClass')
+ cls.addClassCleanup(cleanup, ordering, blowUp=blowUp)
+ def testNothing(self):
+ ordering.append('test')
+ @classmethod
+ def tearDownClass(cls):
+ raise Exception('TearDownClassExc')
+
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
+ with self.assertRaises(Exception) as cm:
+ suite.debug()
+ self.assertEqual(str(cm.exception), 'TearDownClassExc')
+ self.assertEqual(ordering, ['setUpClass', 'test'])
+ self.assertTrue(TestableTest._class_cleanups)
+ TestableTest._class_cleanups.clear()
+
+ ordering = []
+ blowUp = True
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
+ with self.assertRaises(Exception) as cm:
+ suite.debug()
+ self.assertEqual(str(cm.exception), 'TearDownClassExc')
+ self.assertEqual(ordering, ['setUpClass', 'test'])
+ self.assertTrue(TestableTest._class_cleanups)
+ TestableTest._class_cleanups.clear()
+
def test_doClassCleanups_with_errors_addClassCleanUp(self):
class TestableTest(unittest.TestCase):
def testNothing(self):
@@ -332,6 +402,7 @@
self.assertEqual(ordering,
['setUpClass', 'setUp', 'test',
'tearDownClass', 'cleanup_exc'])
+
ordering = []
class_blow_up = True
method_blow_up = False
@@ -355,6 +426,26 @@
['setUpClass', 'setUp', 'tearDownClass',
'cleanup_exc'])
+ def test_with_errors_in_tearDownClass(self):
+ ordering = []
+ class TestableTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ ordering.append('setUpClass')
+ cls.addClassCleanup(cleanup, ordering)
+ def testNothing(self):
+ ordering.append('test')
+ @classmethod
+ def tearDownClass(cls):
+ ordering.append('tearDownClass')
+ raise Exception('TearDownExc')
+
+ result = runTests(TestableTest)
+ self.assertEqual(result.errors[0][1].splitlines()[-1],
+ 'Exception: TearDownExc')
+ self.assertEqual(ordering,
+ ['setUpClass', 'test', 'tearDownClass', 'cleanup_good'])
+
class TestModuleCleanUp(unittest.TestCase):
def test_add_and_do_ModuleCleanup(self):
@@ -532,13 +623,69 @@
'tearDownModule2', 'cleanup_good'])
self.assertEqual(unittest.case._module_cleanups, [])
- def test_debug_module_executes_cleanUp(self):
+ def test_run_module_cleanUp_without_teardown(self):
ordering = []
class Module(object):
@staticmethod
def setUpModule():
ordering.append('setUpModule')
unittest.addModuleCleanup(cleanup, ordering)
+
+ class TestableTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ ordering.append('setUpClass')
+ def testNothing(self):
+ ordering.append('test')
+ @classmethod
+ def tearDownClass(cls):
+ ordering.append('tearDownClass')
+
+ TestableTest.__module__ = 'Module'
+ sys.modules['Module'] = Module
+ runTests(TestableTest)
+ self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
+ 'tearDownClass', 'cleanup_good'])
+ self.assertEqual(unittest.case._module_cleanups, [])
+
+ def test_run_module_cleanUp_when_teardown_exception(self):
+ ordering = []
+ class Module(object):
+ @staticmethod
+ def setUpModule():
+ ordering.append('setUpModule')
+ unittest.addModuleCleanup(cleanup, ordering)
+ @staticmethod
+ def tearDownModule():
+ raise Exception('CleanUpExc')
+
+ class TestableTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ ordering.append('setUpClass')
+ def testNothing(self):
+ ordering.append('test')
+ @classmethod
+ def tearDownClass(cls):
+ ordering.append('tearDownClass')
+
+ TestableTest.__module__ = 'Module'
+ sys.modules['Module'] = Module
+ result = runTests(TestableTest)
+ self.assertEqual(result.errors[0][1].splitlines()[-1],
+ 'Exception: CleanUpExc')
+ self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
+ 'tearDownClass', 'cleanup_good'])
+ self.assertEqual(unittest.case._module_cleanups, [])
+
+ def test_debug_module_executes_cleanUp(self):
+ ordering = []
+ blowUp = False
+ class Module(object):
+ @staticmethod
+ def setUpModule():
+ ordering.append('setUpModule')
+ unittest.addModuleCleanup(cleanup, ordering, blowUp=blowUp)
@staticmethod
def tearDownModule():
ordering.append('tearDownModule')
@@ -562,6 +709,60 @@
'tearDownModule', 'cleanup_good'])
self.assertEqual(unittest.case._module_cleanups, [])
+ ordering = []
+ blowUp = True
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
+ with self.assertRaises(Exception) as cm:
+ suite.debug()
+ self.assertEqual(str(cm.exception), 'CleanUpExc')
+ self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
+ 'tearDownClass', 'tearDownModule', 'cleanup_exc'])
+ self.assertEqual(unittest.case._module_cleanups, [])
+
+ def test_debug_module_cleanUp_when_teardown_exception(self):
+ ordering = []
+ blowUp = False
+ class Module(object):
+ @staticmethod
+ def setUpModule():
+ ordering.append('setUpModule')
+ unittest.addModuleCleanup(cleanup, ordering, blowUp=blowUp)
+ @staticmethod
+ def tearDownModule():
+ raise Exception('TearDownModuleExc')
+
+ class TestableTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ ordering.append('setUpClass')
+ def testNothing(self):
+ ordering.append('test')
+ @classmethod
+ def tearDownClass(cls):
+ ordering.append('tearDownClass')
+
+ TestableTest.__module__ = 'Module'
+ sys.modules['Module'] = Module
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
+ with self.assertRaises(Exception) as cm:
+ suite.debug()
+ self.assertEqual(str(cm.exception), 'TearDownModuleExc')
+ self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
+ 'tearDownClass'])
+ self.assertTrue(unittest.case._module_cleanups)
+ unittest.case._module_cleanups.clear()
+
+ ordering = []
+ blowUp = True
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
+ with self.assertRaises(Exception) as cm:
+ suite.debug()
+ self.assertEqual(str(cm.exception), 'TearDownModuleExc')
+ self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
+ 'tearDownClass'])
+ self.assertTrue(unittest.case._module_cleanups)
+ unittest.case._module_cleanups.clear()
+
def test_addClassCleanup_arg_errors(self):
cleanups = []
def cleanup(*args, **kwargs):
@@ -717,9 +918,9 @@
method_blow_up = False
result = runTests(TestableTest)
self.assertEqual(result.errors[0][1].splitlines()[-1],
- 'Exception: CleanUpExc')
- self.assertEqual(result.errors[1][1].splitlines()[-1],
'Exception: ModuleExc')
+ self.assertEqual(result.errors[1][1].splitlines()[-1],
+ 'Exception: CleanUpExc')
self.assertEqual(ordering, ['setUpModule', 'cleanup_exc'])
ordering = []
diff --git a/common/py3-stdlib/unittest/test/test_skipping.py b/common/py3-stdlib/unittest/test/test_skipping.py
index 1c178a9..7cb9d33 100644
--- a/common/py3-stdlib/unittest/test/test_skipping.py
+++ b/common/py3-stdlib/unittest/test/test_skipping.py
@@ -7,30 +7,50 @@
def test_skipping(self):
class Foo(unittest.TestCase):
+ def defaultTestResult(self):
+ return LoggingResult(events)
def test_skip_me(self):
self.skipTest("skip")
events = []
result = LoggingResult(events)
test = Foo("test_skip_me")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
self.assertEqual(result.skipped, [(test, "skip")])
+ events = []
+ result = test.run()
+ self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
+ 'stopTest', 'stopTestRun'])
+ self.assertEqual(result.skipped, [(test, "skip")])
+ self.assertEqual(result.testsRun, 1)
+
# Try letting setUp skip the test now.
class Foo(unittest.TestCase):
+ def defaultTestResult(self):
+ return LoggingResult(events)
def setUp(self):
self.skipTest("testing")
def test_nothing(self): pass
events = []
result = LoggingResult(events)
test = Foo("test_nothing")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
self.assertEqual(result.skipped, [(test, "testing")])
self.assertEqual(result.testsRun, 1)
+ events = []
+ result = test.run()
+ self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
+ 'stopTest', 'stopTestRun'])
+ self.assertEqual(result.skipped, [(test, "testing")])
+ self.assertEqual(result.testsRun, 1)
+
def test_skipping_subtests(self):
class Foo(unittest.TestCase):
+ def defaultTestResult(self):
+ return LoggingResult(events)
def test_skip_me(self):
with self.subTest(a=1):
with self.subTest(b=2):
@@ -40,7 +60,7 @@
events = []
result = LoggingResult(events)
test = Foo("test_skip_me")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events, ['startTest', 'addSkip', 'addSkip',
'addSkip', 'stopTest'])
self.assertEqual(len(result.skipped), 3)
@@ -54,11 +74,22 @@
self.assertIsNot(subtest, test)
self.assertEqual(result.skipped[2], (test, "skip 3"))
+ events = []
+ result = test.run()
+ self.assertEqual(events,
+ ['startTestRun', 'startTest', 'addSkip', 'addSkip',
+ 'addSkip', 'stopTest', 'stopTestRun'])
+ self.assertEqual([msg for subtest, msg in result.skipped],
+ ['skip 1', 'skip 2', 'skip 3'])
+
def test_skipping_decorators(self):
op_table = ((unittest.skipUnless, False, True),
(unittest.skipIf, True, False))
for deco, do_skip, dont_skip in op_table:
class Foo(unittest.TestCase):
+ def defaultTestResult(self):
+ return LoggingResult(events)
+
@deco(do_skip, "testing")
def test_skip(self): pass
@@ -66,10 +97,11 @@
def test_dont_skip(self): pass
test_do_skip = Foo("test_skip")
test_dont_skip = Foo("test_dont_skip")
+
suite = unittest.TestSuite([test_do_skip, test_dont_skip])
events = []
result = LoggingResult(events)
- suite.run(result)
+ self.assertIs(suite.run(result), result)
self.assertEqual(len(result.skipped), 1)
expected = ['startTest', 'addSkip', 'stopTest',
'startTest', 'addSuccess', 'stopTest']
@@ -78,16 +110,39 @@
self.assertEqual(result.skipped, [(test_do_skip, "testing")])
self.assertTrue(result.wasSuccessful())
+ events = []
+ result = test_do_skip.run()
+ self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
+ 'stopTest', 'stopTestRun'])
+ self.assertEqual(result.skipped, [(test_do_skip, "testing")])
+
+ events = []
+ result = test_dont_skip.run()
+ self.assertEqual(events, ['startTestRun', 'startTest', 'addSuccess',
+ 'stopTest', 'stopTestRun'])
+ self.assertEqual(result.skipped, [])
+
def test_skip_class(self):
@unittest.skip("testing")
class Foo(unittest.TestCase):
+ def defaultTestResult(self):
+ return LoggingResult(events)
def test_1(self):
record.append(1)
+ events = []
record = []
- result = unittest.TestResult()
+ result = LoggingResult(events)
test = Foo("test_1")
suite = unittest.TestSuite([test])
- suite.run(result)
+ self.assertIs(suite.run(result), result)
+ self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
+ self.assertEqual(result.skipped, [(test, "testing")])
+ self.assertEqual(record, [])
+
+ events = []
+ result = test.run()
+ self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
+ 'stopTest', 'stopTestRun'])
self.assertEqual(result.skipped, [(test, "testing")])
self.assertEqual(record, [])
@@ -102,10 +157,62 @@
result = unittest.TestResult()
test = Foo("test_1")
suite = unittest.TestSuite([test])
- suite.run(result)
+ self.assertIs(suite.run(result), result)
self.assertEqual(result.skipped, [(test, "testing")])
self.assertEqual(record, [])
+ def test_skip_in_setup(self):
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ self.skipTest("skip")
+ def test_skip_me(self):
+ self.fail("shouldn't come here")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_skip_me")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
+ self.assertEqual(result.skipped, [(test, "skip")])
+
+ def test_skip_in_cleanup(self):
+ class Foo(unittest.TestCase):
+ def test_skip_me(self):
+ pass
+ def tearDown(self):
+ self.skipTest("skip")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_skip_me")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
+ self.assertEqual(result.skipped, [(test, "skip")])
+
+ def test_failure_and_skip_in_cleanup(self):
+ class Foo(unittest.TestCase):
+ def test_skip_me(self):
+ self.fail("fail")
+ def tearDown(self):
+ self.skipTest("skip")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_skip_me")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events, ['startTest', 'addSkip', 'addFailure', 'stopTest'])
+ self.assertEqual(result.skipped, [(test, "skip")])
+
+ def test_skipping_and_fail_in_cleanup(self):
+ class Foo(unittest.TestCase):
+ def test_skip_me(self):
+ self.skipTest("skip")
+ def tearDown(self):
+ self.fail("fail")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_skip_me")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events, ['startTest', 'addSkip', 'addFailure', 'stopTest'])
+ self.assertEqual(result.skipped, [(test, "skip")])
+
def test_expected_failure(self):
class Foo(unittest.TestCase):
@unittest.expectedFailure
@@ -114,10 +221,12 @@
events = []
result = LoggingResult(events)
test = Foo("test_die")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events,
['startTest', 'addExpectedFailure', 'stopTest'])
+ self.assertFalse(result.failures)
self.assertEqual(result.expectedFailures[0][0], test)
+ self.assertFalse(result.unexpectedSuccesses)
self.assertTrue(result.wasSuccessful())
def test_expected_failure_with_wrapped_class(self):
@@ -129,10 +238,12 @@
events = []
result = LoggingResult(events)
test = Foo("test_1")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events,
['startTest', 'addExpectedFailure', 'stopTest'])
+ self.assertFalse(result.failures)
self.assertEqual(result.expectedFailures[0][0], test)
+ self.assertFalse(result.unexpectedSuccesses)
self.assertTrue(result.wasSuccessful())
def test_expected_failure_with_wrapped_subclass(self):
@@ -147,10 +258,12 @@
events = []
result = LoggingResult(events)
test = Bar("test_1")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events,
['startTest', 'addExpectedFailure', 'stopTest'])
+ self.assertFalse(result.failures)
self.assertEqual(result.expectedFailures[0][0], test)
+ self.assertFalse(result.unexpectedSuccesses)
self.assertTrue(result.wasSuccessful())
def test_expected_failure_subtests(self):
@@ -170,12 +283,52 @@
events = []
result = LoggingResult(events)
test = Foo("test_die")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events,
['startTest', 'addSubTestSuccess',
'addExpectedFailure', 'stopTest'])
+ self.assertFalse(result.failures)
self.assertEqual(len(result.expectedFailures), 1)
self.assertIs(result.expectedFailures[0][0], test)
+ self.assertFalse(result.unexpectedSuccesses)
+ self.assertTrue(result.wasSuccessful())
+
+ def test_expected_failure_and_fail_in_cleanup(self):
+ class Foo(unittest.TestCase):
+ @unittest.expectedFailure
+ def test_die(self):
+ self.fail("help me!")
+ def tearDown(self):
+ self.fail("bad tearDown")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_die")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events,
+ ['startTest', 'addFailure', 'stopTest'])
+ self.assertEqual(len(result.failures), 1)
+ self.assertIn('AssertionError: bad tearDown', result.failures[0][1])
+ self.assertFalse(result.expectedFailures)
+ self.assertFalse(result.unexpectedSuccesses)
+ self.assertFalse(result.wasSuccessful())
+
+ def test_expected_failure_and_skip_in_cleanup(self):
+ class Foo(unittest.TestCase):
+ @unittest.expectedFailure
+ def test_die(self):
+ self.fail("help me!")
+ def tearDown(self):
+ self.skipTest("skip")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_die")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events,
+ ['startTest', 'addSkip', 'stopTest'])
+ self.assertFalse(result.failures)
+ self.assertFalse(result.expectedFailures)
+ self.assertFalse(result.unexpectedSuccesses)
+ self.assertEqual(result.skipped, [(test, "skip")])
self.assertTrue(result.wasSuccessful())
def test_unexpected_success(self):
@@ -186,10 +339,11 @@
events = []
result = LoggingResult(events)
test = Foo("test_die")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events,
['startTest', 'addUnexpectedSuccess', 'stopTest'])
self.assertFalse(result.failures)
+ self.assertFalse(result.expectedFailures)
self.assertEqual(result.unexpectedSuccesses, [test])
self.assertFalse(result.wasSuccessful())
@@ -208,15 +362,54 @@
events = []
result = LoggingResult(events)
test = Foo("test_die")
- test.run(result)
+ self.assertIs(test.run(result), result)
self.assertEqual(events,
['startTest',
'addSubTestSuccess', 'addSubTestSuccess',
'addUnexpectedSuccess', 'stopTest'])
self.assertFalse(result.failures)
+ self.assertFalse(result.expectedFailures)
self.assertEqual(result.unexpectedSuccesses, [test])
self.assertFalse(result.wasSuccessful())
+ def test_unexpected_success_and_fail_in_cleanup(self):
+ class Foo(unittest.TestCase):
+ @unittest.expectedFailure
+ def test_die(self):
+ pass
+ def tearDown(self):
+ self.fail("bad tearDown")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_die")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events,
+ ['startTest', 'addFailure', 'stopTest'])
+ self.assertEqual(len(result.failures), 1)
+ self.assertIn('AssertionError: bad tearDown', result.failures[0][1])
+ self.assertFalse(result.expectedFailures)
+ self.assertFalse(result.unexpectedSuccesses)
+ self.assertFalse(result.wasSuccessful())
+
+ def test_unexpected_success_and_skip_in_cleanup(self):
+ class Foo(unittest.TestCase):
+ @unittest.expectedFailure
+ def test_die(self):
+ pass
+ def tearDown(self):
+ self.skipTest("skip")
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test_die")
+ self.assertIs(test.run(result), result)
+ self.assertEqual(events,
+ ['startTest', 'addSkip', 'stopTest'])
+ self.assertFalse(result.failures)
+ self.assertFalse(result.expectedFailures)
+ self.assertFalse(result.unexpectedSuccesses)
+ self.assertEqual(result.skipped, [(test, "skip")])
+ self.assertTrue(result.wasSuccessful())
+
def test_skip_doesnt_run_setup(self):
class Foo(unittest.TestCase):
wasSetUp = False
@@ -232,7 +425,7 @@
result = unittest.TestResult()
test = Foo("test_1")
suite = unittest.TestSuite([test])
- suite.run(result)
+ self.assertIs(suite.run(result), result)
self.assertEqual(result.skipped, [(test, "testing")])
self.assertFalse(Foo.wasSetUp)
self.assertFalse(Foo.wasTornDown)
@@ -252,7 +445,7 @@
result = unittest.TestResult()
test = Foo("test_1")
suite = unittest.TestSuite([test])
- suite.run(result)
+ self.assertIs(suite.run(result), result)
self.assertEqual(result.skipped, [(test, "testing")])
def test_skip_without_reason(self):
@@ -264,8 +457,74 @@
result = unittest.TestResult()
test = Foo("test_1")
suite = unittest.TestSuite([test])
- suite.run(result)
+ self.assertIs(suite.run(result), result)
self.assertEqual(result.skipped, [(test, "")])
+ def test_debug_skipping(self):
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ events.append("setUp")
+ def tearDown(self):
+ events.append("tearDown")
+ def test1(self):
+ self.skipTest('skipping exception')
+ events.append("test1")
+ @unittest.skip("skipping decorator")
+ def test2(self):
+ events.append("test2")
+
+ events = []
+ test = Foo("test1")
+ with self.assertRaises(unittest.SkipTest) as cm:
+ test.debug()
+ self.assertIn("skipping exception", str(cm.exception))
+ self.assertEqual(events, ["setUp"])
+
+ events = []
+ test = Foo("test2")
+ with self.assertRaises(unittest.SkipTest) as cm:
+ test.debug()
+ self.assertIn("skipping decorator", str(cm.exception))
+ self.assertEqual(events, [])
+
+ def test_debug_skipping_class(self):
+ @unittest.skip("testing")
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ events.append("setUp")
+ def tearDown(self):
+ events.append("tearDown")
+ def test(self):
+ events.append("test")
+
+ events = []
+ test = Foo("test")
+ with self.assertRaises(unittest.SkipTest) as cm:
+ test.debug()
+ self.assertIn("testing", str(cm.exception))
+ self.assertEqual(events, [])
+
+ def test_debug_skipping_subtests(self):
+ class Foo(unittest.TestCase):
+ def setUp(self):
+ events.append("setUp")
+ def tearDown(self):
+ events.append("tearDown")
+ def test(self):
+ with self.subTest(a=1):
+ events.append('subtest')
+ self.skipTest("skip subtest")
+ events.append('end subtest')
+ events.append('end test')
+
+ events = []
+ result = LoggingResult(events)
+ test = Foo("test")
+ with self.assertRaises(unittest.SkipTest) as cm:
+ test.debug()
+ self.assertIn("skip subtest", str(cm.exception))
+ self.assertEqual(events, ['setUp', 'subtest'])
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/unittest/test/testmock/testasync.py b/common/py3-stdlib/unittest/test/testmock/testasync.py
index 690ca4f..e1866a3 100644
--- a/common/py3-stdlib/unittest/test/testmock/testasync.py
+++ b/common/py3-stdlib/unittest/test/testmock/testasync.py
@@ -199,9 +199,9 @@
with self.assertRaises(RuntimeError):
create_autospec(async_func, instance=True)
+ @unittest.skip('Broken test from https://bugs.python.org/issue37251')
def test_create_autospec_awaitable_class(self):
- awaitable_mock = create_autospec(spec=AwaitableClass())
- self.assertIsInstance(create_autospec(awaitable_mock), AsyncMock)
+ self.assertIsInstance(create_autospec(AwaitableClass), AsyncMock)
def test_create_autospec(self):
spec = create_autospec(async_func_args)
diff --git a/common/py3-stdlib/unittest/test/testmock/testmock.py b/common/py3-stdlib/unittest/test/testmock/testmock.py
index ce674e7..fdba543 100644
--- a/common/py3-stdlib/unittest/test/testmock/testmock.py
+++ b/common/py3-stdlib/unittest/test/testmock/testmock.py
@@ -11,7 +11,7 @@
call, DEFAULT, patch, sentinel,
MagicMock, Mock, NonCallableMock,
NonCallableMagicMock, AsyncMock, _Call, _CallList,
- create_autospec
+ create_autospec, InvalidSpecError
)
@@ -38,6 +38,12 @@
def smeth(a, b, c, d=None): pass
+class Typos():
+ autospect = None
+ auto_spec = None
+ set_spec = None
+
+
def something(a): pass
@@ -199,6 +205,28 @@
self.assertRaisesRegex(ValueError, 'Bazinga!', mock)
+ def test_autospec_mock(self):
+ class A(object):
+ class B(object):
+ C = None
+
+ with mock.patch.object(A, 'B'):
+ with self.assertRaisesRegex(InvalidSpecError,
+ "Cannot autospec attr 'B' from target <MagicMock spec='A'"):
+ create_autospec(A).B
+ with self.assertRaisesRegex(InvalidSpecError,
+ "Cannot autospec attr 'B' from target 'A'"):
+ mock.patch.object(A, 'B', autospec=True).start()
+ with self.assertRaisesRegex(InvalidSpecError,
+ "Cannot autospec attr 'C' as the patch target "):
+ mock.patch.object(A.B, 'C', autospec=True).start()
+ with self.assertRaisesRegex(InvalidSpecError,
+ "Cannot spec attr 'B' as the spec "):
+ mock.patch.object(A, 'B', spec=A.B).start()
+ with self.assertRaisesRegex(InvalidSpecError,
+ "Cannot spec attr 'B' as the spec_set "):
+ mock.patch.object(A, 'B', spec_set=A.B).start()
+
def test_reset_mock(self):
parent = Mock()
spec = ["something"]
@@ -1598,14 +1626,23 @@
#Issue21238
def test_mock_unsafe(self):
m = Mock()
- msg = "Attributes cannot start with 'assert' or 'assret'"
+ msg = "is not a valid assertion. Use a spec for the mock"
with self.assertRaisesRegex(AttributeError, msg):
m.assert_foo_call()
with self.assertRaisesRegex(AttributeError, msg):
m.assret_foo_call()
+ with self.assertRaisesRegex(AttributeError, msg):
+ m.asert_foo_call()
+ with self.assertRaisesRegex(AttributeError, msg):
+ m.aseert_foo_call()
+ with self.assertRaisesRegex(AttributeError, msg):
+ m.assrt_foo_call()
m = Mock(unsafe=True)
m.assert_foo_call()
m.assret_foo_call()
+ m.asert_foo_call()
+ m.aseert_foo_call()
+ m.assrt_foo_call()
#Issue21262
def test_assert_not_called(self):
@@ -2156,6 +2193,62 @@
obj = mock(spec=Something)
self.assertIsInstance(obj, Something)
+ def test_bool_not_called_when_passing_spec_arg(self):
+ class Something:
+ def __init__(self):
+ self.obj_with_bool_func = unittest.mock.MagicMock()
+
+ obj = Something()
+ with unittest.mock.patch.object(obj, 'obj_with_bool_func', spec=object): pass
+
+ self.assertEqual(obj.obj_with_bool_func.__bool__.call_count, 0)
+
+ def test_misspelled_arguments(self):
+ class Foo():
+ one = 'one'
+ # patch, patch.object and create_autospec need to check for misspelled
+ # arguments explicitly and throw a RuntimError if found.
+ with self.assertRaises(RuntimeError):
+ with patch(f'{__name__}.Something.meth', autospect=True): pass
+ with self.assertRaises(RuntimeError):
+ with patch.object(Foo, 'one', autospect=True): pass
+ with self.assertRaises(RuntimeError):
+ with patch(f'{__name__}.Something.meth', auto_spec=True): pass
+ with self.assertRaises(RuntimeError):
+ with patch.object(Foo, 'one', auto_spec=True): pass
+ with self.assertRaises(RuntimeError):
+ with patch(f'{__name__}.Something.meth', set_spec=True): pass
+ with self.assertRaises(RuntimeError):
+ with patch.object(Foo, 'one', set_spec=True): pass
+ with self.assertRaises(RuntimeError):
+ m = create_autospec(Foo, set_spec=True)
+ # patch.multiple, on the other hand, should flag misspelled arguments
+ # through an AttributeError, when trying to find the keys from kwargs
+ # as attributes on the target.
+ with self.assertRaises(AttributeError):
+ with patch.multiple(
+ f'{__name__}.Something', meth=DEFAULT, autospect=True): pass
+ with self.assertRaises(AttributeError):
+ with patch.multiple(
+ f'{__name__}.Something', meth=DEFAULT, auto_spec=True): pass
+ with self.assertRaises(AttributeError):
+ with patch.multiple(
+ f'{__name__}.Something', meth=DEFAULT, set_spec=True): pass
+
+ with patch(f'{__name__}.Something.meth', unsafe=True, autospect=True):
+ pass
+ with patch.object(Foo, 'one', unsafe=True, autospect=True): pass
+ with patch(f'{__name__}.Something.meth', unsafe=True, auto_spec=True):
+ pass
+ with patch.object(Foo, 'one', unsafe=True, auto_spec=True): pass
+ with patch(f'{__name__}.Something.meth', unsafe=True, set_spec=True):
+ pass
+ with patch.object(Foo, 'one', unsafe=True, set_spec=True): pass
+ m = create_autospec(Foo, set_spec=True, unsafe=True)
+ with patch.multiple(
+ f'{__name__}.Typos', autospect=True, set_spec=True, auto_spec=True):
+ pass
+
if __name__ == '__main__':
unittest.main()
diff --git a/common/py3-stdlib/unittest/test/testmock/testpatch.py b/common/py3-stdlib/unittest/test/testmock/testpatch.py
index d8c1515..8ab63a1 100644
--- a/common/py3-stdlib/unittest/test/testmock/testpatch.py
+++ b/common/py3-stdlib/unittest/test/testmock/testpatch.py
@@ -1875,9 +1875,10 @@
self.assertEqual(foo(), 1)
self.assertEqual(foo(), 0)
+ orig_doc = foo.__doc__
with patch.object(foo, '__doc__', "FUN"):
self.assertEqual(foo.__doc__, "FUN")
- self.assertEqual(foo.__doc__, "TEST")
+ self.assertEqual(foo.__doc__, orig_doc)
with patch.object(foo, '__module__', "testpatch2"):
self.assertEqual(foo.__module__, "testpatch2")
@@ -1932,8 +1933,13 @@
def test_invalid_target(self):
- with self.assertRaises(TypeError):
- patch('')
+ class Foo:
+ pass
+
+ for target in ['', 12, Foo()]:
+ with self.subTest(target=target):
+ with self.assertRaises(TypeError):
+ patch(target)
def test_cant_set_kwargs_when_passing_a_mock(self):
diff --git a/common/py3-stdlib/unittest/test/testmock/testsealable.py b/common/py3-stdlib/unittest/test/testmock/testsealable.py
index 59f5233..daba2b4 100644
--- a/common/py3-stdlib/unittest/test/testmock/testsealable.py
+++ b/common/py3-stdlib/unittest/test/testmock/testsealable.py
@@ -128,7 +128,7 @@
m.attr_sample2
def test_integration_with_spec_method_definition(self):
- """You need to defin the methods, even if they are in the spec"""
+ """You need to define the methods, even if they are in the spec"""
m = mock.Mock(SampleObject)
m.method_sample1.return_value = 1
@@ -171,6 +171,67 @@
m.test1().test2.test3().test4()
self.assertIn("mock.test1().test2.test3().test4", str(cm.exception))
+ def test_seal_with_autospec(self):
+ # https://bugs.python.org/issue45156
+ class Foo:
+ foo = 0
+ def bar1(self):
+ return 1
+ def bar2(self):
+ return 2
+
+ class Baz:
+ baz = 3
+ def ban(self):
+ return 4
+
+ for spec_set in (True, False):
+ with self.subTest(spec_set=spec_set):
+ foo = mock.create_autospec(Foo, spec_set=spec_set)
+ foo.bar1.return_value = 'a'
+ foo.Baz.ban.return_value = 'b'
+
+ mock.seal(foo)
+
+ self.assertIsInstance(foo.foo, mock.NonCallableMagicMock)
+ self.assertIsInstance(foo.bar1, mock.MagicMock)
+ self.assertIsInstance(foo.bar2, mock.MagicMock)
+ self.assertIsInstance(foo.Baz, mock.MagicMock)
+ self.assertIsInstance(foo.Baz.baz, mock.NonCallableMagicMock)
+ self.assertIsInstance(foo.Baz.ban, mock.MagicMock)
+
+ self.assertEqual(foo.bar1(), 'a')
+ foo.bar1.return_value = 'new_a'
+ self.assertEqual(foo.bar1(), 'new_a')
+ self.assertEqual(foo.Baz.ban(), 'b')
+ foo.Baz.ban.return_value = 'new_b'
+ self.assertEqual(foo.Baz.ban(), 'new_b')
+
+ with self.assertRaises(TypeError):
+ foo.foo()
+ with self.assertRaises(AttributeError):
+ foo.bar = 1
+ with self.assertRaises(AttributeError):
+ foo.bar2()
+
+ foo.bar2.return_value = 'bar2'
+ self.assertEqual(foo.bar2(), 'bar2')
+
+ with self.assertRaises(AttributeError):
+ foo.missing_attr
+ with self.assertRaises(AttributeError):
+ foo.missing_attr = 1
+ with self.assertRaises(AttributeError):
+ foo.missing_method()
+ with self.assertRaises(TypeError):
+ foo.Baz.baz()
+ with self.assertRaises(AttributeError):
+ foo.Baz.missing_attr
+ with self.assertRaises(AttributeError):
+ foo.Baz.missing_attr = 1
+ with self.assertRaises(AttributeError):
+ foo.Baz.missing_method()
+
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/urllib/parse.py b/common/py3-stdlib/urllib/parse.py
index ea897c3..b35997b 100644
--- a/common/py3-stdlib/urllib/parse.py
+++ b/common/py3-stdlib/urllib/parse.py
@@ -78,6 +78,9 @@
'0123456789'
'+-.')
+# Unsafe bytes to be removed per WHATWG spec
+_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
+
# XXX: Consider replacing with functools.lru_cache
MAX_CACHE_SIZE = 20
_parse_cache = {}
@@ -453,6 +456,11 @@
"""
url, scheme, _coerce_result = _coerce_args(url, scheme)
+
+ for b in _UNSAFE_URL_BYTES_TO_REMOVE:
+ url = url.replace(b, "")
+ scheme = scheme.replace(b, "")
+
allow_fragments = bool(allow_fragments)
key = url, scheme, allow_fragments, type(url), type(scheme)
cached = _parse_cache.get(key, None)
@@ -662,7 +670,7 @@
def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
- encoding='utf-8', errors='replace', max_num_fields=None):
+ encoding='utf-8', errors='replace', max_num_fields=None, separator='&'):
"""Parse a query given as a string argument.
Arguments:
@@ -686,12 +694,15 @@
max_num_fields: int. If set, then throws a ValueError if there
are more than n fields read by parse_qsl().
+ separator: str. The symbol to use for separating the query arguments.
+ Defaults to &.
+
Returns a dictionary.
"""
parsed_result = {}
pairs = parse_qsl(qs, keep_blank_values, strict_parsing,
encoding=encoding, errors=errors,
- max_num_fields=max_num_fields)
+ max_num_fields=max_num_fields, separator=separator)
for name, value in pairs:
if name in parsed_result:
parsed_result[name].append(value)
@@ -701,7 +712,7 @@
def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
- encoding='utf-8', errors='replace', max_num_fields=None):
+ encoding='utf-8', errors='replace', max_num_fields=None, separator='&'):
"""Parse a query given as a string argument.
Arguments:
@@ -724,21 +735,27 @@
max_num_fields: int. If set, then throws a ValueError
if there are more than n fields read by parse_qsl().
+ separator: str. The symbol to use for separating the query arguments.
+ Defaults to &.
+
Returns a list, as G-d intended.
"""
qs, _coerce_result = _coerce_args(qs)
+ separator, _ = _coerce_args(separator)
+
+ if not separator or (not isinstance(separator, (str, bytes))):
+ raise ValueError("Separator must be of type string or bytes.")
# If max_num_fields is defined then check that the number of fields
# is less than max_num_fields. This prevents a memory exhaustion DOS
# attack via post bodies with many fields.
if max_num_fields is not None:
- num_fields = 1 + qs.count('&') + qs.count(';')
+ num_fields = 1 + qs.count(separator)
if max_num_fields < num_fields:
raise ValueError('Max number of fields exceeded')
- pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
r = []
- for name_value in pairs:
+ for name_value in qs.split(separator):
if not name_value and not strict_parsing:
continue
nv = name_value.split('=', 1)
diff --git a/common/py3-stdlib/urllib/request.py b/common/py3-stdlib/urllib/request.py
index a8c870b..34b1b0b 100644
--- a/common/py3-stdlib/urllib/request.py
+++ b/common/py3-stdlib/urllib/request.py
@@ -64,7 +64,7 @@
# install it
urllib.request.install_opener(opener)
-f = urllib.request.urlopen('http://www.python.org/')
+f = urllib.request.urlopen('https://www.python.org/')
"""
# XXX issues:
@@ -202,6 +202,8 @@
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH,
cafile=cafile,
capath=capath)
+ # send ALPN extension to indicate HTTP/1.1 protocol
+ context.set_alpn_protocols(['http/1.1'])
https_handler = HTTPSHandler(context=context)
opener = build_opener(https_handler)
elif context:
@@ -771,7 +773,11 @@
raise ValueError("proxy URL with no authority: %r" % proxy)
# We have an authority, so for RFC 3986-compliant URLs (by ss 3.
# and 3.3.), path is empty or starts with '/'
- end = r_scheme.find("/", 2)
+ if '@' in r_scheme:
+ host_separator = r_scheme.find('@')
+ end = r_scheme.find("/", host_separator)
+ else:
+ end = r_scheme.find("/", 2)
if end == -1:
end = None
authority = r_scheme[2:end]
@@ -883,10 +889,10 @@
return True
if base[0] != test[0]:
return False
- common = posixpath.commonprefix((base[1], test[1]))
- if len(common) == len(base[1]):
- return True
- return False
+ prefix = base[1]
+ if prefix[-1:] != '/':
+ prefix += '/'
+ return test[1].startswith(prefix)
class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
@@ -939,7 +945,7 @@
# (single quotes are a violation of the RFC, but appear in the wild)
rx = re.compile('(?:^|,)' # start of the string or ','
'[ \t]*' # optional whitespaces
- '([^ \t]+)' # scheme like "Basic"
+ '([^ \t,]+)' # scheme like "Basic"
'[ \t]+' # mandatory whitespaces
# realm=xxx
# realm='xxx'
diff --git a/common/py3-stdlib/weakref.py b/common/py3-stdlib/weakref.py
index 5fa851d..994ea8a 100644
--- a/common/py3-stdlib/weakref.py
+++ b/common/py3-stdlib/weakref.py
@@ -2,7 +2,7 @@
This module is an implementation of PEP 205:
-http://www.python.org/dev/peps/pep-0205/
+https://www.python.org/dev/peps/pep-0205/
"""
# Naming convention: Variables named "wr" are weak reference objects;
@@ -119,14 +119,17 @@
self.data = {}
self.update(other, **kw)
- def _commit_removals(self):
- l = self._pending_removals
+ def _commit_removals(self, _atomic_removal=_remove_dead_weakref):
+ pop = self._pending_removals.pop
d = self.data
# We shouldn't encounter any KeyError, because this method should
# always be called *before* mutating the dict.
- while l:
- key = l.pop()
- _remove_dead_weakref(d, key)
+ while True:
+ try:
+ key = pop()
+ except IndexError:
+ return
+ _atomic_removal(d, key)
def __getitem__(self, key):
if self._pending_removals:
@@ -370,7 +373,10 @@
if self._iterating:
self._pending_removals.append(k)
else:
- del self.data[k]
+ try:
+ del self.data[k]
+ except KeyError:
+ pass
self._remove = remove
# A list of dead weakrefs (keys to be removed)
self._pending_removals = []
@@ -384,11 +390,16 @@
# because a dead weakref never compares equal to a live weakref,
# even if they happened to refer to equal objects.
# However, it means keys may already have been removed.
- l = self._pending_removals
+ pop = self._pending_removals.pop
d = self.data
- while l:
+ while True:
try:
- del d[l.pop()]
+ key = pop()
+ except IndexError:
+ return
+
+ try:
+ del d[key]
except KeyError:
pass
diff --git a/common/py3-stdlib/webbrowser.py b/common/py3-stdlib/webbrowser.py
index 6023c1e..ec3cece 100755
--- a/common/py3-stdlib/webbrowser.py
+++ b/common/py3-stdlib/webbrowser.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python3
-"""Interfaces for launching and remotely controlling Web browsers."""
+"""Interfaces for launching and remotely controlling web browsers."""
# Maintained by Georg Brandl.
import os
@@ -532,6 +532,10 @@
# OS X can use below Unix support (but we prefer using the OS X
# specific stuff)
+ if sys.platform == "serenityos":
+ # SerenityOS webbrowser, simply called "Browser".
+ register("Browser", None, BackgroundBrowser("Browser"))
+
if sys.platform[:3] == "win":
# First try to use the default Windows browser
register("windows-default", WindowsDefault)
diff --git a/common/py3-stdlib/wsgiref/validate.py b/common/py3-stdlib/wsgiref/validate.py
index 48ac007..6e16578 100644
--- a/common/py3-stdlib/wsgiref/validate.py
+++ b/common/py3-stdlib/wsgiref/validate.py
@@ -137,7 +137,7 @@
"""
When applied between a WSGI server and a WSGI application, this
- middleware will check for WSGI compliancy on a number of levels.
+ middleware will check for WSGI compliance on a number of levels.
This middleware does not modify the request or response in any
way, but will raise an AssertionError if anything seems off
(except for a failure to close the application iterator, which
diff --git a/common/py3-stdlib/xml/etree/ElementInclude.py b/common/py3-stdlib/xml/etree/ElementInclude.py
index 5303062..40a9b22 100644
--- a/common/py3-stdlib/xml/etree/ElementInclude.py
+++ b/common/py3-stdlib/xml/etree/ElementInclude.py
@@ -42,7 +42,7 @@
# --------------------------------------------------------------------
# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/psf/license for licensing details.
+# See https://www.python.org/psf/license for licensing details.
##
# Limited XInclude support for the ElementTree package.
diff --git a/common/py3-stdlib/xml/etree/ElementPath.py b/common/py3-stdlib/xml/etree/ElementPath.py
index d318e65..a1170b5 100644
--- a/common/py3-stdlib/xml/etree/ElementPath.py
+++ b/common/py3-stdlib/xml/etree/ElementPath.py
@@ -48,7 +48,7 @@
# --------------------------------------------------------------------
# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/psf/license for licensing details.
+# See https://www.python.org/psf/license for licensing details.
##
# Implementation module for XPath support. There's usually no reason
@@ -65,8 +65,9 @@
r"//?|"
r"\.\.|"
r"\(\)|"
+ r"!=|"
r"[/.*:\[\]\(\)@=])|"
- r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|"
+ r"((?:\{[^}]+\})?[^/\[\]\(\)@!=\s]+)|"
r"\s+"
)
@@ -253,15 +254,19 @@
if elem.get(key) is not None:
yield elem
return select
- if signature == "@-='":
- # [@attribute='value']
+ if signature == "@-='" or signature == "@-!='":
+ # [@attribute='value'] or [@attribute!='value']
key = predicate[1]
value = predicate[-1]
def select(context, result):
for elem in result:
if elem.get(key) == value:
yield elem
- return select
+ def select_negated(context, result):
+ for elem in result:
+ if (attr_value := elem.get(key)) is not None and attr_value != value:
+ yield elem
+ return select_negated if '!=' in signature else select
if signature == "-" and not re.match(r"\-?\d+$", predicate[0]):
# [tag]
tag = predicate[0]
@@ -270,8 +275,10 @@
if elem.find(tag) is not None:
yield elem
return select
- if signature == ".='" or (signature == "-='" and not re.match(r"\-?\d+$", predicate[0])):
- # [.='value'] or [tag='value']
+ if signature == ".='" or signature == ".!='" or (
+ (signature == "-='" or signature == "-!='")
+ and not re.match(r"\-?\d+$", predicate[0])):
+ # [.='value'] or [tag='value'] or [.!='value'] or [tag!='value']
tag = predicate[0]
value = predicate[-1]
if tag:
@@ -281,12 +288,22 @@
if "".join(e.itertext()) == value:
yield elem
break
+ def select_negated(context, result):
+ for elem in result:
+ for e in elem.iterfind(tag):
+ if "".join(e.itertext()) != value:
+ yield elem
+ break
else:
def select(context, result):
for elem in result:
if "".join(elem.itertext()) == value:
yield elem
- return select
+ def select_negated(context, result):
+ for elem in result:
+ if "".join(elem.itertext()) != value:
+ yield elem
+ return select_negated if '!=' in signature else select
if signature == "-" or signature == "-()" or signature == "-()-":
# [index] or [last()] or [last()-index]
if signature == "-":
diff --git a/common/py3-stdlib/xml/etree/ElementTree.py b/common/py3-stdlib/xml/etree/ElementTree.py
index 7a26900..07be860 100644
--- a/common/py3-stdlib/xml/etree/ElementTree.py
+++ b/common/py3-stdlib/xml/etree/ElementTree.py
@@ -35,7 +35,7 @@
#---------------------------------------------------------------------
# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/psf/license for licensing details.
+# See https://www.python.org/psf/license for licensing details.
#
# ElementTree
# Copyright (c) 1999-2008 by Fredrik Lundh. All rights reserved.
@@ -252,7 +252,7 @@
"""
for element in elements:
self._assert_is_element(element)
- self._children.extend(elements)
+ self._children.append(element)
def insert(self, index, subelement):
"""Insert *subelement* at position *index*."""
@@ -1248,8 +1248,14 @@
# Use the internal, undocumented _parser argument for now; When the
# parser argument of iterparse is removed, this can be killed.
pullparser = XMLPullParser(events=events, _parser=parser)
- def iterator():
+
+ def iterator(source):
+ close_source = False
try:
+ if not hasattr(source, "read"):
+ source = open(source, "rb")
+ close_source = True
+ yield None
while True:
yield from pullparser.read_events()
# load event buffer
@@ -1265,16 +1271,12 @@
source.close()
class IterParseIterator(collections.abc.Iterator):
- __next__ = iterator().__next__
+ __next__ = iterator(source).__next__
it = IterParseIterator()
it.root = None
del iterator, IterParseIterator
- close_source = False
- if not hasattr(source, "read"):
- source = open(source, "rb")
- close_source = True
-
+ next(it)
return it
@@ -1283,7 +1285,7 @@
def __init__(self, events=None, *, _parser=None):
# The _parser argument is for internal use only and must not be relied
# upon in user code. It will be removed in a future release.
- # See http://bugs.python.org/issue17741 for more details.
+ # See https://bugs.python.org/issue17741 for more details.
self._events_queue = collections.deque()
self._parser = _parser or XMLParser(target=TreeBuilder())
@@ -1560,7 +1562,6 @@
# Configure pyexpat: buffering, new-style attribute handling.
parser.buffer_text = 1
parser.ordered_attributes = 1
- parser.specified_attributes = 1
self._doctype = None
self.entity = {}
try:
@@ -1580,7 +1581,6 @@
for event_name in events_to_report:
if event_name == "start":
parser.ordered_attributes = 1
- parser.specified_attributes = 1
def handler(tag, attrib_in, event=event_name, append=append,
start=self._start):
append((event, start(tag, attrib_in)))
diff --git a/common/py3-stdlib/xml/etree/__init__.py b/common/py3-stdlib/xml/etree/__init__.py
index 27fd8f6..e2ec534 100644
--- a/common/py3-stdlib/xml/etree/__init__.py
+++ b/common/py3-stdlib/xml/etree/__init__.py
@@ -30,4 +30,4 @@
# --------------------------------------------------------------------
# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/psf/license for licensing details.
+# See https://www.python.org/psf/license for licensing details.
diff --git a/common/py3-stdlib/xml/sax/handler.py b/common/py3-stdlib/xml/sax/handler.py
index 481733d..e8d417e 100644
--- a/common/py3-stdlib/xml/sax/handler.py
+++ b/common/py3-stdlib/xml/sax/handler.py
@@ -340,3 +340,48 @@
property_xml_string,
property_encoding,
property_interning_dict]
+
+
+class LexicalHandler:
+ """Optional SAX2 handler for lexical events.
+
+ This handler is used to obtain lexical information about an XML
+ document, that is, information about how the document was encoded
+ (as opposed to what it contains, which is reported to the
+ ContentHandler), such as comments and CDATA marked section
+ boundaries.
+
+ To set the LexicalHandler of an XMLReader, use the setProperty
+ method with the property identifier
+ 'http://xml.org/sax/properties/lexical-handler'."""
+
+ def comment(self, content):
+ """Reports a comment anywhere in the document (including the
+ DTD and outside the document element).
+
+ content is a string that holds the contents of the comment."""
+
+ def startDTD(self, name, public_id, system_id):
+ """Report the start of the DTD declarations, if the document
+ has an associated DTD.
+
+ A startEntity event will be reported before declaration events
+ from the external DTD subset are reported, and this can be
+ used to infer from which subset DTD declarations derive.
+
+ name is the name of the document element type, public_id the
+ public identifier of the DTD (or None if none were supplied)
+ and system_id the system identfier of the external subset (or
+ None if none were supplied)."""
+
+ def endDTD(self):
+ """Signals the end of DTD declarations."""
+
+ def startCDATA(self):
+ """Reports the beginning of a CDATA marked section.
+
+ The contents of the CDATA marked section will be reported
+ through the characters event."""
+
+ def endCDATA(self):
+ """Reports the end of a CDATA marked section."""
diff --git a/common/py3-stdlib/xmlrpc/client.py b/common/py3-stdlib/xmlrpc/client.py
index d15d60d..a614cef 100644
--- a/common/py3-stdlib/xmlrpc/client.py
+++ b/common/py3-stdlib/xmlrpc/client.py
@@ -264,16 +264,22 @@
# Issue #13305: different format codes across platforms
_day0 = datetime(1, 1, 1)
-if _day0.strftime('%Y') == '0001': # Mac OS X
+def _try(fmt):
+ try:
+ return _day0.strftime(fmt) == '0001'
+ except ValueError:
+ return False
+if _try('%Y'): # Mac OS X
def _iso8601_format(value):
return value.strftime("%Y%m%dT%H:%M:%S")
-elif _day0.strftime('%4Y') == '0001': # Linux
+elif _try('%4Y'): # Linux
def _iso8601_format(value):
return value.strftime("%4Y%m%dT%H:%M:%S")
else:
def _iso8601_format(value):
return value.strftime("%Y%m%dT%H:%M:%S").zfill(17)
del _day0
+del _try
def _strftime(value):
@@ -1421,11 +1427,13 @@
# establish a "logical" server connection
# get the url
- p = urllib.parse.urlparse(uri)
+ p = urllib.parse.urlsplit(uri)
if p.scheme not in ("http", "https"):
raise OSError("unsupported XML-RPC protocol")
self.__host = p.netloc
- self.__handler = p.path or "/RPC2"
+ self.__handler = urllib.parse.urlunsplit(["", "", *p[2:]])
+ if not self.__handler:
+ self.__handler = "/RPC2"
if transport is None:
if p.scheme == "https":
diff --git a/common/py3-stdlib/xmlrpc/server.py b/common/py3-stdlib/xmlrpc/server.py
index 287e324..69a260f 100644
--- a/common/py3-stdlib/xmlrpc/server.py
+++ b/common/py3-stdlib/xmlrpc/server.py
@@ -750,7 +750,7 @@
url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif pep:
- url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep)
+ url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif text[end:end+1] == '(':
results.append(self.namelink(name, methods, funcs, classes))
diff --git a/common/py3-stdlib/zipfile.py b/common/py3-stdlib/zipfile.py
index 816f858..67cfdfb 100644
--- a/common/py3-stdlib/zipfile.py
+++ b/common/py3-stdlib/zipfile.py
@@ -16,6 +16,7 @@
import threading
import time
import contextlib
+import pathlib
try:
import zlib # We may need its compression method
@@ -1120,8 +1121,15 @@
def write(self, data):
if self.closed:
raise ValueError('I/O operation on closed file.')
- nbytes = len(data)
+
+ # Accept any data that supports the buffer protocol
+ if isinstance(data, (bytes, bytearray)):
+ nbytes = len(data)
+ else:
+ data = memoryview(data)
+ nbytes = data.nbytes
self._file_size += nbytes
+
self._crc = crc32(data, self._crc)
if self._compressor:
data = self._compressor.compress(data)
@@ -2197,13 +2205,12 @@
if not isinstance(source, ZipFile):
return cls(source)
- # Only allow for FastPath when supplied zipfile is read-only
+ # Only allow for FastLookup when supplied zipfile is read-only
if 'r' not in source.mode:
cls = CompleteDirs
- res = cls.__new__(cls)
- vars(res).update(vars(source))
- return res
+ source.__class__ = cls
+ return source
class FastLookup(CompleteDirs):
@@ -2211,6 +2218,7 @@
ZipFile subclass to ensure implicit
dirs exist and are resolved rapidly.
"""
+
def namelist(self):
with contextlib.suppress(AttributeError):
return self.__names
@@ -2242,7 +2250,7 @@
>>> zf.writestr('a.txt', 'content of a')
>>> zf.writestr('b/c.txt', 'content of c')
>>> zf.writestr('b/d/e.txt', 'content of e')
- >>> zf.filename = 'abcde.zip'
+ >>> zf.filename = 'mem/abcde.zip'
Path accepts the zipfile object itself or a filename
@@ -2254,9 +2262,9 @@
>>> a, b = root.iterdir()
>>> a
- Path('abcde.zip', 'a.txt')
+ Path('mem/abcde.zip', 'a.txt')
>>> b
- Path('abcde.zip', 'b/')
+ Path('mem/abcde.zip', 'b/')
name property:
@@ -2267,7 +2275,7 @@
>>> c = b / 'c.txt'
>>> c
- Path('abcde.zip', 'b/c.txt')
+ Path('mem/abcde.zip', 'b/c.txt')
>>> c.name
'c.txt'
@@ -2285,36 +2293,68 @@
Coercion to string:
- >>> str(c)
- 'abcde.zip/b/c.txt'
+ >>> import os
+ >>> str(c).replace(os.sep, posixpath.sep)
+ 'mem/abcde.zip/b/c.txt'
+
+ At the root, ``name``, ``filename``, and ``parent``
+ resolve to the zipfile. Note these attributes are not
+ valid and will raise a ``ValueError`` if the zipfile
+ has no filename.
+
+ >>> root.name
+ 'abcde.zip'
+ >>> str(root.filename).replace(os.sep, posixpath.sep)
+ 'mem/abcde.zip'
+ >>> str(root.parent)
+ 'mem'
"""
__repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})"
def __init__(self, root, at=""):
+ """
+ Construct a Path from a ZipFile or filename.
+
+ Note: When the source is an existing ZipFile object,
+ its type (__class__) will be mutated to a
+ specialized type. If the caller wishes to retain the
+ original type, the caller should either create a
+ separate ZipFile object or pass a filename.
+ """
self.root = FastLookup.make(root)
self.at = at
- def open(self, mode='r', *args, **kwargs):
+ def open(self, mode='r', *args, pwd=None, **kwargs):
"""
Open this entry as text or binary following the semantics
of ``pathlib.Path.open()`` by passing arguments through
to io.TextIOWrapper().
"""
- pwd = kwargs.pop('pwd', None)
+ if self.is_dir():
+ raise IsADirectoryError(self)
zip_mode = mode[0]
+ if not self.exists() and zip_mode == 'r':
+ raise FileNotFoundError(self)
stream = self.root.open(self.at, zip_mode, pwd=pwd)
if 'b' in mode:
if args or kwargs:
raise ValueError("encoding args invalid for binary operation")
return stream
+ else:
+ kwargs["encoding"] = io.text_encoding(kwargs.get("encoding"))
return io.TextIOWrapper(stream, *args, **kwargs)
@property
def name(self):
- return posixpath.basename(self.at.rstrip("/"))
+ return pathlib.Path(self.at).name or self.filename.name
+
+ @property
+ def filename(self):
+ return pathlib.Path(self.root.filename).joinpath(self.at)
def read_text(self, *args, **kwargs):
+ kwargs["encoding"] = io.text_encoding(kwargs.get("encoding"))
with self.open('r', *args, **kwargs) as strm:
return strm.read()
@@ -2326,13 +2366,13 @@
return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/")
def _next(self, at):
- return Path(self.root, at)
+ return self.__class__(self.root, at)
def is_dir(self):
return not self.at or self.at.endswith("/")
def is_file(self):
- return not self.is_dir()
+ return self.exists() and not self.is_dir()
def exists(self):
return self.at in self.root._name_set()
@@ -2349,14 +2389,16 @@
def __repr__(self):
return self.__repr.format(self=self)
- def joinpath(self, add):
- next = posixpath.join(self.at, add)
+ def joinpath(self, *other):
+ next = posixpath.join(self.at, *other)
return self._next(self.root.resolve_dir(next))
__truediv__ = joinpath
@property
def parent(self):
+ if not self.at:
+ return self.filename.parent
parent_at = posixpath.dirname(self.at.rstrip('/'))
if parent_at:
parent_at += '/'
diff --git a/common/py3-stdlib/zipimport.py b/common/py3-stdlib/zipimport.py
index 5ef0a17..25eaee9 100644
--- a/common/py3-stdlib/zipimport.py
+++ b/common/py3-stdlib/zipimport.py
@@ -22,6 +22,7 @@
import marshal # for loads
import sys # for modules
import time # for mktime
+import _warnings # For warn()
__all__ = ['ZipImportError', 'zipimporter']
@@ -42,7 +43,7 @@
STRING_END_ARCHIVE = b'PK\x05\x06'
MAX_COMMENT_LEN = (1 << 16) - 1
-class zipimporter:
+class zipimporter(_bootstrap_external._LoaderBasics):
"""zipimporter(archivepath) -> zipimporter object
Create a new zipimporter instance. 'archivepath' must be a path to
@@ -115,7 +116,12 @@
full path name if it's possibly a portion of a namespace package,
or None otherwise. The optional 'path' argument is ignored -- it's
there for compatibility with the importer protocol.
+
+ Deprecated since Python 3.10. Use find_spec() instead.
"""
+ _warnings.warn("zipimporter.find_loader() is deprecated and slated for "
+ "removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
mi = _get_module_info(self, fullname)
if mi is not None:
# This is a module or package.
@@ -146,15 +152,46 @@
instance itself if the module was found, or None if it wasn't.
The optional 'path' argument is ignored -- it's there for compatibility
with the importer protocol.
+
+ Deprecated since Python 3.10. Use find_spec() instead.
"""
+ _warnings.warn("zipimporter.find_module() is deprecated and slated for "
+ "removal in Python 3.12; use find_spec() instead",
+ DeprecationWarning)
return self.find_loader(fullname, path)[0]
+ def find_spec(self, fullname, target=None):
+ """Create a ModuleSpec for the specified module.
+
+ Returns None if the module cannot be found.
+ """
+ module_info = _get_module_info(self, fullname)
+ if module_info is not None:
+ return _bootstrap.spec_from_loader(fullname, self, is_package=module_info)
+ else:
+ # Not a module or regular package. See if this is a directory, and
+ # therefore possibly a portion of a namespace package.
+
+ # We're only interested in the last path component of fullname
+ # earlier components are recorded in self.prefix.
+ modpath = _get_module_path(self, fullname)
+ if _is_dir(self, modpath):
+ # This is possibly a portion of a namespace
+ # package. Return the string representing its path,
+ # without a trailing separator.
+ path = f'{self.archive}{path_sep}{modpath}'
+ spec = _bootstrap.ModuleSpec(name=fullname, loader=None,
+ is_package=True)
+ spec.submodule_search_locations.append(path)
+ return spec
+ else:
+ return None
def get_code(self, fullname):
"""get_code(fullname) -> code object.
Return the code object for the specified module. Raise ZipImportError
- if the module couldn't be found.
+ if the module couldn't be imported.
"""
code, ispackage, modpath = _get_module_code(self, fullname)
return code
@@ -184,7 +221,8 @@
def get_filename(self, fullname):
"""get_filename(fullname) -> filename string.
- Return the filename for the specified module.
+ Return the filename for the specified module or raise ZipImportError
+ if it couldn't be imported.
"""
# Deciding the filename requires working out where the code
# would come from if the module was actually loaded
@@ -236,8 +274,13 @@
Load the module specified by 'fullname'. 'fullname' must be the
fully qualified (dotted) module name. It returns the imported
- module, or raises ZipImportError if it wasn't found.
+ module, or raises ZipImportError if it could not be imported.
+
+ Deprecated since Python 3.10. Use exec_module() instead.
"""
+ msg = ("zipimport.zipimporter.load_module() is deprecated and slated for "
+ "removal in Python 3.12; use exec_module() instead")
+ _warnings.warn(msg, DeprecationWarning)
code, ispackage, modpath = _get_module_code(self, fullname)
mod = sys.modules.get(fullname)
if mod is None or not isinstance(mod, _module_type):
@@ -280,11 +323,18 @@
return None
except ZipImportError:
return None
- if not _ZipImportResourceReader._registered:
- from importlib.abc import ResourceReader
- ResourceReader.register(_ZipImportResourceReader)
- _ZipImportResourceReader._registered = True
- return _ZipImportResourceReader(self, fullname)
+ from importlib.readers import ZipReader
+ return ZipReader(self, fullname)
+
+
+ def invalidate_caches(self):
+ """Reload the file data of the archive path."""
+ try:
+ self._files = _read_directory(self.archive)
+ _zip_directory_cache[self.archive] = self._files
+ except ZipImportError:
+ _zip_directory_cache.pop(self.archive, None)
+ self._files = {}
def __repr__(self):
@@ -580,20 +630,15 @@
# Given the contents of a .py[co] file, unmarshal the data
-# and return the code object. Return None if it the magic word doesn't
-# match, or if the recorded .py[co] metadata does not match the source,
-# (we do this instead of raising an exception as we fall back
-# to .py if available and we don't want to mask other errors).
+# and return the code object. Raises ImportError it the magic word doesn't
+# match, or if the recorded .py[co] metadata does not match the source.
def _unmarshal_code(self, pathname, fullpath, fullname, data):
exc_details = {
'name': fullname,
'path': fullpath,
}
- try:
- flags = _bootstrap_external._classify_pyc(data, fullname, exc_details)
- except ImportError:
- return None
+ flags = _bootstrap_external._classify_pyc(data, fullname, exc_details)
hash_based = flags & 0b1 != 0
if hash_based:
@@ -607,11 +652,8 @@
source_bytes,
)
- try:
- _bootstrap_external._validate_hash_pyc(
- data, source_hash, fullname, exc_details)
- except ImportError:
- return None
+ _bootstrap_external._validate_hash_pyc(
+ data, source_hash, fullname, exc_details)
else:
source_mtime, source_size = \
_get_mtime_and_size_of_source(self, fullpath)
@@ -697,6 +739,7 @@
# 'fullname'.
def _get_module_code(self, fullname):
path = _get_module_path(self, fullname)
+ import_error = None
for suffix, isbytecode, ispackage in _zip_searchorder:
fullpath = path + suffix
_bootstrap._verbose_message('trying {}{}{}', self.archive, path_sep, fullpath, verbosity=2)
@@ -707,8 +750,12 @@
else:
modpath = toc_entry[0]
data = _get_data(self.archive, toc_entry)
+ code = None
if isbytecode:
- code = _unmarshal_code(self, modpath, fullpath, fullname, data)
+ try:
+ code = _unmarshal_code(self, modpath, fullpath, fullname, data)
+ except ImportError as exc:
+ import_error = exc
else:
code = _compile_source(modpath, data)
if code is None:
@@ -718,75 +765,8 @@
modpath = toc_entry[0]
return code, ispackage, modpath
else:
- raise ZipImportError(f"can't find module {fullname!r}", name=fullname)
-
-
-class _ZipImportResourceReader:
- """Private class used to support ZipImport.get_resource_reader().
-
- This class is allowed to reference all the innards and private parts of
- the zipimporter.
- """
- _registered = False
-
- def __init__(self, zipimporter, fullname):
- self.zipimporter = zipimporter
- self.fullname = fullname
-
- def open_resource(self, resource):
- fullname_as_path = self.fullname.replace('.', '/')
- path = f'{fullname_as_path}/{resource}'
- from io import BytesIO
- try:
- return BytesIO(self.zipimporter.get_data(path))
- except OSError:
- raise FileNotFoundError(path)
-
- def resource_path(self, resource):
- # All resources are in the zip file, so there is no path to the file.
- # Raising FileNotFoundError tells the higher level API to extract the
- # binary data and create a temporary file.
- raise FileNotFoundError
-
- def is_resource(self, name):
- # Maybe we could do better, but if we can get the data, it's a
- # resource. Otherwise it isn't.
- fullname_as_path = self.fullname.replace('.', '/')
- path = f'{fullname_as_path}/{name}'
- try:
- self.zipimporter.get_data(path)
- except OSError:
- return False
- return True
-
- def contents(self):
- # This is a bit convoluted, because fullname will be a module path,
- # but _files is a list of file names relative to the top of the
- # archive's namespace. We want to compare file paths to find all the
- # names of things inside the module represented by fullname. So we
- # turn the module path of fullname into a file path relative to the
- # top of the archive, and then we iterate through _files looking for
- # names inside that "directory".
- from pathlib import Path
- fullname_path = Path(self.zipimporter.get_filename(self.fullname))
- relative_path = fullname_path.relative_to(self.zipimporter.archive)
- # Don't forget that fullname names a package, so its path will include
- # __init__.py, which we want to ignore.
- assert relative_path.name == '__init__.py'
- package_path = relative_path.parent
- subdirs_seen = set()
- for filename in self.zipimporter._files:
- try:
- relative = Path(filename).relative_to(package_path)
- except ValueError:
- continue
- # If the path of the file (which is relative to the top of the zip
- # namespace), relative to the package given when the resource
- # reader was created, has a parent, then it's a name in a
- # subdirectory and thus we skip it.
- parent_name = relative.parent.name
- if len(parent_name) == 0:
- yield relative.name
- elif parent_name not in subdirs_seen:
- subdirs_seen.add(parent_name)
- yield parent_name
+ if import_error:
+ msg = f"module load failed: {import_error}"
+ raise ZipImportError(msg, name=fullname) from import_error
+ else:
+ raise ZipImportError(f"can't find module {fullname!r}", name=fullname)
diff --git a/darwin-x86/bin/acp b/darwin-x86/bin/acp
index 1c69843..d891a8e 100755
--- a/darwin-x86/bin/acp
+++ b/darwin-x86/bin/acp
Binary files differ
diff --git a/darwin-x86/bin/aidl b/darwin-x86/bin/aidl
index 9efd4c3..4727524 100755
--- a/darwin-x86/bin/aidl
+++ b/darwin-x86/bin/aidl
Binary files differ
diff --git a/darwin-x86/bin/bison b/darwin-x86/bin/bison
index 013bbf1..acadff2 100755
--- a/darwin-x86/bin/bison
+++ b/darwin-x86/bin/bison
Binary files differ
diff --git a/darwin-x86/bin/bloaty b/darwin-x86/bin/bloaty
index 102e3b6..2bd6818 100755
--- a/darwin-x86/bin/bloaty
+++ b/darwin-x86/bin/bloaty
Binary files differ
diff --git a/darwin-x86/bin/bzip2 b/darwin-x86/bin/bzip2
index d540d2c..cc890b7 100755
--- a/darwin-x86/bin/bzip2
+++ b/darwin-x86/bin/bzip2
Binary files differ
diff --git a/darwin-x86/bin/ckati b/darwin-x86/bin/ckati
index 6d058ed..7fb74fa 100755
--- a/darwin-x86/bin/ckati
+++ b/darwin-x86/bin/ckati
Binary files differ
diff --git a/darwin-x86/bin/ckati_stamp_dump b/darwin-x86/bin/ckati_stamp_dump
index b198b36..f9ff480 100755
--- a/darwin-x86/bin/ckati_stamp_dump
+++ b/darwin-x86/bin/ckati_stamp_dump
Binary files differ
diff --git a/darwin-x86/bin/flex b/darwin-x86/bin/flex
index 1c33a73..a0f6931 100755
--- a/darwin-x86/bin/flex
+++ b/darwin-x86/bin/flex
Binary files differ
diff --git a/darwin-x86/bin/gavinhoward-bc b/darwin-x86/bin/gavinhoward-bc
index f3ea8f8..b503581 100755
--- a/darwin-x86/bin/gavinhoward-bc
+++ b/darwin-x86/bin/gavinhoward-bc
Binary files differ
diff --git a/darwin-x86/bin/go_extractor b/darwin-x86/bin/go_extractor
index 745862d..2fb241b 100755
--- a/darwin-x86/bin/go_extractor
+++ b/darwin-x86/bin/go_extractor
Binary files differ
diff --git a/darwin-x86/bin/hidl-gen b/darwin-x86/bin/hidl-gen
index 917aaf8..b4fdc44 100755
--- a/darwin-x86/bin/hidl-gen
+++ b/darwin-x86/bin/hidl-gen
Binary files differ
diff --git a/darwin-x86/bin/hidl-lint b/darwin-x86/bin/hidl-lint
index 2d60765..8452bf3 100755
--- a/darwin-x86/bin/hidl-lint
+++ b/darwin-x86/bin/hidl-lint
Binary files differ
diff --git a/darwin-x86/bin/m4 b/darwin-x86/bin/m4
index 7f05ebd..c812415 100755
--- a/darwin-x86/bin/m4
+++ b/darwin-x86/bin/m4
Binary files differ
diff --git a/darwin-x86/bin/make b/darwin-x86/bin/make
index d8c2a3c..ca86fee 100755
--- a/darwin-x86/bin/make
+++ b/darwin-x86/bin/make
Binary files differ
diff --git a/darwin-x86/bin/ninja b/darwin-x86/bin/ninja
index 1cbe036..f002507 100755
--- a/darwin-x86/bin/ninja
+++ b/darwin-x86/bin/ninja
Binary files differ
diff --git a/darwin-x86/bin/one-true-awk b/darwin-x86/bin/one-true-awk
index b9d1990..2f2def1 100755
--- a/darwin-x86/bin/one-true-awk
+++ b/darwin-x86/bin/one-true-awk
Binary files differ
diff --git a/darwin-x86/bin/openssl b/darwin-x86/bin/openssl
index c6c6bdb..e68304a 100755
--- a/darwin-x86/bin/openssl
+++ b/darwin-x86/bin/openssl
Binary files differ
diff --git a/darwin-x86/bin/py2-cmd b/darwin-x86/bin/py2-cmd
index 72d6b67..ec3092f 100755
--- a/darwin-x86/bin/py2-cmd
+++ b/darwin-x86/bin/py2-cmd
Binary files differ
diff --git a/darwin-x86/bin/py3-cmd b/darwin-x86/bin/py3-cmd
index 9d333db..3272ffd 100755
--- a/darwin-x86/bin/py3-cmd
+++ b/darwin-x86/bin/py3-cmd
Binary files differ
diff --git a/darwin-x86/bin/py3-launcher-autorun64 b/darwin-x86/bin/py3-launcher-autorun64
index d714e56..71af69b 100755
--- a/darwin-x86/bin/py3-launcher-autorun64
+++ b/darwin-x86/bin/py3-launcher-autorun64
Binary files differ
diff --git a/darwin-x86/bin/py3-launcher64 b/darwin-x86/bin/py3-launcher64
index 8b43883..1379592 100755
--- a/darwin-x86/bin/py3-launcher64
+++ b/darwin-x86/bin/py3-launcher64
Binary files differ
diff --git a/darwin-x86/bin/runextractor b/darwin-x86/bin/runextractor
index 1f742d1..8936d99 100755
--- a/darwin-x86/bin/runextractor
+++ b/darwin-x86/bin/runextractor
Binary files differ
diff --git a/darwin-x86/bin/toybox b/darwin-x86/bin/toybox
index c00509e..5b2eb3c 100755
--- a/darwin-x86/bin/toybox
+++ b/darwin-x86/bin/toybox
Binary files differ
diff --git a/darwin-x86/bin/xz b/darwin-x86/bin/xz
index 114f888..e5f4df6 100755
--- a/darwin-x86/bin/xz
+++ b/darwin-x86/bin/xz
Binary files differ
diff --git a/darwin-x86/bin/zipalign b/darwin-x86/bin/zipalign
index a817928..7cde13a 100755
--- a/darwin-x86/bin/zipalign
+++ b/darwin-x86/bin/zipalign
Binary files differ
diff --git a/darwin-x86/bin/ziptime b/darwin-x86/bin/ziptime
index 915a659..fb62c95 100755
--- a/darwin-x86/bin/ziptime
+++ b/darwin-x86/bin/ziptime
Binary files differ
diff --git a/darwin-x86/bin/ziptool b/darwin-x86/bin/ziptool
index 3c71260..21903d7 100755
--- a/darwin-x86/bin/ziptool
+++ b/darwin-x86/bin/ziptool
Binary files differ
diff --git a/darwin-x86/lib64/libbase.dylib b/darwin-x86/lib64/libbase.dylib
index 4b405c7..472034f 100755
--- a/darwin-x86/lib64/libbase.dylib
+++ b/darwin-x86/lib64/libbase.dylib
Binary files differ
diff --git a/darwin-x86/lib64/libc++.dylib b/darwin-x86/lib64/libc++.dylib
index 9121ca8..345bdd1 100755
--- a/darwin-x86/lib64/libc++.dylib
+++ b/darwin-x86/lib64/libc++.dylib
Binary files differ
diff --git a/darwin-x86/lib64/libcrypto-host.dylib b/darwin-x86/lib64/libcrypto-host.dylib
index 4d04110..6545040 100755
--- a/darwin-x86/lib64/libcrypto-host.dylib
+++ b/darwin-x86/lib64/libcrypto-host.dylib
Binary files differ
diff --git a/darwin-x86/lib64/libicui18n-host.dylib b/darwin-x86/lib64/libicui18n-host.dylib
index 2344f7b..de58921 100755
--- a/darwin-x86/lib64/libicui18n-host.dylib
+++ b/darwin-x86/lib64/libicui18n-host.dylib
Binary files differ
diff --git a/darwin-x86/lib64/libicuuc-host.dylib b/darwin-x86/lib64/libicuuc-host.dylib
index d63cdc0..3839a74 100755
--- a/darwin-x86/lib64/libicuuc-host.dylib
+++ b/darwin-x86/lib64/libicuuc-host.dylib
Binary files differ
diff --git a/darwin-x86/lib64/liblog.dylib b/darwin-x86/lib64/liblog.dylib
index 9f0f1b1..520818c 100755
--- a/darwin-x86/lib64/liblog.dylib
+++ b/darwin-x86/lib64/liblog.dylib
Binary files differ
diff --git a/darwin-x86/lib64/libsqlite.dylib b/darwin-x86/lib64/libsqlite.dylib
index f52abce..3560cc6 100755
--- a/darwin-x86/lib64/libsqlite.dylib
+++ b/darwin-x86/lib64/libsqlite.dylib
Binary files differ
diff --git a/darwin-x86/lib64/libz-host.dylib b/darwin-x86/lib64/libz-host.dylib
index ee1b38f..d0448ad 100755
--- a/darwin-x86/lib64/libz-host.dylib
+++ b/darwin-x86/lib64/libz-host.dylib
Binary files differ
diff --git a/darwin-x86/lib64/libziparchive.dylib b/darwin-x86/lib64/libziparchive.dylib
index 41d1c41..ab2bbd6 100755
--- a/darwin-x86/lib64/libziparchive.dylib
+++ b/darwin-x86/lib64/libziparchive.dylib
Binary files differ
diff --git a/linux-x86/asan/bin/acp b/linux-x86/asan/bin/acp
index 3366d1f..69e0ce2 100755
--- a/linux-x86/asan/bin/acp
+++ b/linux-x86/asan/bin/acp
Binary files differ
diff --git a/linux-x86/asan/bin/aidl b/linux-x86/asan/bin/aidl
index a583a03..7f6554e 100755
--- a/linux-x86/asan/bin/aidl
+++ b/linux-x86/asan/bin/aidl
Binary files differ
diff --git a/linux-x86/asan/bin/ckati b/linux-x86/asan/bin/ckati
index 03dc93b..7fd3a33 100755
--- a/linux-x86/asan/bin/ckati
+++ b/linux-x86/asan/bin/ckati
Binary files differ
diff --git a/linux-x86/asan/bin/gavinhoward-bc b/linux-x86/asan/bin/gavinhoward-bc
index 79a20eb..aab6c96 100755
--- a/linux-x86/asan/bin/gavinhoward-bc
+++ b/linux-x86/asan/bin/gavinhoward-bc
Binary files differ
diff --git a/linux-x86/asan/bin/ninja b/linux-x86/asan/bin/ninja
index fd27185..f9833db 100755
--- a/linux-x86/asan/bin/ninja
+++ b/linux-x86/asan/bin/ninja
Binary files differ
diff --git a/linux-x86/asan/bin/toybox b/linux-x86/asan/bin/toybox
index 68297a4..5522cda 100755
--- a/linux-x86/asan/bin/toybox
+++ b/linux-x86/asan/bin/toybox
Binary files differ
diff --git a/linux-x86/asan/bin/zipalign b/linux-x86/asan/bin/zipalign
index acf3d57..baef64e 100755
--- a/linux-x86/asan/bin/zipalign
+++ b/linux-x86/asan/bin/zipalign
Binary files differ
diff --git a/linux-x86/asan/bin/ziptime b/linux-x86/asan/bin/ziptime
index 02cdffa..b2094bf 100755
--- a/linux-x86/asan/bin/ziptime
+++ b/linux-x86/asan/bin/ziptime
Binary files differ
diff --git a/linux-x86/asan/bin/ziptool b/linux-x86/asan/bin/ziptool
index d8be5b0..ec9e6ee 100755
--- a/linux-x86/asan/bin/ziptool
+++ b/linux-x86/asan/bin/ziptool
Binary files differ
diff --git a/linux-x86/asan/lib64/libbase.so b/linux-x86/asan/lib64/libbase.so
index da4db9d..49387b2 100755
--- a/linux-x86/asan/lib64/libbase.so
+++ b/linux-x86/asan/lib64/libbase.so
Binary files differ
diff --git a/linux-x86/asan/lib64/libc++.so b/linux-x86/asan/lib64/libc++.so
index 22e34db..cd48320 100755
--- a/linux-x86/asan/lib64/libc++.so
+++ b/linux-x86/asan/lib64/libc++.so
Binary files differ
diff --git a/linux-x86/asan/lib64/libcrypto-host.so b/linux-x86/asan/lib64/libcrypto-host.so
index f8c4e41..f02c02b 100755
--- a/linux-x86/asan/lib64/libcrypto-host.so
+++ b/linux-x86/asan/lib64/libcrypto-host.so
Binary files differ
diff --git a/linux-x86/asan/lib64/libjemalloc5.so b/linux-x86/asan/lib64/libjemalloc5.so
index 2dd72d8..19e7f04 100755
--- a/linux-x86/asan/lib64/libjemalloc5.so
+++ b/linux-x86/asan/lib64/libjemalloc5.so
Binary files differ
diff --git a/linux-x86/asan/lib64/liblog.so b/linux-x86/asan/lib64/liblog.so
index ff93721..86bd5e4 100755
--- a/linux-x86/asan/lib64/liblog.so
+++ b/linux-x86/asan/lib64/liblog.so
Binary files differ
diff --git a/linux-x86/asan/lib64/libz-host.so b/linux-x86/asan/lib64/libz-host.so
index 3881406..a20bff9 100755
--- a/linux-x86/asan/lib64/libz-host.so
+++ b/linux-x86/asan/lib64/libz-host.so
Binary files differ
diff --git a/linux-x86/asan/lib64/libziparchive.so b/linux-x86/asan/lib64/libziparchive.so
index 0ebb85c..74cc38a 100755
--- a/linux-x86/asan/lib64/libziparchive.so
+++ b/linux-x86/asan/lib64/libziparchive.so
Binary files differ
diff --git a/linux-x86/bin/acp b/linux-x86/bin/acp
index 0051c80..be8b230 100755
--- a/linux-x86/bin/acp
+++ b/linux-x86/bin/acp
Binary files differ
diff --git a/linux-x86/bin/aidl b/linux-x86/bin/aidl
index 6d31a2d..b83b03b 100755
--- a/linux-x86/bin/aidl
+++ b/linux-x86/bin/aidl
Binary files differ
diff --git a/linux-x86/bin/bison b/linux-x86/bin/bison
index dff701c..a6242cb 100755
--- a/linux-x86/bin/bison
+++ b/linux-x86/bin/bison
Binary files differ
diff --git a/linux-x86/bin/bloaty b/linux-x86/bin/bloaty
index 1c2cb05..10de301 100755
--- a/linux-x86/bin/bloaty
+++ b/linux-x86/bin/bloaty
Binary files differ
diff --git a/linux-x86/bin/bzip2 b/linux-x86/bin/bzip2
index 0debfe5..b0f216e 100755
--- a/linux-x86/bin/bzip2
+++ b/linux-x86/bin/bzip2
Binary files differ
diff --git a/linux-x86/bin/ckati b/linux-x86/bin/ckati
index f404f55..48c08d2 100755
--- a/linux-x86/bin/ckati
+++ b/linux-x86/bin/ckati
Binary files differ
diff --git a/linux-x86/bin/ckati_stamp_dump b/linux-x86/bin/ckati_stamp_dump
index 76457fc..0d97177 100755
--- a/linux-x86/bin/ckati_stamp_dump
+++ b/linux-x86/bin/ckati_stamp_dump
Binary files differ
diff --git a/linux-x86/bin/create_minidebuginfo b/linux-x86/bin/create_minidebuginfo
index f9a354e..99dc10c 100755
--- a/linux-x86/bin/create_minidebuginfo
+++ b/linux-x86/bin/create_minidebuginfo
Binary files differ
diff --git a/linux-x86/bin/flex b/linux-x86/bin/flex
index 5caef46..0fafe34 100755
--- a/linux-x86/bin/flex
+++ b/linux-x86/bin/flex
Binary files differ
diff --git a/linux-x86/bin/gavinhoward-bc b/linux-x86/bin/gavinhoward-bc
index e5cdc51..657fbf8 100755
--- a/linux-x86/bin/gavinhoward-bc
+++ b/linux-x86/bin/gavinhoward-bc
Binary files differ
diff --git a/linux-x86/bin/go_extractor b/linux-x86/bin/go_extractor
index c9ae12e..c8d1902 100755
--- a/linux-x86/bin/go_extractor
+++ b/linux-x86/bin/go_extractor
Binary files differ
diff --git a/linux-x86/bin/hidl-gen b/linux-x86/bin/hidl-gen
index 24914e0..ae6816c 100755
--- a/linux-x86/bin/hidl-gen
+++ b/linux-x86/bin/hidl-gen
Binary files differ
diff --git a/linux-x86/bin/hidl-lint b/linux-x86/bin/hidl-lint
index 81bf260..b2044f8 100755
--- a/linux-x86/bin/hidl-lint
+++ b/linux-x86/bin/hidl-lint
Binary files differ
diff --git a/linux-x86/bin/m4 b/linux-x86/bin/m4
index e97ae15..60b5f3c 100755
--- a/linux-x86/bin/m4
+++ b/linux-x86/bin/m4
Binary files differ
diff --git a/linux-x86/bin/make b/linux-x86/bin/make
index 6a17969..d7a0d6a 100755
--- a/linux-x86/bin/make
+++ b/linux-x86/bin/make
Binary files differ
diff --git a/linux-x86/bin/ninja b/linux-x86/bin/ninja
index e1461bf..f83ed92 100755
--- a/linux-x86/bin/ninja
+++ b/linux-x86/bin/ninja
Binary files differ
diff --git a/linux-x86/bin/nsjail b/linux-x86/bin/nsjail
index 4773679..990593b 100755
--- a/linux-x86/bin/nsjail
+++ b/linux-x86/bin/nsjail
Binary files differ
diff --git a/linux-x86/bin/one-true-awk b/linux-x86/bin/one-true-awk
index c8c546b..8c52d40 100755
--- a/linux-x86/bin/one-true-awk
+++ b/linux-x86/bin/one-true-awk
Binary files differ
diff --git a/linux-x86/bin/openssl b/linux-x86/bin/openssl
index 8ff7907..6f5b5b2 100755
--- a/linux-x86/bin/openssl
+++ b/linux-x86/bin/openssl
Binary files differ
diff --git a/linux-x86/bin/py2-cmd b/linux-x86/bin/py2-cmd
index c625eb5..33d09db 100755
--- a/linux-x86/bin/py2-cmd
+++ b/linux-x86/bin/py2-cmd
Binary files differ
diff --git a/linux-x86/bin/py3-cmd b/linux-x86/bin/py3-cmd
index 0904311..53e21c4 100755
--- a/linux-x86/bin/py3-cmd
+++ b/linux-x86/bin/py3-cmd
Binary files differ
diff --git a/linux-x86/bin/py3-launcher-autorun64 b/linux-x86/bin/py3-launcher-autorun64
index af3df5e..1a324f2 100755
--- a/linux-x86/bin/py3-launcher-autorun64
+++ b/linux-x86/bin/py3-launcher-autorun64
Binary files differ
diff --git a/linux-x86/bin/py3-launcher64 b/linux-x86/bin/py3-launcher64
index 1705135..cb62b83 100755
--- a/linux-x86/bin/py3-launcher64
+++ b/linux-x86/bin/py3-launcher64
Binary files differ
diff --git a/linux-x86/bin/runextractor b/linux-x86/bin/runextractor
index 49ff947..5850ada 100755
--- a/linux-x86/bin/runextractor
+++ b/linux-x86/bin/runextractor
Binary files differ
diff --git a/linux-x86/bin/toybox b/linux-x86/bin/toybox
index 59e9b56..1c34b73 100755
--- a/linux-x86/bin/toybox
+++ b/linux-x86/bin/toybox
Binary files differ
diff --git a/linux-x86/bin/xz b/linux-x86/bin/xz
index 6f5461b..d71e3c5 100755
--- a/linux-x86/bin/xz
+++ b/linux-x86/bin/xz
Binary files differ
diff --git a/linux-x86/bin/zipalign b/linux-x86/bin/zipalign
index e83aeb0..7deaa45 100755
--- a/linux-x86/bin/zipalign
+++ b/linux-x86/bin/zipalign
Binary files differ
diff --git a/linux-x86/bin/ziptime b/linux-x86/bin/ziptime
index e0a198a..08a3b23 100755
--- a/linux-x86/bin/ziptime
+++ b/linux-x86/bin/ziptime
Binary files differ
diff --git a/linux-x86/bin/ziptool b/linux-x86/bin/ziptool
index 169b8bf..e9b2e8a 100755
--- a/linux-x86/bin/ziptool
+++ b/linux-x86/bin/ziptool
Binary files differ
diff --git a/linux-x86/lib64/libbase.so b/linux-x86/lib64/libbase.so
index d4529e2..6b7af65 100755
--- a/linux-x86/lib64/libbase.so
+++ b/linux-x86/lib64/libbase.so
Binary files differ
diff --git a/linux-x86/lib64/libc++.so b/linux-x86/lib64/libc++.so
index 17b14c9..e22c3b1 100755
--- a/linux-x86/lib64/libc++.so
+++ b/linux-x86/lib64/libc++.so
Binary files differ
diff --git a/linux-x86/lib64/libcrypto-host.so b/linux-x86/lib64/libcrypto-host.so
index cc60f1a..68db0b3 100755
--- a/linux-x86/lib64/libcrypto-host.so
+++ b/linux-x86/lib64/libcrypto-host.so
Binary files differ
diff --git a/linux-x86/lib64/libicui18n-host.so b/linux-x86/lib64/libicui18n-host.so
index 3f1fb13..cc89f7a 100755
--- a/linux-x86/lib64/libicui18n-host.so
+++ b/linux-x86/lib64/libicui18n-host.so
Binary files differ
diff --git a/linux-x86/lib64/libicuuc-host.so b/linux-x86/lib64/libicuuc-host.so
index d0e83fd..278cef3 100755
--- a/linux-x86/lib64/libicuuc-host.so
+++ b/linux-x86/lib64/libicuuc-host.so
Binary files differ
diff --git a/linux-x86/lib64/libjemalloc5.so b/linux-x86/lib64/libjemalloc5.so
index c85b8a4..3697338 100755
--- a/linux-x86/lib64/libjemalloc5.so
+++ b/linux-x86/lib64/libjemalloc5.so
Binary files differ
diff --git a/linux-x86/lib64/liblog.so b/linux-x86/lib64/liblog.so
index eaf85b2..fd8498a 100755
--- a/linux-x86/lib64/liblog.so
+++ b/linux-x86/lib64/liblog.so
Binary files differ
diff --git a/linux-x86/lib64/libnl.so b/linux-x86/lib64/libnl.so
index 4df2a47..7c73cc6 100755
--- a/linux-x86/lib64/libnl.so
+++ b/linux-x86/lib64/libnl.so
Binary files differ
diff --git a/linux-x86/lib64/libprotobuf-cpp-full.so b/linux-x86/lib64/libprotobuf-cpp-full.so
index 3b5b9ba..b693784 100755
--- a/linux-x86/lib64/libprotobuf-cpp-full.so
+++ b/linux-x86/lib64/libprotobuf-cpp-full.so
Binary files differ
diff --git a/linux-x86/lib64/libsqlite.so b/linux-x86/lib64/libsqlite.so
index 7db8697..3fae08e 100755
--- a/linux-x86/lib64/libsqlite.so
+++ b/linux-x86/lib64/libsqlite.so
Binary files differ
diff --git a/linux-x86/lib64/libz-host.so b/linux-x86/lib64/libz-host.so
index ea62227..e29e1ec 100755
--- a/linux-x86/lib64/libz-host.so
+++ b/linux-x86/lib64/libz-host.so
Binary files differ
diff --git a/linux-x86/lib64/libziparchive.so b/linux-x86/lib64/libziparchive.so
index aca2b7b..bd1ffeb 100755
--- a/linux-x86/lib64/libziparchive.so
+++ b/linux-x86/lib64/libziparchive.so
Binary files differ
diff --git a/linux_musl-x86/asan/bin/acp b/linux_musl-x86/asan/bin/acp
index 3366d1f..69e0ce2 100755
--- a/linux_musl-x86/asan/bin/acp
+++ b/linux_musl-x86/asan/bin/acp
Binary files differ
diff --git a/linux_musl-x86/asan/bin/aidl b/linux_musl-x86/asan/bin/aidl
index a583a03..7f6554e 100755
--- a/linux_musl-x86/asan/bin/aidl
+++ b/linux_musl-x86/asan/bin/aidl
Binary files differ
diff --git a/linux_musl-x86/asan/bin/ckati b/linux_musl-x86/asan/bin/ckati
index 03dc93b..7fd3a33 100755
--- a/linux_musl-x86/asan/bin/ckati
+++ b/linux_musl-x86/asan/bin/ckati
Binary files differ
diff --git a/linux_musl-x86/asan/bin/gavinhoward-bc b/linux_musl-x86/asan/bin/gavinhoward-bc
index 79a20eb..aab6c96 100755
--- a/linux_musl-x86/asan/bin/gavinhoward-bc
+++ b/linux_musl-x86/asan/bin/gavinhoward-bc
Binary files differ
diff --git a/linux_musl-x86/asan/bin/ninja b/linux_musl-x86/asan/bin/ninja
index fd27185..f9833db 100755
--- a/linux_musl-x86/asan/bin/ninja
+++ b/linux_musl-x86/asan/bin/ninja
Binary files differ
diff --git a/linux_musl-x86/asan/bin/toybox b/linux_musl-x86/asan/bin/toybox
index 68297a4..5522cda 100755
--- a/linux_musl-x86/asan/bin/toybox
+++ b/linux_musl-x86/asan/bin/toybox
Binary files differ
diff --git a/linux_musl-x86/asan/bin/zipalign b/linux_musl-x86/asan/bin/zipalign
index acf3d57..baef64e 100755
--- a/linux_musl-x86/asan/bin/zipalign
+++ b/linux_musl-x86/asan/bin/zipalign
Binary files differ
diff --git a/linux_musl-x86/asan/bin/ziptime b/linux_musl-x86/asan/bin/ziptime
index 02cdffa..b2094bf 100755
--- a/linux_musl-x86/asan/bin/ziptime
+++ b/linux_musl-x86/asan/bin/ziptime
Binary files differ
diff --git a/linux_musl-x86/asan/bin/ziptool b/linux_musl-x86/asan/bin/ziptool
index d8be5b0..ec9e6ee 100755
--- a/linux_musl-x86/asan/bin/ziptool
+++ b/linux_musl-x86/asan/bin/ziptool
Binary files differ
diff --git a/linux_musl-x86/asan/lib64/libbase.so b/linux_musl-x86/asan/lib64/libbase.so
index da4db9d..49387b2 100755
--- a/linux_musl-x86/asan/lib64/libbase.so
+++ b/linux_musl-x86/asan/lib64/libbase.so
Binary files differ
diff --git a/linux_musl-x86/asan/lib64/libc++.so b/linux_musl-x86/asan/lib64/libc++.so
index 22e34db..cd48320 100755
--- a/linux_musl-x86/asan/lib64/libc++.so
+++ b/linux_musl-x86/asan/lib64/libc++.so
Binary files differ
diff --git a/linux_musl-x86/asan/lib64/libcrypto-host.so b/linux_musl-x86/asan/lib64/libcrypto-host.so
index f8c4e41..f02c02b 100755
--- a/linux_musl-x86/asan/lib64/libcrypto-host.so
+++ b/linux_musl-x86/asan/lib64/libcrypto-host.so
Binary files differ
diff --git a/linux_musl-x86/asan/lib64/libjemalloc5.so b/linux_musl-x86/asan/lib64/libjemalloc5.so
index 2dd72d8..19e7f04 100755
--- a/linux_musl-x86/asan/lib64/libjemalloc5.so
+++ b/linux_musl-x86/asan/lib64/libjemalloc5.so
Binary files differ
diff --git a/linux_musl-x86/asan/lib64/liblog.so b/linux_musl-x86/asan/lib64/liblog.so
index ff93721..86bd5e4 100755
--- a/linux_musl-x86/asan/lib64/liblog.so
+++ b/linux_musl-x86/asan/lib64/liblog.so
Binary files differ
diff --git a/linux_musl-x86/asan/lib64/libz-host.so b/linux_musl-x86/asan/lib64/libz-host.so
index 3881406..a20bff9 100755
--- a/linux_musl-x86/asan/lib64/libz-host.so
+++ b/linux_musl-x86/asan/lib64/libz-host.so
Binary files differ
diff --git a/linux_musl-x86/asan/lib64/libziparchive.so b/linux_musl-x86/asan/lib64/libziparchive.so
index 0ebb85c..74cc38a 100755
--- a/linux_musl-x86/asan/lib64/libziparchive.so
+++ b/linux_musl-x86/asan/lib64/libziparchive.so
Binary files differ
diff --git a/linux_musl-x86/bin/acp b/linux_musl-x86/bin/acp
index 59ea459..7d32a42 100755
--- a/linux_musl-x86/bin/acp
+++ b/linux_musl-x86/bin/acp
Binary files differ
diff --git a/linux_musl-x86/bin/aidl b/linux_musl-x86/bin/aidl
index 37da92d..824b78c 100755
--- a/linux_musl-x86/bin/aidl
+++ b/linux_musl-x86/bin/aidl
Binary files differ
diff --git a/linux_musl-x86/bin/bison b/linux_musl-x86/bin/bison
index a86b84c..314016a 100755
--- a/linux_musl-x86/bin/bison
+++ b/linux_musl-x86/bin/bison
Binary files differ
diff --git a/linux_musl-x86/bin/bloaty b/linux_musl-x86/bin/bloaty
index cbcc70d..4494f94 100755
--- a/linux_musl-x86/bin/bloaty
+++ b/linux_musl-x86/bin/bloaty
Binary files differ
diff --git a/linux_musl-x86/bin/bzip2 b/linux_musl-x86/bin/bzip2
index 6f24cbd..6d3480b 100755
--- a/linux_musl-x86/bin/bzip2
+++ b/linux_musl-x86/bin/bzip2
Binary files differ
diff --git a/linux_musl-x86/bin/ckati b/linux_musl-x86/bin/ckati
index e09bf25..58726da 100755
--- a/linux_musl-x86/bin/ckati
+++ b/linux_musl-x86/bin/ckati
Binary files differ
diff --git a/linux_musl-x86/bin/ckati_stamp_dump b/linux_musl-x86/bin/ckati_stamp_dump
index f59bb1c..e9b4621 100755
--- a/linux_musl-x86/bin/ckati_stamp_dump
+++ b/linux_musl-x86/bin/ckati_stamp_dump
Binary files differ
diff --git a/linux_musl-x86/bin/create_minidebuginfo b/linux_musl-x86/bin/create_minidebuginfo
index 2687185..a79bc8c 100755
--- a/linux_musl-x86/bin/create_minidebuginfo
+++ b/linux_musl-x86/bin/create_minidebuginfo
Binary files differ
diff --git a/linux_musl-x86/bin/flex b/linux_musl-x86/bin/flex
index 9effa62..f17c128 100755
--- a/linux_musl-x86/bin/flex
+++ b/linux_musl-x86/bin/flex
Binary files differ
diff --git a/linux_musl-x86/bin/gavinhoward-bc b/linux_musl-x86/bin/gavinhoward-bc
index aaf8803..811f5fd 100755
--- a/linux_musl-x86/bin/gavinhoward-bc
+++ b/linux_musl-x86/bin/gavinhoward-bc
Binary files differ
diff --git a/linux_musl-x86/bin/go_extractor b/linux_musl-x86/bin/go_extractor
index c9ae12e..c8d1902 100755
--- a/linux_musl-x86/bin/go_extractor
+++ b/linux_musl-x86/bin/go_extractor
Binary files differ
diff --git a/linux_musl-x86/bin/hidl-gen b/linux_musl-x86/bin/hidl-gen
index 3a8fd31..b79c462 100755
--- a/linux_musl-x86/bin/hidl-gen
+++ b/linux_musl-x86/bin/hidl-gen
Binary files differ
diff --git a/linux_musl-x86/bin/hidl-lint b/linux_musl-x86/bin/hidl-lint
index bbba576..13d81d8 100755
--- a/linux_musl-x86/bin/hidl-lint
+++ b/linux_musl-x86/bin/hidl-lint
Binary files differ
diff --git a/linux_musl-x86/bin/m4 b/linux_musl-x86/bin/m4
index 7cad0eb..30a0921 100755
--- a/linux_musl-x86/bin/m4
+++ b/linux_musl-x86/bin/m4
Binary files differ
diff --git a/linux_musl-x86/bin/make b/linux_musl-x86/bin/make
index d6e60d6..74f9d26 100755
--- a/linux_musl-x86/bin/make
+++ b/linux_musl-x86/bin/make
Binary files differ
diff --git a/linux_musl-x86/bin/ninja b/linux_musl-x86/bin/ninja
index 59d2ccf..cad8c09 100755
--- a/linux_musl-x86/bin/ninja
+++ b/linux_musl-x86/bin/ninja
Binary files differ
diff --git a/linux_musl-x86/bin/nsjail b/linux_musl-x86/bin/nsjail
index 2fceca2..1c0242a 100755
--- a/linux_musl-x86/bin/nsjail
+++ b/linux_musl-x86/bin/nsjail
Binary files differ
diff --git a/linux_musl-x86/bin/one-true-awk b/linux_musl-x86/bin/one-true-awk
index 90a39d8..89b0405 100755
--- a/linux_musl-x86/bin/one-true-awk
+++ b/linux_musl-x86/bin/one-true-awk
Binary files differ
diff --git a/linux_musl-x86/bin/openssl b/linux_musl-x86/bin/openssl
index efd8157..84e59d9 100755
--- a/linux_musl-x86/bin/openssl
+++ b/linux_musl-x86/bin/openssl
Binary files differ
diff --git a/linux_musl-x86/bin/py2-cmd b/linux_musl-x86/bin/py2-cmd
index 262f8a4..fbb02ea 100755
--- a/linux_musl-x86/bin/py2-cmd
+++ b/linux_musl-x86/bin/py2-cmd
Binary files differ
diff --git a/linux_musl-x86/bin/py3-cmd b/linux_musl-x86/bin/py3-cmd
index 982c41e..8ed8473 100755
--- a/linux_musl-x86/bin/py3-cmd
+++ b/linux_musl-x86/bin/py3-cmd
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher-autorun-static64 b/linux_musl-x86/bin/py3-launcher-autorun-static64
index b744124..5afaa77 100755
--- a/linux_musl-x86/bin/py3-launcher-autorun-static64
+++ b/linux_musl-x86/bin/py3-launcher-autorun-static64
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher-autorun64 b/linux_musl-x86/bin/py3-launcher-autorun64
index 74635d9..4b465c5 100755
--- a/linux_musl-x86/bin/py3-launcher-autorun64
+++ b/linux_musl-x86/bin/py3-launcher-autorun64
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher-static64 b/linux_musl-x86/bin/py3-launcher-static64
index 94fc036..6c443e6 100755
--- a/linux_musl-x86/bin/py3-launcher-static64
+++ b/linux_musl-x86/bin/py3-launcher-static64
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher64 b/linux_musl-x86/bin/py3-launcher64
index 8841d22..351fc1b 100755
--- a/linux_musl-x86/bin/py3-launcher64
+++ b/linux_musl-x86/bin/py3-launcher64
Binary files differ
diff --git a/linux_musl-x86/bin/runextractor b/linux_musl-x86/bin/runextractor
index 49ff947..5850ada 100755
--- a/linux_musl-x86/bin/runextractor
+++ b/linux_musl-x86/bin/runextractor
Binary files differ
diff --git a/linux_musl-x86/bin/toybox b/linux_musl-x86/bin/toybox
index edf6da2..fc56e15 100755
--- a/linux_musl-x86/bin/toybox
+++ b/linux_musl-x86/bin/toybox
Binary files differ
diff --git a/linux_musl-x86/bin/xz b/linux_musl-x86/bin/xz
index d4be008..0cebd19 100755
--- a/linux_musl-x86/bin/xz
+++ b/linux_musl-x86/bin/xz
Binary files differ
diff --git a/linux_musl-x86/bin/zipalign b/linux_musl-x86/bin/zipalign
index acd7147..499011c 100755
--- a/linux_musl-x86/bin/zipalign
+++ b/linux_musl-x86/bin/zipalign
Binary files differ
diff --git a/linux_musl-x86/bin/ziptime b/linux_musl-x86/bin/ziptime
index b8238c4..2256adb 100755
--- a/linux_musl-x86/bin/ziptime
+++ b/linux_musl-x86/bin/ziptime
Binary files differ
diff --git a/linux_musl-x86/bin/ziptool b/linux_musl-x86/bin/ziptool
index 247e902..ef9d14a 100755
--- a/linux_musl-x86/bin/ziptool
+++ b/linux_musl-x86/bin/ziptool
Binary files differ
diff --git a/linux_musl-x86/lib64/libbase.so b/linux_musl-x86/lib64/libbase.so
index 0d700b7..c1acbe1 100755
--- a/linux_musl-x86/lib64/libbase.so
+++ b/linux_musl-x86/lib64/libbase.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libc++.so b/linux_musl-x86/lib64/libc++.so
index 212292b..b5eac52 100755
--- a/linux_musl-x86/lib64/libc++.so
+++ b/linux_musl-x86/lib64/libc++.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libc_musl.so b/linux_musl-x86/lib64/libc_musl.so
index fcf6794..113c4ad 100755
--- a/linux_musl-x86/lib64/libc_musl.so
+++ b/linux_musl-x86/lib64/libc_musl.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libclang_rt.ubsan_standalone-x86_64.so b/linux_musl-x86/lib64/libclang_rt.ubsan_standalone-x86_64.so
new file mode 100644
index 0000000..f509763
--- /dev/null
+++ b/linux_musl-x86/lib64/libclang_rt.ubsan_standalone-x86_64.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libcrypto-host.so b/linux_musl-x86/lib64/libcrypto-host.so
index 25c63f9..adba547 100755
--- a/linux_musl-x86/lib64/libcrypto-host.so
+++ b/linux_musl-x86/lib64/libcrypto-host.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libicui18n-host.so b/linux_musl-x86/lib64/libicui18n-host.so
index e5938c4..d090557 100755
--- a/linux_musl-x86/lib64/libicui18n-host.so
+++ b/linux_musl-x86/lib64/libicui18n-host.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libicuuc-host.so b/linux_musl-x86/lib64/libicuuc-host.so
index d954908..8a00535 100755
--- a/linux_musl-x86/lib64/libicuuc-host.so
+++ b/linux_musl-x86/lib64/libicuuc-host.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libjemalloc5.so b/linux_musl-x86/lib64/libjemalloc5.so
index 806168d..280ae8e 100755
--- a/linux_musl-x86/lib64/libjemalloc5.so
+++ b/linux_musl-x86/lib64/libjemalloc5.so
Binary files differ
diff --git a/linux_musl-x86/lib64/liblog.so b/linux_musl-x86/lib64/liblog.so
index 34df19d..f233d02 100755
--- a/linux_musl-x86/lib64/liblog.so
+++ b/linux_musl-x86/lib64/liblog.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libnl.so b/linux_musl-x86/lib64/libnl.so
index 648772a..4ca0795 100755
--- a/linux_musl-x86/lib64/libnl.so
+++ b/linux_musl-x86/lib64/libnl.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libprotobuf-cpp-full.so b/linux_musl-x86/lib64/libprotobuf-cpp-full.so
index a2d1ecf..e95ccc4 100755
--- a/linux_musl-x86/lib64/libprotobuf-cpp-full.so
+++ b/linux_musl-x86/lib64/libprotobuf-cpp-full.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libsqlite.so b/linux_musl-x86/lib64/libsqlite.so
index 4f3d8f6..f3477ad 100755
--- a/linux_musl-x86/lib64/libsqlite.so
+++ b/linux_musl-x86/lib64/libsqlite.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libz-host.so b/linux_musl-x86/lib64/libz-host.so
index 013a5b2..7732607 100755
--- a/linux_musl-x86/lib64/libz-host.so
+++ b/linux_musl-x86/lib64/libz-host.so
Binary files differ
diff --git a/linux_musl-x86/lib64/libziparchive.so b/linux_musl-x86/lib64/libziparchive.so
index 015fc25..003fb8f 100755
--- a/linux_musl-x86/lib64/libziparchive.so
+++ b/linux_musl-x86/lib64/libziparchive.so
Binary files differ
diff --git a/manifest.xml b/manifest.xml
index e9bcb5f..3431e49 100644
--- a/manifest.xml
+++ b/manifest.xml
@@ -5,25 +5,25 @@
<default remote="aosp" revision="build-tools-release" sync-j="4" />
- <project name="platform/build" path="build/make" revision="12f80f0c34752030f67d11d2497543b40af725e1" upstream="build-tools-release">
+ <project name="platform/build" path="build/make" revision="cadeaee25b5a011fca9798bb2291830e046db48f" upstream="build-tools-release">
<linkfile dest="build/tools" src="tools" />
</project>
- <project name="platform/build/blueprint" path="build/blueprint" revision="534ae5cc77676e734f9a6658f48f6b55357c29fa" upstream="build-tools-release" />
+ <project name="platform/build/blueprint" path="build/blueprint" revision="8c59bccaf2bf9588bb6c73ab4db6da201f1e3e8d" upstream="build-tools-release" />
- <project name="platform/build/soong" path="build/soong" revision="81801da24fc8a1873a785d9e19581fed6725b34e" upstream="build-tools-release">
+ <project name="platform/build/soong" path="build/soong" revision="b70001f468f398a5c70285ed69161072b441c88f" upstream="build-tools-release">
<linkfile dest="Android.bp" src="root.bp" />
<linkfile dest="bootstrap.bash" src="bootstrap.bash" />
</project>
- <project name="platform/external/golang-protobuf" path="external/golang-protobuf" revision="1d4a1b807962dbec87fc593e68b0ed20619504ce" upstream="build-tools-release" />
+ <project name="platform/external/golang-protobuf" path="external/golang-protobuf" revision="eb5d78d10d73c678a091d42b6aef043b40a09d8a" upstream="build-tools-release" />
- <project clone-depth="1" name="platform/prebuilts/build-tools" path="prebuilts/build-tools" revision="a3471315830206a36b9e4f7c79392233723513af" upstream="build-tools-release" />
+ <project clone-depth="1" name="platform/prebuilts/build-tools" path="prebuilts/build-tools" revision="0e437a690017f5a52d3105c15f071b9286aef7a4" upstream="build-tools-release" />
- <project clone-depth="1" groups="pdk" name="platform/prebuilts/remoteexecution-client" path="prebuilts/remoteexecution-client" revision="5dcada3170c979d617c46192c44c55a7762aabdb" upstream="build-tools-release" />
+ <project clone-depth="1" groups="pdk" name="platform/prebuilts/remoteexecution-client" path="prebuilts/remoteexecution-client" revision="4cc7e61a95415d2f5e6ed2247c39c7c9dc54b46a" upstream="build-tools-release" />
- <project clone-depth="1" groups="linux" name="platform/prebuilts/clang/host/linux-x86" path="prebuilts/clang/host/linux-x86" revision="e18e8609b72d001a36ed210ec3895fc55c89db82" upstream="build-tools-release" />
+ <project clone-depth="1" groups="linux" name="platform/prebuilts/clang/host/linux-x86" path="prebuilts/clang/host/linux-x86" revision="f7adefc90364180e268930602628c55246ec7f98" upstream="build-tools-release" />
<project clone-depth="1" name="platform/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8" path="prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8" revision="7454fb3d3249c268b7e61a6551e4a17dfd031ea9" upstream="build-tools-release" />
@@ -35,7 +35,7 @@
<project clone-depth="1" groups="linux" name="platform/prebuilts/ninja/linux-x86" path="prebuilts/ninja/linux-x86" revision="cfaa2ca50f534ec2945c2587ffa119519268efad" upstream="build-tools-release" />
- <project clone-depth="1" groups="darwin" name="platform/prebuilts/clang/host/darwin-x86" path="prebuilts/clang/host/darwin-x86" revision="531b1668cffbc49b5a0705d16521ab9d2318784b" upstream="build-tools-release" />
+ <project clone-depth="1" groups="darwin" name="platform/prebuilts/clang/host/darwin-x86" path="prebuilts/clang/host/darwin-x86" revision="68442395633f4adb1dda46287af789fe699c2c75" upstream="build-tools-release" />
<project clone-depth="1" groups="darwin" name="platform/prebuilts/gcc/darwin-x86/host/headers" path="prebuilts/gcc/darwin-x86/host/headers" revision="4ac4f7cc41cf3c9e36fc3d6cf37fd1cfa9587a68" upstream="build-tools-release" />
@@ -45,7 +45,7 @@
<project clone-depth="1" groups="darwin" name="platform/prebuilts/ninja/darwin-x86" path="prebuilts/ninja/darwin-x86" revision="2433f0ca209251e67a89bbaf6c34da9fe54edeac" upstream="build-tools-release" />
- <project clone-depth="1" groups="linux" name="platform/prebuilts/clang/host/windows-x86" path="prebuilts/clang/host/windows-x86" revision="6113c650a597695ccc3ff7a8d7cf99a74a842fea" upstream="build-tools-release" />
+ <project clone-depth="1" groups="linux" name="platform/prebuilts/clang/host/windows-x86" path="prebuilts/clang/host/windows-x86" revision="6403bfef08d6a381529bfeab3bb6aeb64fa2a495" upstream="build-tools-release" />
<project clone-depth="1" groups="linux" name="platform/prebuilts/gcc/linux-x86/host/x86_64-w64-mingw32-4.8" path="prebuilts/gcc/linux-x86/host/x86_64-w64-mingw32-4.8" revision="394eb6414b099b017e355ceae71d5bd22938c46e" upstream="build-tools-release" />
@@ -57,13 +57,13 @@
<project clone-depth="1" name="platform/prebuilts/clang-tools" path="prebuilts/clang-tools" revision="acf69ef3af49f628d230a3075f861b2d624a43a8" upstream="build-tools-release" />
- <project clone-depth="1" name="platform/prebuilts/misc" path="prebuilts/misc" revision="e7c40b8cf082b2a3f832c17af3e81ff87d5fd8e9" upstream="build-tools-release" />
+ <project clone-depth="1" name="platform/prebuilts/misc" path="prebuilts/misc" revision="babbbc42fe6acb9c7c220f0a76d3af6cd4a6017b" upstream="build-tools-release" />
<project clone-depth="1" groups="pdk,tools" name="platform/prebuilts/tools" path="prebuilts/tools" revision="c3c160abf40cad1f0b12ae8ecba5acecb99e9674" upstream="build-tools-release" />
- <project name="platform/bionic" path="bionic" revision="ffb60205090aa3d688f1420a8528acf31d3f70c5" upstream="build-tools-release" />
+ <project name="platform/bionic" path="bionic" revision="c093e0919f6d48f404d8d9b7b45963c030ad1924" upstream="build-tools-release" />
- <project name="platform/development" path="development" revision="8614161df8e891bdd1ec09a2a401b91fee80387c" upstream="build-tools-release" />
+ <project name="platform/development" path="development" revision="d145c61bcfbc87d09172470c86f513b322576d83" upstream="build-tools-release" />
<project name="platform/external/arm-optimized-routines" path="external/arm-optimized-routines" revision="1c9813378c5dd9770241459eb61a4a039e026c6b" upstream="build-tools-release" />
@@ -89,11 +89,11 @@
<project name="platform/external/fmtlib" path="external/fmtlib" revision="4f9e441d30ad1c41a065f4e644cdac798154e97c" upstream="build-tools-release" />
- <project name="platform/external/go-cmp" path="external/go-cmp" revision="3f05150be7afae350fed0f3cf3b9ff0799bf9d3d" upstream="build-tools-release" />
+ <project name="platform/external/go-cmp" path="external/go-cmp" revision="dfdff05c1f6e6efdf3c47fdc63cd6ceb4c6a5d4a" upstream="build-tools-release" />
<project name="platform/external/go-creachadair-shell" path="external/go-creachadair-shell" revision="bbbc3604ea8929caee86551ea0028880316c4be6" upstream="build-tools-release" />
- <project name="platform/external/go-creachadair-stringset" path="external/go-creachadair-stringset" revision="f97446c79177a482bcb3bb2a0e95acb21052507d" upstream="build-tools-release" />
+ <project name="platform/external/go-creachadair-stringset" path="external/go-creachadair-stringset" revision="7cc3132c8ccb3d58640625858427e0d228c3cf65" upstream="build-tools-release" />
<project name="platform/external/go-etree" path="external/go-etree" revision="7fa46d9c9eb9134443a7fc56a354f07b15fc3a76" upstream="build-tools-release" />
@@ -101,17 +101,17 @@
<project name="platform/external/golang-x-sync" path="external/golang-x-sync" revision="9831fe029e5158a1c094ba12ca461c2d7c5f2589" upstream="build-tools-release" />
- <project name="platform/external/golang-x-sys" path="external/golang-x-sys" revision="b7fed96c1ad0b91e858531accb49d819b48ab618" upstream="build-tools-release" />
+ <project name="platform/external/golang-x-sys" path="external/golang-x-sys" revision="a02e6bb6ecaba8aa3d3f9336f4faca3bac5106e5" upstream="build-tools-release" />
- <project name="platform/external/golang-x-tools" path="external/golang-x-tools" revision="457aa020f87e3a763226dc76aa3459fd23d0aa11" upstream="build-tools-release" />
+ <project name="platform/external/golang-x-tools" path="external/golang-x-tools" revision="3225eca48f7ce16eb31b2dd5a170806c1214a49e" upstream="build-tools-release" />
<project name="platform/external/google-java-format" path="external/google-java-format" revision="2e807a09ae0a6425158ce9f6dd5c973a18000c69" upstream="build-tools-release" />
- <project name="platform/external/googletest" path="external/googletest" revision="23658bb9f8c596f2e2cc57082f59d117b46d123e" upstream="build-tools-release" />
+ <project name="platform/external/googletest" path="external/googletest" revision="d2d9eae964f8a1c9431c908d4a5dc502801c42ed" upstream="build-tools-release" />
<project name="platform/external/guava" path="external/guava" revision="cea8b35c5ef589dc9ceff02bc8e85f6e8b721fb9" upstream="build-tools-release" />
- <project name="platform/external/icu" path="external/icu" revision="64da1a181fb00b6b376e9a640dffd3ab6a89c095" upstream="build-tools-release" />
+ <project name="platform/external/icu" path="external/icu" revision="2fe8eaded815eebf7c309435224140af71b37dc2" upstream="build-tools-release" />
<project name="platform/external/javapoet" path="external/javapoet" revision="d274ed4f3e04713ec752625c446b9f59daaa80bb" upstream="build-tools-release" />
@@ -139,7 +139,7 @@
<project name="platform/external/lzma" path="external/lzma" revision="3e8a4d1d81b7e8e046ad1f5e79dcbe4ff839404e" upstream="build-tools-release" />
- <project name="platform/external/musl" path="external/musl" revision="18ee9bc2a7b68ff95c7f4d082a4caf2758107764" upstream="build-tools-release" />
+ <project name="platform/external/musl" path="external/musl" revision="0ec25d0fad188a9ed3c294e5528fefd25463c1aa" upstream="build-tools-release" />
<project name="platform/external/pcre" path="external/pcre" revision="fe81096baddfe4d58f69306f123dae70654b4af6" upstream="build-tools-release" />
@@ -147,17 +147,17 @@
<project name="platform/external/safe-iop" path="external/safe-iop" revision="b805514f31a231a0e78a18f296c0454fcadead1a" upstream="build-tools-release" />
- <project name="platform/external/selinux" path="external/selinux" revision="d77195538b0f4e28e444f7d3b98a6382d9f9c69d" upstream="build-tools-release" />
+ <project name="platform/external/selinux" path="external/selinux" revision="2fe4fad491aa904a84e093f44a807931fed46625" upstream="build-tools-release" />
<project name="platform/external/sqlite" path="external/sqlite" revision="dec725a9d4d6a4da79ccf554bda224a461eeded5" upstream="build-tools-release" />
<project name="platform/external/starlark-go" path="external/starlark-go" revision="dfe047847e867d4b297b4524a129e169484d8c42" upstream="build-tools-release" />
- <project name="platform/external/zlib" path="external/zlib" revision="813594e30898f98ccc7bfc87b7d9819ce9665b72" upstream="build-tools-release" />
+ <project name="platform/external/zlib" path="external/zlib" revision="de355433d43a3449a3e77f63b074a10c9197aaec" upstream="build-tools-release" />
<project name="platform/external/zopfli" path="external/zopfli" revision="cf3f34689a36a959d4578e82adb46532e38c2eaa" upstream="build-tools-release" />
- <project name="platform/system/core" path="system/core" revision="0c9a655738d44648293801900b9c8ac5c485474c" upstream="build-tools-release" />
+ <project name="platform/system/core" path="system/core" revision="6fdcc57e3d8c3761a242ab8c3c9ff03df7c5a1a2" upstream="build-tools-release" />
<project name="platform/system/libbase" path="system/libbase" revision="a6c75ad3b52211a9368fda383b535c30f5c5ca58" upstream="build-tools-release" />
@@ -167,13 +167,13 @@
<project name="platform/system/logging" path="system/logging" revision="838f03c47c3b891f4febcd6a5b70c0038e07cb7d" upstream="build-tools-release" />
- <project name="platform/system/unwinding" path="system/unwinding" revision="c16e46be3ab8d4878f414a0b316406690d946a22" upstream="build-tools-release" />
+ <project name="platform/system/unwinding" path="system/unwinding" revision="aa737839c449735c70f355ffaf33833e141ada5b" upstream="build-tools-release" />
- <project name="platform/system/tools/xsdc" path="system/tools/xsdc" revision="3ba8aa48392ab0559b4cb4e2c440eec0ae39ae00" upstream="build-tools-release" />
+ <project name="platform/system/tools/xsdc" path="system/tools/xsdc" revision="ad69f814f6456bf07799cb0576ed7d4d3f554a71" upstream="build-tools-release" />
- <project name="platform/test/app_compat/csuite" path="test/app_compat/csuite" revision="4b95fc80b1f366280b2e1a62f6a8a31b850b7e8b" upstream="build-tools-release" />
+ <project name="platform/test/app_compat/csuite" path="test/app_compat/csuite" revision="5387d15c213eb1a3366bae2545ad8d3f790643d3" upstream="build-tools-release" />
- <project name="platform/art" path="art" revision="f814566140865b64ac4e084d3f2f0bd619a607da" upstream="build-tools-release" />
+ <project name="platform/art" path="art" revision="16ae04de14db430c8ec239e75cd9d0553a27c311" upstream="build-tools-release" />
<project name="platform/build/kati" path="build/kati" revision="6ad4268491b968b4cb257aeeb8c6a605ea8af8bf" upstream="build-tools-release" />
@@ -181,7 +181,7 @@
<project name="platform/external/bc" path="external/bc" revision="830c78bfc070f6f019571b2682c7892604561cba" upstream="build-tools-release" />
- <project name="platform/external/bison" path="external/bison" revision="661d941d054e51ed48d0683a643f5842051f4bba" upstream="build-tools-release" />
+ <project name="platform/external/bison" path="external/bison" revision="d868ebf0b94600d99735f2bd4a77bc9245ce11c4" upstream="build-tools-release" />
<project name="platform/external/bloaty" path="external/bloaty" revision="4e97ca09798800c00d8887d6b63992669fb1a631" upstream="build-tools-release" />
@@ -193,27 +193,27 @@
<project name="platform/external/ninja" path="external/ninja" revision="0c2101e9f1e0b9f85b1c6032148a57f887c1475e" upstream="build-tools-release" />
- <project name="platform/external/nsjail" path="external/nsjail" revision="9c38c3715e4dfacda98a65a0f5606d9059eedf43" upstream="build-tools-release" />
+ <project name="platform/external/nsjail" path="external/nsjail" revision="52fd46b555cb427199d5d44e49287151244a4cd8" upstream="build-tools-release" />
<project name="platform/external/one-true-awk" path="external/one-true-awk" revision="539f77df4407fddf23dce84e20a59b1475b20e3f" upstream="build-tools-release" />
- <project name="platform/external/openssl" path="external/openssl" revision="005e88f9deb1e50b98694404bb85a688e119abeb" upstream="build-tools-release" />
+ <project name="platform/external/openssl" path="external/openssl" revision="1998e09b0111afe2318e3c9b47be510c9ce6f07a" upstream="build-tools-release" />
<project name="platform/external/python/cpython2" path="external/python/cpython2" revision="66a1bd5b30cf3915a4e087c40aaceb5561ff9e79" upstream="build-tools-release" />
- <project name="platform/external/python/cpython3" path="external/python/cpython3" revision="711bec18e70680e202068d42885052f8b47257b3" upstream="build-tools-release" />
+ <project name="platform/external/python/cpython3" path="external/python/cpython3" revision="69d927acd83d5b74e046bc7b55a18423b5a049d8" upstream="build-tools-release" />
<project name="platform/external/toybox" path="external/toybox" revision="5b39beb4a0780e107d42c46e78c42b1d9399953a" upstream="build-tools-release" />
<project name="platform/external/turbine" path="external/turbine" revision="097e956e80b1ae0668ea105618d71087159f4647" upstream="build-tools-release" />
- <project name="platform/system/tools/aidl" path="system/tools/aidl" revision="46c272b63d5ba1d952b3cef1defcfe0128a91b40" upstream="build-tools-release" />
+ <project name="platform/system/tools/aidl" path="system/tools/aidl" revision="ec2a6e39f6a40e5c51c5aead3b953dda77cd33ea" upstream="build-tools-release" />
- <project name="platform/system/tools/hidl" path="system/tools/hidl" revision="3de18409b3f9d3cff57b4b7875cf80160a78ce83" upstream="build-tools-release" />
+ <project name="platform/system/tools/hidl" path="system/tools/hidl" revision="66102d7eeaf4f1206d8f3b81071b9e76736a0097" upstream="build-tools-release" />
<project name="toolchain/go" revision="01b90271bfe5ae1e2766f2c8f1272e6458f3918e" upstream="build-tools-release" />
- <project name="toolchain/m4" revision="5defa77e9da37c7ff71db3fc2675efdafa692108" upstream="build-tools-release" />
+ <project name="toolchain/m4" revision="2880c62e70c98965ec978ae229a8cf379573599f" upstream="build-tools-release" />
<project name="toolchain/make" revision="98b2bd218f82dd5f3ecb054846ae4fdd01a76588" upstream="build-tools-release" />
diff --git a/sysroots/i686-linux-musl/lib/Scrt1-real.o b/sysroots/i686-linux-musl/lib/Scrt1-real.o
index ab5abc4..0b924ce 100644
--- a/sysroots/i686-linux-musl/lib/Scrt1-real.o
+++ b/sysroots/i686-linux-musl/lib/Scrt1-real.o
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/crt1.o b/sysroots/i686-linux-musl/lib/crt1.o
index caff96e..85b7f9b 100644
--- a/sysroots/i686-linux-musl/lib/crt1.o
+++ b/sysroots/i686-linux-musl/lib/crt1.o
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/crti.o b/sysroots/i686-linux-musl/lib/crti.o
index abe46e3..463d185 100644
--- a/sysroots/i686-linux-musl/lib/crti.o
+++ b/sysroots/i686-linux-musl/lib/crti.o
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/crtn.o b/sysroots/i686-linux-musl/lib/crtn.o
index bdab425..23f14a8 100644
--- a/sysroots/i686-linux-musl/lib/crtn.o
+++ b/sysroots/i686-linux-musl/lib/crtn.o
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/libc++.so b/sysroots/i686-linux-musl/lib/libc++.so
index a532cd9..7c97c9c 100755
--- a/sysroots/i686-linux-musl/lib/libc++.so
+++ b/sysroots/i686-linux-musl/lib/libc++.so
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/libc++abi.a b/sysroots/i686-linux-musl/lib/libc++abi.a
index e2d459a..ce740ec 100644
--- a/sysroots/i686-linux-musl/lib/libc++abi.a
+++ b/sysroots/i686-linux-musl/lib/libc++abi.a
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/libc.a b/sysroots/i686-linux-musl/lib/libc.a
index 080cd09..2137af5 100644
--- a/sysroots/i686-linux-musl/lib/libc.a
+++ b/sysroots/i686-linux-musl/lib/libc.a
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/libc_musl.so b/sysroots/i686-linux-musl/lib/libc_musl.so
index 8d8677d..af34cab 100755
--- a/sysroots/i686-linux-musl/lib/libc_musl.so
+++ b/sysroots/i686-linux-musl/lib/libc_musl.so
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/libc_musl_linker_object.o b/sysroots/i686-linux-musl/lib/libc_musl_linker_object.o
index ef58116..042cc87 100644
--- a/sysroots/i686-linux-musl/lib/libc_musl_linker_object.o
+++ b/sysroots/i686-linux-musl/lib/libc_musl_linker_object.o
Binary files differ
diff --git a/sysroots/i686-linux-musl/lib/linker.script b/sysroots/i686-linux-musl/lib/linker.script
index 03bb310..d9f8b86 100755
--- a/sysroots/i686-linux-musl/lib/linker.script
+++ b/sysroots/i686-linux-musl/lib/linker.script
@@ -3,13 +3,13 @@
.linker_r 0x1000 : {
KEEP(*(.linker_r));
}
- .linker_rx 0x48aa0 : {
+ .linker_rx 0x4b330 : {
KEEP(*(.linker_rx));
}
- .linker_rw 0x124450 : {
+ .linker_rw 0x127c40 : {
KEEP(*(.linker_rw));
}
- .linker_rw2 0x1258b0 : {
+ .linker_rw2 0x129108 : {
KEEP(*(.linker_rw2));
}
/DISCARD/ : { *(.interp) }
diff --git a/sysroots/i686-linux-musl/lib/rcrt1.o b/sysroots/i686-linux-musl/lib/rcrt1.o
index 4f52e92..d851a77 100644
--- a/sysroots/i686-linux-musl/lib/rcrt1.o
+++ b/sysroots/i686-linux-musl/lib/rcrt1.o
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/Scrt1-real.o b/sysroots/x86_64-linux-musl/lib/Scrt1-real.o
index 17e6a60..513deaf 100644
--- a/sysroots/x86_64-linux-musl/lib/Scrt1-real.o
+++ b/sysroots/x86_64-linux-musl/lib/Scrt1-real.o
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/crt1.o b/sysroots/x86_64-linux-musl/lib/crt1.o
index 99b5f7d..4b30451 100644
--- a/sysroots/x86_64-linux-musl/lib/crt1.o
+++ b/sysroots/x86_64-linux-musl/lib/crt1.o
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/crti.o b/sysroots/x86_64-linux-musl/lib/crti.o
index ad6a291..55a4ffc 100644
--- a/sysroots/x86_64-linux-musl/lib/crti.o
+++ b/sysroots/x86_64-linux-musl/lib/crti.o
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/crtn.o b/sysroots/x86_64-linux-musl/lib/crtn.o
index b397495..af02170 100644
--- a/sysroots/x86_64-linux-musl/lib/crtn.o
+++ b/sysroots/x86_64-linux-musl/lib/crtn.o
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/libc++.so b/sysroots/x86_64-linux-musl/lib/libc++.so
index 212292b..b5eac52 100755
--- a/sysroots/x86_64-linux-musl/lib/libc++.so
+++ b/sysroots/x86_64-linux-musl/lib/libc++.so
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/libc++abi.a b/sysroots/x86_64-linux-musl/lib/libc++abi.a
index 84c1384..2747b58 100644
--- a/sysroots/x86_64-linux-musl/lib/libc++abi.a
+++ b/sysroots/x86_64-linux-musl/lib/libc++abi.a
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/libc.a b/sysroots/x86_64-linux-musl/lib/libc.a
index aefcc39..162cfc6 100644
--- a/sysroots/x86_64-linux-musl/lib/libc.a
+++ b/sysroots/x86_64-linux-musl/lib/libc.a
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/libc_musl.so b/sysroots/x86_64-linux-musl/lib/libc_musl.so
index fcf6794..113c4ad 100755
--- a/sysroots/x86_64-linux-musl/lib/libc_musl.so
+++ b/sysroots/x86_64-linux-musl/lib/libc_musl.so
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/libc_musl_linker_object.o b/sysroots/x86_64-linux-musl/lib/libc_musl_linker_object.o
index 4104f6c..20cc55c 100644
--- a/sysroots/x86_64-linux-musl/lib/libc_musl_linker_object.o
+++ b/sysroots/x86_64-linux-musl/lib/libc_musl_linker_object.o
Binary files differ
diff --git a/sysroots/x86_64-linux-musl/lib/linker.script b/sysroots/x86_64-linux-musl/lib/linker.script
index 1680ee3..950b948 100755
--- a/sysroots/x86_64-linux-musl/lib/linker.script
+++ b/sysroots/x86_64-linux-musl/lib/linker.script
@@ -3,13 +3,13 @@
.linker_r 0x1000 : {
KEEP(*(.linker_r));
}
- .linker_rx 0x4df44 : {
+ .linker_rx 0x500fc : {
KEEP(*(.linker_rx));
}
- .linker_rw 0x116eb0 : {
+ .linker_rw 0x11a090 : {
KEEP(*(.linker_rw));
}
- .linker_rw2 0x118580 : {
+ .linker_rw2 0x11b7f0 : {
KEEP(*(.linker_rw2));
}
/DISCARD/ : { *(.interp) }
diff --git a/sysroots/x86_64-linux-musl/lib/rcrt1.o b/sysroots/x86_64-linux-musl/lib/rcrt1.o
index 00825a7..1716ec2 100644
--- a/sysroots/x86_64-linux-musl/lib/rcrt1.o
+++ b/sysroots/x86_64-linux-musl/lib/rcrt1.o
Binary files differ