diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000000..82694d81f276b2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +*.pck binary +Lib/test/cjkencodings/* binary +Lib/test/decimaltestdata/*.decTest binary +Lib/test/sndhdrdata/sndhdr.* binary +Lib/test/test_email/data/msg_26.txt binary +Lib/test/xmltestdata/* binary +Lib/venv/scripts/nt/* binary +Lib/test/coding20731.py binary diff --git a/.github/appveyor.yml b/.github/appveyor.yml new file mode 100644 index 00000000000000..d8bfb9adf9378d --- /dev/null +++ b/.github/appveyor.yml @@ -0,0 +1,28 @@ +version: 3.6.1+.{build} +clone_depth: 5 +branches: + only: + - master + - /\d\.\d/ + - buildbot-custom +build_script: +- cmd: PCbuild\build.bat -e +test_script: +- cmd: PCbuild\rt.bat -q -uall -rwW --slowest --timeout=1200 -j0 + +# Only trigger AppVeyor if actual code or its configuration changes +only_commits: + files: + - .github/appveyor.yml + - .gitattributes + - Grammar/ + - Include/ + - Lib/ + - Modules/ + - Objects/ + - PC/ + - PCBuild/ + - Parser/ + - Programs/ + - Python/ + - Tools/ diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 00000000000000..fcf9df6a7a698e --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,35 @@ +codecov: + notify: + require_ci_to_pass: true +comment: off +ignore: + - "Doc/**/*" + - "Misc/*" + - "Mac/**/*" + - "PC/**/*" + - "PCbuild/**/*" + - "Tools/**/*" + - "Grammar/*" +coverage: + precision: 2 + range: + - 70.0 + - 100.0 + round: down + status: + changes: off + project: off + patch: + default: + target: 100% + only_pulls: true + threshold: 0.05 +parsers: + gcov: + branch_detection: + conditional: true + loop: true + macro: false + method: false + javascript: + enable_partials: false diff --git a/.gitignore b/.gitignore index ed4ebfbbd9de74..9392fe5defd6f5 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ .gdb_history Doc/build/ Doc/venv/ +Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* @@ -51,6 +52,8 @@ PCbuild/*.suo PCbuild/*.*sdf PCbuild/*-pgi PCbuild/*-pgo +PCbuild/*.VC.db +PCbuild/*.VC.opendb PCbuild/.vs/ PCbuild/amd64/ PCbuild/obj/ diff --git a/.hgtouch b/.hgtouch deleted file mode 100644 index b9be0f11fdb829..00000000000000 --- a/.hgtouch +++ /dev/null @@ -1,17 +0,0 @@ -# -*- Makefile -*- -# Define dependencies of generated files that are checked into hg. -# The syntax of this file uses make rule dependencies, without actions - -Python/importlib.h: Lib/importlib/_bootstrap.py Programs/_freeze_importlib.c - -Include/opcode.h: Lib/opcode.py Tools/scripts/generate_opcode_h.py - -Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py -Python/Python-ast.c: Include/Python-ast.h - -Python/opcode_targets.h: Python/makeopcodetargets.py Lib/opcode.py - -Objects/typeslots.inc: Include/typeslots.h Objects/typeslots.py - -Include/graminit.h: Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c -Python/graminit.c: Include/graminit.h Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c diff --git a/.mention-bot b/.mention-bot new file mode 100644 index 00000000000000..cb53b993fb1e8a --- /dev/null +++ b/.mention-bot @@ -0,0 +1,3 @@ +{ + "findPotentialReviewers": false +} diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000000..2b4556fc6fc462 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,96 @@ +language: c +dist: trusty +sudo: false +group: beta + +# To cache doc-building dependencies. +cache: pip + +branches: + only: + - master + - /^\d\.\d$/ + +matrix: + fast_finish: true + allow_failures: + - env: OPTIONAL=true + include: + - os: linux + language: c + compiler: clang + # gcc also works, but to keep the # of concurrent builds down, we use one C + # compiler here and the other to run the coverage build. Clang is preferred + # in this instance for its better error messages. + env: TESTING=cpython + - os: osx + language: c + compiler: clang + # Testing under macOS is optional until testing stability has been demonstrated. + env: OPTIONAL=true + before_install: + - brew install openssl xz + - export CPPFLAGS="-I$(brew --prefix openssl)/include" + - export LDFLAGS="-L$(brew --prefix openssl)/lib" + - os: linux + language: python + python: 3.6 + env: TESTING=docs + before_script: + - cd Doc + # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. + # (Updating the version is fine as long as no warnings are raised by doing so.) + - python -m pip install sphinx~=1.6.1 + script: + - make check suspicious html SPHINXOPTS="-q -W -j4" + - os: linux + language: c + compiler: gcc + env: OPTIONAL=true + before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure + make -s -j4 + # Need a venv that can parse covered code. + ./python -m venv venv + ./venv/bin/python -m pip install -U coverage + script: + # Skip tests that re-run the entire test suite. + - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu,-tzdata -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn + after_script: # Probably should be after_success once test suite updated to run under coverage.py. + # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. + - source ./venv/bin/activate + - bash <(curl -s https://codecov.io/bash) + +# Travis provides only 2 cores, so don't overdo the parallelism and waste memory. +before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure --with-pydebug + make -j4 + +script: + # `-r -w` implicitly provided through `make buildbottest`. + - make buildbottest TESTOPTS="-j4 -uall,-cpu,-tzdata" + +notifications: + email: false + irc: + channels: + # This is set to a secure variable to prevent forks from notifying the + # IRC channel whenever they fail a build. This can be removed when travis + # implements https://github.com/travis-ci/travis-ci/issues/1094. + # The actual value here is: irc.freenode.net#python-dev + - secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU=" + on_success: change + on_failure: always + skip_join: true diff --git a/Doc/Makefile b/Doc/Makefile index 91f937f985831a..ae59f3294f1c06 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -4,13 +4,13 @@ # # You can set these variables from the command line. -PYTHON = python +PYTHON = python3 SPHINXBUILD = sphinx-build PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) .PHONY: help build html htmlhelp latex text changes linkcheck \ @@ -153,21 +153,26 @@ dist: cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: - $(PYTHON) tools/rstlint.py -i tools -i venv + $(PYTHON) tools/rstlint.py -i tools -i venv -i README.rst serve: ../Tools/scripts/serve.py build/html # Targets for daily automated doc build +# By default, Sphinx only rebuilds pages where the page content has changed. +# This means it doesn't always pick up changes to preferred link targets, etc +# To ensure such changes are picked up, we build the published docs with +# `-E` (to ignore the cached environment) and `-a` (to ignore already existing +# output files) # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' -make suspicious # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/README.txt b/Doc/README.rst similarity index 83% rename from Doc/README.txt rename to Doc/README.rst index 4f8e9f8f1417fb..dcd3d6e80ff3c4 100644 --- a/Doc/README.txt +++ b/Doc/README.rst @@ -2,20 +2,21 @@ Python Documentation README ~~~~~~~~~~~~~~~~~~~~~~~~~~~ This directory contains the reStructuredText (reST) sources to the Python -documentation. You don't need to build them yourself, prebuilt versions are -available at . +documentation. You don't need to build them yourself, `prebuilt versions are +available `_. Documentation on authoring Python documentation, including information about -both style and markup, is available in the "Documenting Python" chapter of the -developers guide . +both style and markup, is available in the "`Documenting Python +`_" chapter of the +developers guide. Building the docs ================= -You need to have Sphinx installed; it is the toolset +You need to have `Sphinx `_ installed; it is the toolset used to build the docs. It is not included in this tree, but maintained -separately and available from PyPI . +separately and `available from PyPI `_. Using make @@ -108,11 +109,11 @@ see the make targets above). Contributing ============ -Bugs in the content should be reported to the Python bug tracker at -https://bugs.python.org. +Bugs in the content should be reported to the +`Python bug tracker `_. -Bugs in the toolset should be reported in the Sphinx bug tracker at -https://www.bitbucket.org/birkenfeld/sphinx/issues/. +Bugs in the toolset should be reported in the +`Sphinx bug tracker `_. You can also send a mail to the Python Documentation Team at docs@python.org, and we will process your request as soon as possible. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 037b85cfd11754..2bc1bd876a2fe2 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -291,16 +291,11 @@ an error value). is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, and so forth. - Warning categories must be subclasses of :c:data:`Warning`; the default warning - category is :c:data:`RuntimeWarning`. The standard Python warning categories are - available as global variables whose names are ``PyExc_`` followed by the Python - exception name. These have the type :c:type:`PyObject\*`; they are all class - objects. Their names are :c:data:`PyExc_Warning`, :c:data:`PyExc_UserWarning`, - :c:data:`PyExc_UnicodeWarning`, :c:data:`PyExc_DeprecationWarning`, - :c:data:`PyExc_SyntaxWarning`, :c:data:`PyExc_RuntimeWarning`, and - :c:data:`PyExc_FutureWarning`. :c:data:`PyExc_Warning` is a subclass of - :c:data:`PyExc_Exception`; the other warning categories are subclasses of - :c:data:`PyExc_Warning`. + Warning categories must be subclasses of :c:data:`PyExc_Warning`; + :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; + the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard + Python warning categories are available as global variables whose names are + enumerated at :ref:`standardwarningcategories`. For information about warning control, see the documentation for the :mod:`warnings` module and the :option:`-W` option in the command line @@ -750,6 +745,61 @@ All standard Python exceptions are available as global variables whose names are :c:type:`PyObject\*`; they are all class objects. For completeness, here are all the variables: +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +=========================================+=================================+==========+ @@ -759,8 +809,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | @@ -769,27 +817,31 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportError` | :exc:`ImportError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_IndexError` | :exc:`IndexError` | | +-----------------------------------------+---------------------------------+----------+ @@ -801,8 +853,12 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NameError` | :exc:`NameError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | @@ -823,16 +879,32 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemError` | :exc:`SystemError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ValueError` | :exc:`ValueError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | @@ -849,11 +921,18 @@ the variables: and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. .. versionadded:: 3.5 - :c:data:`PyExc_RecursionError`. + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. These are compatibility aliases to :c:data:`PyExc_OSError`: +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ @@ -867,52 +946,6 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: .. versionchanged:: 3.3 These aliases used to be separate exception types. - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_LookupError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_ConnectionError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_ImportError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_MemoryError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - Notes: (1) @@ -924,3 +957,60 @@ Notes: (3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. + +.. _standardwarningcategories: + +Standard Warning Categories +=========================== + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DeprecationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f592cb65c3e3a9..f50680b3d29558 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -85,13 +85,12 @@ All integers are implemented as "long" integer objects of arbitrary size. Return a new :c:type:`PyLongObject` based on the string value in *str*, which is interpreted according to the radix in *base*. If *pend* is non-*NULL*, *\*pend* will point to the first character in *str* which follows the - representation of the number. If *base* is ``0``, the radix will be - determined based on the leading characters of *str*: if *str* starts with - ``'0x'`` or ``'0X'``, radix 16 will be used; if *str* starts with ``'0o'`` or - ``'0O'``, radix 8 will be used; if *str* starts with ``'0b'`` or ``'0B'``, - radix 2 will be used; otherwise radix 10 will be used. If *base* is not - ``0``, it must be between ``2`` and ``36``, inclusive. Leading spaces are - ignored. If there are no digits, :exc:`ValueError` will be raised. + representation of the number. If *base* is ``0``, *str* is interpreted using + the :ref:`integers` definition; in this case, leading zeros in a + non-zero decimal number raises a :exc:`ValueError`. If *base* is not ``0``, + it must be between ``2`` and ``36``, inclusive. Leading spaces and single + underscores after a base specifier and between digits are ignored. If there + are no digits, :exc:`ValueError` will be raised. .. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index a6d0f4688d1b78..c6d1d02a2fa510 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -34,7 +34,7 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - Return a string object containing the marshalled representation of *value*. + Return a bytes object containing the marshalled representation of *value*. *version* indicates the file format. @@ -88,10 +88,10 @@ written using these routines? :exc:`TypeError`) and returns *NULL*. -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len) +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - Return a Python object from the data stream in a character buffer - containing *len* bytes pointed to by *string*. + Return a Python object from the data stream in a byte buffer + containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 3ff545275fb3b8..873fb2ac1d3cad 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -391,7 +391,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: +:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index a825164918f446..8b695e065aeffd 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -56,3 +56,14 @@ Slice Objects .. versionchanged:: 3.2 The parameter type for the *slice* parameter was ``PySliceObject*`` before. + + +Ellipsis Object +--------------- + + +.. c:var:: PyObject *Py_Ellipsis + + The Python ``Ellipsis`` object. This object has no methods. It needs to be + treated just like any other object with respect to reference counts. Like + :c:data:`Py_None` it is a singleton object. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f48119391f2bbd..c080f317bee9d9 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -241,7 +241,7 @@ definition with the same method name. +==================+=============+===============================+ | :attr:`name` | char \* | name of the member | +------------------+-------------+-------------------------------+ - | :attr:`type` | int | the type of the member in the | + | :attr:`!type` | int | the type of the member in the | | | | C struct | +------------------+-------------+-------------------------------+ | :attr:`offset` | Py_ssize_t | the offset in bytes that the | @@ -256,7 +256,7 @@ definition with the same method name. | | | docstring | +------------------+-------------+-------------------------------+ - :attr:`type` can be one of many ``T_`` macros corresponding to various C + :attr:`!type` can be one of many ``T_`` macros corresponding to various C types. When the member is accessed in Python, it will be converted to the equivalent Python type. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 02f7ada7be7ee4..6e91576ee8f004 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1393,77 +1393,78 @@ Character Map Codecs This codec is special in that it can be used to implement many different codecs (and this is in fact what was done to obtain most of the standard codecs included in the :mod:`encodings` package). The codec uses mapping to encode and -decode characters. - -Decoding mappings must map single string characters to single Unicode -characters, integers (which are then interpreted as Unicode ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -Encoding mappings must map single Unicode characters to single string -characters, integers (which are then interpreted as Latin-1 ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -The mapping objects provided must only support the __getitem__ mapping -interface. - -If a character lookup fails with a LookupError, the character is copied as-is -meaning that its ordinal value will be interpreted as Unicode or Latin-1 ordinal -resp. Because of this, mappings only need to contain those mappings which map -characters to different code points. +decode characters. The mapping objects provided must support the +:meth:`__getitem__` mapping interface; dictionaries and sequences work well. These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* using - the given *mapping* object. Return *NULL* if an exception was raised by the - codec. If *mapping* is *NULL* latin-1 decoding will be done. Else it can be a - dictionary mapping byte or a unicode string, which is treated as a lookup table. - Byte values greater that the length of the string and U+FFFE "characters" are - treated as "undefined mapping". + Create a Unicode object by decoding *size* bytes of the encoded string *s* + using the given *mapping* object. Return *NULL* if an exception was raised + by the codec. + + If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + *mapping* must map bytes ordinals (integers in the range from 0 to 255) + to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or ``None``. Unmapped data bytes -- ones which cause a + :exc:`LookupError`, as well as ones which get mapped to ``None``, + ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause + an error. .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - Encode a Unicode object using the given *mapping* object and return the result - as Python string object. Error handling is "strict". Return *NULL* if an + Encode a Unicode object using the given *mapping* object and return the + result as a bytes object. Error handling is "strict". Return *NULL* if an exception was raised by the codec. -The following codec API is special in that maps Unicode to Unicode. - + The *mapping* object must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or ``None``. Unmapped character + ordinals (ones which cause a :exc:`LookupError`) as well as mapped to + ``None`` are treated as "undefined mapping" and cause an error. -.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *table, const char *errors) - - Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a - character mapping *table* to it and return the resulting Unicode object. Return - *NULL* when an exception was raised by the codec. - The *mapping* table must map Unicode ordinal integers to Unicode ordinal - integers or ``None`` (causing deletion of the character). +.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. + Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given + *mapping* object and return the result as a bytes object. Return *NULL* if + an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API - ` + :c:func:`PyUnicode_AsCharmapString` or + :c:func:`PyUnicode_AsEncodedString`. -.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ +The following codec API is special in that maps Unicode to Unicode. + +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ PyObject *mapping, const char *errors) - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return a Python string object. Return *NULL* if an - exception was raised by the codec. + Translate a Unicode object using the given *mapping* object and return the + resulting Unicode object. Return *NULL* if an exception was raised by the + codec. + + The *mapping* object must map Unicode ordinal integers to Unicode strings, + integers (which are then interpreted as Unicode ordinals) or ``None`` + (causing deletion of the character). Unmapped character ordinals (ones + which cause a :exc:`LookupError`) are left untouched and are copied as-is. + + +.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) + + Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a + character *mapping* table to it and return the resulting Unicode object. + Return *NULL* when an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsCharmapString` or - :c:func:`PyUnicode_AsEncodedString`. + :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API + ` MBCS codecs for Windows diff --git a/Doc/conf.py b/Doc/conf.py index b1bb6208bb4b8e..18aebb68a8d8df 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -37,7 +37,7 @@ needs_sphinx = '1.2' # Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*'] +exclude_patterns = ['venv/*', 'README.rst'] # Options for HTML output @@ -88,11 +88,24 @@ # Options for LaTeX output # ------------------------ +# Get LaTeX to handle Unicode correctly +latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} + +# Additional stuff for the LaTeX preamble. +latex_elements['preamble'] = r''' +\authoraddress{ + \strong{Python Software Foundation}\\ + Email: \email{docs@python.org} +} +\let\Verbatim=\OriginalVerbatim +\let\endVerbatim=\endOriginalVerbatim +''' + # The paper size ('letter' or 'a4'). -latex_paper_size = 'a4' +latex_elements['papersize'] = 'a4' # The font size ('10pt', '11pt' or '12pt'). -latex_font_size = '10pt' +latex_elements['font_size'] = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -125,22 +138,9 @@ for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst') -# Additional stuff for the LaTeX preamble. -latex_preamble = r''' -\authoraddress{ - \strong{Python Software Foundation}\\ - Email: \email{docs@python.org} -} -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim -''' - # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] -# Get LaTeX to handle Unicode correctly -latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} - # Options for Epub output # ----------------------- diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 1f5be9cdb29a97..4e2761d8a7d046 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -321,7 +321,7 @@ You can read back this static file, by using the >>> metadata.description 'Easily download, build, install, upgrade, and uninstall Python packages' -Notice that the class can also be instanciated with a metadata file path to +Notice that the class can also be instantiated with a metadata file path to loads its values:: >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index b8ce4377877e70..003b4e505d3247 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -124,7 +124,7 @@ our objects and in some error messages, for example:: >>> "" + noddy.new_noddy() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: cannot add type "noddy.Noddy" to string Note that the name is a dotted name that includes both the module name and the diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index f1e33afdabf8a5..8f6a907a8a2fda 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -159,7 +159,7 @@ How do I obtain a copy of the Python source? The latest Python source distribution is always available from python.org, at https://www.python.org/downloads/. The latest development sources can be obtained -via anonymous Mercurial access at https://hg.python.org/cpython. +at https://github.com/python/cpython/. The source distribution is a gzipped tar file containing the complete C source, Sphinx-formatted documentation, Python library modules, example programs, and @@ -222,8 +222,8 @@ releases are announced on the comp.lang.python and comp.lang.python.announce newsgroups and on the Python home page at https://www.python.org/; an RSS feed of news is available. -You can also access the development version of Python through Mercurial. See -https://docs.python.org/devguide/faq.html for details. +You can also access the development version of Python through Git. See +`The Python Developer's Guide `_ for details. How do I submit bug reports and patches for Python? diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d7253436beaf61..6ac83e45d2e815 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -300,9 +300,10 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools --> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent size" to 4, and select the "Insert spaces" radio button. -If you suspect mixed tabs and spaces are causing problems in leading whitespace, -run Python with the :option:`-t` switch or run ``Tools/Scripts/tabnanny.py`` to -check a directory tree in batch mode. +Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs +and spaces are causing problems in leading whitespace. +You may also run the :mod:`tabnanny` module to check a directory tree +in batch mode. How do I check for a keypress without blocking? diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 41ee3d83b311fa..dba9186d935a6a 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -131,6 +131,10 @@ Glossary binary file A :term:`file object` able to read and write :term:`bytes-like objects `. + Examples of binary files are files opened in binary mode (``'rb'``, + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, + :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and + :class:`gzip.GzipFile`. .. seealso:: A :term:`text file` reads and writes :class:`str` objects. @@ -155,7 +159,7 @@ Glossary bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` and ``.pyo`` files so that executing the same file is + cached in ``.pyc`` files so that executing the same file is faster the second time (recompilation from source to bytecode can be avoided). This "intermediate language" is said to run on a :term:`virtual machine` that executes the machine code corresponding to @@ -316,6 +320,11 @@ Glossary A module written in C or C++, using Python's C API to interact with the core and with user code. + f-string + String literals prefixed with ``'f'`` or ``'F'`` are commonly called + "f-strings" which is short for + :ref:`formatted string literals `. See also :pep:`498`. + file object An object exposing a file-oriented API (with methods such as :meth:`read()` or :meth:`write()`) to an underlying resource. Depending @@ -458,9 +467,9 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable, while no mutable - containers (such as lists or dictionaries) are. Objects which are - instances of user-defined classes are hashable by default; they all + All of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not. Objects which are + instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. @@ -966,6 +975,9 @@ Glossary A :term:`file object` able to read and write :class:`str` objects. Often, a text file actually accesses a byte-oriented datastream and handles the :term:`text encoding` automatically. + Examples of text files are files opened in text mode (``'r'`` or ``'w'``), + :data:`sys.stdin`, :data:`sys.stdout`, and instances of + :class:`io.StringIO`. .. seealso:: A :term:`binary file` reads and write :class:`bytes` objects. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 7e161a59add8ae..9d770f5232b440 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -221,7 +221,7 @@ before proceeding. Introducing Optional arguments ============================== -So far we, have been playing with positional arguments. Let us +So far we have been playing with positional arguments. Let us have a look on how to add optional ones:: import argparse diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index c2bf473e1ff9ea..2dd6c34e2ced02 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -282,7 +282,7 @@ this:: . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" - return types.MethodType(self, obj, objtype) + return types.MethodType(self, obj) Running the interpreter shows how the function descriptor works in practice:: diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 8ae9679894a578..40601812a77cb5 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -210,7 +210,7 @@ You can experiment with the iteration interface manually: 3 >>> next(it) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in StopIteration >>> @@ -474,7 +474,7 @@ Here's a sample usage of the ``generate_ints()`` generator: 2 >>> next(gen) Traceback (most recent call last): - File "stdin", line 1, in ? + File "stdin", line 1, in File "stdin", line 2, in generate_ints StopIteration @@ -577,7 +577,7 @@ And here's an example of changing the counter: 9 >>> next(it) #doctest: +SKIP Traceback (most recent call last): - File "t.py", line 15, in ? + File "t.py", line 15, in it.next() StopIteration @@ -653,8 +653,9 @@ This can also be written as a list comprehension: [0, 2, 4, 6, 8] -:func:`enumerate(iter) ` counts off the elements in the iterable, -returning 2-tuples containing the count and each element. :: +:func:`enumerate(iter, start=0) ` counts off the elements in the +iterable returning 2-tuples containing the count (from *start*) and +each element. :: >>> for item in enumerate(['subject', 'verb', 'object']): ... print(item) @@ -747,14 +748,16 @@ The module's functions fall into a few broad classes: Creating new iterators ---------------------- -:func:`itertools.count(n) ` returns an infinite stream of -integers, increasing by 1 each time. You can optionally supply the starting -number, which defaults to 0:: +:func:`itertools.count(start, step) ` returns an infinite +stream of evenly spaced values. You can optionally supply the starting number, +which defaults to 0, and the interval between numbers, which defaults to 1:: itertools.count() => 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... itertools.count(10) => 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... + itertools.count(10, 5) => + 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, ... :func:`itertools.cycle(iter) ` saves a copy of the contents of a provided iterable and returns a new iterator that returns its elements from @@ -1060,10 +1063,10 @@ write the obvious :keyword:`for` loop:: for i in [1,2,3]: product *= i -A related function is `itertools.accumulate(iterable, func=operator.add) `. It performs the same calculation, but instead of +returning only the final result, :func:`accumulate` returns an iterator that +also yields each partial result:: itertools.accumulate([1,2,3,4,5]) => 1, 3, 6, 10, 15 @@ -1235,6 +1238,8 @@ Python documentation Documentation for the :mod:`itertools` module. +Documentation for the :mod:`functools` module. + Documentation for the :mod:`operator` module. :pep:`289`: "Generator Expressions" diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index bb79bb1748fb9c..6498ea56957719 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1683,7 +1683,7 @@ Implementing structured logging ------------------------------- Although most logging messages are intended for reading by humans, and thus not -readily machine-parseable, there might be cirumstances where you want to output +readily machine-parseable, there might be circumstances where you want to output messages in a structured format which *is* capable of being parsed by a program (without needing complex regular expressions to parse the log message). This is straightforward to achieve using the logging package. There are a number of diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index a48ae1f5faba7e..9649b9c609c255 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -43,9 +43,9 @@ hold values ranging from 0 to 255. ASCII codes only went up to 127, so some machines assigned values between 128 and 255 to accented characters. Different machines had different codes, however, which led to problems exchanging files. Eventually various commonly used sets of values for the 128--255 range emerged. -Some were true standards, defined by the International Standards Organization, -and some were *de facto* conventions that were invented by one company or -another and managed to catch on. +Some were true standards, defined by the International Organization for +Standardization, and some were *de facto* conventions that were invented by one +company or another and managed to catch on. 255 characters aren't very many. For example, you can't fit both the accented characters used in Western Europe and the Cyrillic alphabet used for Russian diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 18b5c6556be1c9..8d383e03ee8a28 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -34,8 +34,8 @@ handling common situations - like basic authentication, cookies, proxies and so on. These are provided by objects called handlers and openers. urllib.request supports fetching URLs for many "URL schemes" (identified by the string -before the ":" in URL - for example "ftp" is the URL scheme of -"ftp://python.org/") using their associated network protocols (e.g. FTP, HTTP). +before the ``":"`` in URL - for example ``"ftp"`` is the URL scheme of +``"ftp://python.org/"``) using their associated network protocols (e.g. FTP, HTTP). This tutorial focuses on the most common case, HTTP. For straightforward situations *urlopen* is very easy to use. But as soon as you @@ -511,10 +511,10 @@ than the URL you pass to .add_password() will also match. :: ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) -e.g. "http://example.com/" *or* an "authority" (i.e. the hostname, -optionally including the port number) e.g. "example.com" or "example.com:8080" +e.g. ``"http://example.com/"`` *or* an "authority" (i.e. the hostname, +optionally including the port number) e.g. ``"example.com"`` or ``"example.com:8080"`` (the latter example includes a port number). The authority, if present, must -NOT contain the "userinfo" component - for example "joe:password@example.com" is +NOT contain the "userinfo" component - for example ``"joe:password@example.com"`` is not correct. diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py index b853d23b170985..a38a39de3e7c86 100644 --- a/Doc/includes/setup.py +++ b/Doc/includes/setup.py @@ -5,4 +5,5 @@ Extension("noddy2", ["noddy2.c"]), Extension("noddy3", ["noddy3.c"]), Extension("noddy4", ["noddy4.c"]), + Extension("shoddy", ["shoddy.c"]), ]) diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c index 07a272124ceaba..0c6d412b3c4cda 100644 --- a/Doc/includes/shoddy.c +++ b/Doc/includes/shoddy.c @@ -31,7 +31,7 @@ Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) static PyTypeObject ShoddyType = { - PyObject_HEAD_INIT(NULL) + PyVarObject_HEAD_INIT(NULL, 0) "shoddy.Shoddy", /* tp_name */ sizeof(Shoddy), /* tp_basicsize */ 0, /* tp_itemsize */ diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index ace1bfaf8cb9d6..4c9a528d42e703 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -199,13 +199,6 @@ and off individually. They are described here in more detail. because the :class:`memoryview` API is similar but not exactly the same as that of :class:`buffer`. -.. 2to3fixer:: callable - - Converts ``callable(x)`` to ``isinstance(x, collections.Callable)``, adding - an import to :mod:`collections` if needed. Note ``callable(x)`` has returned - in Python 3.2, so if you do not intend to support Python 3.1, you can disable - this fixer. - .. 2to3fixer:: dict Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3fbf51058673d9..3f55506c669a29 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -36,7 +36,7 @@ BaseTransport Base class for transports. - .. method:: close(self) + .. method:: close() Close the transport. If the transport has a buffer for outgoing data, buffered data will be flushed asynchronously. No more data @@ -44,7 +44,7 @@ BaseTransport protocol's :meth:`connection_lost` method will be called with :const:`None` as its argument. - .. method:: is_closing(self) + .. method:: is_closing() Return ``True`` if the transport is closing or is closed. @@ -251,7 +251,7 @@ BaseSubprocessTransport if it hasn't returned, similarly to the :attr:`subprocess.Popen.returncode` attribute. - .. method:: kill(self) + .. method:: kill() Kill the subprocess, as in :meth:`subprocess.Popen.kill`. @@ -384,7 +384,7 @@ The following callbacks are called on :class:`Protocol` instances: .. method:: Protocol.eof_received() - Calls when the other end signals it won't send any more data + Called when the other end signals it won't send any more data (for example by calling :meth:`write_eof`, if the other end also uses asyncio). diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index dc93a74c6dee12..16ba9a3cd6c7f2 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -80,7 +80,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. however, where :class:`~subprocess.Popen` takes a single argument which is list of strings, :func:`subprocess_exec` takes multiple string arguments. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. Other parameters: @@ -123,7 +123,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. using the platform's "shell" syntax. This is similar to the standard library :class:`subprocess.Popen` class called with ``shell=True``. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. See :meth:`~AbstractEventLoop.subprocess_exec` for more details about diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 080d9d77ec984e..ceecf17cba23e3 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -237,14 +237,18 @@ The legacy interface: .. function:: decodebytes(s) - decodestring(s) Decode the :term:`bytes-like object` *s*, which must contain one or more lines of base64 encoded data, and return the decoded :class:`bytes`. - ``decodestring`` is a deprecated alias. .. versionadded:: 3.1 +.. function:: decodestring(s) + + Deprecated alias of :func:`decodebytes`. + + .. deprecated:: 3.1 + .. function:: encode(input, output) @@ -257,14 +261,19 @@ The legacy interface: .. function:: encodebytes(s) - encodestring(s) Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary data, and return :class:`bytes` containing the base64-encoded data, with newlines (``b'\n'``) inserted after every 76 bytes of output, and ensuring that there is a trailing newline, as per :rfc:`2045` (MIME). - ``encodestring`` is a deprecated alias. + .. versionadded:: 3.1 + +.. function:: encodestring(s) + + Deprecated alias of :func:`encodebytes`. + + .. deprecated:: 3.1 An example usage of the module: diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst index 359ab23b2f9787..2966e0dbfbcfe8 100644 --- a/Doc/library/binhex.rst +++ b/Doc/library/binhex.rst @@ -55,5 +55,3 @@ the source for details. If you code or decode textfiles on non-Macintosh platforms they will still use the old Macintosh newline convention (carriage-return as end of line). -As of this writing, :func:`hexbin` appears to not work in all cases. - diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index f40cfdfd592163..3b4a8ff440e7ae 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -266,10 +266,10 @@ immediate playback:: 'Draw circle with given radius an options extent and steps: CIRCLE 50' circle(*parse(arg)) def do_position(self, arg): - 'Print the current turle position: POSITION' + 'Print the current turtle position: POSITION' print('Current position is %d %d\n' % position()) def do_heading(self, arg): - 'Print the current turle heading in degrees: HEADING' + 'Print the current turtle heading in degrees: HEADING' print('Current heading is %d\n' % (heading(),)) def do_color(self, arg): 'Set the color: COLOR BLUE' diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 8e2eb4d9b0382f..2d51f0cfdc094d 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -771,9 +771,9 @@ they add the ability to access fields by name instead of position index. helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` method which lists the tuple contents in a ``name=value`` format. - The *field_names* are a single string with each fieldname separated by whitespace - and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names* - can be a sequence of strings such as ``['x', 'y']``. + The *field_names* are a sequence of strings such as ``['x', 'y']``. + Alternatively, *field_names* can be a single string with each fieldname + separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. Any valid Python identifier may be used for a fieldname except for names starting with an underscore. Valid identifiers consist of letters, digits, diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index e09562dc9046c7..04c2a820921759 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -34,8 +34,8 @@ can be customized by end users easily. .. seealso:: Module :mod:`shlex` - Support for a creating Unix shell-like mini-languages which can be used - as an alternate format for application configuration files. + Support for creating Unix shell-like mini-languages which can be used as + an alternate format for application configuration files. Module :mod:`json` The json module implements a subset of JavaScript syntax which can also @@ -983,13 +983,16 @@ ConfigParser Objects .. method:: read(filenames, encoding=None) Attempt to read and parse a list of filenames, returning a list of - filenames which were successfully parsed. If *filenames* is a string, it - is treated as a single filename. If a file named in *filenames* cannot - be opened, that file will be ignored. This is designed so that you can - specify a list of potential configuration file locations (for example, - the current directory, the user's home directory, and some system-wide - directory), and all existing configuration files in the list will be - read. If none of the named files exist, the :class:`ConfigParser` + filenames which were successfully parsed. + + If *filenames* is a string or :term:`path-like object`, it is treated as + a single filename. If a file named in *filenames* cannot be opened, that + file will be ignored. This is designed so that you can specify a list of + potential configuration file locations (for example, the current + directory, the user's home directory, and some system-wide directory), + and all existing configuration files in the list will be read. + + If none of the named files exist, the :class:`ConfigParser` instance will contain an empty dataset. An application which requires initial values to be loaded from a file should load the required file or files using :meth:`read_file` before calling :meth:`read` for any @@ -1006,6 +1009,9 @@ ConfigParser Objects The *encoding* parameter. Previously, all files were read using the default encoding for :func:`open`. + .. versionadded:: 3.6.1 + The *filenames* parameter accepts a :term:`path-like object`. + .. method:: read_file(f, source=None) diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index f0742cee55bd55..469a3eed606ff0 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -46,7 +46,7 @@ A small number of constants live in the built-in namespace. They are: .. note:: - ``NotImplentedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index d0b861d469bc05..2041d9175ea587 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -47,8 +47,8 @@ copy operations: * Recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop. -* Because deep copy copies *everything* it may copy too much, e.g., - even administrative data structures that should be shared even between copies. +* Because deep copy copies everything it may copy too much, such as data + which is intended to be shared between copies. The :func:`deepcopy` function avoids these problems by: diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 52a8a310ec9d62..43714f7479283d 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -401,8 +401,10 @@ Reader objects (:class:`DictReader` instances and objects returned by the .. method:: csvreader.__next__() - Return the next row of the reader's iterable object as a list, parsed according - to the current dialect. Usually you should call this as ``next(reader)``. + Return the next row of the reader's iterable object as a list (if the object + was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` + instance), parsed according to the current dialect. Usually you should call + this as ``next(reader)``. Reader objects have the following public attributes: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3840935ce04eb6..cdcbefa4e8084a 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -97,7 +97,7 @@ Functions are accessed as attributes of dll objects:: <_FuncPtr object at 0x...> >>> print(windll.kernel32.MyOwnFunction) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 239, in __getattr__ func = _StdcallFuncPtr(name, self) AttributeError: function 'MyOwnFunction' not found @@ -135,7 +135,7 @@ functions can be accessed by indexing the dll object with the ordinal number:: <_FuncPtr object at 0x...> >>> cdll.kernel32[0] # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 310, in __getitem__ func = _StdcallFuncPtr(name, self) AttributeError: function ordinal 0 not found @@ -161,33 +161,25 @@ as the NULL pointer):: 0x1d000000 >>> -:mod:`ctypes` tries to protect you from calling functions with the wrong number -of arguments or the wrong calling convention. Unfortunately this only works on -Windows. It does this by examining the stack after the function returns, so -although an error is raised the function *has* been called:: +.. note:: - >>> windll.kernel32.GetModuleHandleA() # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in ? - ValueError: Procedure probably called with not enough arguments (4 bytes missing) - >>> windll.kernel32.GetModuleHandleA(0, 0) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in ? - ValueError: Procedure probably called with too many arguments (4 bytes in excess) - >>> + :mod:`ctypes` may raise a :exc:`ValueError` after calling the function, if + it detects that an invalid number of arguments were passed. This behavior + should not be relied upon. It is deprecated in 3.6.2, and will be removed + in 3.7. -The same exception is raised when you call an ``stdcall`` function with the +:exc:`ValueError` is raised when you call an ``stdcall`` function with the ``cdecl`` calling convention, or vice versa:: >>> cdll.kernel32.GetModuleHandleA(None) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with not enough arguments (4 bytes missing) >>> >>> windll.msvcrt.printf(b"spam") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with too many arguments (4 bytes in excess) >>> @@ -200,7 +192,7 @@ argument values:: >>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in OSError: exception: access violation reading 0x00000020 >>> @@ -284,7 +276,7 @@ the correct type and value:: >>> c_int() c_long(0) >>> c_wchar_p("Hello, World") - c_wchar_p('Hello, World') + c_wchar_p(140018365411392) >>> c_ushort(-3) c_ushort(65533) >>> @@ -309,11 +301,15 @@ bytes objects are immutable):: >>> s = "Hello, World" >>> c_s = c_wchar_p(s) >>> print(c_s) - c_wchar_p('Hello, World') + c_wchar_p(139966785747344) + >>> print(c_s.value) + Hello World >>> c_s.value = "Hi, there" - >>> print(c_s) - c_wchar_p('Hi, there') - >>> print(s) # first object is unchanged + >>> print(c_s) # the memory location has changed + c_wchar_p(139966783348904) + >>> print(c_s.value) + Hi, there + >>> print(s) # first object is unchanged Hello, World >>> @@ -369,7 +365,7 @@ from within *IDLE* or *PythonWin*:: 19 >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: Don't know how to convert parameter 2 >>> @@ -432,7 +428,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: wrong type >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 @@ -482,7 +478,7 @@ single character Python bytes object into a C char:: 'def' >>> strchr(b"abcdef", b"def") Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: one character string expected >>> print(strchr(b"abcdef", b"x")) None @@ -508,7 +504,7 @@ useful to check for error return values and automatically raise an exception:: 486539264 >>> GetModuleHandle("something silly") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 3, in ValidHandle OSError: [Errno 126] The specified module could not be found. >>> @@ -579,7 +575,7 @@ Here is a simple example of a POINT structure, which contains two integers named 0 5 >>> POINT(1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: too many initializers >>> @@ -782,7 +778,7 @@ new type:: >>> PI(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: expected c_long instead of int >>> PI(c_int(42)) @@ -858,7 +854,7 @@ but not instances of other types:: >>> bar.values = (c_byte * 4)() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: incompatible types, c_byte_Array_4 instance instead of LP_c_long instance >>> @@ -909,7 +905,7 @@ work:: ... ("next", POINTER(cell))] ... Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 2, in cell NameError: name 'cell' is not defined >>> diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index d746eafaea9b5c..3442e4e75a3e45 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -290,8 +290,8 @@ The module :mod:`curses` defines the following functions: .. function:: initscr() - Initialize the library. Return a :class:`WindowObject` which represents the - whole screen. + Initialize the library. Return a :ref:`window ` object + which represents the whole screen. .. note:: @@ -383,8 +383,8 @@ The module :mod:`curses` defines the following functions: .. function:: newwin(nlines, ncols) newwin(nlines, ncols, begin_y, begin_x) - Return a new window, whose left-upper corner is at ``(begin_y, begin_x)``, and - whose height/width is *nlines*/*ncols*. + Return a new :ref:`window `, whose left-upper corner + is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. By default, the window will extend from the specified position to the lower right corner of the screen. @@ -1443,7 +1443,7 @@ The exact keycaps available are system dependent. +-------------------+--------------------------------------------+ | ``KEY_SEOL`` | Shifted Clear line | +-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Dxit | +| ``KEY_SEXIT`` | Shifted Exit | +-------------------+--------------------------------------------+ | ``KEY_SFIND`` | Shifted Find | +-------------------+--------------------------------------------+ @@ -1679,10 +1679,10 @@ You can instantiate a :class:`Textbox` object as follows: .. class:: Textbox(win) Return a textbox widget object. The *win* argument should be a curses - :class:`WindowObject` in which the textbox is to be contained. The edit cursor - of the textbox is initially located at the upper left hand corner of the - containing window, with coordinates ``(0, 0)``. The instance's - :attr:`stripspaces` flag is initially on. + :ref:`window ` object in which the textbox is to + be contained. The edit cursor of the textbox is initially located at the + upper left hand corner of the containing window, with coordinates ``(0, 0)``. + The instance's :attr:`stripspaces` flag is initially on. :class:`Textbox` objects have the following methods: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a15690ba489a66..c795782034a464 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -20,6 +20,10 @@ interpreter. between versions of Python. Use of this module should not be considered to work across Python VMs or Python releases. + .. versionchanged:: 3.6 + Use 2 bytes for each instruction. Previously the number of bytes varied + by instruction. + Example: Given the function :func:`myfunc`:: @@ -210,6 +214,11 @@ operation is being performed, so the intermediate analysis object isn't useful: This generator function uses the ``co_firstlineno`` and ``co_lnotab`` attributes of the code object *code* to find the offsets which are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. + See :source:`Objects/lnotab_notes.txt` for the ``co_lnotab`` format and + how to decode it. + + .. versionchanged:: 3.6 + Line numbers can be decreasing. Before, they were always increasing. .. function:: findlabels(code) @@ -772,8 +781,13 @@ All of the following opcodes use their arguments. .. opcode:: BUILD_MAP (count) - Pushes a new dictionary object onto the stack. The dictionary is pre-sized - to hold *count* entries. + Pushes a new dictionary object onto the stack. Pops ``2 * count`` items + so that the dictionary holds *count* entries: + ``{..., TOS3: TOS2, TOS1: TOS}``. + + .. versionchanged:: 3.5 + The dictionary is created from stack items instead of creating an + empty dictionary pre-sized to hold *count* items. .. opcode:: BUILD_CONST_KEY_MAP (count) @@ -793,6 +807,63 @@ All of the following opcodes use their arguments. .. versionadded:: 3.6 +.. opcode:: BUILD_TUPLE_UNPACK (count) + + Pops *count* iterables from the stack, joins them in a single tuple, + and pushes the result. Implements iterable unpacking in tuple + displays ``(*x, *y, *z)``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, + but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position + ``count + 1`` should be the corresponding callable ``f``. + + .. versionadded:: 3.6 + + +.. opcode:: BUILD_LIST_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list + instead of tuple. Implements iterable unpacking in list + displays ``[*x, *y, *z]``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_SET_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set + instead of tuple. Implements iterable unpacking in set + displays ``{*x, *y, *z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK (count) + + Pops *count* mappings from the stack, merges them into a single dictionary, + and pushes the result. Implements dictionary unpacking in dictionary + displays ``{**x, **y, **z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_MAP_UNPACK`, + but is used for ``f(**x, **y, **z)`` call syntax. The stack item at + position ``count + 2`` should be the corresponding callable ``f``. + + .. versionadded:: 3.5 + .. versionchanged:: 3.6 + The position of the callable is determined by adding 2 to the opcode + argument instead of encoding it in the second byte of the argument. + + .. opcode:: LOAD_ATTR (namei) Replaces TOS with ``getattr(TOS, co_names[namei])``. @@ -947,14 +1018,45 @@ All of the following opcodes use their arguments. .. opcode:: CALL_FUNCTION (argc) - Calls a function. The low byte of *argc* indicates the number of positional - parameters, the high byte the number of keyword parameters. On the stack, the - opcode finds the keyword parameters first. For each keyword argument, the - value is on top of the key. Below the keyword parameters, the positional - parameters are on the stack, with the right-most parameter on top. Below the - parameters, the function object to call is on the stack. Pops all function - arguments, and the function itself off the stack, and pushes the return - value. + Calls a function. *argc* indicates the number of positional arguments. + The positional arguments are on the stack, with the right-most argument + on top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + This opcode is used only for calls with positional arguments. + + +.. opcode:: CALL_FUNCTION_KW (argc) + + Calls a function. *argc* indicates the number of arguments (positional + and keyword). The top element on the stack contains a tuple of keyword + argument names. Below the tuple, keyword arguments are on the stack, in + the order corresponding to the tuple. Below the keyword arguments, the + positional arguments are on the stack, with the right-most parameter on + top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + Keyword arguments are packed in a tuple instead of a dictionary, + *argc* indicates the total number of arguments + + +.. opcode:: CALL_FUNCTION_EX (flags) + + Calls a function. The lowest bit of *flags* indicates whether the + var-keyword argument is placed at the top of the stack. Below the + var-keyword argument, the var-positional argument is on the stack. + Below the arguments, the function object to call is placed. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. Note that this opcode pops at most three items + from the stack. Var-positional and var-keyword arguments are packed + by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. + + .. versionadded:: 3.6 .. opcode:: MAKE_FUNCTION (argc) @@ -987,28 +1089,6 @@ All of the following opcodes use their arguments. two most-significant bytes. -.. opcode:: CALL_FUNCTION_VAR (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the variable argument list, followed by - keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by explicit keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_VAR_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by the variable-arguments tuple, followed by explicit keyword and positional - arguments. - - .. opcode:: FORMAT_VALUE (flags) Used for implementing formatted literal strings (f-strings). Pops @@ -1034,8 +1114,13 @@ All of the following opcodes use their arguments. .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - opcodes which don't take arguments ``< HAVE_ARGUMENT`` and those which do - ``>= HAVE_ARGUMENT``. + opcodes which don't use their argument and those that do + (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + + .. versionchanged:: 3.6 + Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. + .. _opcode_collections: diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 15b12f7aa786ea..587a0a09a94791 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -408,7 +408,7 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: list.remove(x): x not in list That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): @@ -432,7 +432,7 @@ multi-line detail:: >>> raise ValueError('multi\n line\ndetail') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: multi line detail @@ -591,7 +591,7 @@ doctest decides whether actual output matches an example's expected output: >>> (1, 2)[3] = 'moo' Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: object doesn't support item assignment passes under Python 2.3 and later Python versions with the flag specified, diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 2c65079ebdc3f1..7e11782face074 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -33,11 +33,11 @@ having a MIME type such as :mimetype:`multipart/\*` or The conceptual model provided by a :class:`Message` object is that of an ordered dictionary of headers with additional methods for accessing both specialized information from the headers, for accessing the payload, for -generating a serialized version of the mssage, and for recursively walking over -the object tree. Note that duplicate headers are supported but special methods -must be used to access them. +generating a serialized version of the message, and for recursively walking +over the object tree. Note that duplicate headers are supported but special +methods must be used to access them. -The :class:`Message` psuedo-dictionary is indexed by the header names, which +The :class:`Message` pseudo-dictionary is indexed by the header names, which must be ASCII values. The values of the dictionary are strings that are supposed to contain only ASCII characters; there is some special handling for non-ASCII input, but it doesn't always produce the correct results. Headers @@ -67,7 +67,7 @@ Here are the methods of the :class:`Message` class: Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. For backward compabitility reasons, + *unixfrom* defaults to ``False``. For backward compatibility reasons, *maxheaderlen* defaults to ``0``, so if you want a different value you must override it explicitly (the value specified for *max_line_length* in the policy will be ignored by this method). The *policy* argument may be @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: This is a legacy method. On the :class:`~email.emailmessage.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the - realted ``make`` and ``add`` methods. + related ``make`` and ``add`` methods. .. method:: get_payload(i=None, decode=False) diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 57743d5a29f1dd..f56836ae2d2781 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -157,7 +157,7 @@ Currently the email package provides only one concrete content manager, MIME charset name, use the standard charset instead. If *cte* is set, encode the payload using the specified content transfer - encoding, and set the :mailheader:`Content-Transfer-Endcoding` header to + encoding, and set the :mailheader:`Content-Transfer-Encoding` header to that value. Possible values for *cte* are ``quoted-printable``, ``base64``, ``7bit``, ``8bit``, and ``binary``. If the input cannot be encoded in the specified encoding (for example, specifying a *cte* of @@ -203,5 +203,5 @@ Currently the email package provides only one concrete content manager, .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module ` diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 2d0d1923cd25c4..5838767b18f742 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -102,9 +102,9 @@ All defect classes are subclassed from :class:`email.errors.MessageDefect`. return false even though its content type claims to be :mimetype:`multipart`. * :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 - enocded bytes, the padding was not correct. Enough padding is added to + encoded bytes, the padding was not correct. Enough padding is added to perform the decode, but the resulting decoded bytes may be invalid. * :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 - enocded bytes, characters outside the base64 alphebet were encountered. + encoded bytes, characters outside the base64 alphabet were encountered. The characters are ignored, but the resulting decoded bytes may be invalid. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index ab0fbc29d1ec85..1e64e1066c7da5 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -88,8 +88,8 @@ over channels that are not "8 bit clean". If ``cte_type`` is ``7bit``, convert the bytes with the high bit set as needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, transform parts with non-ASCII - :mailheader:`Cotnent-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatibile + :mailheader:`Content-Transfer-Encoding` + (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII bytes in headers using the MIME ``unknown-8bit`` character set, thus rendering them RFC-compliant. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 2c830cfd81fb47..ce283c6b596cf2 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -451,5 +451,5 @@ construct structured values to assign to specific headers. .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional module ` diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 32852e706986b4..261d0d62cfe618 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -52,7 +52,7 @@ message objects. .. class:: EmailMessage(policy=default) - If *policy* is specified use the rules it specifies to udpate and serialize + If *policy* is specified use the rules it specifies to update and serialize the representation of the message. If *policy* is not set, use the :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses @@ -63,7 +63,7 @@ message objects. Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. For backward compabitility + string. *unixfrom* defaults to ``False``. For backward compatibility with the base :class:`~email.message.Message` class *maxheaderlen* is accepted, but defaults to ``None``, which means that by default the line length is controlled by the @@ -213,7 +213,7 @@ message objects. del msg['subject'] msg['subject'] = 'Python roolz!' - If the :mod:`policy` defines certain haders to be unique (as the standard + If the :mod:`policy` defines certain headers to be unique (as the standard policies do), this method may raise a :exc:`ValueError` when an attempt is made to assign a value to such a header when one already exists. This behavior is intentional for consistency's sake, but do not depend on it @@ -364,7 +364,7 @@ message objects. *header* specifies an alternative header to :mailheader:`Content-Type`. If the value contains non-ASCII characters, the charset and language may - be explicity specified using the optional *charset* and *language* + be explicitly specified using the optional *charset* and *language* parameters. Optional *language* specifies the :rfc:`2231` language, defaulting to the empty string. Both *charset* and *language* should be strings. The default is to use the ``utf8`` *charset* and ``None`` for @@ -558,7 +558,7 @@ message objects. the part a candidate match if the value of the header is ``inline``. If none of the candidates matches any of the preferences in - *preferneclist*, return ``None``. + *preferencelist*, return ``None``. Notes: (1) For most applications the only *preferencelist* combinations that really make sense are ``('plain',)``, ``('html', 'plain')``, and the @@ -746,6 +746,6 @@ message objects. .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index d9dae9f0b993c4..f37f6aa28dec7d 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -242,7 +242,7 @@ Here are the classes: Unless the *_charset* argument is explicitly set to ``None``, the MIMEText object created will have both a :mailheader:`Content-Type` header - with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Endcoding` + with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Encoding` header. This means that a subsequent ``set_payload`` call will not result in an encoded payload, even if a charset is passed in the ``set_payload`` command. You can "reset" this behavior by deleting the diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index c323ebc6401b99..dea409d223da4c 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -139,7 +139,7 @@ message body, instead setting the payload to the raw body. .. class:: BytesParser(_class=None, *, policy=policy.compat32) Create a :class:`BytesParser` instance. The *_class* and *policy* - arguments have the same meaning and sematnics as the *_factory* + arguments have the same meaning and semantics as the *_factory* and *policy* arguments of :class:`BytesFeedParser`. Note: **The policy keyword should always be specified**; The default will diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8a418778b8e0fc..8e7076259810f5 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -276,7 +276,7 @@ added matters. To illustrate:: Called when a header is added to an :class:`~email.message.EmailMessage` or :class:`~email.message.Message` object. If the returned value is not ``0`` or ``None``, and there are already a number of headers with the - name *name* greather than or equal to the value returned, a + name *name* greater than or equal to the value returned, a :exc:`ValueError` is raised. Because the default behavior of ``Message.__setitem__`` is to append the @@ -533,7 +533,7 @@ more closely to the RFCs relevant to their domains. The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``. Useful for serializing messages to a message store without using encoded - words in the headers. Should only be used for SMTP trasmission if the + words in the headers. Should only be used for SMTP transmission if the sender or recipient addresses have non-ASCII characters (the :meth:`smtplib.SMTP.send_message` method handles this automatically). @@ -647,5 +647,5 @@ The header objects and their attributes are described in .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional feature `. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 5cd6472f3e2f45..6548adf789da17 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -24,8 +24,8 @@ Module Contents --------------- This module defines four enumeration classes that can be used to define unique -sets of names and values: :class:`Enum`, :class:`IntEnum`, and -:class:`IntFlags`. It also defines one decorator, :func:`unique`, and one +sets of names and values: :class:`Enum`, :class:`IntEnum`, :class:`Flag`, and +:class:`IntFlag`. It also defines one decorator, :func:`unique`, and one helper, :class:`auto`. .. class:: Enum diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index a428f5165fc8d2..a6b20a5ac95b1b 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -243,7 +243,7 @@ The following exceptions are the exceptions that are usually raised. .. note:: - It should not be used to indicate that an operater or method is not + It should not be used to indicate that an operator or method is not meant to be supported at all -- in that case either leave the operator / method undefined or, if a subclass, set it to :data:`None`. diff --git a/Doc/library/fpectl.rst b/Doc/library/fpectl.rst index e4b528cf0b0b6f..96607165ba4e3e 100644 --- a/Doc/library/fpectl.rst +++ b/Doc/library/fpectl.rst @@ -89,7 +89,7 @@ The following example demonstrates how to start up and test operation of the >>> import math >>> math.exp(1000) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in FloatingPointError: in math_1 diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index b8c1dcfef2d98e..7291dfe84811c6 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -235,7 +235,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Retrieve a file in binary transfer mode. *cmd* should be an appropriate ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single string argument giving the data + each block of data received, with a single bytes argument giving the data block. The optional *blocksize* argument specifies the maximum chunk size to read on the low-level socket object created to do the actual transfer (which will also be the largest size of the data blocks passed to *callback*). A @@ -255,9 +255,9 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. prints the line to ``sys.stdout``. -.. method:: FTP.set_pasv(boolean) +.. method:: FTP.set_pasv(val) - Enable "passive" mode if *boolean* is true, other disable passive mode. + Enable "passive" mode if *val* is true, otherwise disable passive mode. Passive mode is on by default. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c26037bd9eef19..6621f4aabf7058 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -16,8 +16,8 @@ are always available. They are listed here in alphabetical order. :func:`ascii` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` :func:`bin` :func:`eval` :func:`int` :func:`open` |func-str|_ :func:`bool` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` -:func:`bytearray` :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -:func:`bytes` :func:`float` :func:`iter` :func:`print` |func-tuple|_ +|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` +|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ :func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` :func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` :func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` @@ -37,7 +37,8 @@ are always available. They are listed here in alphabetical order. .. |func-str| replace:: ``str()`` .. |func-tuple| replace:: ``tuple()`` .. |func-range| replace:: ``range()`` - +.. |func-bytearray| replace:: ``bytearray()`` +.. |func-bytes| replace:: ``bytes()`` .. function:: abs(x) @@ -99,6 +100,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytearray: .. class:: bytearray([source[, encoding[, errors]]]) + :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable sequence of integers in the range 0 <= x < 256. It has most of the usual @@ -128,6 +130,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: .. class:: bytes([source[, encoding[, errors]]]) + :noindex: Return a new "bytes" object, which is an immutable sequence of integers in the range ``0 <= x < 256``. :class:`bytes` is an immutable version of @@ -610,7 +613,7 @@ are always available. They are listed here in alphabetical order. .. note:: - For object's with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`__hash__` methods, note that :func:`hash` truncates the return value based on the bit width of the host machine. See :meth:`__hash__` for details. @@ -1072,7 +1075,7 @@ are always available. They are listed here in alphabetical order. * The ``'x'`` mode was added. * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. * :exc:`FileExistsError` is now raised if the file opened in exclusive - * creation mode (``'x'``) already exists. + creation mode (``'x'``) already exists. .. versionchanged:: 3.4 @@ -1125,7 +1128,7 @@ are always available. They are listed here in alphabetical order. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end* and *file*, if present, must be given as keyword + by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword arguments. All non-keyword arguments are converted to strings like :func:`str` does and @@ -1254,17 +1257,21 @@ are always available. They are listed here in alphabetical order. .. function:: round(number[, ndigits]) - Return the floating point value *number* rounded to *ndigits* digits after - the decimal point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. Delegates to ``number.__round__(ndigits)``. + Return *number* rounded to *ndigits* precision after the decimal + point. If *ndigits* is omitted or is ``None``, it returns the + nearest integer to its input. For the built-in types supporting :func:`round`, values are rounded to the closest multiple of 10 to the power minus *ndigits*; if two multiples are equally close, rounding is done toward the even choice (so, for example, both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). The return value is an integer if called with one argument, + ``2``). Any integer value is valid for *ndigits* (positive, zero, or + negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. + For a general Python object ``number``, ``round(number, ndigits)`` delegates to + ``number.__round__(ndigits)``. + .. note:: The behavior of :func:`round` for floats can be surprising: for example, diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index fb5c1df611d8f2..b29020bc7ca5ca 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -105,7 +105,8 @@ of which this module provides three different variants: Contains the output stream for writing a response back to the client. Proper adherence to the HTTP protocol must be used when writing to - this stream. + this stream in order to achieve successful interoperation with HTTP + clients. .. versionchanged:: 3.6 This is an :class:`io.BufferedIOBase` stream. diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index a629bc50dbc749..1a2dac0233c4d2 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -244,7 +244,7 @@ Go to File/Line single: stack viewer Debugger (toggle) - When actived, code entered in the Shell or run from an Editor will run + When activated, code entered in the Shell or run from an Editor will run under the debugger. In the Editor, breakpoints can be set with the context menu. This feature is still incomplete and somewhat experimental. @@ -372,7 +372,7 @@ the :kbd:`Command` key on Mac OSX. * :kbd:`C-l` center window around the insertion point - * :kbd:`C-b` go backwards one character without deleting (usually you can + * :kbd:`C-b` go backward one character without deleting (usually you can also use the cursor key for this) * :kbd:`C-f` go forward one character without deleting (usually you can @@ -394,7 +394,7 @@ After a block-opening statement, the next line is indented by 4 spaces (in the Python Shell window by one tab). After certain keywords (break, return etc.) the next line is dedented. In leading indentation, :kbd:`Backspace` deletes up to 4 spaces if they are there. :kbd:`Tab` inserts spaces (in the Python -Shell window one tab), number depends on Indent width. Currently tabs +Shell window one tab), number depends on Indent width. Currently, tabs are restricted to four spaces due to Tcl/Tk limitations. See also the indent/dedent region commands in the edit menu. @@ -418,7 +418,7 @@ If there is only one possible completion for the characters entered, a :kbd:`C-space` will open a completions window. In an empty string, this will contain the files in the current directory. On a blank line, it will contain the built-in and user-defined functions and -classes in the current name spaces, plus any modules imported. If some +classes in the current namespaces, plus any modules imported. If some characters have been entered, the ACW will attempt to be more specific. If a string of characters is typed, the ACW selection will jump to the @@ -446,7 +446,7 @@ longer or disable the extension. Calltips ^^^^^^^^ -A calltip is shown when one types :kbd:`(` after the name of an *acccessible* +A calltip is shown when one types :kbd:`(` after the name of an *accessible* function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, or :kbd:`)` is typed. When the cursor is in the argument part of a definition, @@ -557,7 +557,7 @@ IDLE-console differences As much as possible, the result of executing Python code with IDLE is the same as executing the same code in a console window. However, the different -interface and operation occasionally affects visible results. For instance, +interface and operation occasionally affect visible results. For instance, ``sys.modules`` starts with more entries. IDLE also replaces ``sys.stdin``, ``sys.stdout``, and ``sys.stderr`` with @@ -583,7 +583,7 @@ If firewall software complains anyway, you can ignore it. If the attempt to make the socket connection fails, Idle will notify you. Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of +may be either a firewall blocking the connection or misconfiguration of a particular system. Until the problem is fixed, one can run Idle with the -n command line switch. @@ -619,14 +619,14 @@ Setting preferences The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a +IDLE ships with four built-in key sets. In addition, a user can create a custom key set in the Configure IDLE dialog under the keys tab. Extensions ^^^^^^^^^^ -IDLE contains an extension facility. Peferences for extensions can be +IDLE contains an extension facility. Preferences for extensions can be changed with Configure Extensions. See the beginning of config-extensions.def in the idlelib directory for further information. The default extensions are currently: diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 41a784d982d6c2..6be28a2b31cb66 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -34,185 +34,198 @@ provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special attributes: -+-----------+-----------------+---------------------------+ -| Type | Attribute | Description | -+===========+=================+===========================+ -| module | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-----------------+---------------------------+ -| class | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-----------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-----------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-----------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-----------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-----------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-----------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-----------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-----------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-----------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-----------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-----------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_restricted | 0 or 1 if frame is in | -| | | restricted execution mode | -+-----------+-----------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-----------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including \* or \*\* | -| | | args) | -+-----------+-----------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-----------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-----------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-----------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-----------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-----------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-----------------+---------------------------+ -| | co_names | tuple of names of local | -| | | variables | -+-----------+-----------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-----------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-----------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-----------------+---------------------------+ -| generator | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | gi_frame | frame | -+-----------+-----------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-----------------+---------------------------+ -| | gi_code | code | -+-----------+-----------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-----------------+---------------------------+ -| | cr_frame | frame | -+-----------+-----------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-----------------+---------------------------+ -| | cr_code | code | -+-----------+-----------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ ++-----------+-------------------+---------------------------+ +| Type | Attribute | Description | ++===========+===================+===========================+ +| module | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __file__ | filename (missing for | +| | | built-in modules) | ++-----------+-------------------+---------------------------+ +| class | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_restricted | 0 or 1 if frame is in | +| | | restricted execution mode | ++-----------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------+-------------------+---------------------------+ +| | co_names | tuple of names of local | +| | | variables | ++-----------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------+-------------------+---------------------------+ +| | gi_code | code | ++-----------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------+-------------------+---------------------------+ +| | cr_code | code | ++-----------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -442,7 +455,9 @@ Retrieving source code Return in a single string any lines of comments immediately preceding the object's source code (for a class, function, or method), or at the top of the - Python source file (if the object is a module). + Python source file (if the object is a module). If the object's source code + is unavailable, return ``None``. This could happen if the object has been + defined in C or the interactive shell. .. function:: getfile(object) @@ -905,10 +920,8 @@ Classes and functions are the names of the ``*`` and ``**`` arguments or ``None``. *locals* is the locals dictionary of the given frame. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) @@ -944,10 +957,8 @@ Classes and functions :func:`getargvalues`. The format\* arguments are the corresponding optional formatting functions that are called to turn names and values into strings. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: getmro(cls) @@ -1270,6 +1281,10 @@ Code Objects Bit Flags Python code objects have a ``co_flags`` attribute, which is a bitmap of the following flags: +.. data:: CO_OPTIMIZED + + The code object is optimized, using fast locals. + .. data:: CO_NEWLOCALS If set, a new dict will be created for the frame's ``f_locals`` when @@ -1283,6 +1298,10 @@ the following flags: The code object has a variable keyword parameter (``**kwargs``-like). +.. data:: CO_NESTED + + The flag is set when the code object is a nested function. + .. data:: CO_GENERATOR The flag is set when the code object is a generator function, i.e. diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8103c614aaf450..fdbdcb169fab09 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -503,7 +503,7 @@ Encoders and Decoders Exceptions ---------- -.. exception:: JSONDecodeError(msg, doc, pos, end=None) +.. exception:: JSONDecodeError(msg, doc, pos) Subclass of :exc:`ValueError` with the following additional attributes: diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 5edb23de83bcff..cce6c23e611e31 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -39,7 +39,7 @@ Reading and writing compressed files object`. The *filename* argument can be either an actual file name (given as a - :class:`str`, :class:`bytes` or :term:`path-like object` object), in + :class:`str`, :class:`bytes` or :term:`path-like ` object), in which case the named file is opened, or it can be an existing file object to read from or write to. @@ -76,7 +76,7 @@ Reading and writing compressed files An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate directly on a named file. The *filename* argument specifies either the file object to wrap, or the name of the file to open (as a :class:`str`, - :class:`bytes` or :term:`path-like object` object). When wrapping an + :class:`bytes` or :term:`path-like ` object). When wrapping an existing file object, the wrapped file will not be closed when the :class:`LZMAFile` is closed. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 1ffc6effc7142d..d65afc20041133 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -49,7 +49,7 @@ For format *version* lower than 3, recursive lists, sets and dictionaries cannot be written (see below). There are functions that read/write files as well as functions operating on -strings. +bytes-like objects. The module defines these functions: @@ -57,9 +57,7 @@ The module defines these functions: .. function:: dump(value, file[, version]) Write the value on the open file. The value must be a supported type. The - file must be an open file object such as ``sys.stdout`` or returned by - :func:`open` or :func:`os.popen`. It must be opened in binary mode (``'wb'`` - or ``'w+b'``). + file must be a writeable :term:`binary file`. If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written @@ -74,8 +72,7 @@ The module defines these functions: Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be an open file object opened in binary mode (``'rb'`` or - ``'r+b'``). + file must be a readable :term:`binary file`. .. note:: @@ -85,7 +82,7 @@ The module defines these functions: .. function:: dumps(value[, version]) - Return the string that would be written to a file by ``dump(value, file)``. The + Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. @@ -93,11 +90,11 @@ The module defines these functions: (see below). -.. function:: loads(string) +.. function:: loads(bytes) - Convert the string to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra characters in the - string are ignored. + Convert the :term:`bytes-like object` to a value. If no valid value is found, raise + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the + input are ignored. In addition, the following constants are defined: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 464248c3ea7990..67b7a7178534d3 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -186,78 +186,78 @@ than one MIME-type database; it provides an interface similar to the one of the loaded "on top" of the default database. -.. attribute:: MimeTypes.suffix_map + .. attribute:: MimeTypes.suffix_map - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. + Dictionary mapping suffixes to suffixes. This is used to allow recognition of + encoded files for which the encoding and the type are indicated by the same + extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` + to allow the encoding and type to be recognized separately. This is initially a + copy of the global :data:`suffix_map` defined in the module. -.. attribute:: MimeTypes.encodings_map + .. attribute:: MimeTypes.encodings_map - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. + Dictionary mapping filename extensions to encoding types. This is initially a + copy of the global :data:`encodings_map` defined in the module. -.. attribute:: MimeTypes.types_map + .. attribute:: MimeTypes.types_map - Tuple containing two dictionaries, mapping filename extensions to MIME types: - the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. + Tuple containing two dictionaries, mapping filename extensions to MIME types: + the first dictionary is for the non-standards types and the second one is for + the standard types. They are initialized by :data:`common_types` and + :data:`types_map`. -.. attribute:: MimeTypes.types_map_inv + .. attribute:: MimeTypes.types_map_inv - Tuple containing two dictionaries, mapping MIME types to a list of filename - extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. + Tuple containing two dictionaries, mapping MIME types to a list of filename + extensions: the first dictionary is for the non-standards types and the + second one is for the standard types. They are initialized by + :data:`common_types` and :data:`types_map`. -.. method:: MimeTypes.guess_extension(type, strict=True) + .. method:: MimeTypes.guess_extension(type, strict=True) - Similar to the :func:`guess_extension` function, using the tables stored as part - of the object. + Similar to the :func:`guess_extension` function, using the tables stored as part + of the object. -.. method:: MimeTypes.guess_type(url, strict=True) + .. method:: MimeTypes.guess_type(url, strict=True) - Similar to the :func:`guess_type` function, using the tables stored as part of - the object. + Similar to the :func:`guess_type` function, using the tables stored as part of + the object. -.. method:: MimeTypes.guess_all_extensions(type, strict=True) + .. method:: MimeTypes.guess_all_extensions(type, strict=True) - Similar to the :func:`guess_all_extensions` function, using the tables stored - as part of the object. + Similar to the :func:`guess_all_extensions` function, using the tables stored + as part of the object. -.. method:: MimeTypes.read(filename, strict=True) + .. method:: MimeTypes.read(filename, strict=True) - Load MIME information from a file named *filename*. This uses :meth:`readfp` to - parse the file. + Load MIME information from a file named *filename*. This uses :meth:`readfp` to + parse the file. - If *strict* is ``True``, information will be added to list of standard types, - else to the list of non-standard types. + If *strict* is ``True``, information will be added to list of standard types, + else to the list of non-standard types. -.. method:: MimeTypes.readfp(fp, strict=True) + .. method:: MimeTypes.readfp(fp, strict=True) - Load MIME type information from an open file *fp*. The file must have the format of - the standard :file:`mime.types` files. + Load MIME type information from an open file *fp*. The file must have the format of + the standard :file:`mime.types` files. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. -.. method:: MimeTypes.read_windows_registry(strict=True) + .. method:: MimeTypes.read_windows_registry(strict=True) - Load MIME type information from the Windows registry. Availability: Windows. + Load MIME type information from the Windows registry. Availability: Windows. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. - .. versionadded:: 3.2 + .. versionadded:: 3.2 diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 37fa2a2868dac9..974ab2d481e210 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2859,7 +2859,7 @@ These functions are all available on Linux only. :ref:`not following symlinks `. .. versionchanged:: 3.6 - Accepts a :term:`path-like object` fpr *path* and *attribute*. + Accepts a :term:`path-like object` for *path* and *attribute*. .. function:: listxattr(path=None, *, follow_symlinks=True) diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index ee1ce50b6bd5e1..d40abe167653b7 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -9,14 +9,15 @@ available for Python: .. seealso:: `PyGObject `_ - provides introspection bindings for C libraries using + PyGObject provides introspection bindings for C libraries using `GObject `_. One of these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online `Python GTK+ 3 Tutorial `_ is available. - `PyGTK `_ provides bindings for an older version + `PyGTK `_ + PyGTK provides bindings for an older version of the library, GTK+ 2. It provides an object oriented interface that is slightly higher level than the C one. There are also bindings to `GNOME `_. An online `tutorial @@ -27,15 +28,10 @@ available for Python: extensive C++ GUI application development framework that is available for Unix, Windows and Mac OS X. :program:`sip` is a tool for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. The *PyQt3* bindings have a - book, `GUI Programming with Python: QT Edition - `_ by Boudewijn - Rempt. The *PyQt4* bindings also have a book, `Rapid GUI Programming - with Python and Qt `_, by Mark - Summerfield. + is specifically designed for Python. `PySide `_ - is a newer binding to the Qt toolkit, provided by Nokia. + PySide is a newer binding to the Qt toolkit, provided by Nokia. Compared to PyQt, its licensing scheme is friendlier to non-open source applications. @@ -49,9 +45,7 @@ available for Python: documentation and context sensitive help, printing, HTML viewing, low-level device context drawing, drag and drop, system clipboard access, an XML-based resource format and more, including an ever growing library - of user-contributed modules. wxPython has a book, `wxPython in Action - `_, by Noel Rappin and - Robin Dunn. + of user-contributed modules. PyGTK, PyQt, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7c37bb7d248c91..6225a3a1f07996 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -76,7 +76,7 @@ The typical usage to inspect a crashed program is:: >>> import mymodule >>> mymodule.test() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "./mymodule.py", line 4, in test test2() File "./mymodule.py", line 3, in test2 diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 959d9b98a84691..5796e3acb6a797 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -85,11 +85,11 @@ next line: ``Ordered by: standard name``, indicates that the text string in the far right column was used to sort the output. The column headings include: ncalls - for the number of calls, + for the number of calls. tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) + for the total time spent in the given function (and excluding time made in + calls to sub-functions) percall is the quotient of ``tottime`` divided by ``ncalls`` @@ -215,11 +215,11 @@ functions: and gathers profiling statistics from the execution. If no file name is present, then this function automatically creates a :class:`~pstats.Stats` - instance and prints a simple profiling report. If the sort value is specified + instance and prints a simple profiling report. If the sort value is specified, it is passed to this :class:`~pstats.Stats` instance to control how the results are sorted. -.. function:: runctx(command, globals, locals, filename=None) +.. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the globals and locals dictionaries for the *command* string. This routine @@ -444,9 +444,10 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. significant entries. Initially, the list is taken to be the complete set of profiled functions. Each restriction is either an integer (to select a count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a regular expression (to pattern match - the standard name that is printed. If several restrictions are provided, - then they are applied sequentially. For example:: + select a percentage of lines), or a string that will interpreted as a + regular expression (to pattern match the standard name that is printed). + If several restrictions are provided, then they are applied sequentially. + For example:: print_stats(.1, 'foo:') diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 075a8b5139b196..e43b9aecd86835 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -869,7 +869,7 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index fe5ebcc67c840b..5ae304d25e1399 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -42,6 +42,12 @@ module-level functions and methods on that don't require you to compile a regex object first, but miss some fine-tuning parameters. +.. seealso:: + + The third-party `regex `_ module, + which has an API compatible with the standard library :mod:`re` module, + but offers additional functionality and a more thorough Unicode support. + .. _re-syntax: @@ -778,11 +784,22 @@ form. Unmatched groups are replaced with an empty string. -.. function:: escape(string) +.. function:: escape(pattern) - Escape all the characters in pattern except ASCII letters, numbers and ``'_'``. + Escape all the characters in *pattern* except ASCII letters, numbers and ``'_'``. This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. + have regular expression metacharacters in it. For example:: + + >>> print(re.escape('python.exe')) + python\.exe + + >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" + >>> print('[%s]+' % re.escape(legal_chars)) + [abcdefghijklmnopqrstuvwxyz0123456789\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\:]+ + + >>> operators = ['+', '-', '*', '/', '**'] + >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) + \/|\-|\+|\*\*|\* .. versionchanged:: 3.3 The ``'_'`` character is no longer escaped. @@ -811,15 +828,15 @@ form. .. attribute:: pos - The index of *pattern* where compilation failed. + The index in *pattern* where compilation failed (may be ``None``). .. attribute:: lineno - The line corresponding to *pos*. + The line corresponding to *pos* (may be ``None``). .. attribute:: colno - The column corresponding to *pos*. + The column corresponding to *pos* (may be ``None``). .. versionchanged:: 3.5 Added additional attributes. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f97118ebe05788..bd5442c6a27aa0 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -290,7 +290,7 @@ Edge and Level Trigger Polling (epoll) Objects | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | | | associated fd has an event. The default (if | | | this flag is not set) is to wake all epoll | - | | objects polling on on a fd. | + | | objects polling on a fd. | +-------------------------+-----------------------------------------------+ | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | | | down writing half of connection. | diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 1624d88aaed38d..6d864a836de075 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -68,7 +68,7 @@ constants below: .. class:: SelectorKey A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to - associate a file object to its underlying file decriptor, selected event + associate a file object to its underlying file descriptor, selected event mask and attached data. It is returned by several :class:`BaseSelector` methods. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index a85bf339ae1bc8..41e5bafa53d468 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -153,7 +153,7 @@ Directory and files operations is true and *src* is a symbolic link, *dst* will be a copy of the file *src* refers to. - :func:`copy` copies the file data and the file's permission + :func:`~shutil.copy` copies the file data and the file's permission mode (see :func:`os.chmod`). Other metadata, like the file's creation and modification times, is not preserved. To preserve all file metadata from the original, use @@ -302,7 +302,7 @@ Directory and files operations *src* and *dst*, and will be used to copy *src* to *dest* if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`copy` as the + default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index e383201aab1461..85ee8a75cf77a9 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -13,6 +13,12 @@ This module offers several classes to implement SMTP (email) servers. +.. seealso:: + + The `aiosmtpd `_ package is a recommended + replacement for this module. It is based on :mod:`asyncio` and provides a + more straightforward API. :mod:`smtpd` should be considered deprecated. + Several server implementations are present; one is a generic do-nothing implementation, which can be overridden, while the other two offer specific mail-sending strategies. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5635577da0f946..9fef7d7f03f1f0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -640,6 +640,11 @@ Cursor Objects .. versionchanged:: 3.6 Added support for the ``REPLACE`` statement. + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + .. attribute:: description This read-only attribute provides the column names of the last query. To @@ -927,14 +932,6 @@ By default, the :mod:`sqlite3` module opens transactions implicitly before a Data Modification Language (DML) statement (i.e. ``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). -So if you are within a transaction and issue a command like ``CREATE TABLE -...``, ``VACUUM``, ``PRAGMA``, the :mod:`sqlite3` module will commit implicitly -before executing that command. There are two reasons for doing that. The first -is that some of these commands don't work within transactions. The other reason -is that sqlite3 needs to keep track of the transaction state (if a transaction -is active or not). The current transaction state is exposed through the -:attr:`Connection.in_transaction` attribute of the connection object. - You can control which kind of ``BEGIN`` statements sqlite3 implicitly executes (or none at all) via the *isolation_level* parameter to the :func:`connect` call, or via the :attr:`isolation_level` property of connections. @@ -945,6 +942,9 @@ Otherwise leave it at its default, which will result in a plain "BEGIN" statement, or set it to one of SQLite's supported isolation levels: "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". +The current transaction state is exposed through the +:attr:`Connection.in_transaction` attribute of the connection object. + .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL statements. This is no longer the case. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 07b97715a985b5..0ce73c14095480 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -610,13 +610,13 @@ Constants .. data:: PROTOCOL_TLS Selects the highest protocol version that both the client and server support. - Despite the name, this option can select "TLS" protocols as well as "SSL". + Despite the name, this option can select both "SSL" and "TLS" protocols. .. versionadded:: 3.6 .. data:: PROTOCOL_TLS_CLIENT - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support client-side :class:`SSLSocket` connections. The protocol enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by default. @@ -625,7 +625,7 @@ Constants .. data:: PROTOCOL_TLS_SERVER - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support server-side :class:`SSLSocket` connections. .. versionadded:: 3.6 @@ -846,7 +846,7 @@ Constants The version string of the OpenSSL library loaded by the interpreter:: >>> ssl.OPENSSL_VERSION - 'OpenSSL 0.9.8k 25 Mar 2009' + 'OpenSSL 1.0.2k 26 Jan 2017' .. versionadded:: 3.2 @@ -856,7 +856,7 @@ Constants OpenSSL library:: >>> ssl.OPENSSL_VERSION_INFO - (0, 9, 8, 11, 15) + (1, 0, 2, 11, 15) .. versionadded:: 3.2 @@ -865,9 +865,9 @@ Constants The raw version number of the OpenSSL library, as a single integer:: >>> ssl.OPENSSL_VERSION_NUMBER - 9470143 + 268443839 >>> hex(ssl.OPENSSL_VERSION_NUMBER) - '0x9080bf' + '0x100020bf' .. versionadded:: 3.2 @@ -948,7 +948,7 @@ SSL Sockets :ref:`notes on non-blocking sockets `. Usually, :class:`SSLSocket` are not created directly, but using the - the :meth:`SSLContext.wrap_socket` method. + :meth:`SSLContext.wrap_socket` method. .. versionchanged:: 3.5 The :meth:`sendfile` method was added. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9a4f42caa408a5..ca3404aa21e555 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -108,11 +108,11 @@ Notes: (1) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`False`. + argument if the first one is false. (2) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`True`. + argument if the first one is true. (3) ``not`` has a lower priority than non-Boolean operators, so ``not a == b`` is @@ -829,7 +829,7 @@ restrictions imposed by *s*. The ``in`` and ``not in`` operations have the same priorities as the comparison operations. The ``+`` (concatenation) and ``*`` (repetition) -operations have the same priority as the corresponding numeric operations. +operations have the same priority as the corresponding numeric operations. [3]_ .. index:: triple: operations on; sequence; types @@ -1719,10 +1719,10 @@ expression support in the :mod:`re` module). .. method:: str.join(iterable) - Return a string which is the concatenation of the strings in the - :term:`iterable` *iterable*. A :exc:`TypeError` will be raised if there are - any non-string values in *iterable*, including :class:`bytes` objects. The - separator between elements is the string providing this method. + Return a string which is the concatenation of the strings in *iterable*. + A :exc:`TypeError` will be raised if there are any non-string values in + *iterable*, including :class:`bytes` objects. The separator between + elements is the string providing this method. .. method:: str.ljust(width[, fillchar]) @@ -2264,8 +2264,8 @@ The :mod:`array` module supports efficient storage of basic data types like .. _typebytes: -Bytes ------ +Bytes Objects +------------- .. index:: object: bytes @@ -2274,65 +2274,67 @@ binary protocols are based on the ASCII text encoding, bytes objects offer several methods that are only valid when working with ASCII compatible data and are closely related to string objects in a variety of other ways. -Firstly, the syntax for bytes literals is largely the same as that for string -literals, except that a ``b`` prefix is added: +.. class:: bytes([source[, encoding[, errors]]]) -* Single quotes: ``b'still allows embedded "double" quotes'`` -* Double quotes: ``b"still allows embedded 'single' quotes"``. -* Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` + Firstly, the syntax for bytes literals is largely the same as that for string + literals, except that a ``b`` prefix is added: -Only ASCII characters are permitted in bytes literals (regardless of the -declared source code encoding). Any binary values over 127 must be entered -into bytes literals using the appropriate escape sequence. + * Single quotes: ``b'still allows embedded "double" quotes'`` + * Double quotes: ``b"still allows embedded 'single' quotes"``. + * Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` -As with string literals, bytes literals may also use a ``r`` prefix to disable -processing of escape sequences. See :ref:`strings` for more about the various -forms of bytes literal, including supported escape sequences. + Only ASCII characters are permitted in bytes literals (regardless of the + declared source code encoding). Any binary values over 127 must be entered + into bytes literals using the appropriate escape sequence. -While bytes literals and representations are based on ASCII text, bytes -objects actually behave like immutable sequences of integers, with each -value in the sequence restricted such that ``0 <= x < 256`` (attempts to -violate this restriction will trigger :exc:`ValueError`. This is done -deliberately to emphasise that while many binary formats include ASCII based -elements and can be usefully manipulated with some text-oriented algorithms, -this is not generally the case for arbitrary binary data (blindly applying -text processing algorithms to binary data formats that are not ASCII -compatible will usually lead to data corruption). + As with string literals, bytes literals may also use a ``r`` prefix to disable + processing of escape sequences. See :ref:`strings` for more about the various + forms of bytes literal, including supported escape sequences. -In addition to the literal forms, bytes objects can be created in a number of -other ways: + While bytes literals and representations are based on ASCII text, bytes + objects actually behave like immutable sequences of integers, with each + value in the sequence restricted such that ``0 <= x < 256`` (attempts to + violate this restriction will trigger :exc:`ValueError`. This is done + deliberately to emphasise that while many binary formats include ASCII based + elements and can be usefully manipulated with some text-oriented algorithms, + this is not generally the case for arbitrary binary data (blindly applying + text processing algorithms to binary data formats that are not ASCII + compatible will usually lead to data corruption). -* A zero-filled bytes object of a specified length: ``bytes(10)`` -* From an iterable of integers: ``bytes(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytes(obj)`` + In addition to the literal forms, bytes objects can be created in a number of + other ways: -Also see the :ref:`bytes ` built-in. + * A zero-filled bytes object of a specified length: ``bytes(10)`` + * From an iterable of integers: ``bytes(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytes(obj)`` -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytes type has an additional class method to read data in that format: + Also see the :ref:`bytes ` built-in. -.. classmethod:: bytes.fromhex(string) + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytes type has an additional class method to read data in that format: - This :class:`bytes` class method returns a bytes object, decoding the - given string object. The string must contain two hexadecimal digits per - byte, with ASCII spaces being ignored. + .. classmethod:: fromhex(string) - >>> bytes.fromhex('2Ef0 F1f2 ') - b'.\xf0\xf1\xf2' + This :class:`bytes` class method returns a bytes object, decoding the + given string object. The string must contain two hexadecimal digits per + byte, with ASCII whitespace being ignored. -A reverse conversion function exists to transform a bytes object into its -hexadecimal representation. + >>> bytes.fromhex('2Ef0 F1f2 ') + b'.\xf0\xf1\xf2' -.. method:: bytes.hex() + A reverse conversion function exists to transform a bytes object into its + hexadecimal representation. - Return a string object containing two hexadecimal digits for each - byte in the instance. + .. method:: hex() - >>> b'\xf0\xf1\xf2'.hex() - 'f0f1f2' + Return a string object containing two hexadecimal digits for each + byte in the instance. - .. versionadded:: 3.5 + >>> b'\xf0\xf1\xf2'.hex() + 'f0f1f2' + + .. versionadded:: 3.5 Since bytes objects are sequences of integers (akin to a tuple), for a bytes object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes @@ -2362,45 +2364,49 @@ Bytearray Objects .. index:: object: bytearray :class:`bytearray` objects are a mutable counterpart to :class:`bytes` -objects. There is no dedicated literal syntax for bytearray objects, instead -they are always created by calling the constructor: +objects. -* Creating an empty instance: ``bytearray()`` -* Creating a zero-filled instance with a given length: ``bytearray(10)`` -* From an iterable of integers: ``bytearray(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` +.. class:: bytearray([source[, encoding[, errors]]]) -As bytearray objects are mutable, they support the -:ref:`mutable ` sequence operations in addition to the -common bytes and bytearray operations described in :ref:`bytes-methods`. + There is no dedicated literal syntax for bytearray objects, instead + they are always created by calling the constructor: -Also see the :ref:`bytearray ` built-in. + * Creating an empty instance: ``bytearray()`` + * Creating a zero-filled instance with a given length: ``bytearray(10)`` + * From an iterable of integers: ``bytearray(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytearray type has an additional class method to read data in that format: + As bytearray objects are mutable, they support the + :ref:`mutable ` sequence operations in addition to the + common bytes and bytearray operations described in :ref:`bytes-methods`. -.. classmethod:: bytearray.fromhex(string) + Also see the :ref:`bytearray ` built-in. - This :class:`bytearray` class method returns bytearray object, decoding - the given string object. The string must contain two hexadecimal digits - per byte, with ASCII spaces being ignored. + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytearray type has an additional class method to read data in that format: - >>> bytearray.fromhex('2Ef0 F1f2 ') - bytearray(b'.\xf0\xf1\xf2') + .. classmethod:: fromhex(string) -A reverse conversion function exists to transform a bytearray object into its -hexadecimal representation. + This :class:`bytearray` class method returns bytearray object, decoding + the given string object. The string must contain two hexadecimal digits + per byte, with ASCII whitespace being ignored. -.. method:: bytearray.hex() + >>> bytearray.fromhex('2Ef0 F1f2 ') + bytearray(b'.\xf0\xf1\xf2') - Return a string object containing two hexadecimal digits for each - byte in the instance. + A reverse conversion function exists to transform a bytearray object into its + hexadecimal representation. - >>> bytearray(b'\xf0\xf1\xf2').hex() - 'f0f1f2' + .. method:: hex() + + Return a string object containing two hexadecimal digits for each + byte in the instance. - .. versionadded:: 3.5 + >>> bytearray(b'\xf0\xf1\xf2').hex() + 'f0f1f2' + + .. versionadded:: 3.5 Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be @@ -2539,11 +2545,11 @@ arbitrary binary data. bytearray.join(iterable) Return a bytes or bytearray object which is the concatenation of the - binary data sequences in the :term:`iterable` *iterable*. A - :exc:`TypeError` will be raised if there are any values in *iterable* - that are not :term:`bytes-like objects `, including - :class:`str` objects. The separator between elements is the contents - of the bytes or bytearray object providing this method. + binary data sequences in *iterable*. A :exc:`TypeError` will be raised + if there are any values in *iterable* that are not :term:`bytes-like + objects `, including :class:`str` objects. The + separator between elements is the contents of the bytes or + bytearray object providing this method. .. staticmethod:: bytes.maketrans(from, to) @@ -3982,9 +3988,7 @@ The constructors for both classes work the same: Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, the *elem* set is temporarily mutated during the search and then - restored. During the search, the *elem* set should not be read or mutated - since it does not have a meaningful value. + frozenset, a temporary one is created from *elem*. .. _typesmapping: diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ad2abe82453622..548e4a68cbf1d1 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -466,9 +466,13 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. In particular, the function looks for - *executable* (or for the first item in *args*) relative to *cwd* if the - executable path is a relative path. + *cwd* before executing the child. *cwd* can be a :class:`str` and + :term:`path-like ` object. In particular, the function + looks for *executable* (or for the first item in *args*) relative to *cwd* + if the executable path is a relative path. + + .. versionchanged:: 3.6 + *cwd* parameter accepts a :term:`path-like object`. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index 1ecc7a7cf92bd0..c8357e4fcc85e2 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -118,7 +118,7 @@ AU_read objects, as returned by :func:`.open` above, have the following methods: .. method:: AU_read.getnchannels() - Returns number of audio channels (1 for mone, 2 for stereo). + Returns number of audio channels (1 for mono, 2 for stereo). .. method:: AU_read.getsampwidth() diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 08b74a9affd24f..f066a765d00ec7 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -255,7 +255,6 @@ You can use :mod:`sysconfig` as a script with Python's *-m* option: AIX_GENUINE_CPLUSPLUS = "0" AR = "ar" ARFLAGS = "rc" - ASDLGEN = "./Parser/asdl_c.py" ... This call will print in the standard output the information returned by diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 1edb0fbabb2023..dfe688a2f93e0c 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -48,14 +48,14 @@ described below. .. exception:: NannyNag - Raised by :func:`tokeneater` if detecting an ambiguous indent. Captured and + Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and handled in :func:`check`. -.. function:: tokeneater(type, token, start, end, line) +.. function:: process_tokens(tokens) - This function is used by :func:`check` as a callback parameter to the function - :func:`tokenize.tokenize`. + This function is used by :func:`check` to process tokens generated by the + :mod:`tokenize` module. .. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, reset_globals diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index d8f809753dfdd7..337c061107299c 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -146,6 +146,10 @@ Some facts and figures: .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. class:: TarFile Class for reading and writing tar archives. Do not use this class directly: @@ -266,7 +270,8 @@ be finalized; only the internally used file object will be closed. See the All following arguments are optional and can be accessed as instance attributes as well. - *name* is the pathname of the archive. It can be omitted if *fileobj* is given. + *name* is the pathname of the archive. *name* may be a :term:`path-like object`. + It can be omitted if *fileobj* is given. In this case, the file object's :attr:`name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append @@ -319,6 +324,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. classmethod:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a @@ -390,14 +399,17 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. File attributes (owner, mtime, mode) are set unless - *set_attrs* is false. + directory using *path*. *path* may be a :term:`path-like object`. + File attributes (owner, mtime, mode) are set unless *set_attrs* is false. If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile are used to set the owner/group for the extracted files. Otherwise, the named @@ -418,6 +430,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + + .. method:: TarFile.extractfile(member) Extract a member from the archive as a file object. *member* may be a filename @@ -464,7 +480,8 @@ be finalized; only the internally used file object will be closed. See the Create a :class:`TarInfo` object from the result of :func:`os.stat` or equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. If + specified as a :term:`file object` *fileobj* with a file descriptor. + *name* may be a :term:`path-like object`. If given, *arcname* specifies an alternative name for the file in the archive, otherwise, the name is taken from *fileobj*’s :attr:`~io.FileIO.name` attribute, or the *name* argument. The name @@ -478,6 +495,9 @@ be finalized; only the internally used file object will be closed. See the The :attr:`~TarInfo.name` may also be modified, in which case *arcname* could be a dummy string. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + .. method:: TarFile.close() diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 665261ffb21f94..c59aca1e189086 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -245,12 +245,12 @@ The module uses a global variable to store the name of the directory used for temporary files returned by :func:`gettempdir`. It can be set directly to override the selection process, but this is discouraged. All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommend approach. +to specify the directory and this is the recommended approach. .. data:: tempdir When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to all the functions defined in this + default value for the *dir* argument to the functions defined in this module. If ``tempdir`` is unset or ``None`` at any call to any of the above diff --git a/Doc/library/test.rst b/Doc/library/test.rst index fab3e1fe4cc6ad..9d4ff7ad8b45a3 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -570,7 +570,8 @@ The :mod:`test.support` module defines the following functions: def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) -.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()): + +.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) Returns the set of attributes, functions or methods of *ref_api* not found on *other_api*, except for a defined list of items to be diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 2792dfdce04c6c..cda859fe4cbd3a 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -371,8 +371,9 @@ All methods are executed atomically. lock, subsequent attempts to acquire it block, until it is released; any thread may release it. - .. versionchanged:: 3.3 - Changed from a factory function to a class. + Note that ``Lock`` is actually a factory function which returns an instance + of the most efficient version of the concrete Lock class that is supported + by the platform. .. method:: acquire(blocking=True, timeout=-1) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ae17f6f4f0d236..d2e2ac4410aba3 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -17,11 +17,23 @@ semantics of these functions varies among platforms. An explanation of some terminology and conventions is in order. +.. _epoch: + .. index:: single: epoch -* The :dfn:`epoch` is the point where the time starts. On January 1st of that - year, at 0 hours, the "time since the epoch" is zero. For Unix, the epoch is - 1970. To find out what the epoch is, look at ``gmtime(0)``. +* The :dfn:`epoch` is the point where the time starts, and is platform + dependent. For Unix, the epoch is January 1, 1970, 00:00:00 (UTC). + To find out what the epoch is on a given platform, look at + ``time.gmtime(0)``. + +.. _leap seconds: https://en.wikipedia.org/wiki/Leap_second + +.. index:: seconds since the epoch + +* The term :dfn:`seconds since the epoch` refers to the total number + of elapsed seconds since the epoch, typically excluding + `leap seconds`_. Leap seconds are excluded from this total on all + POSIX-compliant platforms. .. index:: single: Year 2038 @@ -467,7 +479,7 @@ The module defines the following functions and data items: (2) The range really is ``0`` to ``61``; value ``60`` is valid in - timestamps representing leap seconds and value ``61`` is supported + timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. (3) @@ -572,12 +584,28 @@ The module defines the following functions and data items: .. function:: time() - Return the time in seconds since the epoch as a floating point number. + Return the time in seconds since the epoch_ as a floating point + number. The specific date of the epoch and the handling of + `leap seconds`_ is platform dependent. + On Windows and most Unix systems, the epoch is January 1, 1970, + 00:00:00 (UTC) and leap seconds are not counted towards the time + in seconds since the epoch. This is commonly referred to as + `Unix time `_. + To find out what the epoch is on a given platform, look at + ``gmtime(0)``. + Note that even though the time is always returned as a floating point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a - lower value than a previous call if the system clock has been set back between - the two calls. + lower value than a previous call if the system clock has been set back + between the two calls. + + The number returned by :func:`.time` may be converted into a more common + time format (i.e. year, month, day, hour, etc...) in UTC by passing it to + :func:`gmtime` function or in local time by passing it to the + :func:`localtime` function. In both cases a + :class:`struct_time` object is returned, from which the components + of the calendar date may be accessed as attributes. .. data:: timezone diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 3b772765aca260..5793c54ae8d1a3 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -134,21 +134,21 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None) - Automatically determine how many times to call :meth:`.timeit`. + Automatically determine how many times to call :meth:`.timeit`. - This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual - (number of loops, time taken for that number of loops). It calls - :meth:`.timeit` with *number* set to successive powers of ten (10, - 100, 1000, ...) up to a maximum of one billion, until the time taken - is at least 0.2 second, or the maximum is reached. + This is a convenience function that calls :meth:`.timeit` repeatedly + so that the total time >= 0.2 second, returning the eventual + (number of loops, time taken for that number of loops). It calls + :meth:`.timeit` with *number* set to successive powers of ten (10, + 100, 1000, ...) up to a maximum of one billion, until the time taken + is at least 0.2 second, or the maximum is reached. - If *callback* is given and is not ``None``, it will be called after - each trial with two arguments: ``callback(number, time_taken)``. + If *callback* is given and is not ``None``, it will be called after + each trial with two arguments: ``callback(number, time_taken)``. - .. versionadded:: 3.6 + .. versionadded:: 3.6 .. method:: Timer.repeat(repeat=3, number=1000000) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index b0ac81271c5c18..5cb7029adf5e9e 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -13,6 +13,12 @@ annotated statement coverage listings, print caller/callee relationships and list functions executed during a program run. It can be used in another program or from the command line. +.. seealso:: + + `Coverage.py `_ + A popular third-party coverage tool that provides HTML + output along with advanced features such as branch coverage. + .. _trace-cli: Command-Line Usage diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 066ee96fc004ab..55d331c996a187 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -45,9 +45,9 @@ The module defines the following functions: * if *tb* is not ``None``, it prints a header ``Traceback (most recent call last):`` * it prints the exception *etype* and *value* after the stack trace - * if *etype* is :exc:`SyntaxError` and *value* has the appropriate format, it - prints the line where the syntax error occurred with a caret indicating the - approximate position of the error. + * if *type(value)* is :exc:`SyntaxError` and *value* has the appropriate + format, it prints the line where the syntax error occurred with a caret + indicating the approximate position of the error. The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the @@ -55,6 +55,9 @@ The module defines the following functions: printed as well, like the interpreter itself does when printing an unhandled exception. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: print_exc(limit=None, file=None, chain=True) @@ -131,6 +134,9 @@ The module defines the following functions: containing internal newlines. When these lines are concatenated and printed, exactly the same text is printed as does :func:`print_exception`. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: format_exc(limit=None, chain=True) @@ -372,6 +378,7 @@ exception and traceback: print("*** print_tb:") traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) print("*** print_exception:") + # exc_type below is ignored on 3.5 and later traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stdout) print("*** print_exc:") @@ -381,6 +388,7 @@ exception and traceback: print(formatted_lines[0]) print(formatted_lines[-1]) print("*** format_exception:") + # exc_type below is ignored on 3.5 and later print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback))) print("*** extract_tb:") diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 1986972549c503..31761be02b78ee 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -1797,7 +1797,7 @@ Input methods :param prompt: string Pop up a dialog window for input of a string. Parameter title is - the title of the dialog window, propmt is a text mostly describing + the title of the dialog window, prompt is a text mostly describing what information to input. Return the string input. If the dialog is canceled, return ``None``. :: diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 6c8982ba743526..1780739ad1dd43 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -508,6 +508,14 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__float__``. +.. class:: SupportsComplex + + An ABC with one abstract method ``__complex__``. + +.. class:: SupportsBytes + + An ABC with one abstract method ``__bytes__``. + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -574,6 +582,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.6.1 + .. class:: List(list, MutableSequence[T]) Generic version of :class:`list`. @@ -656,7 +666,19 @@ The module defines the following classes, functions and decorators: .. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - A generic version of :class:`collections.defaultdict` + A generic version of :class:`collections.defaultdict`. + +.. class:: Counter(collections.Counter, Dict[T, int]) + + A generic version of :class:`collections.Counter`. + + .. versionadded:: 3.6.1 + +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + + A generic version of :class:`collections.ChainMap`. + + .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -740,9 +762,12 @@ The module defines the following classes, functions and decorators: This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These representing the types of I/O streams such as returned by + These represent the types of I/O streams such as returned by :func:`open`. + These types are also accessible directly as ``typing.IO``, + ``typing.TextIO``, and ``typing.BinaryIO``. + .. class:: re Wrapper namespace for regular expression matching types. @@ -754,6 +779,9 @@ The module defines the following classes, functions and decorators: ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. + These types are also accessible directly as ``typing.Pattern`` + and ``typing.Match``. + .. class:: NamedTuple Typed version of namedtuple. @@ -780,10 +808,20 @@ The module defines the following classes, functions and decorators: Fields with a default value must come after any fields without a default. The resulting class has two extra attributes: ``_field_types``, - giving a dict mapping field names to types, and ``field_defaults``, a dict + giving a dict mapping field names to types, and ``_field_defaults``, a dict mapping field names to default values. (The field names are in the ``_fields`` attribute, which is part of the namedtuple API.) + ``NamedTuple`` subclasses can also have docstrings and methods:: + + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 + + def __repr__(self) -> str: + return f'' + Backward-compatible usage:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) @@ -792,7 +830,7 @@ The module defines the following classes, functions and decorators: Added support for :pep:`526` variable annotation syntax. .. versionchanged:: 3.6.1 - Added support for default values. + Added support for default values, methods, and docstrings. .. function:: NewType(typ) @@ -970,9 +1008,9 @@ The module defines the following classes, functions and decorators: :data:`ClassVar` is not a class itself, and should not be used with :func:`isinstance` or :func:`issubclass`. - Note that :data:`ClassVar` does not change Python runtime behavior; - it can be used by 3rd party type checkers, so that the following - code might flagged as an error by those:: + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: enterprise_d = Starship(3000) enterprise_d.stats = {} # Error, setting class variable on instance @@ -1003,5 +1041,10 @@ The module defines the following classes, functions and decorators: if TYPE_CHECKING: import expensive_mod - def fun(): - local_var: expensive_mod.some_type = other_fun() + def fun(arg: 'expensive_mod.SomeType') -> None: + local_var: expensive_mod.AnotherType = other_fun() + + Note that the first type annotation must be enclosed in quotes, making it a + "forward reference", to hide the ``expensive_mod`` reference from the + interpreter runtime. Type annotations for local variables are not + evaluated, so the second annotation does not need to be enclosed in quotes. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 643180953fd23d..2a9777609527d0 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -158,7 +158,7 @@ Examples: 9 >>> unicodedata.decimal('a') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: not a decimal >>> unicodedata.category('A') # 'L'etter, 'u'ppercase 'Lu' diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index c6d0ec92b6e9ab..a552cbfc70ad2e 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -303,14 +303,14 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_once_with(*args, **kwargs) - Assert that the mock was called exactly once and with the specified - arguments. + Assert that the mock was called exactly once and that that call was + with the specified arguments. >>> mock = Mock(return_value=None) >>> mock('foo', bar='baz') >>> mock.assert_called_once_with('foo', bar='baz') - >>> mock('foo', bar='baz') - >>> mock.assert_called_once_with('foo', bar='baz') + >>> mock('other', bar='values') + >>> mock.assert_called_once_with('other', bar='values') Traceback (most recent call last): ... AssertionError: Expected 'mock' to be called once. Called 2 times. @@ -322,7 +322,8 @@ the *new_callable* argument to :func:`patch`. The assert passes if the mock has *ever* been called, unlike :meth:`assert_called_with` and :meth:`assert_called_once_with` that - only pass if the call is the most recent one. + only pass if the call is the most recent one, and in the case of + :meth:`assert_called_once_with` it must also be the only call. >>> mock = Mock(return_value=None) >>> mock(1, 2, arg='thing') diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 676321b46a2232..1cc69e62e63318 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -118,6 +118,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -236,6 +239,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index e289b971e7c269..b02a006d733b6e 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -166,8 +166,8 @@ Extension types can easily be made to support weak references; see performed by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakKeyDictionary` objects have the following additional methods. These -expose the internal references directly. The references are not guaranteed to +:class:`WeakKeyDictionary` objects have an additional method that +exposes the internal references directly. The references are not guaranteed to be "live" at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer @@ -192,9 +192,9 @@ than needed. by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakValueDictionary` objects have the following additional methods. -These method have the same issues as the and :meth:`keyrefs` method of -:class:`WeakKeyDictionary` objects. +:class:`WeakValueDictionary` objects have an additional method that has the +same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` +objects. .. method:: WeakValueDictionary.valuerefs() diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2e9e814693df57..40470e8736e7e4 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -248,7 +248,7 @@ utility to most DOM users. .. rubric:: Footnotes -.. [#] The encoding name included in the XML output should conform to +.. [1] The encoding name included in the XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not valid in an XML document's declaration, even though Python accepts it as an encoding name. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index b50255434de945..5c0f469ad7a5cf 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -108,7 +108,7 @@ DOMEventStream Objects :class:`xml.dom.minidom.Element` if event equals :data:`START_ELEMENT` or :data:`END_ELEMENT` or :class:`xml.dom.minidom.Text` if event equals :data:`CHARACTERS`. - The current node does not contain informations about its children, unless + The current node does not contain information about its children, unless :func:`expandNode` is called. .. method:: expandNode(node) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index b54eace41188b0..7d814ad406eb1b 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1192,7 +1192,7 @@ Exceptions .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5eb6f103380661..a5d42118ba5176 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -132,8 +132,9 @@ ZipFile Objects .. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) - Open a ZIP file, where *file* can be either a path to a file (a string) or a - file-like object. The *mode* parameter should be ``'r'`` to read an existing + Open a ZIP file, where *file* can be a path to a file (a string), a + file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. If *mode* is ``'x'`` and *file* refers to an existing file, @@ -183,6 +184,9 @@ ZipFile Objects Previously, a plain :exc:`RuntimeError` was raised for unrecognized compression values. + .. versionchanged:: 3.6.2 + The *file* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.close() @@ -284,6 +288,9 @@ ZipFile Objects Calling :meth:`extract` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.extractall(path=None, members=None, pwd=None) @@ -304,6 +311,9 @@ ZipFile Objects Calling :meth:`extractall` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.printdir() @@ -403,6 +413,9 @@ ZipFile Objects The following data attributes are also available: +.. attribute:: ZipFile.filename + + Name of the ZIP file. .. attribute:: ZipFile.debug @@ -451,12 +464,12 @@ The :class:`PyZipFile` constructor takes the same parameters as the added to the archive, compiling if necessary. If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.py[co]`) file is added at the top level + just the (corresponding :file:`\*.pyc`) file is added at the top level (no path information). If *pathname* is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files - :file:`\*.py[co]` are added at the top level. If the directory is a - package directory, then all :file:`\*.py[co]` are added under the package + :file:`\*.pyc` are added at the top level. If the directory is a + package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, all of these are added recursively. @@ -488,6 +501,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionadded:: 3.4 The *filterfunc* parameter. + .. versionchanged:: 3.6.2 + The *pathname* parameter accepts a :term:`path-like object`. + .. _zipinfo-objects: @@ -514,6 +530,10 @@ file: .. versionadded:: 3.6 + .. versionchanged:: 3.6.2 + The *filename* parameter accepts a :term:`path-like object`. + + Instances have the following methods and attributes: .. method:: ZipInfo.is_dir() diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 46b8c245f7b3fe..eaae2bb04b7482 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -9,7 +9,7 @@ -------------- This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.py[co]`) and packages from ZIP-format archives. It is usually not +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not needed to use the :mod:`zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. diff --git a/Doc/make.bat b/Doc/make.bat index da1f8765a4f358..b1a3738f309d3b 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -74,7 +74,7 @@ echo. Provided by this script: echo. clean, check, serve, htmlview echo. echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.txt in this directory or +echo.filenames to build or are ignored. See README.rst in this directory or echo.the documentation for your version of Sphinx for more exhaustive lists echo.of available targets and descriptions of each. echo. @@ -86,7 +86,7 @@ goto end :build if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% + set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% ) cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 095a2380b379bc..7c140a3bc86cb8 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -320,9 +320,9 @@ Sequences A bytes object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in function :func:`bytes` can be used to - construct bytes objects. Also, bytes objects can be decoded to strings - via the :meth:`~bytes.decode` method. + (like ``b'abc'``) and the built-in :func:`bytes()` constructor + can be used to create bytes objects. Also, bytes objects can be + decoded to strings via the :meth:`~bytes.decode` method. Mutable sequences .. index:: @@ -349,9 +349,9 @@ Sequences .. index:: bytearray A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable (and hence - unhashable), byte arrays otherwise provide the same interface and - functionality as immutable bytes objects. + :func:`bytearray` constructor. Aside from being mutable + (and hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable :class:`bytes` objects. .. index:: module: array @@ -1119,9 +1119,9 @@ Basic customization (usually an instance of *cls*). Typical implementations create a new instance of the class by invoking the - superclass's :meth:`__new__` method using ``super(currentclass, - cls).__new__(cls[, ...])`` with appropriate arguments and then modifying the - newly-created instance as necessary before returning it. + superclass's :meth:`__new__` method using ``super().__new__(cls[, ...])`` + with appropriate arguments and then modifying the newly-created instance + as necessary before returning it. If :meth:`__new__` returns an instance of *cls*, then the new instance's :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where @@ -1145,7 +1145,7 @@ Basic customization class constructor expression. If a base class has an :meth:`__init__` method, the derived class's :meth:`__init__` method, if any, must explicitly call it to ensure proper initialization of the base class part of the - instance; for example: ``BaseClass.__init__(self, [args...])``. + instance; for example: ``super().__init__([args...])``. Because :meth:`__new__` and :meth:`__init__` work together in constructing objects (:meth:`__new__` to create it, and :meth:`__init__` to customize it), @@ -1253,8 +1253,8 @@ Basic customization .. index:: builtin: bytes - Called by :func:`bytes` to compute a byte-string representation of an - object. This should return a ``bytes`` object. + Called by :ref:`bytes ` to compute a byte-string representation + of an object. This should return a :class:`bytes` object. .. index:: single: string; __format__() (object method) @@ -1578,8 +1578,8 @@ Class Binding ``A.__dict__['x'].__get__(None, A)``. Super Binding - If ``a`` is an instance of :class:`super`, then the binding ``super(B, - obj).m()`` searches ``obj.__class__.__mro__`` for the base class ``A`` + If ``a`` is an instance of :class:`super`, then the binding ``super(B, obj).m()`` + searches ``obj.__class__.__mro__`` for the base class ``A`` immediately preceding ``B`` and then invokes the descriptor with the call: ``A.__dict__['m'].__get__(obj, obj.__class__)``. @@ -2011,6 +2011,14 @@ through the container; for mappings, :meth:`__iter__` should be the same as :meth:`__bool__` method and whose :meth:`__len__` method returns zero is considered to be false in a Boolean context. + .. impl-detail:: + + In CPython, the length is required to be at most :attr:`sys.maxsize`. + If the length is larger than :attr:`!sys.maxsize` some features (such as + :func:`len`) may raise :exc:`OverflowError`. To prevent raising + :exc:`!OverflowError` by truth value testing, an object must define a + :meth:`__bool__` method. + .. method:: object.__length_hint__(self) @@ -2021,6 +2029,7 @@ through the container; for mappings, :meth:`__iter__` should be the same as .. versionadded:: 3.4 + .. note:: Slicing is done exclusively with the following three methods. A call like :: diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 5f1ea92ed460f3..d08abdf3343fe4 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -164,15 +164,6 @@ Builtins and restricted execution .. index:: pair: restricted; execution -The builtins namespace associated with the execution of a code block is actually -found by looking up the name ``__builtins__`` in its global namespace; this -should be a dictionary or a module (in the latter case the module's dictionary -is used). By default, when in the :mod:`__main__` module, ``__builtins__`` is -the built-in module :mod:`builtins`; when in any other module, -``__builtins__`` is an alias for the dictionary of the :mod:`builtins` module -itself. ``__builtins__`` can be set to a user-created dictionary to create a -weak form of restricted execution. - .. impl-detail:: Users should not touch ``__builtins__``; it is strictly an implementation @@ -180,6 +171,15 @@ weak form of restricted execution. :keyword:`import` the :mod:`builtins` module and modify its attributes appropriately. +The builtins namespace associated with the execution of a code block +is actually found by looking up the name ``__builtins__`` in its +global namespace; this should be a dictionary or a module (in the +latter case the module's dictionary is used). By default, when in the +:mod:`__main__` module, ``__builtins__`` is the built-in module +:mod:`builtins`; when in any other module, ``__builtins__`` is an +alias for the dictionary of the :mod:`builtins` module itself. + + .. _dynamic-features: Interaction with dynamic features @@ -194,12 +194,6 @@ This means that the following code will print 42:: i = 42 f() -There are several cases where Python statements are illegal when used in -conjunction with nested scopes that contain free variables. - -If a variable is referenced in an enclosing scope, it is illegal to delete the -name. An error will be reported at compile time. - .. XXX from * also invalid with relative imports (at least currently) The :func:`eval` and :func:`exec` functions do not have access to the full diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3a4b80557caf67..d92be975aacef3 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -190,7 +190,7 @@ Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. A comprehension in an :keyword:`async def` function may consist of either a :keyword:`for` or :keyword:`async for` clause following the leading -expression, may contan additonal :keyword:`for` or :keyword:`async for` +expression, may contain additional :keyword:`for` or :keyword:`async for` clauses, and may also use :keyword:`await` expressions. If a comprehension contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an @@ -636,7 +636,7 @@ which are used to control the execution of a generator function. without yielding another value, an :exc:`StopAsyncIteration` exception is raised by the awaitable. If the generator function does not catch the passed-in exception, or - raises a different exception, then when the awaitalbe is run that exception + raises a different exception, then when the awaitable is run that exception propagates to the caller of the awaitable. .. index:: exception: GeneratorExit @@ -905,7 +905,7 @@ keyword arguments (and any ``**expression`` arguments -- see below). So:: 2 1 >>> f(a=1, *(2,)) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: f() got multiple values for keyword argument 'a' >>> f(1, *(2,)) 1 2 @@ -1317,7 +1317,7 @@ built-in types. * Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can be compared only within each of their types, with the restriction that ranges do not support order comparison. Equality comparison across these types - results in unequality, and ordering comparison across these types raises + results in inequality, and ordering comparison across these types raises :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding @@ -1355,7 +1355,7 @@ built-in types. true). * Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and elements + equal `(key, value)` pairs. Equality comparison of the keys and values enforces reflexivity. Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. @@ -1431,28 +1431,29 @@ Membership test operations -------------------------- The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not -in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether the -dictionary has a given key. For container types such as list, tuple, set, -frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent +s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. +``x not in s`` returns the negation of ``x in s``. All built-in sequences and +set types support this as well as dictionary, for which :keyword:`in` tests +whether the dictionary has a given key. For container types such as list, tuple, +set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. -For the string and bytes types, ``x in y`` is true if and only if *x* is a +For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` is true if and only if ``y.__contains__(x)`` is true. +y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and +``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is true if some value ``z`` with ``x == z`` is +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is true if and only if there is a non-negative +:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x == y[i]``, and all lower integer indices do not raise :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 5e2c1c8b0758d8..d504f37387422e 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -36,7 +36,7 @@ implement import semantics. When a module is first imported, Python searches for the module and if found, it creates a module object [#fnmo]_, initializing it. If the named module -cannot be found, an :exc:`ModuleNotFoundError` is raised. Python implements various +cannot be found, a :exc:`ModuleNotFoundError` is raised. Python implements various strategies to search for the named module when the import machinery is invoked. These strategies can be modified and extended by using various hooks described in the sections below. @@ -167,7 +167,7 @@ arguments to the :keyword:`import` statement, or from the parameters to the This name will be used in various phases of the import search, and it may be the dotted path to a submodule, e.g. ``foo.bar.baz``. In this case, Python first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``. -If any of the intermediate imports fail, an :exc:`ModuleNotFoundError` is raised. +If any of the intermediate imports fail, a :exc:`ModuleNotFoundError` is raised. The module cache @@ -185,7 +185,7 @@ object. During import, the module name is looked up in :data:`sys.modules` and if present, the associated value is the module satisfying the import, and the -process completes. However, if the value is ``None``, then an +process completes. However, if the value is ``None``, then a :exc:`ModuleNotFoundError` is raised. If the module name is missing, Python will continue searching for the module. @@ -194,7 +194,7 @@ associated module (as other modules may hold references to it), but it will invalidate the cache entry for the named module, causing Python to search anew for the named module upon its next import. The key can also be assigned to ``None``, forcing the next import -of the module to result in an :exc:`ModuleNotFoundError`. +of the module to result in a :exc:`ModuleNotFoundError`. Beware though, as if you keep a reference to the module object, invalidate its cache entry in :data:`sys.modules`, and then re-import the @@ -298,7 +298,7 @@ The second argument is the path entries to use for the module search. For top-level modules, the second argument is ``None``, but for submodules or subpackages, the second argument is the value of the parent package's ``__path__`` attribute. If the appropriate ``__path__`` attribute cannot -be accessed, an :exc:`ModuleNotFoundError` is raised. The third argument +be accessed, a :exc:`ModuleNotFoundError` is raised. The third argument is an existing module object that will be the target of loading later. The import system passes in a target module only during reload. @@ -431,7 +431,7 @@ on the module object. If the method returns ``None``, the import machinery will create the new module itself. .. versionadded:: 3.4 - The create_module() method of loaders. + The :meth:`~importlib.abc.Loader.create_module` method of loaders. .. versionchanged:: 3.4 The :meth:`~importlib.abc.Loader.load_module` method was replaced by @@ -464,8 +464,11 @@ import machinery will create the new module itself. .. versionchanged:: 3.5 A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. Starting in Python 3.6 it will be an error to not - define ``create_module()`` on a loader attached to a ModuleSpec. + ``create_module()`` is not. + +.. versionchanged:: 3.6 + An :exc:`ImportError` is raised when ``exec_module()`` is defined but + ``create_module()`` is not. Submodules ---------- @@ -613,7 +616,7 @@ the module. module.__path__ --------------- -By definition, if a module has an ``__path__`` attribute, it is a package, +By definition, if a module has a ``__path__`` attribute, it is a package, regardless of its value. A package's ``__path__`` attribute is used during imports of its subpackages. @@ -887,7 +890,7 @@ import statements within that module. To selectively prevent import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), -it is sufficient to raise :exc:`ModuleNoFoundError` directly from +it is sufficient to raise :exc:`ModuleNotFoundError` directly from :meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning ``None``. The latter indicates that the meta path search should continue, while raising an exception terminates it immediately. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index da7017afff0a3d..7f9c66481785c1 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -696,6 +696,17 @@ a temporary variable. >>> f"newline: {newline}" 'newline: 10' +Formatted string literals cannot be used as docstrings, even if they do not +include expressions. + +:: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> foo.__doc__ is None + True + See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index e152b16ee32d4f..8786d73f68a369 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -587,7 +587,7 @@ printed:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero The above exception was the direct cause of the following exception: @@ -606,7 +606,7 @@ attached as the new exception's :attr:`__context__` attribute:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero During handling of the above exception, another exception occurred: @@ -614,9 +614,27 @@ attached as the new exception's :attr:`__context__` attribute:: File "", line 4, in RuntimeError: Something bad happened +Exception chaining can be explicitly suppressed by specifying :const:`None` in +the ``from`` clause:: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError: Something bad happened + Additional information on exceptions can be found in section :ref:`exceptions`, and information about handling exceptions is in section :ref:`try`. +.. versionchanged:: 3.3 + :const:`None` is now permitted as ``Y`` in ``raise X from Y``. + +.. versionadded:: 3.3 + The ``__suppress_context__`` attribute to suppress automatic display of the + exception context. .. _break: @@ -922,7 +940,7 @@ annotation. builtin: eval builtin: compile -**Programmer's note:** the :keyword:`global` is a directive to the parser. It +**Programmer's note:** :keyword:`global` is a directive to the parser. It applies only to code parsed at the same time as the :keyword:`global` statement. In particular, a :keyword:`global` statement contained in a string or code object supplied to the built-in :func:`exec` function does not affect the code diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 273191bbd3c025..865f2edee02205 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -21,6 +21,7 @@ from sphinx import addnodes from sphinx.builders import Builder +from sphinx.locale import translators from sphinx.util.nodes import split_explicit_title from sphinx.util.compat import Directive from sphinx.writers.html import HTMLTranslator @@ -34,7 +35,7 @@ ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://hg.python.org/cpython/file/3.6/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.6/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -79,7 +80,7 @@ def new_depart_literal_block(self, node): def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): issue = utils.unescape(text) - text = 'issue ' + issue + text = 'bpo-' + issue refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) return [refnode], [] @@ -103,16 +104,25 @@ class ImplementationDetail(Directive): optional_arguments = 1 final_argument_whitespace = True + # This text is copied to templates/dummy.html + label_text = 'CPython implementation detail:' + def run(self): pnode = nodes.compound(classes=['impl-detail']) + label = translators['sphinx'].gettext(self.label_text) content = self.content - add_text = nodes.strong('CPython implementation detail:', - 'CPython implementation detail:') + add_text = nodes.strong(label, label) if self.arguments: n, m = self.state.inline_text(self.arguments[0], self.lineno) pnode.append(nodes.paragraph('', '', *(n + m))) self.state.nested_parse(content, self.content_offset, pnode) if pnode.children and isinstance(pnode[0], nodes.paragraph): + content = nodes.inline(pnode[0].rawsource, translatable=True) + content.source = pnode[0].source + content.line = pnode[0].line + content += pnode[0].children + pnode[0].replace_self(nodes.paragraph('', '', content, + translatable=False)) pnode[0].insert(0, add_text) pnode[0].insert(1, nodes.Text(' ')) else: @@ -225,7 +235,7 @@ def run(self): # Support for including Misc/NEWS -issue_re = re.compile('([Ii])ssue #([0-9]+)') +issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -253,7 +263,7 @@ def run(self): text = 'The NEWS file is not available.' node = nodes.strong(text, text) return [node] - content = issue_re.sub(r'`\1ssue #\2 `__', + content = issue_re.sub(r'`bpo-\1 `__', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index c6e03119ae9730..2dc540459020a6 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -106,7 +106,7 @@ howto/pyporting,,::,Programming Language :: Python :: 2 howto/pyporting,,::,Programming Language :: Python :: 3 howto/regex,,::, howto/regex,,:foo,(?:foo) -howto/urllib2,,:password,"for example ""joe:password@example.com""" +howto/urllib2,,:password,"""joe:password@example.com""" library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," library/bisect,32,:hi,all(val >= x for val in a[i:hi]) library/bisect,42,:hi,all(val > x for val in a[i:hi]) @@ -302,6 +302,8 @@ whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe: whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf +library/re,,`,!#$%&'*+-.^_`|~: +library/re,,`,\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\: library/tarfile,,:xz,'x:xz' library/xml.etree.elementtree,,:sometag,prefix:sometag library/xml.etree.elementtree,,:fictional,">> m[::2].tolist() library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: whatsnew/3.5,,:root,'WARNING:root:warning\n' @@ -324,6 +326,3 @@ whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') whatsnew/3.5,,:root,ERROR:root:exception whatsnew/3.5,,:exception,ERROR:root:exception whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')] -whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" -whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" -whatsnew/changelog,,`,"for readability (was ""`"")." diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 243d8107779361..71d0bba683074e 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -3,8 +3,11 @@

{{ _('This Page') }}

{%- endif %} diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index de84ae3abc81c7..3a05cb697938fd 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -42,7 +42,7 @@

Download Python {{ release }} Documentation

These archives contain all the content in the documentation.

HTML Help (.chm) files are made available in the "Windows" section -on the Python +on the Python download page.

diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html new file mode 100644 index 00000000000000..6e43be23230b54 --- /dev/null +++ b/Doc/tools/templates/dummy.html @@ -0,0 +1,6 @@ +This file is not an actual template, but used to add some +texts in extensions to sphinx.pot file. + +In extensions/pyspecific.py: + +{% trans %}CPython implementation detail:{% endtrans %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1076c1f51b7d19..d795c0a5586bc8 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -1,5 +1,12 @@ -{% extends "defindex.html" %} -{% block tables %} +{% extends "layout.html" %} +{%- block htmltitle -%} +{{ shorttitle }} +{%- endblock -%} +{% block body %} +

{{ docstitle|e }}

+

+ {% trans %}Welcome! This is the documentation for Python {{ release }}.{% endtrans %} +

{% trans %}Parts of the documentation:{% endtrans %}

diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index e134d5d62ea667..073444cf8b3985 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -784,7 +784,7 @@ using the :func:`next` built-in function; this example shows how it all works:: 'c' >>> next(it) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in next(it) StopIteration diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index d43461886e7933..54171bc96f0f08 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -475,7 +475,7 @@ Here's an example that fails due to this restriction:: ... >>> function(0, a=0) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: function() got multiple values for keyword argument 'a' When a final formal parameter of the form ``**name`` is present, it receives a @@ -492,8 +492,7 @@ function like this:: for arg in arguments: print(arg) print("-" * 40) - keys = sorted(keywords.keys()) - for kw in keys: + for kw in keywords: print(kw, ":", keywords[kw]) It could be called like this:: @@ -513,13 +512,13 @@ and of course it would print: It's very runny, sir. It's really very, VERY runny, sir. ---------------------------------------- - client : John Cleese shopkeeper : Michael Palin + client : John Cleese sketch : Cheese Shop Sketch -Note that the list of keyword argument names is created by sorting the result -of the keywords dictionary's ``keys()`` method before printing its contents; -if this is not done, the order in which the arguments are printed is undefined. +Note that the order in which the keyword arguments are printed is guaranteed +to match the order in which they were provided in the function call. + .. _tut-arbitraryargs: diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 953a68b44af7bb..1a73ac9d05936a 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -22,11 +22,11 @@ objects: Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. -.. method:: list.extend(L) +.. method:: list.extend(iterable) :noindex: - Extend the list by appending all the items in the given list. Equivalent to - ``a[len(a):] = L``. + Extend the list by appending all the items from the iterable. Equivalent to + ``a[len(a):] = iterable``. .. method:: list.insert(i, x) @@ -68,7 +68,7 @@ objects: The optional arguments *start* and *end* are interpreted as in the slice notation and are used to limit the search to a particular subsequence of - *x*. The returned index is computed relative to the beginning of the full + the list. The returned index is computed relative to the beginning of the full sequence rather than the *start* argument. @@ -261,7 +261,7 @@ it must be parenthesized. :: [(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (5, 25)] >>> # the tuple must be parenthesized, otherwise an error is raised >>> [x, x**2 for x in range(6)] - File "", line 1, in ? + File "", line 1, in [x, x**2 for x in range(6)] ^ SyntaxError: invalid syntax diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index beeaac36b9bd24..bad0302b037861 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -362,7 +362,7 @@ attempts to use the file object will automatically fail. :: >>> f.close() >>> f.read() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: I/O operation on closed file It is good practice to use the :keyword:`with` keyword when dealing with file diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 7e8ee3e5ea19bc..8956aa5a261350 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -100,10 +100,8 @@ give you an error:: There is full support for floating point; operators with mixed type operands convert the integer operand to floating point:: - >>> 3 * 3.75 / 1.5 - 7.5 - >>> 7.0 / 2 - 3.5 + >>> 4 * 3.75 - 1 + 14.0 In interactive mode, the last printed expression is assigned to the variable ``_``. This means that when you are using Python as a desk calculator, it is @@ -359,7 +357,7 @@ The built-in function :func:`len` returns the length of a string:: Information about string formatting with :meth:`str.format`. :ref:`old-string-formatting` - The old formatting operations invoked when strings and Unicode strings are + The old formatting operations invoked when strings are the left operand of the ``%`` operator are described in more detail here. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 195f63f0a319da..40a06b9adc06ef 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -532,8 +532,8 @@ conflict. .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set to a non-empty string, Python won't try to write ``.pyc`` or - ``.pyo`` files on the import of source modules. This is equivalent to + If this is set to a non-empty string, Python won't try to write ``.pyc`` + files on the import of source modules. This is equivalent to specifying the :option:`-B` option. @@ -571,7 +571,7 @@ conflict. .. versionchanged:: 3.6 On Windows, the encoding specified by this variable is ignored for interactive - console buffers unless :envvar:`PYTHONLEGACYWINDOWSIOENCODING` is also specified. + console buffers unless :envvar:`PYTHONLEGACYWINDOWSSTDIO` is also specified. Files and pipes redirected through the standard streams are not affected. .. envvar:: PYTHONNOUSERSITE @@ -700,7 +700,7 @@ conflict. .. versionadded:: 3.6 See :pep:`529` for more details. -.. envvar:: PYTHONLEGACYWINDOWSIOENCODING +.. envvar:: PYTHONLEGACYWINDOWSSTDIO If set to a non-empty string, does not use the new console reader and writer. This means that Unicode characters will be encoded according to diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 97f0a49ca771e5..604688ce94cc32 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -77,7 +77,7 @@ The build process consists in the usual :: make install invocations. Configuration options and caveats for specific Unix platforms are -extensively documented in the :source:`README` file in the root of the Python +extensively documented in the :source:`README.rst` file in the root of the Python source tree. .. warning:: diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 3e4b70e8a17ef7..68687e9f3ec367 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -550,9 +550,9 @@ Shebang Lines If the first line of a script file starts with ``#!``, it is known as a "shebang" line. Linux and other Unix like operating systems have native -support for such lines and are commonly used on such systems to indicate how -a script should be executed. This launcher allows the same facilities to be -using with Python scripts on Windows and the examples above demonstrate their +support for such lines and they are commonly used on such systems to indicate +how a script should be executed. This launcher allows the same facilities to +be used with Python scripts on Windows and the examples above demonstrate their use. To allow shebang lines in Python scripts to be portable between Unix and diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index edb74f043e0b64..a6ba5bbb720ca2 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2327,11 +2327,12 @@ The :func:`inspect.getargspec` function is deprecated and scheduled to be removed in Python 3.6. (See :issue:`20438` for details.) The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getargvalues`, :func:`~inspect.getcallargs`, -:func:`~inspect.getargvalues`, :func:`~inspect.formatargspec`, and -:func:`~inspect.formatargvalues` functions are deprecated in favor of -the :func:`inspect.signature` API. -(Contributed by Yury Selivanov in :issue:`20438`.) +:func:`~inspect.getcallargs`, and :func:`~inspect.formatargspec` functions are +deprecated in favor of the :func:`inspect.signature` API. (Contributed by Yury +Selivanov in :issue:`20438`.) + +:func:`~inspect.getargvalues` and :func:`~inspect.formatargvalues` functions +were inadvertently marked as deprecated with the release of Python 3.5.0. Use of :const:`re.LOCALE` flag with str patterns or :const:`re.ASCII` is now deprecated. (Contributed by Serhiy Storchaka in :issue:`22407`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 96fd256b991b76..a701cafa610788 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2,8 +2,6 @@ What's New In Python 3.6 **************************** -:Release: |release| -:Date: |today| :Editors: Elvis Pranskevichus , Yury Selivanov .. Rules for maintenance: @@ -536,7 +534,7 @@ provide correctly read str objects to Python code. ``sys.stdin``, This change only applies when using an interactive console, and not when redirecting files or pipes. To revert to the previous behaviour for interactive -console use, set :envvar:`PYTHONLEGACYWINDOWSIOENCODING`. +console use, set :envvar:`PYTHONLEGACYWINDOWSSTDIO`. .. seealso:: diff --git a/Include/ceval.h b/Include/ceval.h index 89c6062f11e17c..1e482729a1cc98 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -216,6 +216,7 @@ PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif diff --git a/Include/fileutils.h b/Include/fileutils.h index b933e985392e73..900c70faad719f 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -22,7 +22,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #ifdef MS_WINDOWS struct _Py_stat_struct { unsigned long st_dev; - __int64 st_ino; + uint64_t st_ino; unsigned short st_mode; int st_nlink; int st_uid; diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 49930e85565349..a658e9f5840e1c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 0 +#define PY_MICRO_VERSION 1 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.0+" +#define PY_VERSION "3.6.1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 5a67666874da94..01abfa9fcd6fb7 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -70,8 +70,8 @@ PyAPI_FUNC(const char *) Py_GetCopyright(void); PyAPI_FUNC(const char *) Py_GetCompiler(void); PyAPI_FUNC(const char *) Py_GetBuildInfo(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(const char *) _Py_hgidentifier(void); -PyAPI_FUNC(const char *) _Py_hgversion(void); +PyAPI_FUNC(const char *) _Py_gitidentifier(void); +PyAPI_FUNC(const char *) _Py_gitversion(void); #endif /* Internal -- various one-time initializations */ diff --git a/Include/pyport.h b/Include/pyport.h index 52a91a0d11d056..426822a81f075a 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -37,7 +37,7 @@ Used in: Py_SAFE_DOWNCAST * integral synonyms. Only define the ones we actually need. */ -// long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. +/* long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. */ #ifndef HAVE_LONG_LONG #define HAVE_LONG_LONG 1 #endif diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 36263544607096..579ac073d0f241 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,11 +45,13 @@ PyAPI_FUNC(int) PySlice_GetIndicesEx(PyObject *r, Py_ssize_t length, Py_ssize_t *step, Py_ssize_t *slicelength); #if !defined(Py_LIMITED_API) || (Py_LIMITED_API+0 >= 0x03050400 && Py_LIMITED_API+0 < 0x03060000) || Py_LIMITED_API+0 >= 0x03060100 +#ifdef Py_LIMITED_API #define PySlice_GetIndicesEx(slice, length, start, stop, step, slicelen) ( \ PySlice_Unpack((slice), (start), (stop), (step)) < 0 ? \ ((*(slicelen) = 0), -1) : \ ((*(slicelen) = PySlice_AdjustIndices((length), (start), (stop), *(step))), \ 0)) +#endif PyAPI_FUNC(int) PySlice_Unpack(PyObject *slice, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 587cf03e3697eb..5b877185f258d3 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1609,50 +1609,41 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( This codec uses mappings to encode and decode characters. - Decoding mappings must map single string characters to single - Unicode characters, integers (which are then interpreted as Unicode - ordinals) or None (meaning "undefined mapping" and causing an - error). - - Encoding mappings must map single Unicode characters to single - string characters, integers (which are then interpreted as Latin-1 - ordinals) or None (meaning "undefined mapping" and causing an - error). - - If a character lookup fails with a LookupError, the character is - copied as-is meaning that its ordinal value will be interpreted as - Unicode or Latin-1 ordinal resp. Because of this mappings only need - to contain those mappings which map characters to different code - points. + Decoding mappings must map byte ordinals (integers in the range from 0 to + 255) to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or None. Unmapped data bytes (ones which cause a LookupError) + as well as mapped to None, 0xFFFE or '\ufffe' are treated as "undefined + mapping" and cause an error. + + Encoding mappings must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or None. Unmapped character + ordinals (ones which cause a LookupError) as well as mapped to + None are treated as "undefined mapping" and cause an error. */ PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap( const char *string, /* Encoded string */ Py_ssize_t length, /* size of string */ - PyObject *mapping, /* character mapping - (char ordinal -> unicode ordinal) */ + PyObject *mapping, /* decoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString( PyObject *unicode, /* Unicode object */ - PyObject *mapping /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping /* encoding mapping */ ); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( PyObject *unicode, /* Unicode object */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); #endif @@ -1661,8 +1652,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -1960,8 +1951,8 @@ PyAPI_FUNC(PyObject*) PyUnicode_RSplit( /* Translate a string by applying a character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -2318,6 +2309,10 @@ PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy( PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); +#elif !defined(NDEBUG) +/* For asserts that call _PyUnicode_CheckConsistency(), which would + * otherwise be a problem when building with asserts but without Py_DEBUG. */ +#define _PyUnicode_CheckConsistency(op, check_content) PyUnicode_Check(op) #endif #ifndef Py_LIMITED_API diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b172f3f360e6ef..005d884572465f 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -908,7 +908,8 @@ def index(self, value, start=0, stop=None): i = start while stop is None or i < stop: try: - if self[i] == value: + v = self[i] + if v is value or v == value: return i except IndexError: break @@ -917,7 +918,7 @@ def index(self, value, start=0, stop=None): def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' - return sum(1 for v in self if v == value) + return sum(1 for v in self if v is value or v == value) Sequence.register(tuple) Sequence.register(str) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index eadf06f20e2507..e37852e2536c33 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -210,7 +210,7 @@ def _remove_universal_flags(_config_vars): # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub(r'-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) flags = re.sub('-isysroot [^ \t]*', ' ', flags) _save_modified_value(_config_vars, cv, flags) diff --git a/Lib/aifc.py b/Lib/aifc.py index 692d0bfd272bf0..13ad7dc5ca3d62 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -303,6 +303,8 @@ class Aifc_read: # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk # _framesize -- size of one frame in the file + _file = None # Set here since __del__ checks it + def initfp(self, file): self._version = 0 self._convert = None @@ -344,9 +346,15 @@ def initfp(self, file): def __init__(self, f): if isinstance(f, str): - f = builtins.open(f, 'rb') - # else, assume it is an open file object already - self.initfp(f) + file_object = builtins.open(f, 'rb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + else: + # assume it is an open file object already + self.initfp(f) def __enter__(self): return self @@ -541,18 +549,23 @@ class Aifc_write: # _datalength -- the size of the audio samples written to the header # _datawritten -- the size of the audio samples actually written + _file = None # Set here since __del__ checks it + def __init__(self, f): if isinstance(f, str): - filename = f - f = builtins.open(f, 'wb') - else: - # else, assume it is an open file object already - filename = '???' - self.initfp(f) - if filename[-5:] == '.aiff': - self._aifc = 0 + file_object = builtins.open(f, 'wb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + + # treat .aiff file extensions as non-compressed audio + if f.endswith('.aiff'): + self._aifc = 0 else: - self._aifc = 1 + # assume it is an open file object already + self.initfp(f) def initfp(self, file): self._file = file diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 0df58c5f873d88..a4967b854c5e68 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -459,7 +459,8 @@ def run_until_complete(self, future): # local task. future.exception() raise - future.remove_done_callback(_run_until_complete_cb) + finally: + future.remove_done_callback(_run_until_complete_cb) if not future.done(): raise RuntimeError('Event loop stopped before Future completed.') diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 28a45fc3cc5aee..e85634e588f5a5 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -11,6 +11,7 @@ import functools import inspect +import os import reprlib import socket import subprocess @@ -611,6 +612,9 @@ def new_event_loop(self): # A TLS for the running event loop, used by _get_running_loop. class _RunningLoop(threading.local): _loop = None + _pid = None + + _running_loop = _RunningLoop() @@ -620,7 +624,9 @@ def _get_running_loop(): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - return _running_loop._loop + running_loop = _running_loop._loop + if running_loop is not None and _running_loop._pid == os.getpid(): + return running_loop def _set_running_loop(loop): @@ -629,6 +635,7 @@ def _set_running_loop(loop): This is a low-level function intended to be used by event loops. This function is thread-specific. """ + _running_loop._pid = os.getpid() _running_loop._loop = loop diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7ad28d6aa0089a..ab7ff0bf93d076 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -543,8 +543,10 @@ def eof_received(self): def _get_extra_info(self, name, default=None): if name in self._extra: return self._extra[name] - else: + elif self._transport is not None: return self._transport.get_extra_info(name, default) + else: + return default def _start_shutdown(self): if self._in_shutdown: diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index b2f5304f772121..4c85466859f8f0 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -24,6 +24,8 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._process_exited = False + self._pipe_fds = [] def __repr__(self): info = [self.__class__.__name__] @@ -43,12 +45,14 @@ def connection_made(self, transport): self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -86,9 +90,18 @@ def pipe_connection_lost(self, fd, exc): else: reader.set_exception(exc) + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + self._maybe_close_transport() + def process_exited(self): - self._transport.close() - self._transport = None + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: + self._transport.close() + self._transport = None class Process: diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 4d79367d5cb600..d7867d128a8afe 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -180,7 +180,12 @@ def _step(self, exc=None): else: result = coro.throw(exc) except StopIteration as exc: - self.set_result(exc.value) + if self._must_cancel: + # Task is cancelled right before coro stops. + self._must_cancel = False + self.set_exception(futures.CancelledError()) + else: + self.set_result(exc.value) except futures.CancelledError: super().cancel() # I.e., Future.cancel(self). except Exception as exc: @@ -517,7 +522,8 @@ def ensure_future(coro_or_future, *, loop=None): elif compat.PY35 and inspect.isawaitable(coro_or_future): return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) else: - raise TypeError('A Future, a coroutine or an awaitable is required') + raise TypeError('An asyncio.Future, a coroutine or an awaitable is ' + 'required') @coroutine diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py index 99e3839f456858..b12d5db2a9755d 100644 --- a/Lib/asyncio/test_utils.py +++ b/Lib/asyncio/test_utils.py @@ -449,12 +449,15 @@ def new_test_loop(self, gen=None): self.set_event_loop(loop) return loop + def unpatch_get_running_loop(self): + events._get_running_loop = self._get_running_loop + def setUp(self): self._get_running_loop = events._get_running_loop events._get_running_loop = lambda: None def tearDown(self): - events._get_running_loop = self._get_running_loop + self.unpatch_get_running_loop() events.set_event_loop(None) diff --git a/Lib/base64.py b/Lib/base64.py index 58f6ad6816ee2f..eb8f258a2d1977 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -541,7 +541,8 @@ def encodebytes(s): def encodestring(s): """Legacy alias of encodebytes().""" import warnings - warnings.warn("encodestring() is a deprecated alias, use encodebytes()", + warnings.warn("encodestring() is a deprecated alias since 3.1, " + "use encodebytes()", DeprecationWarning, 2) return encodebytes(s) @@ -554,7 +555,8 @@ def decodebytes(s): def decodestring(s): """Legacy alias of decodebytes().""" import warnings - warnings.warn("decodestring() is a deprecated alias, use decodebytes()", + warnings.warn("decodestring() is a deprecated alias since Python 3.1, " + "use decodebytes()", DeprecationWarning, 2) return decodebytes(s) diff --git a/Lib/configparser.py b/Lib/configparser.py index af5aca1feae34a..230ab2b017eade 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -143,6 +143,7 @@ import functools import io import itertools +import os import re import sys import warnings @@ -687,7 +688,7 @@ def read(self, filenames, encoding=None): Return list of successfully read files. """ - if isinstance(filenames, str): + if isinstance(filenames, (str, os.PathLike)): filenames = [filenames] read_ok = [] for filename in filenames: @@ -696,6 +697,8 @@ def read(self, filenames, encoding=None): self._read(fp, filename) except OSError: continue + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) read_ok.append(filename) return read_ok diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 7d94a579c872b7..5e47054954ba5a 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -88,7 +88,7 @@ def __exit__(self, type, value, traceback): try: next(self.gen) except StopIteration: - return + return False else: raise RuntimeError("generator didn't stop") else: @@ -98,20 +98,19 @@ def __exit__(self, type, value, traceback): value = type() try: self.gen.throw(type, value, traceback) - raise RuntimeError("generator didn't stop after throw()") except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration # raised inside the "with" statement from being suppressed. return exc is not value except RuntimeError as exc: - # Don't re-raise the passed in exception. (issue27112) + # Don't re-raise the passed in exception. (issue27122) if exc is value: return False # Likewise, avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError # (see PEP 479). - if exc.__cause__ is value: + if type is StopIteration and exc.__cause__ is value: return False raise except: @@ -122,8 +121,10 @@ def __exit__(self, type, value, traceback): # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. # - if sys.exc_info()[1] is not value: - raise + if sys.exc_info()[1] is value: + return False + raise + raise RuntimeError("generator didn't stop after throw()") def contextmanager(func): diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py index 8eac58f0262caf..f622093df61da5 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/ctypes/test/test_callbacks.py @@ -244,6 +244,7 @@ def callback(a, b, c, d, e): def test_callback_large_struct(self): class Check: pass + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c class X(Structure): _fields_ = [ ('first', c_ulong), @@ -255,6 +256,11 @@ def callback(check, s): check.first = s.first check.second = s.second check.third = s.third + # See issue #29565. + # The structure should be passed by value, so + # any changes to it should not be reflected in + # the value passed + s.first = s.second = s.third = 0x0badf00d check = Check() s = X() @@ -275,6 +281,11 @@ def callback(check, s): self.assertEqual(check.first, 0xdeadbeef) self.assertEqual(check.second, 0xcafebabe) self.assertEqual(check.third, 0x0bad1dea) + # See issue #29565. + # Ensure that the original struct is unchanged. + self.assertEqual(s.first, check.first) + self.assertEqual(s.second, check.second) + self.assertEqual(s.third, check.third) ################################################################ diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 8f6fe5f25429e5..2e778fb1b43740 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -2,7 +2,8 @@ from ctypes import * from ctypes.test import need_symbol from struct import calcsize -import _testcapi +import _ctypes_test +import test.support class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -201,7 +202,10 @@ class X(Structure): "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + @test.support.cpython_only + def test_packed_c_limits(self): # Issue 15989 + import _testcapi d = {"_fields_": [("a", c_byte)], "_pack_": _testcapi.INT_MAX + 1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) @@ -391,6 +395,28 @@ class Z(Y): (1, 0, 0, 0, 0, 0)) self.assertRaises(TypeError, lambda: Z(1, 2, 3, 4, 5, 6, 7)) + def test_pass_by_value(self): + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c + class X(Structure): + _fields_ = [ + ('first', c_ulong), + ('second', c_ulong), + ('third', c_ulong), + ] + + s = X() + s.first = 0xdeadbeef + s.second = 0xcafebabe + s.third = 0x0bad1dea + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_large_struct_update_value + func.argtypes = (X,) + func.restype = None + func(s) + self.assertEqual(s.first, 0xdeadbeef) + self.assertEqual(s.second, 0xcafebabe) + self.assertEqual(s.third, 0x0bad1dea) + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Lib/ctypes/test/test_win32.py b/Lib/ctypes/test/test_win32.py index da1624015e198c..5d85ad6200b319 100644 --- a/Lib/ctypes/test/test_win32.py +++ b/Lib/ctypes/test/test_win32.py @@ -41,15 +41,19 @@ class FunctionCallTestCase(unittest.TestCase): @unittest.skipIf(sys.executable.lower().endswith('_d.exe'), "SEH not enabled in debug builds") def test_SEH(self): - # Call functions with invalid arguments, and make sure - # that access violations are trapped and raise an - # exception. - self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) + # Disable faulthandler to prevent logging the warning: + # "Windows fatal exception: access violation" + with support.disable_faulthandler(): + # Call functions with invalid arguments, and make sure + # that access violations are trapped and raise an + # exception. + self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) def test_noargs(self): # This is a special case on win32 x64 windll.user32.GetDesktopWindow() + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWintypes(unittest.TestCase): def test_HWND(self): diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 8bf1a7016bdf42..2bcd1dd2885991 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -93,14 +93,11 @@ def get_python_inc(plat_specific=0, prefix=None): # the build directory may not be the source directory, we # must use "srcdir" from the makefile to find the "Include" # directory. - base = _sys_home or project_base if plat_specific: - return base - if _sys_home: - incdir = os.path.join(_sys_home, get_config_var('AST_H_DIR')) + return _sys_home or project_base else: incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) + return os.path.normpath(incdir) python_dir = 'python' + get_python_version() + build_flags return os.path.join(prefix, "include", python_dir) elif os.name == "nt": diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index be7f5f38aafda7..c6502d61d54d0e 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -37,6 +37,13 @@ def setUp(self): from distutils.command import build_ext build_ext.USER_BASE = site.USER_BASE + # bpo-30132: On Windows, a .pdb file may be created in the current + # working directory. Create a temporary working directory to cleanup + # everything at the end of the test. + self.temp_cwd = support.temp_cwd() + self.temp_cwd.__enter__() + self.addCleanup(self.temp_cwd.__exit__, None, None, None) + def build_ext(self, *args, **kwargs): return build_ext(*args, **kwargs) diff --git a/Lib/email/architecture.rst b/Lib/email/architecture.rst index 78572ae63b4d2b..fcd10bde1325bb 100644 --- a/Lib/email/architecture.rst +++ b/Lib/email/architecture.rst @@ -66,7 +66,7 @@ data payloads. Message Lifecycle ----------------- -The general lifecyle of a message is: +The general lifecycle of a message is: Creation A `Message` object can be created by a Parser, or it can be diff --git a/Lib/getpass.py b/Lib/getpass.py index be511211585a48..36e17e4cb6965d 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -7,7 +7,6 @@ echoing of the password contents while reading. On Windows, the msvcrt module will be used. -On the Mac EasyDialogs.AskPassword is used, if available. """ diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index ffc03c4112f073..f10cd345e886c8 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -90,7 +90,7 @@

Navigation

25.5. IDLE

-

Source code: Lib/idlelib/

+

Source code: Lib/idlelib/

IDLE is Python’s Integrated Development and Learning Environment.

IDLE has the following features:

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index f3ee391ca006fc..dd6c997abc587b 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -1,5 +1,7 @@ #! /usr/bin/env python3 +import sys + try: from tkinter import * except ImportError: @@ -25,7 +27,6 @@ import re import socket import subprocess -import sys import threading import time import tokenize diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 2fa90120e7decc..1c0b03bff8a379 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -318,9 +318,12 @@ def shutdown(self): self.file.close() try: self.sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: self.sock.close() diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index b6a9f82e05f268..8b11d22b024ffa 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -136,7 +136,7 @@ def reload(module): """ if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") + raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: diff --git a/Lib/inspect.py b/Lib/inspect.py index 4d56ef5d41b47f..a2dcb888a0c609 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -253,18 +253,24 @@ def iscode(object): """Return true if the object is a code object. Code objects provide these attributes: - co_argcount number of arguments (not including * or ** args) - co_code string of raw compiled bytecode - co_consts tuple of constants used in the bytecode - co_filename name of file in which this code object was created - co_firstlineno number of first line in Python source code - co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg - co_lnotab encoded mapping of line numbers to bytecode indices - co_name name with which this code object was defined - co_names tuple of names of local variables - co_nlocals number of local variables - co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_argcount number of arguments (not including *, ** args + or keyword only arguments) + co_code string of raw compiled bytecode + co_cellvars tuple of names of cell variables + co_consts tuple of constants used in the bytecode + co_filename name of file in which this code object was created + co_firstlineno number of first line in Python source code + co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg + | 16=nested | 32=generator | 64=nofree | 128=coroutine + | 256=iterable_coroutine | 512=async_generator + co_freevars tuple of names of free variables + co_kwonlyargcount number of keyword only arguments (not including ** arg) + co_lnotab encoded mapping of line numbers to bytecode indices + co_name name with which this code object was defined + co_names tuple of names of local variables + co_nlocals number of local variables + co_stacksize virtual machine stack space required + co_varnames tuple of names of arguments and local variables""" return isinstance(object, types.CodeType) def isbuiltin(object): diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89e..45afc5f4e53fcf 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. @@ -74,10 +74,11 @@ def maybe(*choices): return group(*choices) + '?' Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -Triple = group("[ubUB]?[rR]?'''", '[ubUB]?[rR]?"""') +_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?" +Triple = group(_litprefix + "'''", _litprefix + '"""') # Single-line ' or " string. -String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') +String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get @@ -95,9 +96,9 @@ def maybe(*choices): return group(*choices) + '?' Token = Ignore + PlainToken # First (or only) line of ' or " string. -ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + +ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) @@ -109,19 +110,26 @@ def maybe(*choices): return group(*choices) + '?' "r'''": single3prog, 'r"""': double3prog, "u'''": single3prog, 'u"""': double3prog, "b'''": single3prog, 'b"""': double3prog, + "f'''": single3prog, 'f"""': double3prog, "ur'''": single3prog, 'ur"""': double3prog, "br'''": single3prog, 'br"""': double3prog, + "rb'''": single3prog, 'rb"""': double3prog, "R'''": single3prog, 'R"""': double3prog, "U'''": single3prog, 'U"""': double3prog, "B'''": single3prog, 'B"""': double3prog, + "F'''": single3prog, 'F"""': double3prog, "uR'''": single3prog, 'uR"""': double3prog, "Ur'''": single3prog, 'Ur"""': double3prog, "UR'''": single3prog, 'UR"""': double3prog, "bR'''": single3prog, 'bR"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog, + "rB'''": single3prog, 'rB"""': double3prog, + "Rb'''": single3prog, 'Rb"""': double3prog, + "RB'''": single3prog, 'RB"""': double3prog, 'r': None, 'R': None, 'u': None, 'U': None, + 'f': None, 'F': None, 'b': None, 'B': None} triple_quoted = {} @@ -129,20 +137,26 @@ def maybe(*choices): return group(*choices) + '?' "r'''", 'r"""', "R'''", 'R"""', "u'''", 'u"""', "U'''", 'U"""', "b'''", 'b"""', "B'''", 'B"""', + "f'''", 'f"""', "F'''", 'F"""', "ur'''", 'ur"""', "Ur'''", 'Ur"""', "uR'''", 'uR"""', "UR'''", 'UR"""', "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""',): + "bR'''", 'bR"""', "BR'''", 'BR"""', + "rb'''", 'rb"""', "Rb'''", 'Rb"""', + "rB'''", 'rB"""', "RB'''", 'RB"""',): triple_quoted[t] = t single_quoted = {} for t in ("'", '"', "r'", 'r"', "R'", 'R"', "u'", 'u"', "U'", 'U"', "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', "ur'", 'ur"', "Ur'", 'Ur"', "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', ): + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',): single_quoted[t] = t tabsize = 8 diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a5411a855a..0b9bee0ab42ea6 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 9a969e8ec04741..3f7ab9714e38f9 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -320,6 +320,7 @@ def test_5(self): def test_6(self): self.validate("lst: List[int] = []") + class TestExcept(GrammarTest): def test_new(self): s = """ @@ -338,6 +339,27 @@ def test_old(self): self.validate(s) +class TestStringLiterals(GrammarTest): + prefixes = ("'", '"', + "r'", 'r"', "R'", 'R"', + "u'", 'u"', "U'", 'U"', + "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', + "ur'", 'ur"', "Ur'", 'Ur"', + "uR'", 'uR"', "UR'", 'UR"', + "br'", 'br"', "Br'", 'Br"', + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',) + + def test_lit(self): + for pre in self.prefixes: + single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) + self.validate(single) + triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) + self.validate(triple) + + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms class TestSetLiteral(GrammarTest): def test_1(self): diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 7d779734f37b0d..2f934b33071563 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -815,16 +815,38 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), if isinstance(address, str): self.unixsocket = True - self._connect_unixsocket(address) + # Syslog server may be unavailable during handler initialisation. + # C's openlog() function also ignores connection errors. + # Moreover, we ignore these errors while logging, so it not worse + # to ignore it also here. + try: + self._connect_unixsocket(address) + except OSError: + pass else: self.unixsocket = False if socktype is None: socktype = socket.SOCK_DGRAM - self.socket = socket.socket(socket.AF_INET, socktype) - if socktype == socket.SOCK_STREAM: - self.socket.connect(address) + host, port = address + ress = socket.getaddrinfo(host, port, 0, socktype) + if not ress: + raise OSError("getaddrinfo returns an empty list") + for res in ress: + af, socktype, proto, _, sa = res + err = sock = None + try: + sock = socket.socket(af, socktype, proto) + if socktype == socket.SOCK_STREAM: + sock.connect(sa) + break + except OSError as exc: + err = exc + if sock is not None: + sock.close() + if err is not None: + raise err + self.socket = sock self.socktype = socktype - self.formatter = None def _connect_unixsocket(self, address): use_socktype = self.socktype @@ -863,7 +885,7 @@ def encodePriority(self, facility, priority): priority = self.priority_names[priority] return (facility << 3) | priority - def close (self): + def close(self): """ Closes the socket. """ diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index f2c179e4e0afaa..d5ce6257456766 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -149,8 +149,15 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): util._close_stdin() - # ignoring SIGCHLD means no need to reap zombie processes - handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) + # ignoring SIGCHLD means no need to reap zombie processes; + # letting SIGINT through avoids KeyboardInterrupt tracebacks + handlers = { + signal.SIGCHLD: signal.SIG_IGN, + signal.SIGINT: signal.SIG_DFL, + } + old_handlers = {sig: signal.signal(sig, val) + for (sig, val) in handlers.items()} + with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ selectors.DefaultSelector() as selector: _forkserver._forkserver_address = listener.getsockname() @@ -175,7 +182,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): code = 1 if os.fork() == 0: try: - _serve_one(s, listener, alive_r, handler) + _serve_one(s, listener, alive_r, old_handlers) except Exception: sys.excepthook(*sys.exc_info()) sys.stderr.flush() @@ -186,11 +193,12 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): if e.errno != errno.ECONNABORTED: raise -def _serve_one(s, listener, alive_r, handler): - # close unnecessary stuff and reset SIGCHLD handler +def _serve_one(s, listener, alive_r, handlers): + # close unnecessary stuff and reset signal handlers listener.close() os.close(alive_r) - signal.signal(signal.SIGCHLD, handler) + for sig, val in handlers.items(): + signal.signal(sig, val) # receive fds from parent process fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ffdf42614d59eb..a545f3c1a18961 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -118,7 +118,7 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, try: result = (True, func(*args, **kwds)) except Exception as e: - if wrap_exception: + if wrap_exception and func is not _helper_reraises_exception: e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: @@ -128,9 +128,15 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) +def _helper_reraises_exception(ex): + 'Pickle-able helper function for use by _guarded_task_generation.' + raise ex + # # Class representing a process pool # @@ -275,6 +281,17 @@ def starmap_async(self, func, iterable, chunksize=None, callback=None, return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) + def _guarded_task_generation(self, result_job, func, iterable): + '''Provides a generator of tasks for imap and imap_unordered with + appropriate handling for iterables which throw exceptions during + iteration.''' + try: + i = -1 + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + except Exception as e: + yield (result_job, i+1, _helper_reraises_exception, (e,), {}) + def imap(self, func, iterable, chunksize=1): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. @@ -283,15 +300,23 @@ def imap(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def imap_unordered(self, func, iterable, chunksize=1): @@ -302,15 +327,23 @@ def imap_unordered(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def apply_async(self, func, args=(), kwds={}, callback=None, @@ -321,7 +354,7 @@ def apply_async(self, func, args=(), kwds={}, callback=None, if self._state != RUN: raise ValueError("Pool not running") result = ApplyResult(self._cache, callback, error_callback) - self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) + self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) return result def map_async(self, func, iterable, chunksize=None, callback=None, @@ -352,8 +385,14 @@ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, task_batches = Pool._get_tasks(func, iterable, chunksize) result = MapResult(self._cache, chunksize, len(iterable), callback, error_callback=error_callback) - self._taskqueue.put((((result._job, i, mapper, (x,), {}) - for i, x in enumerate(task_batches)), None)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapper, + task_batches), + None + ) + ) return result @staticmethod @@ -375,37 +414,32 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): for taskseq, set_length in iter(taskqueue.get, None): task = None - i = -1 try: - for i, task in enumerate(taskseq): + # iterating taskseq cannot fail + for task in taskseq: if thread._state: util.debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: - job, ind = task[:2] + job, idx = task[:2] try: - cache[job]._set(ind, (False, e)) + cache[job]._set(idx, (False, e)) except KeyError: pass else: if set_length: util.debug('doing set_length()') - set_length(i+1) + idx = task[1] if task else -1 + set_length(idx + 1) continue break - except Exception as ex: - job, ind = task[:2] if task else (0, 0) - if job in cache: - cache[job]._set(ind + 1, (False, ex)) - if set_length: - util.debug('doing set_length()') - set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +479,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +496,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index dda03ddf5425ce..7f77837a74acb6 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -221,8 +221,8 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): else: wacquire = None - try: - while 1: + while 1: + try: nacquire() try: if not buffer: @@ -249,21 +249,19 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): wrelease() except IndexError: pass - except Exception as e: - if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: - return - # Since this runs in a daemon thread the resources it uses - # may be become unusable while the process is cleaning up. - # We ignore errors which happen after the process has - # started to cleanup. - try: + except Exception as e: + if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: + return + # Since this runs in a daemon thread the resources it uses + # may be become unusable while the process is cleaning up. + # We ignore errors which happen after the process has + # started to cleanup. if is_exiting(): info('error in queue thread: %s', e) + return else: import traceback traceback.print_exc() - except Exception: - pass _sentinel = object() @@ -337,6 +335,7 @@ def __getstate__(self): def __setstate__(self, state): (self._reader, self._writer, self._rlock, self._wlock) = state + self._poll = self._reader.poll def get(self): with self._rlock: diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 1a2c0db40b9cc6..0ce274ceca6057 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -386,7 +386,7 @@ def _close_stdin(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - passfds = sorted(passfds) + passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9f347216b183a9..70f5cba76ffa8a 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1220,25 +1220,23 @@ def touch(self, mode=0o666, exist_ok=True): os.close(fd) def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ if self._closed: self._raise_closed() - if not parents: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - else: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - except OSError as e: - if e.errno != ENOENT or self.parent == self: - raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + try: + self._accessor.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise def chmod(self, mode): """ diff --git a/Lib/platform.py b/Lib/platform.py index e48ad0b6e7bdea..cc2db9870d84b6 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -110,7 +110,7 @@ """ -__version__ = '1.0.7' +__version__ = '1.0.8' import collections import sys, os, re, subprocess @@ -1198,7 +1198,9 @@ def _sys_version(sys_version=None): elif buildtime: builddate = builddate + ' ' + buildtime - if hasattr(sys, '_mercurial'): + if hasattr(sys, '_git'): + _, branch, revision = sys._git + elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial elif hasattr(sys, 'subversion'): # sys.subversion was added in Python 2.5 diff --git a/Lib/poplib.py b/Lib/poplib.py index cae6950eb6d2d6..6bcfa5cfeba37b 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -288,9 +288,12 @@ def close(self): if sock is not None: try: sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: sock.close() diff --git a/Lib/pstats.py b/Lib/pstats.py index d861413d4195f7..2c5bf981b85cf8 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -48,11 +48,14 @@ class Stats: printed. The sort_stats() method now processes some additional options (i.e., in - addition to the old -1, 0, 1, or 2). It takes an arbitrary number of - quoted strings to select the sort order. For example sort_stats('time', - 'name') sorts on the major key of 'internal function time', and on the - minor key of 'the name of the function'. Look at the two tables in - sort_stats() and get_sort_arg_defs(self) for more examples. + addition to the old -1, 0, 1, or 2 that are respectively interpreted as + 'stdname', 'calls', 'time', and 'cumulative'). It takes an arbitrary number + of quoted strings to select the sort order. + + For example sort_stats('time', 'name') sorts on the major key of 'internal + function time', and on the minor key of 'the name of the function'. Look at + the two tables in sort_stats() and get_sort_arg_defs(self) for more + examples. All methods return self, so you can string together commands like: Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index c7fac3395b3be0..2caab630978fa0 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Dec 6 18:51:51 2016 -topics = {'assert': '\n' - 'The "assert" statement\n' +# Autogenerated by Sphinx on Sat Mar 4 12:14:44 2017 +topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' 'Assert statements are a convenient way to insert debugging ' @@ -39,8 +38,7 @@ 'Assignments to "__debug__" are illegal. The value for the ' 'built-in\n' 'variable is determined when the interpreter starts.\n', - 'assignment': '\n' - 'Assignment statements\n' + 'assignment': 'Assignment statements\n' '*********************\n' '\n' 'Assignment statements are used to (re)bind names to values and ' @@ -405,8 +403,7 @@ 'See also: **PEP 526** - Variable and attribute annotation ' 'syntax\n' ' **PEP 484** - Type hints\n', - 'atom-identifiers': '\n' - 'Identifiers (Names)\n' + 'atom-identifiers': 'Identifiers (Names)\n' '*******************\n' '\n' 'An identifier occurring as an atom is a name. See ' @@ -446,8 +443,7 @@ 'happen. If the class name consists only of underscores, ' 'no\n' 'transformation is done.\n', - 'atom-literals': '\n' - 'Literals\n' + 'atom-literals': 'Literals\n' '********\n' '\n' 'Python supports string and bytes literals and various ' @@ -476,8 +472,7 @@ 'may obtain\n' 'the same object or a different object with the same ' 'value.\n', - 'attribute-access': '\n' - 'Customizing attribute access\n' + 'attribute-access': 'Customizing attribute access\n' '****************************\n' '\n' 'The following methods can be defined to customize the ' @@ -851,8 +846,7 @@ '* *__class__* assignment works only if both classes have ' 'the same\n' ' *__slots__*.\n', - 'attribute-references': '\n' - 'Attribute references\n' + 'attribute-references': 'Attribute references\n' '********************\n' '\n' 'An attribute reference is a primary followed by a ' @@ -875,8 +869,7 @@ 'determined by the object. Multiple evaluations of ' 'the same attribute\n' 'reference may yield different objects.\n', - 'augassign': '\n' - 'Augmented assignment statements\n' + 'augassign': 'Augmented assignment statements\n' '*******************************\n' '\n' 'Augmented assignment is the combination, in a single statement, ' @@ -940,8 +933,7 @@ 'about\n' 'class and instance attributes applies as for regular ' 'assignments.\n', - 'binary': '\n' - 'Binary arithmetic operations\n' + 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' 'The binary arithmetic operations have the conventional priority\n' @@ -1029,8 +1021,7 @@ 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' 'The numeric arguments are first converted to a common type.\n', - 'bitwise': '\n' - 'Binary bitwise operations\n' + 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' 'Each of the three bitwise operations has a different priority ' @@ -1050,8 +1041,7 @@ 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' 'which must be integers.\n', - 'bltin-code-objects': '\n' - 'Code Objects\n' + 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' 'Code objects are used by the implementation to ' @@ -1074,8 +1064,7 @@ '\n' 'See The standard type hierarchy for more ' 'information.\n', - 'bltin-ellipsis-object': '\n' - 'The Ellipsis Object\n' + 'bltin-ellipsis-object': 'The Ellipsis Object\n' '*******************\n' '\n' 'This object is commonly used by slicing (see ' @@ -1087,8 +1076,7 @@ '"Ellipsis" singleton.\n' '\n' 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': '\n' - 'The Null Object\n' + 'bltin-null-object': 'The Null Object\n' '***************\n' '\n' "This object is returned by functions that don't " @@ -1100,8 +1088,7 @@ 'same singleton.\n' '\n' 'It is written as "None".\n', - 'bltin-type-objects': '\n' - 'Type Objects\n' + 'bltin-type-objects': 'Type Objects\n' '************\n' '\n' 'Type objects represent the various object types. An ' @@ -1113,8 +1100,7 @@ 'all standard built-in types.\n' '\n' 'Types are written like this: "".\n', - 'booleans': '\n' - 'Boolean operations\n' + 'booleans': 'Boolean operations\n' '******************\n' '\n' ' or_test ::= and_test | or_test "or" and_test\n' @@ -1163,8 +1149,7 @@ 'its\n' 'argument (for example, "not \'foo\'" produces "False" rather ' 'than "\'\'".)\n', - 'break': '\n' - 'The "break" statement\n' + 'break': 'The "break" statement\n' '*********************\n' '\n' ' break_stmt ::= "break"\n' @@ -1185,8 +1170,7 @@ 'clause, that "finally" clause is executed before really leaving ' 'the\n' 'loop.\n', - 'callable-types': '\n' - 'Emulating callable objects\n' + 'callable-types': 'Emulating callable objects\n' '**************************\n' '\n' 'object.__call__(self[, args...])\n' @@ -1195,8 +1179,7 @@ 'this method\n' ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' ' "x.__call__(arg1, arg2, ...)".\n', - 'calls': '\n' - 'Calls\n' + 'calls': 'Calls\n' '*****\n' '\n' 'A call calls a callable object (e.g., a *function*) with a ' @@ -1217,7 +1200,8 @@ ' ("," "*" expression | "," ' 'keyword_item)*\n' ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "**" expression)*\n' + ' ("," keyword_item | "," "**" ' + 'expression)*\n' ' keyword_item ::= identifier "=" expression\n' '\n' 'An optional trailing comma may be present after the positional and\n' @@ -1382,8 +1366,7 @@ ' The class must define a "__call__()" method; the effect is then ' 'the\n' ' same as if that method was called.\n', - 'class': '\n' - 'Class definitions\n' + 'class': 'Class definitions\n' '*****************\n' '\n' 'A class definition defines a class object (see section The ' @@ -1469,8 +1452,7 @@ '\n' 'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n' ' Class Decorators\n', - 'comparisons': '\n' - 'Comparisons\n' + 'comparisons': 'Comparisons\n' '***********\n' '\n' 'Unlike C, all comparison operations in Python have the same ' @@ -1623,7 +1605,7 @@ 'restriction that\n' ' ranges do not support order comparison. Equality ' 'comparison across\n' - ' these types results in unequality, and ordering comparison ' + ' these types results in inequality, and ordering comparison ' 'across\n' ' these types raises "TypeError".\n' '\n' @@ -1762,6 +1744,12 @@ ' to sequences, but not to sets or mappings). See also the\n' ' "total_ordering()" decorator.\n' '\n' + '* The "hash()" result should be consistent with equality. ' + 'Objects\n' + ' that are equal should either have the same hash value, or ' + 'be marked\n' + ' as unhashable.\n' + '\n' 'Python does not enforce these consistency rules. In fact, ' 'the\n' 'not-a-number values are an example for not following these ' @@ -1833,8 +1821,7 @@ 'is determined using the "id()" function. "x is not y" yields ' 'the\n' 'inverse truth value. [4]\n', - 'compound': '\n' - 'Compound statements\n' + 'compound': 'Compound statements\n' '*******************\n' '\n' 'Compound statements contain (groups of) other statements; they ' @@ -2613,7 +2600,8 @@ 'functions, even if they do not contain "await" or "async" ' 'keywords.\n' '\n' - 'It is a "SyntaxError" to use "yield" expressions in "async def"\n' + 'It is a "SyntaxError" to use "yield from" expressions in "async ' + 'def"\n' 'coroutines.\n' '\n' 'An example of a coroutine function:\n' @@ -2724,8 +2712,7 @@ ' body is transformed into the namespace\'s "__doc__" item ' 'and\n' " therefore the class's *docstring*.\n", - 'context-managers': '\n' - 'With Statement Context Managers\n' + 'context-managers': 'With Statement Context Managers\n' '*******************************\n' '\n' 'A *context manager* is an object that defines the ' @@ -2787,8 +2774,7 @@ ' The specification, background, and examples for the ' 'Python "with"\n' ' statement.\n', - 'continue': '\n' - 'The "continue" statement\n' + 'continue': 'The "continue" statement\n' '************************\n' '\n' ' continue_stmt ::= "continue"\n' @@ -2805,8 +2791,7 @@ '"finally" clause, that "finally" clause is executed before ' 'really\n' 'starting the next loop cycle.\n', - 'conversions': '\n' - 'Arithmetic conversions\n' + 'conversions': 'Arithmetic conversions\n' '**********************\n' '\n' 'When a description of an arithmetic operator below uses the ' @@ -2832,8 +2817,7 @@ "left argument to the '%' operator). Extensions must define " 'their own\n' 'conversion behavior.\n', - 'customization': '\n' - 'Basic customization\n' + 'customization': 'Basic customization\n' '*******************\n' '\n' 'object.__new__(cls[, ...])\n' @@ -3152,15 +3136,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only ' - 'required property\n' + ' "__hash__()" should return an integer. The only required ' + 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive ' - 'or) the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the ' + 'components of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -3272,8 +3259,7 @@ ' neither "__len__()" nor "__bool__()", all its instances ' 'are\n' ' considered true.\n', - 'debugger': '\n' - '"pdb" --- The Python Debugger\n' + 'debugger': '"pdb" --- The Python Debugger\n' '*****************************\n' '\n' '**Source code:** Lib/pdb.py\n' @@ -3938,8 +3924,7 @@ '[1] Whether a frame is considered to originate in a certain ' 'module\n' ' is determined by the "__name__" in the frame globals.\n', - 'del': '\n' - 'The "del" statement\n' + 'del': 'The "del" statement\n' '*******************\n' '\n' ' del_stmt ::= "del" target_list\n' @@ -3968,8 +3953,7 @@ 'Changed in version 3.2: Previously it was illegal to delete a name\n' 'from the local namespace if it occurs as a free variable in a nested\n' 'block.\n', - 'dict': '\n' - 'Dictionary displays\n' + 'dict': 'Dictionary displays\n' '*******************\n' '\n' 'A dictionary display is a possibly empty series of key/datum pairs\n' @@ -4013,8 +3997,7 @@ 'should be *hashable*, which excludes all mutable objects.) Clashes\n' 'between duplicate keys are not detected; the last datum (textually\n' 'rightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': '\n' - 'Interaction with dynamic features\n' + 'dynamic-features': 'Interaction with dynamic features\n' '*********************************\n' '\n' 'Name resolution of free variables occurs at runtime, not ' @@ -4050,8 +4033,7 @@ 'override the global and local namespace. If only one ' 'namespace is\n' 'specified, it is used for both.\n', - 'else': '\n' - 'The "if" statement\n' + 'else': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -4068,8 +4050,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'exceptions': '\n' - 'Exceptions\n' + 'exceptions': 'Exceptions\n' '**********\n' '\n' 'Exceptions are a means of breaking out of the normal flow of ' @@ -4145,8 +4126,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'execmodel': '\n' - 'Execution model\n' + 'execmodel': 'Execution model\n' '***************\n' '\n' '\n' @@ -4477,8 +4457,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'exprlists': '\n' - 'Expression lists\n' + 'exprlists': 'Expression lists\n' '****************\n' '\n' ' expression_list ::= expression ( "," expression )* [","]\n' @@ -4515,8 +4494,7 @@ 'value of that expression. (To create an empty tuple, use an ' 'empty pair\n' 'of parentheses: "()".)\n', - 'floating': '\n' - 'Floating point literals\n' + 'floating': 'Floating point literals\n' '***********************\n' '\n' 'Floating point literals are described by the following lexical\n' @@ -4552,8 +4530,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'for': '\n' - 'The "for" statement\n' + 'for': 'The "for" statement\n' '*******************\n' '\n' 'The "for" statement is used to iterate over the elements of a ' @@ -4625,8 +4602,7 @@ '\n' ' for x in a[:]:\n' ' if x < 0: a.remove(x)\n', - 'formatstrings': '\n' - 'Format String Syntax\n' + 'formatstrings': 'Format String Syntax\n' '********************\n' '\n' 'The "str.format()" method and the "Formatter" class share ' @@ -5345,8 +5321,7 @@ ' 9 9 11 1001\n' ' 10 A 12 1010\n' ' 11 B 13 1011\n', - 'function': '\n' - 'Function definitions\n' + 'function': 'Function definitions\n' '********************\n' '\n' 'A function definition defines a user-defined function object ' @@ -5515,8 +5490,7 @@ '\n' ' **PEP 3107** - Function Annotations\n' ' The original specification for function annotations.\n', - 'global': '\n' - 'The "global" statement\n' + 'global': 'The "global" statement\n' '**********************\n' '\n' ' global_stmt ::= "global" identifier ("," identifier)*\n' @@ -5560,8 +5534,7 @@ 'code containing the function call. The same applies to the ' '"eval()"\n' 'and "compile()" functions.\n', - 'id-classes': '\n' - 'Reserved classes of identifiers\n' + 'id-classes': 'Reserved classes of identifiers\n' '*******************************\n' '\n' 'Certain classes of identifiers (besides keywords) have ' @@ -5609,8 +5582,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'identifiers': '\n' - 'Identifiers and keywords\n' + 'identifiers': 'Identifiers and keywords\n' '************************\n' '\n' 'Identifiers (also referred to as *names*) are described by ' @@ -5758,8 +5730,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'if': '\n' - 'The "if" statement\n' + 'if': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -5775,8 +5746,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'imaginary': '\n' - 'Imaginary literals\n' + 'imaginary': 'Imaginary literals\n' '******************\n' '\n' 'Imaginary literals are described by the following lexical ' @@ -5796,8 +5766,7 @@ '\n' ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' '3.14_15_93j\n', - 'import': '\n' - 'The "import" statement\n' + 'import': 'The "import" statement\n' '**********************\n' '\n' ' import_stmt ::= "import" module ["as" name] ( "," module ' @@ -6058,8 +6027,7 @@ '\n' ' **PEP 236** - Back to the __future__\n' ' The original proposal for the __future__ mechanism.\n', - 'in': '\n' - 'Membership test operations\n' + 'in': 'Membership test operations\n' '**************************\n' '\n' 'The operators "in" and "not in" test for membership. "x in s"\n' @@ -6094,8 +6062,7 @@ '\n' 'The operator "not in" is defined to have the inverse true value of\n' '"in".\n', - 'integers': '\n' - 'Integer literals\n' + 'integers': 'Integer literals\n' '****************\n' '\n' 'Integer literals are described by the following lexical ' @@ -6141,8 +6108,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'lambda': '\n' - 'Lambdas\n' + 'lambda': 'Lambdas\n' '*******\n' '\n' ' lambda_expr ::= "lambda" [parameter_list]: expression\n' @@ -6165,8 +6131,7 @@ 'Note that functions created with lambda expressions cannot ' 'contain\n' 'statements or annotations.\n', - 'lists': '\n' - 'List displays\n' + 'lists': 'List displays\n' '*************\n' '\n' 'A list display is a possibly empty series of expressions enclosed ' @@ -6183,8 +6148,7 @@ 'from left to right and placed into the list object in that order.\n' 'When a comprehension is supplied, the list is constructed from the\n' 'elements resulting from the comprehension.\n', - 'naming': '\n' - 'Naming and binding\n' + 'naming': 'Naming and binding\n' '******************\n' '\n' '\n' @@ -6397,8 +6361,7 @@ 'override the global and local namespace. If only one namespace ' 'is\n' 'specified, it is used for both.\n', - 'nonlocal': '\n' - 'The "nonlocal" statement\n' + 'nonlocal': 'The "nonlocal" statement\n' '************************\n' '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' @@ -6429,8 +6392,7 @@ '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' ' The specification for the "nonlocal" statement.\n', - 'numbers': '\n' - 'Numeric literals\n' + 'numbers': 'Numeric literals\n' '****************\n' '\n' 'There are three types of numeric literals: integers, floating ' @@ -6444,8 +6406,7 @@ 'is actually an expression composed of the unary operator \'"-"\' ' 'and the\n' 'literal "1".\n', - 'numeric-types': '\n' - 'Emulating numeric types\n' + 'numeric-types': 'Emulating numeric types\n' '***********************\n' '\n' 'The following methods can be defined to emulate numeric ' @@ -6621,8 +6582,7 @@ ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' ' both should return the same value.\n', - 'objects': '\n' - 'Objects, values and types\n' + 'objects': 'Objects, values and types\n' '*************************\n' '\n' "*Objects* are Python's abstraction for data. All data in a " @@ -6750,8 +6710,7 @@ 'created empty lists. (Note that "c = d = []" assigns the same ' 'object\n' 'to both "c" and "d".)\n', - 'operator-summary': '\n' - 'Operator precedence\n' + 'operator-summary': 'Operator precedence\n' '*******************\n' '\n' 'The following table summarizes the operator precedence ' @@ -6924,8 +6883,7 @@ 'arithmetic\n' ' or bitwise unary operator on its right, that is, ' '"2**-1" is "0.5".\n', - 'pass': '\n' - 'The "pass" statement\n' + 'pass': 'The "pass" statement\n' '********************\n' '\n' ' pass_stmt ::= "pass"\n' @@ -6938,8 +6896,7 @@ ' def f(arg): pass # a function that does nothing (yet)\n' '\n' ' class C: pass # a class with no methods (yet)\n', - 'power': '\n' - 'The power operator\n' + 'power': 'The power operator\n' '******************\n' '\n' 'The power operator binds more tightly than unary operators on its\n' @@ -6973,8 +6930,7 @@ 'Raising a negative number to a fractional power results in a ' '"complex"\n' 'number. (In earlier versions it raised a "ValueError".)\n', - 'raise': '\n' - 'The "raise" statement\n' + 'raise': 'The "raise" statement\n' '*********************\n' '\n' ' raise_stmt ::= "raise" [expression ["from" expression]]\n' @@ -7059,8 +7015,7 @@ 'Exceptions, and information about handling exceptions is in ' 'section\n' 'The try statement.\n', - 'return': '\n' - 'The "return" statement\n' + 'return': 'The "return" statement\n' '**********************\n' '\n' ' return_stmt ::= "return" [expression_list]\n' @@ -7087,9 +7042,15 @@ 'generator is done and will cause "StopIteration" to be raised. ' 'The\n' 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n', - 'sequence-types': '\n' - 'Emulating container types\n' + '"StopIteration" and becomes the "StopIteration.value" attribute.\n' + '\n' + 'In an asynchronous generator function, an empty "return" ' + 'statement\n' + 'indicates that the asynchronous generator is done and will cause\n' + '"StopAsyncIteration" to be raised. A non-empty "return" statement ' + 'is\n' + 'a syntax error in an asynchronous generator function.\n', + 'sequence-types': 'Emulating container types\n' '*************************\n' '\n' 'The following methods can be defined to implement ' @@ -7310,8 +7271,7 @@ ' iteration protocol via "__getitem__()", see this ' 'section in the\n' ' language reference.\n', - 'shifting': '\n' - 'Shifting operations\n' + 'shifting': 'Shifting operations\n' '*******************\n' '\n' 'The shifting operations have lower priority than the arithmetic\n' @@ -7335,8 +7295,7 @@ 'operand is\n' ' larger than "sys.maxsize" an "OverflowError" exception is ' 'raised.\n', - 'slicings': '\n' - 'Slicings\n' + 'slicings': 'Slicings\n' '********\n' '\n' 'A slicing selects a range of items in a sequence object (e.g., ' @@ -7387,8 +7346,7 @@ 'as lower bound, upper bound and stride, respectively, ' 'substituting\n' '"None" for missing expressions.\n', - 'specialattrs': '\n' - 'Special Attributes\n' + 'specialattrs': 'Special Attributes\n' '******************\n' '\n' 'The implementation adds a few special read-only attributes ' @@ -7473,8 +7431,7 @@ '[5] To format only a tuple you should therefore provide a\n' ' singleton tuple whose only element is the tuple to be ' 'formatted.\n', - 'specialnames': '\n' - 'Special method names\n' + 'specialnames': 'Special method names\n' '********************\n' '\n' 'A class can implement certain operations that are invoked by ' @@ -7835,15 +7792,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only required ' + ' "__hash__()" should return an integer. The only required ' 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive or) ' - 'the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the components ' + 'of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -9262,8 +9222,7 @@ 'special method *must* be set on the class object itself in ' 'order to be\n' 'consistently invoked by the interpreter).\n', - 'string-methods': '\n' - 'String Methods\n' + 'string-methods': 'String Methods\n' '**************\n' '\n' 'Strings implement all of the common sequence operations, ' @@ -9500,12 +9459,11 @@ 'characters\n' ' and there is at least one character, false otherwise. ' 'Decimal\n' - ' characters are those from general category "Nd". This ' - 'category\n' - ' includes digit characters, and all characters that can ' - 'be used to\n' - ' form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC ' - 'DIGIT ZERO.\n' + ' characters are those that can be used to form numbers ' + 'in base 10,\n' + ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' + 'decimal character\n' + ' is a character in the Unicode General Category "Nd".\n' '\n' 'str.isdigit()\n' '\n' @@ -9515,10 +9473,13 @@ 'include decimal\n' ' characters and digits that need special handling, such ' 'as the\n' - ' compatibility superscript digits. Formally, a digit is ' - 'a character\n' - ' that has the property value Numeric_Type=Digit or\n' - ' Numeric_Type=Decimal.\n' + ' compatibility superscript digits. This covers digits ' + 'which cannot\n' + ' be used to form numbers in base 10, like the Kharosthi ' + 'numbers.\n' + ' Formally, a digit is a character that has the property ' + 'value\n' + ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' '\n' 'str.isidentifier()\n' '\n' @@ -10064,8 +10025,7 @@ " '00042'\n" ' >>> "-42".zfill(5)\n' " '-0042'\n", - 'strings': '\n' - 'String and Bytes literals\n' + 'strings': 'String and Bytes literals\n' '*************************\n' '\n' 'String literals are described by the following lexical ' @@ -10299,8 +10259,7 @@ 'followed by a newline is interpreted as those two characters as ' 'part\n' 'of the literal, *not* as a line continuation.\n', - 'subscriptions': '\n' - 'Subscriptions\n' + 'subscriptions': 'Subscriptions\n' '*************\n' '\n' 'A subscription selects an item of a sequence (string, tuple ' @@ -10357,8 +10316,7 @@ "A string's items are characters. A character is not a " 'separate data\n' 'type but a string of exactly one character.\n', - 'truth': '\n' - 'Truth Value Testing\n' + 'truth': 'Truth Value Testing\n' '*******************\n' '\n' 'Any object can be tested for truth value, for use in an "if" or\n' @@ -10390,8 +10348,7 @@ 'otherwise stated. (Important exception: the Boolean operations ' '"or"\n' 'and "and" always return one of their operands.)\n', - 'try': '\n' - 'The "try" statement\n' + 'try': 'The "try" statement\n' '*******************\n' '\n' 'The "try" statement specifies exception handlers and/or cleanup code\n' @@ -10538,8 +10495,7 @@ 'Exceptions, and information on using the "raise" statement to ' 'generate\n' 'exceptions may be found in section The raise statement.\n', - 'types': '\n' - 'The standard type hierarchy\n' + 'types': 'The standard type hierarchy\n' '***************************\n' '\n' 'Below is a list of the types that are built into Python. ' @@ -11097,6 +11053,27 @@ 'statements.\n' ' See also the Coroutine Objects section.\n' '\n' + ' Asynchronous generator functions\n' + ' A function or method which is defined using "async def" and\n' + ' which uses the "yield" statement is called a *asynchronous\n' + ' generator function*. Such a function, when called, returns ' + 'an\n' + ' asynchronous iterator object which can be used in an "async ' + 'for"\n' + ' statement to execute the body of the function.\n' + '\n' + ' Calling the asynchronous iterator\'s "aiterator.__anext__()"\n' + ' method will return an *awaitable* which when awaited will\n' + ' execute until it provides a value using the "yield" ' + 'expression.\n' + ' When the function executes an empty "return" statement or ' + 'falls\n' + ' off the end, a "StopAsyncIteration" exception is raised and ' + 'the\n' + ' asynchronous iterator will have reached the end of the set ' + 'of\n' + ' values to be yielded.\n' + '\n' ' Built-in functions\n' ' A built-in function object is a wrapper around a C function.\n' ' Examples of built-in functions are "len()" and "math.sin()"\n' @@ -11233,14 +11210,14 @@ 'the\n' ' dictionary containing the class\'s namespace; "__bases__" is a ' 'tuple\n' - ' (possibly empty or a singleton) containing the base classes, in ' - 'the\n' - ' order of their occurrence in the base class list; "__doc__" is ' - 'the\n' - ' class\'s documentation string, or "None" if undefined;\n' - ' "__annotations__" (optional) is a dictionary containing ' - '*variable\n' - ' annotations* collected during class body execution.\n' + ' containing the base classes, in the order of their occurrence ' + 'in\n' + ' the base class list; "__doc__" is the class\'s documentation ' + 'string,\n' + ' or "None" if undefined; "__annotations__" (optional) is a\n' + ' dictionary containing *variable annotations* collected during ' + 'class\n' + ' body execution.\n' '\n' 'Class instances\n' ' A class instance is created by calling a class object (see ' @@ -11520,8 +11497,7 @@ ' under "User-defined methods". Class method objects are ' 'created\n' ' by the built-in "classmethod()" constructor.\n', - 'typesfunctions': '\n' - 'Functions\n' + 'typesfunctions': 'Functions\n' '*********\n' '\n' 'Function objects are created by function definitions. The ' @@ -11538,8 +11514,7 @@ 'different object types.\n' '\n' 'See Function definitions for more information.\n', - 'typesmapping': '\n' - 'Mapping Types --- "dict"\n' + 'typesmapping': 'Mapping Types --- "dict"\n' '************************\n' '\n' 'A *mapping* object maps *hashable* values to arbitrary ' @@ -11896,8 +11871,7 @@ " {'bacon'}\n" " >>> keys ^ {'sausage', 'juice'}\n" " {'juice', 'sausage', 'bacon', 'spam'}\n", - 'typesmethods': '\n' - 'Methods\n' + 'typesmethods': 'Methods\n' '*******\n' '\n' 'Methods are functions that are called using the attribute ' @@ -11954,8 +11928,7 @@ " 'my name is method'\n" '\n' 'See The standard type hierarchy for more information.\n', - 'typesmodules': '\n' - 'Modules\n' + 'typesmodules': 'Modules\n' '*******\n' '\n' 'The only special operation on a module is attribute access: ' @@ -11992,8 +11965,7 @@ 'written as\n' '"".\n', - 'typesseq': '\n' - 'Sequence Types --- "list", "tuple", "range"\n' + 'typesseq': 'Sequence Types --- "list", "tuple", "range"\n' '*******************************************\n' '\n' 'There are three basic sequence types: lists, tuples, and range\n' @@ -12141,9 +12113,9 @@ '\n' '3. If *i* or *j* is negative, the index is relative to the end ' 'of\n' - ' the string: "len(s) + i" or "len(s) + j" is substituted. But ' - 'note\n' - ' that "-0" is still "0".\n' + ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' + 'But\n' + ' note that "-0" is still "0".\n' '\n' '4. The slice of *s* from *i* to *j* is defined as the sequence ' 'of\n' @@ -12162,12 +12134,17 @@ ' (j-i)/k". In other words, the indices are "i", "i+k", ' '"i+2*k",\n' ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). If *i* or *j* is greater than "len(s)", use\n' - ' "len(s)". If *i* or *j* are omitted or "None", they become ' - '"end"\n' - ' values (which end depends on the sign of *k*). Note, *k* ' - 'cannot be\n' - ' zero. If *k* is "None", it is treated like "1".\n' + ' including *j*). When *k* is positive, *i* and *j* are ' + 'reduced to\n' + ' "len(s)" if they are greater. When *k* is negative, *i* and ' + '*j* are\n' + ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' + 'are\n' + ' omitted or "None", they become "end" values (which end ' + 'depends on\n' + ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' + '"None", it\n' + ' is treated like "1".\n' '\n' '6. Concatenating immutable sequences always results in a new\n' ' object. This means that building up a sequence by repeated\n' @@ -12685,8 +12662,7 @@ ' * The linspace recipe shows how to implement a lazy version ' 'of\n' ' range that suitable for floating point applications.\n', - 'typesseq-mutable': '\n' - 'Mutable Sequence Types\n' + 'typesseq-mutable': 'Mutable Sequence Types\n' '**********************\n' '\n' 'The operations in the following table are defined on ' @@ -12826,8 +12802,7 @@ 'referenced multiple\n' ' times, as explained for "s * n" under Common Sequence ' 'Operations.\n', - 'unary': '\n' - 'Unary arithmetic and bitwise operations\n' + 'unary': 'Unary arithmetic and bitwise operations\n' '***************************************\n' '\n' 'All unary arithmetic and bitwise operations have the same ' @@ -12849,8 +12824,7 @@ 'In all three cases, if the argument does not have the proper type, ' 'a\n' '"TypeError" exception is raised.\n', - 'while': '\n' - 'The "while" statement\n' + 'while': 'The "while" statement\n' '*********************\n' '\n' 'The "while" statement is used for repeated execution as long as an\n' @@ -12874,8 +12848,7 @@ 'executed in the first suite skips the rest of the suite and goes ' 'back\n' 'to testing the expression.\n', - 'with': '\n' - 'The "with" statement\n' + 'with': 'The "with" statement\n' '********************\n' '\n' 'The "with" statement is used to wrap the execution of a block with\n' @@ -12948,8 +12921,7 @@ ' The specification, background, and examples for the Python ' '"with"\n' ' statement.\n', - 'yield': '\n' - 'The "yield" statement\n' + 'yield': 'The "yield" statement\n' '*********************\n' '\n' ' yield_stmt ::= yield_expression\n' diff --git a/Lib/smtplib.py b/Lib/smtplib.py index f7c2c77ab42075..5e422b704ad4dc 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -250,6 +250,7 @@ def __init__(self, host='', port=0, local_hostname=None, if host: (code, msg) = self.connect(host, port) if code != 220: + self.close() raise SMTPConnectError(code, msg) if local_hostname is not None: self.local_hostname = local_hostname diff --git a/Lib/sqlite3/test/transactions.py b/Lib/sqlite3/test/transactions.py index 45f1b04c69648f..b8a13de55bc720 100644 --- a/Lib/sqlite3/test/transactions.py +++ b/Lib/sqlite3/test/transactions.py @@ -179,6 +179,15 @@ def CheckDdlDoesNotAutostartTransaction(self): result = self.con.execute("select * from test").fetchall() self.assertEqual(result, []) + def CheckImmediateTransactionalDDL(self): + # You can achieve transactional DDL by issuing a BEGIN + # statement manually. + self.con.execute("begin immediate") + self.con.execute("create table test(i)") + self.con.rollback() + with self.assertRaises(sqlite.OperationalError): + self.con.execute("select * from test") + def CheckTransactionalDDL(self): # You can achieve transactional DDL by issuing a BEGIN # statement manually. diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index fc684ae96fd30a..a6e8a1f08d68a8 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -21,6 +21,17 @@ # should this really be here? class error(Exception): + """Exception raised for invalid regular expressions. + + Attributes: + + msg: The unformatted error message + pattern: The regular expression pattern + pos: The index in the pattern where compilation failed (may be None) + lineno: The line corresponding to pos (may be None) + colno: The column corresponding to pos (may be None) + """ + def __init__(self, msg, pattern=None, pos=None): self.msg = msg self.pattern = pattern diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 6aa49c3bf6f8ed..608f9a26642f05 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -404,7 +404,7 @@ def _escape(source, escape, state): pass raise source.error("bad escape %s" % escape, len(escape)) -def _parse_sub(source, state, verbose, nested=True): +def _parse_sub(source, state, verbose, nested): # parse an alternation: a|b|c items = [] @@ -412,7 +412,8 @@ def _parse_sub(source, state, verbose, nested=True): sourcematch = source.match start = source.tell() while True: - itemsappend(_parse(source, state, verbose)) + itemsappend(_parse(source, state, verbose, nested + 1, + not nested and not items)) if not sourcematch("|"): break @@ -454,10 +455,10 @@ def _parse_sub(source, state, verbose, nested=True): subpattern.append((BRANCH, (None, items))) return subpattern -def _parse_sub_cond(source, state, condgroup, verbose): - item_yes = _parse(source, state, verbose) +def _parse_sub_cond(source, state, condgroup, verbose, nested): + item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): - item_no = _parse(source, state, verbose) + item_no = _parse(source, state, verbose, nested + 1) if source.next == "|": raise source.error("conditional backref with more than two branches") else: @@ -466,7 +467,7 @@ def _parse_sub_cond(source, state, condgroup, verbose): subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern -def _parse(source, state, verbose): +def _parse(source, state, verbose, nested, first=False): # parse a simple pattern subpattern = SubPattern(state) @@ -692,7 +693,7 @@ def _parse(source, state, verbose): lookbehindgroups = state.lookbehindgroups if lookbehindgroups is None: state.lookbehindgroups = state.groups - p = _parse_sub(source, state, verbose) + p = _parse_sub(source, state, verbose, nested + 1) if dir < 0: if lookbehindgroups is None: state.lookbehindgroups = None @@ -730,18 +731,19 @@ def _parse(source, state, verbose): state.checklookbehindgroup(condgroup, source) elif char in FLAGS or char == "-": # flags - pos = source.pos flags = _parse_flags(source, state, char) if flags is None: # global flags - if pos != 3: # "(?x" + if not first or subpattern: import warnings warnings.warn( 'Flags not at the start of the expression %s%s' % ( source.string[:20], # truncate long regexes ' (truncated)' if len(source.string) > 20 else '', ), - DeprecationWarning, stacklevel=7 + DeprecationWarning, stacklevel=nested + 6 ) + if (state.flags & SRE_FLAG_VERBOSE) and not verbose: + raise Verbose continue add_flags, del_flags = flags group = None @@ -756,11 +758,11 @@ def _parse(source, state, verbose): except error as err: raise source.error(err.msg, len(name) + 1) from None if condgroup: - p = _parse_sub_cond(source, state, condgroup, verbose) + p = _parse_sub_cond(source, state, condgroup, verbose, nested + 1) else: sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and not (del_flags & SRE_FLAG_VERBOSE)) - p = _parse_sub(source, state, sub_verbose) + p = _parse_sub(source, state, sub_verbose, nested + 1) if not source.match(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) @@ -795,9 +797,6 @@ def _parse_flags(source, state, char): msg = "unknown flag" if char.isalpha() else "missing -, : or )" raise source.error(msg, len(char)) if char == ")": - if ((add_flags & SRE_FLAG_VERBOSE) and - not (state.flags & SRE_FLAG_VERBOSE)): - raise Verbose state.flags |= add_flags return None if add_flags & GLOBAL_FLAGS: @@ -853,7 +852,7 @@ def parse(str, flags=0, pattern=None): pattern.str = str try: - p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, False) + p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0) except Verbose: # the VERBOSE flag was switched on inside the pattern. to be # on the safe side, we'll parse the whole thing again... @@ -861,7 +860,7 @@ def parse(str, flags=0, pattern=None): pattern.flags = flags | SRE_FLAG_VERBOSE pattern.str = str source.seek(0) - p = _parse_sub(source, pattern, True, False) + p = _parse_sub(source, pattern, True, 0) p.pattern.flags = fix_flags(str, p.pattern.flags) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 822ddb459e2615..172126929160bf 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -986,7 +986,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, int(not close_fds), creationflags, env, - cwd, + os.fspath(cwd) if cwd is not None else None, startupinfo) finally: # Child is launched. Close the parent's copy of those pipe @@ -1253,7 +1253,8 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, fds_to_keep.add(errpipe_write) self.pid = _posixsubprocess.fork_exec( args, executable_list, - close_fds, sorted(fds_to_keep), cwd, env_list, + close_fds, tuple(sorted(map(int, fds_to_keep))), + cwd, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 46e0f56a392ae6..bfb670c9027d76 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -59,7 +59,7 @@ def main(): class NannyNag(Exception): """ - Raised by tokeneater() if detecting an ambiguous indent. + Raised by process_tokens() if detecting an ambiguous indent. Captured and handled in check(). """ def __init__(self, lineno, msg, line): diff --git a/Lib/test/185test.db b/Lib/test/185test.db deleted file mode 100644 index 14cb5e258bc096..00000000000000 Binary files a/Lib/test/185test.db and /dev/null differ diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b5f47825466de6..f1f93674935e7a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -18,6 +18,7 @@ import logging import struct import operator +import weakref import test.support import test.support.script_helper @@ -751,6 +752,20 @@ def test_timeout(self): # Windows (usually 15.6 ms) self.assertGreaterEqual(delta, 0.170) + def test_queue_feeder_donot_stop_onexc(self): + # bpo-30414: verify feeder handles exceptions correctly + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + class NotSerializable(object): + def __reduce__(self): + raise AttributeError + with test.support.captured_stderr(): + q = self.Queue() + q.put(NotSerializable()) + q.put(True) + self.assertTrue(q.get(timeout=0.1)) + # # # @@ -1738,14 +1753,30 @@ def raise_large_valuerror(wait): time.sleep(wait) raise ValueError("x" * 1024**2) +def identity(x): + return x + +class CountedObject(object): + n_instances = 0 + + def __new__(cls): + cls.n_instances += 1 + return object.__new__(cls) + + def __del__(self): + type(self).n_instances -= 1 + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): + if when == -1: + raise SayWhenError("Somebody said when") for i in range(total): if i == when: raise SayWhenError("Somebody said when") yield i + class _TestPool(BaseTestCase): @classmethod @@ -1818,6 +1849,32 @@ def test_map_chunksize(self): except multiprocessing.TimeoutError: self.fail("pool.map_async with chunksize stalled on null list") + def test_map_handle_iterable_exception(self): + if self.TYPE == 'manager': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + # SayWhenError seen at the very first of the iterable + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + # again, make sure it's reentrant + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(10, 3), 1) + + class SpecialIterable: + def __iter__(self): + return self + def __next__(self): + raise SayWhenError + def __len__(self): + return 1 + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + def test_async(self): res = self.pool.apply_async(sqr, (7, TIMEOUT1,)) get = TimingWrapper(res.get) @@ -1848,6 +1905,13 @@ def test_imap_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap(sqr, exception_throwing_generator(10, 3), 1) for i in range(3): self.assertEqual(next(it), i*i) @@ -1874,6 +1938,17 @@ def test_imap_unordered_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap_unordered(sqr, exception_throwing_generator(10, 3), 1) @@ -1955,7 +2030,7 @@ def test_traceback(self): except Exception as e: exc = e else: - raise AssertionError('expected RuntimeError') + self.fail('expected RuntimeError') self.assertIs(type(exc), RuntimeError) self.assertEqual(exc.args, (123,)) cause = exc.__cause__ @@ -1969,6 +2044,17 @@ def test_traceback(self): sys.excepthook(*sys.exc_info()) self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + # _helper_reraises_exception should not make the error + # a remote exception + with self.Pool(1) as p: + try: + p.map(sqr, exception_throwing_generator(1, -1), 1) + except Exception as e: + exc = e + else: + self.fail('expected SayWhenError') + self.assertIs(type(exc), SayWhenError) + self.assertIs(exc.__cause__, None) @classmethod def _test_wrapped_exception(cls): @@ -2000,6 +2086,20 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.time() - t_start, 0.9) + def test_release_task_refs(self): + # Issue #29861: task arguments and results should not be kept + # alive after we are done with them. + objs = [CountedObject() for i in range(10)] + refs = [weakref.ref(o) for o in objs] + self.pool.map(identity, objs) + + del objs + time.sleep(DELTA) # let threaded cleanup code run + self.assertEqual(set(wr() for wr in refs), {None}) + # With a process pool, copies of the objects are returned, check + # they were released too. + self.assertEqual(CountedObject.n_instances, 0) + def raising(): raise KeyError("key") @@ -3872,6 +3972,42 @@ def test_semaphore_tracker(self): self.assertRegex(err, expected) self.assertRegex(err, r'semaphore_tracker: %r: \[Errno' % name1) +class TestSimpleQueue(unittest.TestCase): + + @classmethod + def _test_empty(cls, queue, child_can_start, parent_can_continue): + child_can_start.wait() + # issue 30301, could fail under spawn and forkserver + try: + queue.put(queue.empty()) + queue.put(queue.empty()) + finally: + parent_can_continue.set() + + def test_empty(self): + queue = multiprocessing.SimpleQueue() + child_can_start = multiprocessing.Event() + parent_can_continue = multiprocessing.Event() + + proc = multiprocessing.Process( + target=self._test_empty, + args=(queue, child_can_start, parent_can_continue) + ) + proc.daemon = True + proc.start() + + self.assertTrue(queue.empty()) + + child_can_start.set() + parent_can_continue.wait() + + self.assertFalse(queue.empty()) + self.assertEqual(queue.get(), True) + self.assertEqual(queue.get(), False) + self.assertTrue(queue.empty()) + + proc.join() + # # Mixins # diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2350125f6d6dbe..bccd97aa3c7f60 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -4313,6 +4313,11 @@ def test_replace(self): dt = dt.replace(fold=1, tzinfo=Eastern) self.assertEqual(t.replace(tzinfo=None).fold, 1) self.assertEqual(dt.replace(tzinfo=None).fold, 1) + # Out of bounds. + with self.assertRaises(ValueError): + t.replace(fold=2) + with self.assertRaises(ValueError): + dt.replace(fold=2) # Check that fold is a keyword-only argument with self.assertRaises(TypeError): t.replace(1, 1, 1, None, 1) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index d194e775eaeb8f..1dbe88efe70ffd 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -371,59 +371,55 @@ def test_sleep(self): @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +# bpo-30320: Need pthread_sigmask() to block the signal, otherwise the test +# is vulnerable to a race condition between the child and the parent processes. +@unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') class SignalEINTRTest(EINTRBaseTest): """ EINTR tests for the signal module. """ - @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), - 'need signal.sigtimedwait()') - def test_sigtimedwait(self): - t0 = time.monotonic() - signal.sigtimedwait([signal.SIGUSR1], self.sleep_time) - dt = time.monotonic() - t0 - self.assertGreaterEqual(dt, self.sleep_time) - - @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), - 'need signal.sigwaitinfo()') - def test_sigwaitinfo(self): - # Issue #25277, #25868: give a few milliseconds to the parent process - # between os.write() and signal.sigwaitinfo() to works around a race - # condition - self.sleep_time = 0.100 - + def check_sigwait(self, wait_func): signum = signal.SIGUSR1 pid = os.getpid() old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) - rpipe, wpipe = os.pipe() - code = '\n'.join(( 'import os, time', 'pid = %s' % os.getpid(), 'signum = %s' % int(signum), 'sleep_time = %r' % self.sleep_time, - 'rpipe = %r' % rpipe, - 'os.read(rpipe, 1)', - 'os.close(rpipe)', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) + old_mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum]) + t0 = time.monotonic() - proc = self.subprocess(code, pass_fds=(rpipe,)) - os.close(rpipe) + proc = self.subprocess(code) with kill_on_error(proc): - # sync child-parent - os.write(wpipe, b'x') - os.close(wpipe) + wait_func(signum) + dt = time.monotonic() - t0 + + self.assertEqual(proc.wait(), 0) - # parent + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + def wait_func(signum): signal.sigwaitinfo([signum]) - dt = time.monotonic() - t0 - self.assertEqual(proc.wait(), 0) - self.assertGreaterEqual(dt, self.sleep_time) + self.check_sigwait(wait_func) + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigwaitinfo()') + def test_sigtimedwait(self): + def wait_func(signum): + signal.sigtimedwait([signum], 120.0) + + self.check_sigwait(wait_func) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") @@ -437,6 +433,8 @@ def test_select(self): self.stop_alarm() self.assertGreaterEqual(dt, self.sleep_time) + @unittest.skipIf(sys.platform == "darwin", + "poll may fail on macOS; see issue #28087") @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') def test_poll(self): poller = select.poll() diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index d621f5f9f3ee1f..cdbd1b8b92553c 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -343,5 +343,10 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.huntrleaks and ns.verbose3: + ns.verbose3 = False + print("WARNING: Disable --verbose3 because it's incompatible with " + "--huntrleaks: see http://bugs.python.org/issue27103", + file=sys.stderr) return ns diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 3b3d2458b1f2e0..e2e58bd872f094 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -143,9 +143,14 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + abs_classes = filter(isabstract, abs_classes) + if 'typing' in sys.modules: + t = sys.modules['typing'] + # These classes require special treatment because they do not appear + # in direct subclasses of collections.abc classes + abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] + for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() obj._abc_cache.clear() diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 96ad3af8df4cf7..8309f266bfbe9b 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -279,7 +279,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): if not self.quiet and not self.pgo: print(f"Warning -- {name} was modified by {self.testname}", file=sys.stderr, flush=True) - if self.verbose > 1: - print(f" Before: {original}\n After: {current} ", - file=sys.stderr, flush=True) + print(f" Before: {original}\n After: {current} ", + file=sys.stderr, flush=True) return False diff --git a/Lib/test/mod_generics_cache.py b/Lib/test/mod_generics_cache.py new file mode 100644 index 00000000000000..d9a60b4b28c325 --- /dev/null +++ b/Lib/test/mod_generics_cache.py @@ -0,0 +1,14 @@ +"""Module for testing the behavior of generics across different modules.""" + +from typing import TypeVar, Generic + +T = TypeVar('T') + + +class A(Generic[T]): + pass + + +class B(Generic[T]): + class A(Generic[T]): + pass diff --git a/Lib/test/sndhdrdata/README b/Lib/test/sndhdrdata/README index 8a17c0041a463f..b2cb664615b9c7 100644 --- a/Lib/test/sndhdrdata/README +++ b/Lib/test/sndhdrdata/README @@ -3,10 +3,3 @@ following commands: dd if=/dev/zero of=sndhdr.raw bs=20 count=1 sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - -Sound file samples used by Lib/test/test_sndhdr.py and generated using the -following commands: - - dd if=/dev/zero of=sndhdr.raw bs=20 count=1 - sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 15d8fc849b9a0a..f8d177990bfb03 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2105,12 +2105,15 @@ def swap_attr(obj, attr, new_val): restoring the old value at the end of the block. If `attr` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if hasattr(obj, attr): real_val = getattr(obj, attr) setattr(obj, attr, new_val) try: - yield + yield real_val finally: setattr(obj, attr, real_val) else: @@ -2118,7 +2121,8 @@ def swap_attr(obj, attr, new_val): try: yield finally: - delattr(obj, attr) + if hasattr(obj, attr): + delattr(obj, attr) @contextlib.contextmanager def swap_item(obj, item, new_val): @@ -2132,12 +2136,15 @@ def swap_item(obj, item, new_val): restoring the old value at the end of the block. If `item` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if item in obj: real_val = obj[item] obj[item] = new_val try: - yield + yield real_val finally: obj[item] = real_val else: @@ -2145,7 +2152,8 @@ def swap_item(obj, item, new_val): try: yield finally: - del obj[item] + if item in obj: + del obj[item] def strip_python_stderr(stderr): """Strip the stderr of a Python process from potential debug output @@ -2433,6 +2441,7 @@ def __enter__(self): (0, self.old_value[1])) except (ValueError, OSError): pass + if sys.platform == 'darwin': # Check if the 'Crash Reporter' on OSX was configured # in 'Developer' mode and warn that it will get triggered @@ -2440,10 +2449,14 @@ def __enter__(self): # # This assumes that this context manager is used in tests # that might trigger the next manager. - value = subprocess.Popen(['/usr/bin/defaults', 'read', - 'com.apple.CrashReporter', 'DialogType'], - stdout=subprocess.PIPE).communicate()[0] - if value.strip() == b'developer': + cmd = ['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with proc: + stdout = proc.communicate()[0] + if stdout.strip() == b'developer': print("this test triggers the Crash Reporter, " "that is intentional", end='', flush=True) @@ -2581,3 +2594,19 @@ def setswitchinterval(interval): if _is_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) + + +@contextlib.contextmanager +def disable_faulthandler(): + # use sys.__stderr__ instead of sys.stderr, since regrtest replaces + # sys.stderr with a StringIO which has no file descriptor when a test + # is run with -W/--verbose3. + fd = sys.__stderr__.fileno() + + is_enabled = faulthandler.is_enabled() + try: + faulthandler.disable() + yield + finally: + if is_enabled: + faulthandler.enable(file=fd, all_threads=True) diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 1bd1f89c8aa609..a731a5136ba5fd 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -1,5 +1,6 @@ -from test.support import findfile, TESTFN, unlink +from test.support import check_no_resource_warning, findfile, TESTFN, unlink import unittest +from unittest import mock from test import audiotests from audioop import byteswap import io @@ -149,6 +150,21 @@ def test_skipunknown(self): #This file contains chunk types aifc doesn't recognize. self.f = aifc.open(findfile('Sine-1000Hz-300ms.aif')) + def test_close_opened_files_on_error(self): + non_aifc_file = findfile('pluck-pcm8.wav', subdir='audiodata') + with check_no_resource_warning(self): + with self.assertRaises(aifc.Error): + # Try opening a non-AIFC file, with the expectation that + # `aifc.open` will fail (without raising a ResourceWarning) + self.f = aifc.open(non_aifc_file, 'rb') + + # Aifc_write.initfp() won't raise in normal case. But some errors + # (e.g. MemoryError, KeyboardInterrupt, etc..) can happen. + with mock.patch.object(aifc.Aifc_write, 'initfp', + side_effect=RuntimeError): + with self.assertRaises(RuntimeError): + self.fout = aifc.open(TESTFN, 'wb') + def test_params_added(self): f = self.f = aifc.open(TESTFN, 'wb') f.aiff() diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 1f8967ccff3e7c..d67f9195eba04c 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -14,14 +14,6 @@ import array from array import _array_reconstructor as array_reconstructor -try: - # Try to determine availability of long long independently - # of the array module under test - struct.calcsize('@q') - have_long_long = True -except struct.error: - have_long_long = False - sizeof_wchar = array.array('u').itemsize @@ -32,9 +24,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = "ubBhHiIlLfd" -if have_long_long: - typecodes += 'qQ' +typecodes = 'ubBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -1240,7 +1230,26 @@ def test_frombytearray(self): b = array.array(self.typecode, a) self.assertEqual(a, b) -class SignedNumberTest(NumberTest): +class IntegerNumberTest(NumberTest): + def test_type_error(self): + a = array.array(self.typecode) + a.append(42) + with self.assertRaises(TypeError): + a.append(42.0) + with self.assertRaises(TypeError): + a[0] = 42.0 + +class Intable: + def __init__(self, num): + self._num = num + def __int__(self): + return self._num + def __sub__(self, other): + return Intable(int(self) - int(other)) + def __add__(self, other): + return Intable(int(self) + int(other)) + +class SignedNumberTest(IntegerNumberTest): example = [-1, 0, 1, 42, 0x7f] smallerexample = [-1, 0, 1, 42, 0x7e] biggerexample = [-1, 0, 1, 43, 0x7f] @@ -1251,8 +1260,9 @@ def test_overflow(self): lower = -1 * int(pow(2, a.itemsize * 8 - 1)) upper = int(pow(2, a.itemsize * 8 - 1)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) -class UnsignedNumberTest(NumberTest): +class UnsignedNumberTest(IntegerNumberTest): example = [0, 1, 17, 23, 42, 0xff] smallerexample = [0, 1, 17, 23, 42, 0xfe] biggerexample = [0, 1, 17, 23, 43, 0xff] @@ -1263,6 +1273,7 @@ def test_overflow(self): lower = 0 upper = int(pow(2, a.itemsize * 8)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) def test_bytes_extend(self): s = bytes(self.example) @@ -1314,12 +1325,10 @@ class UnsignedLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'L' minitemsize = 4 -@unittest.skipIf(not have_long_long, 'need long long support') class LongLongTest(SignedNumberTest, unittest.TestCase): typecode = 'q' minitemsize = 8 -@unittest.skipIf(not have_long_long, 'need long long support') class UnsignedLongLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'Q' minitemsize = 8 diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 28d92a9f4e3eac..492a84a2313baf 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1,6 +1,7 @@ """Tests for events.py.""" import collections.abc +import concurrent.futures import functools import gc import io @@ -57,6 +58,15 @@ def osx_tiger(): return version < (10, 5) +def _test_get_event_loop_new_process__sub_proc(): + async def doit(): + return 'hello' + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop.run_until_complete(doit()) + + ONLYCERT = data_file('ssl_cert.pem') ONLYKEY = data_file('ssl_key.pem') SIGNED_CERTFILE = data_file('keycert3.pem') @@ -2181,6 +2191,20 @@ def tearDown(self): asyncio.set_child_watcher(None) super().tearDown() + def test_get_event_loop_new_process(self): + async def main(): + pool = concurrent.futures.ProcessPoolExecutor() + result = await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + pool.shutdown() + return result + + self.unpatch_get_running_loop() + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + if hasattr(selectors, 'KqueueSelector'): class KqueueEventLoopTests(UnixEventLoopTestsMixin, SubprocessTestsMixin, diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 89afdcaff28665..99336f86ab824e 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -569,6 +569,35 @@ def test_remove_done_callback(self): self.assertEqual(bag, [2]) self.assertEqual(f.result(), 'foo') + def test_remove_done_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_schedule_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + def mut(f): + f.remove_done_callback(str) + + fut = self._new_future() + fut.add_done_callback(mut) + fut.add_done_callback(str) + fut.add_done_callback(str) + fut.set_result(1) + test_utils.run_briefly(self.loop) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 59ff0f6967e5b5..f1771c5561afea 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -95,5 +95,17 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_get_extra_info_on_closed_connection(self): + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bba688bb5a53c7..2e14a8a9735535 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -459,6 +459,30 @@ def test_popen_error(self): self.loop.run_until_complete(create) self.assertEqual(warns, []) + def test_read_stdout_after_process_exit(self): + @asyncio.coroutine + def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + fut = asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + loop=self.loop) + + process = yield from fut + while True: + data = yield from process.stdout.read(65536) + if data: + yield from asyncio.sleep(0.3, loop=self.loop) + else: + break + + self.loop.run_until_complete(execute()) + if sys.platform != 'win32': # Unix diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a18d49ae374112..5462c80ad3013a 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -588,6 +588,24 @@ def task(): self.assertFalse(t._must_cancel) # White-box test. self.assertFalse(t.cancel()) + def test_cancel_at_end(self): + """coroutine end right after task is cancelled""" + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + return 12 + + t = self.new_task(loop, task()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + def test_stop_while_run_in_complete(self): def gen(): @@ -1462,6 +1480,14 @@ def test_current_task(self): def coro(loop): self.assertTrue(Task.current_task(loop=loop) is task) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(loop) + try: + self.assertIs(Task.current_task(None), task) + self.assertIs(Task.current_task(), task) + finally: + asyncio.set_event_loop(None) + task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) self.assertIsNone(Task.current_task(loop=self.loop)) @@ -1806,8 +1832,17 @@ def kill_me(loop): # schedule the task coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) + self.assertEqual(Task.all_tasks(loop=self.loop), {task}) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(self.loop) + try: + self.assertEqual(Task.all_tasks(), {task}) + self.assertEqual(Task.all_tasks(None), {task}) + finally: + asyncio.set_event_loop(None) + # execute the task so it waits for future self.loop._run_once() self.assertEqual(len(self.loop._ready), 0) diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 51c65737a441de..dc2f716e0bb828 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -502,7 +502,7 @@ def handle_connect(self): class BaseTestAPI: def tearDown(self): - asyncore.close_all() + asyncore.close_all(ignore_all=True) def loop_waiting_for_flag(self, instance, timeout=5): timeout = float(timeout) / 100 @@ -661,6 +661,9 @@ def test_handle_expt(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + class TestClient(BaseClient): def handle_expt(self): self.socket.recv(1024, socket.MSG_OOB) @@ -752,50 +755,50 @@ def test_bind(self): def test_set_reuse_addr(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") - sock = socket.socket(self.family) - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except OSError: - unittest.skip("SO_REUSEADDR not supported on this platform") - else: - # if SO_REUSEADDR succeeded for sock we expect asyncore - # to do the same - s = asyncore.dispatcher(socket.socket(self.family)) - self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - s.socket.close() - s.create_socket(self.family) - s.set_reuse_addr() - self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - finally: - sock.close() + + with socket.socket(self.family) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) @unittest.skipUnless(threading, 'Threading required for this test.') @support.reap_threads def test_quick_connect(self): # see: http://bugs.python.org/issue10340 - if self.family in (socket.AF_INET, getattr(socket, "AF_INET6", object())): - server = BaseServer(self.family, self.addr) - t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, - count=500)) - t.start() - def cleanup(): - t.join(timeout=TIMEOUT) - if t.is_alive(): - self.fail("join() timed out") - self.addCleanup(cleanup) - - s = socket.socket(self.family, socket.SOCK_STREAM) - s.settimeout(.2) - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) - try: - s.connect(server.address) - except OSError: - pass - finally: - s.close() + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + self.skipTest("test specific to AF_INET and AF_INET6") + + server = BaseServer(self.family, self.addr) + # run the thread 500 ms: the socket should be connected in 200 ms + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=5)) + t.start() + try: + with socket.socket(self.family, socket.SOCK_STREAM) as s: + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + + try: + s.connect(server.address) + except OSError: + pass + finally: + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") class TestAPI_UseIPv4Sockets(BaseTestAPI): family = socket.AF_INET diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 172bd25419cea7..c761076c4a0225 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -143,6 +143,7 @@ def test_bound_methods(self): self.assertEqual(l, [5]) +@support.cpython_only class SubinterpreterTest(unittest.TestCase): def test_callbacks_leak(self): diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 4f86aaa0c0eb49..47547396b8cb54 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -18,6 +18,14 @@ def check_type_errors(self, f): int_data = memoryview(b"1234").cast('I') self.assertRaises(TypeError, f, int_data) + def test_encodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.encodestring(b"www.python.org") + + def test_decodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.decodestring(b"d3d3LnB5dGhvbi5vcmc=\n") + def test_encodebytes(self): eq = self.assertEqual eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a103a7d39cae24..cd82fa64570cb5 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -507,6 +507,11 @@ def test_mod(self): a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') self.assertIs(type(a), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b = b % b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_imod(self): b = self.type2test(b'hello, %b!') @@ -519,6 +524,11 @@ def test_imod(self): b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') self.assertIs(type(b), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b %= b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_rmod(self): with self.assertRaises(TypeError): diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6c3625d4879138..216851c2d36c8b 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -98,7 +98,7 @@ class Z(object): def __len__(self): return 1 self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -106,7 +106,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -116,7 +116,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") @@ -490,9 +490,8 @@ def test_skipitem(self): # test the format unit when not skipped format = c + "i" try: - # (note: the format string must be bytes!) _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, - format.encode("ascii"), keywords) + format, keywords) when_not_skipped = False except SystemError as e: s = "argument 1 (impossible)" @@ -504,7 +503,7 @@ def test_skipitem(self): optional_format = "|" + format try: _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, - optional_format.encode("ascii"), keywords) + optional_format, keywords) when_skipped = False except SystemError as e: s = "impossible: '{}'".format(format) @@ -517,40 +516,64 @@ def test_skipitem(self): self.assertIs(when_skipped, when_not_skipped, message) def test_parse_tuple_and_keywords(self): - # parse_tuple_and_keywords error handling tests + # Test handling errors in the parse_tuple_and_keywords helper itself self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords, (), {}, 42, []) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', 42) + (), {}, '', 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [''] * 42) + (), {}, '', [''] * 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [42]) + (), {}, '', [42]) + + def test_bad_use(self): + # Test handling invalid format and keywords in + # PyArg_ParseTupleAndKeywords() + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '||O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1, 2), {}, '|O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$$O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$|O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|OO', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|$O', ['']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|OO', ['a', '']) def test_positional_only(self): parse = _testcapi.parse_tuple_and_keywords - parse((1, 2, 3), {}, b'OOO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OOO', ['', '', 'a']) + parse((1, 2, 3), {}, 'OOO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OOO', ['', '', 'a']) - parse((1,), {}, b'O|OO', ['', '', 'a']) + parse((1,), {'a': 3}, 'OOO', ['', '', 'a']) + parse((1,), {}, 'O|OO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|OO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OO$O', ['', '', 'a']) + parse((), {}, 'O|OO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes exactly 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OO$O', ['', '', 'a']) - parse((1,), {}, b'O|O$O', ['', '', 'a']) + parse((1,), {'a': 3}, 'OO$O', ['', '', 'a']) + parse((1,), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|O$O', ['', '', 'a']) + parse((), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(SystemError, r'Empty parameter name after \$'): - parse((1,), {}, b'O|$OO', ['', '', 'a']) + parse((1,), {}, 'O|$OO', ['', '', 'a']) with self.assertRaisesRegex(SystemError, 'Empty keyword'): - parse((1,), {}, b'O|OO', ['', 'a', '']) + parse((1,), {}, 'O|OO', ['', 'a', '']) @unittest.skipUnless(threading, 'Threading required for this test.') diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 4d554a397b4a51..ecc01f277954d5 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -568,5 +568,32 @@ class B(A): a = A(hash(A.f)^(-1)) hash(a.f) + def testSetattrWrapperNameIntern(self): + # Issue #25794: __setattr__ should intern the attribute name + class A: + pass + + def add(self, other): + return 'summa' + + name = str(b'__add__', 'ascii') # shouldn't be optimized + self.assertIsNot(name, '__add__') # not interned + type.__setattr__(A, name, add) + self.assertEqual(A() + 1, 'summa') + + name2 = str(b'__add__', 'ascii') + self.assertIsNot(name2, '__add__') + self.assertIsNot(name2, name) + type.__delattr__(A, name2) + with self.assertRaises(TypeError): + A() + 1 + + def testSetattrNonStringName(self): + class A: + pass + + with self.assertRaises(TypeError): + type.__setattr__(A, b'x', None) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e058ecd086df3c..1587daf8f582dd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -572,6 +572,73 @@ def test_syntaxerror_indented_caret_position(self): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^", text) + def test_consistent_sys_path_for_direct_execution(self): + # This test case ensures that the following all give the same + # sys.path configuration: + # + # ./python -s script_dir/__main__.py + # ./python -s script_dir + # ./python -I script_dir + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir, support.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from directly executing __main__.py + # We omit PYTHONPATH and user site to align with isolated mode + p = spawn_python("-Es", script_name, cwd=work_dir) + out_by_name = kill_python(p).decode().splitlines() + self.assertEqual(out_by_name[0], script_dir) + self.assertNotIn(work_dir, out_by_name) + # Directory execution should give the same output + p = spawn_python("-Es", script_dir, cwd=work_dir) + out_by_dir = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir, out_by_name) + # As should directory execution in isolated mode + p = spawn_python("-I", script_dir, cwd=work_dir) + out_by_dir_isolated = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir_isolated, out_by_dir, out_by_name) + + def test_consistent_sys_path_for_module_execution(self): + # This test case ensures that the following both give the same + # sys.path configuration: + # ./python -sm script_pkg.__main__ + # ./python -sm script_pkg + # + # And that this fails as unable to find the package: + # ./python -Im script_pkg + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from `-m script_pkg.__main__` + # We omit PYTHONPATH and user site to better align with the + # direct execution test cases + p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) + out_by_module = kill_python(p).decode().splitlines() + self.assertEqual(out_by_module[0], '') + self.assertNotIn(script_dir, out_by_module) + # Package execution should give the same output + p = spawn_python("-sm", "script_pkg", cwd=work_dir) + out_by_package = kill_python(p).decode().splitlines() + self.assertEqual(out_by_package, out_by_module) + # Isolated mode should fail with an import error + exitcode, stdout, stderr = assert_python_failure( + "-Im", "script_pkg", cwd=work_dir + ) + traceback_lines = stderr.decode().splitlines() + self.assertIn("No module named script_pkg", traceback_lines[-1]) def test_main(): support.run_unittest(CmdLineTest) diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index 3bdf7d0e14b3b7..2a450718d5aae5 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -48,6 +48,12 @@ class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase): (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), + # issue29990 + (b"\xff\x30\x81\x30", "strict", None), + (b"\x81\x30\xff\x30", "strict", None), + (b"abc\x81\x39\xff\x39\xc1\xc4", "replace", "abc\ufffd\x39\ufffd\x39\u804a"), + (b"abc\xab\x36\xff\x30def", "replace", 'abc\ufffd\x36\ufffd\x30def'), + (b"abc\xbf\x38\xff\x32\xc1\xc4", "ignore", "abc\x38\x32\u804a"), ) has_iso10646 = True @@ -80,6 +86,10 @@ class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), + # issue 30003 + ('ab~cd', 'strict', b'ab~~cd'), # escape ~ + (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode + (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode ) if __name__ == "__main__": diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 87454cc6704cb0..47f756213ddf00 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1310,20 +1310,29 @@ def test_issue26915(self): class CustomEqualObject: def __eq__(self, other): return False - class CustomSequence(list): - def __contains__(self, value): - return Sequence.__contains__(self, value) + class CustomSequence(Sequence): + def __init__(self, seq): + self._seq = seq + def __getitem__(self, index): + return self._seq[index] + def __len__(self): + return len(self._seq) nan = float('nan') obj = CustomEqualObject() + seq = CustomSequence([nan, obj, nan]) containers = [ - CustomSequence([nan, obj]), + seq, ItemsView({1: nan, 2: obj}), ValuesView({1: nan, 2: obj}) ] for container in containers: for elem in container: self.assertIn(elem, container) + self.assertEqual(seq.index(nan), 0) + self.assertEqual(seq.index(obj), 1) + self.assertEqual(seq.count(nan), 2) + self.assertEqual(seq.count(obj), 1) def assertSameSet(self, s1, s2): # coerce both to a real set then check equality diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index c249ca724bf0ce..cee49343e268a2 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -387,6 +387,29 @@ def __complex__(self): self.assertAlmostEqual(complex(complex1(1j)), 2j) self.assertRaises(TypeError, complex, complex2(1j)) + @support.requires_IEEE_754 + def test_constructor_special_numbers(self): + class complex2(complex): + pass + for x in 0.0, -0.0, INF, -INF, NAN: + for y in 0.0, -0.0, INF, -INF, NAN: + with self.subTest(x=x, y=y): + z = complex(x, y) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(x, y) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex(complex2(x, y)) + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(complex(x, y)) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + def test_underscores(self): # check underscores for lit in VALID_UNDERSCORE_LITERALS: diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 0d06080da33c52..72c3f19fb41f56 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,7 @@ import configparser import io import os -import sys +import pathlib import textwrap import unittest import warnings @@ -721,6 +721,16 @@ def test_read_returns_file_list(self): parsed_files = cf.read(file1) self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we pass only a Path object: + cf = self.newconfig() + parsed_files = cf.read(pathlib.Path(file1)) + self.assertEqual(parsed_files, [file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we passed both a filename and a Path object: + cf = self.newconfig() + parsed_files = cf.read([pathlib.Path(file1), file1]) + self.assertEqual(parsed_files, [file1, file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: cf = self.newconfig() parsed_files = cf.read(["nonexistent-file"]) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index c04c804af57047..b1a467d952da0f 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -152,6 +152,29 @@ def woohoo(): else: self.fail('StopIteration was suppressed') + def test_contextmanager_do_not_unchain_non_stopiteration_exceptions(self): + @contextmanager + def test_issue29692(): + try: + yield + except Exception as exc: + raise RuntimeError('issue29692:Chained') from exc + try: + with test_issue29692(): + raise ZeroDivisionError + except Exception as ex: + self.assertIs(type(ex), RuntimeError) + self.assertEqual(ex.args[0], 'issue29692:Chained') + self.assertIsInstance(ex.__cause__, ZeroDivisionError) + + try: + with test_issue29692(): + raise StopIteration('issue29692:Unchained') + except Exception as ex: + self.assertIs(type(ex), StopIteration) + self.assertEqual(ex.args[0], 'issue29692:Unchained') + self.assertIsNone(ex.__cause__) + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 78439a2acae2d8..2b79a17ea703f5 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1103,6 +1103,21 @@ async def waiter(coro): "coroutine is being awaited already"): waiter(coro).send(None) + def test_await_16(self): + # See https://bugs.python.org/issue29600 for details. + + async def f(): + return ValueError() + + async def g(): + try: + raise KeyError + except: + return await f() + + _, result = run_async(g()) + self.assertIsNone(result.__context__) + def test_with_1(self): class Manager: def __init__(self, name): @@ -1680,6 +1695,44 @@ async def foo(): warnings.simplefilter("error") run_async(foo()) + def test_for_11(self): + class F: + def __aiter__(self): + return self + def __anext__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __anext__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + + def test_for_12(self): + class F: + def __aiter__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __aiter__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + def test_for_tuple(self): class Done(Exception): pass @@ -2064,6 +2117,7 @@ def wrap(gen): sys.set_coroutine_wrapper(None) +@support.cpython_only class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 832bb9c8e2dbcc..8013f37c88da0a 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1085,6 +1085,91 @@ def test_free_after_iterating(self): support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) + def test_equal_operator_modifying_operand(self): + # test fix for seg fault reported in issue 27945 part 3. + class X(): + def __del__(self): + dict_b.clear() + + def __eq__(self, other): + dict_a.clear() + return True + + def __hash__(self): + return 13 + + dict_a = {X(): 0} + dict_b = {X(): X()} + self.assertTrue(dict_a == dict_b) + + def test_fromkeys_operator_modifying_dict_operand(self): + # test fix for seg fault reported in issue 27945 part 4a. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1): 1, X(2): 2} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_fromkeys_operator_modifying_set_operand(self): + # test fix for seg fault reported in issue 27945 part 4b. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1), X(2)} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_dictitems_contains_use_after_free(self): + class X: + def __eq__(self, other): + d.clear() + return NotImplemented + + d = {0: set()} + (0, X()) in d.items() + + def test_init_use_after_free(self): + class X: + def __hash__(self): + pair[:] = [] + return 13 + + pair = [X(), 123] + dict([pair]) + + def test_oob_indexing_dictiter_iternextitem(self): + class X(int): + def __del__(self): + d.clear() + + d = {i: X(i) for i in range(8)} + + def iter_and_mutate(): + for result in d.items(): + if result[0] == 2: + d[2] = None # free d[2] --> X(2).__del__ was called + + self.assertRaises(RuntimeError, iter_and_mutate) + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 48379222c37659..960fc0f1bf9486 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1,5 +1,6 @@ # Python test set -- part 5, built-in exceptions +import copy import os import sys import unittest @@ -1119,6 +1120,25 @@ def test_non_str_argument(self): exc = ImportError(arg) self.assertEqual(str(arg), str(exc)) + def test_copy_pickle(self): + for kwargs in (dict(), + dict(name='somename'), + dict(path='somepath'), + dict(name='somename', path='somepath')): + orig = ImportError('test', **kwargs) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + exc = pickle.loads(pickle.dumps(orig, proto)) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + for c in copy.copy, copy.deepcopy: + exc = c(orig) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index bdd8d1a2a6163f..01cbae3f200b1a 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -755,6 +755,18 @@ def test_raise_exception(self): 3, name) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_disable_windows_exc_handler(self): + code = dedent(""" + import faulthandler + faulthandler.enable() + faulthandler.disable() + code = faulthandler._EXCEPTION_ACCESS_VIOLATION + faulthandler._raise_exception(code) + """) + output, exitcode = self.get_output(code) + self.assertEqual(output, []) + self.assertEqual(exitcode, 0xC0000005) if __name__ == "__main__": diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 3da210ae1997ad..57a02656206ff4 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from functools import wraps from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest, - make_bad_fd, cpython_only) + make_bad_fd, cpython_only, swap_attr) from collections import UserList import _io # C implementation of io @@ -176,6 +176,12 @@ def testReprNoCloseFD(self): finally: os.close(fd) + def testRecursiveRepr(self): + # Issue #25455 + with swap_attr(self.f, 'name', self.f): + with self.assertRaises(RuntimeError): + repr(self.f) # Should not crash + def testErrors(self): f = self.f self.assertFalse(f.isatty()) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index ac8473db503b76..6491f458c3acc4 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -119,15 +119,27 @@ def test_float_memoryview(self): self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3) def test_error_message(self): - testlist = ('\xbd', '123\xbd', ' 123 456 ') - for s in testlist: - try: + def check(s): + with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm: float(s) - except ValueError as e: - self.assertIn(s.strip(), e.args[0]) - else: - self.fail("Expected int(%r) to raise a ValueError", s) - + self.assertEqual(str(cm.exception), + 'could not convert string to float: %r' % (s,)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + check(b' 123 456 ') + + # non-ascii digits (error came from non-digit '!') + check('\u0663\u0661\u0664!') + # embedded NUL + check('123\x00') + check('123\x00 245') + check('123\x00245') + # byte string with embedded NUL + check(b'123\x00') + # non-UTF-8 byte string + check(b'123\xa0') @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') def test_float_with_comma(self): diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index fb7424624bb041..78245c3ab1a176 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -1,18 +1,19 @@ """Test cases for the fnmatch module.""" import unittest +import os from fnmatch import fnmatch, fnmatchcase, translate, filter class FnmatchTestCase(unittest.TestCase): - def check_match(self, filename, pattern, should_match=1, fn=fnmatch): + def check_match(self, filename, pattern, should_match=True, fn=fnmatch): if should_match: self.assertTrue(fn(filename, pattern), "expected %r to match pattern %r" % (filename, pattern)) else: - self.assertTrue(not fn(filename, pattern), + self.assertFalse(fn(filename, pattern), "expected %r not to match pattern %r" % (filename, pattern)) @@ -26,15 +27,15 @@ def test_fnmatch(self): check('abc', '*') check('abc', 'ab[cd]') check('abc', 'ab[!de]') - check('abc', 'ab[de]', 0) - check('a', '??', 0) - check('a', 'b', 0) + check('abc', 'ab[de]', False) + check('a', '??', False) + check('a', 'b', False) # these test that '\' is handled correctly in character sets; # see SF bug #409651 check('\\', r'[\]') check('a', r'[!\]') - check('\\', r'[!\]', 0) + check('\\', r'[!\]', False) # test that filenames with newlines in them are handled correctly. # http://bugs.python.org/issue6665 @@ -51,14 +52,38 @@ def test_mix_bytes_str(self): def test_fnmatchcase(self): check = self.check_match - check('AbC', 'abc', 0, fnmatchcase) - check('abc', 'AbC', 0, fnmatchcase) + check('abc', 'abc', True, fnmatchcase) + check('AbC', 'abc', False, fnmatchcase) + check('abc', 'AbC', False, fnmatchcase) + check('AbC', 'AbC', True, fnmatchcase) + + check('usr/bin', 'usr/bin', True, fnmatchcase) + check('usr\\bin', 'usr/bin', False, fnmatchcase) + check('usr/bin', 'usr\\bin', False, fnmatchcase) + check('usr\\bin', 'usr\\bin', True, fnmatchcase) def test_bytes(self): self.check_match(b'test', b'te*') self.check_match(b'test\xff', b'te*\xff') self.check_match(b'foo\nbar', b'foo*') + def test_case(self): + ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') + check = self.check_match + check('abc', 'abc') + check('AbC', 'abc', ignorecase) + check('abc', 'AbC', ignorecase) + check('AbC', 'AbC') + + def test_sep(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + check = self.check_match + check('usr/bin', 'usr/bin') + check('usr\\bin', 'usr/bin', normsep) + check('usr/bin', 'usr\\bin', normsep) + check('usr\\bin', 'usr\\bin') + + class TranslateTestCase(unittest.TestCase): def test_translate(self): @@ -75,7 +100,28 @@ def test_translate(self): class FilterTestCase(unittest.TestCase): def test_filter(self): - self.assertEqual(filter(['a', 'b'], 'a'), ['a']) + self.assertEqual(filter(['Python', 'Ruby', 'Perl', 'Tcl'], 'P*'), + ['Python', 'Perl']) + self.assertEqual(filter([b'Python', b'Ruby', b'Perl', b'Tcl'], b'P*'), + [b'Python', b'Perl']) + + def test_mix_bytes_str(self): + self.assertRaises(TypeError, filter, ['test'], b'*') + self.assertRaises(TypeError, filter, [b'test'], '*') + + def test_case(self): + ignorecase = os.path.normcase('P') == os.path.normcase('p') + self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.p*'), + ['Test.py', 'Test.PL'] if ignorecase else ['Test.py']) + self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.P*'), + ['Test.py', 'Test.PL'] if ignorecase else ['Test.PL']) + + def test_sep(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr/*'), + ['usr/bin', 'usr\\lib'] if normsep else ['usr/bin']) + self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr\\*'), + ['usr/bin', 'usr\\lib'] if normsep else ['usr\\lib']) if __name__ == "__main__": diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 83eb29faf88d6c..b6ba2e566b295f 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -315,10 +315,12 @@ def __bytes__(self): testcommon(b"%b", b"abc", b"abc") testcommon(b"%b", bytearray(b"def"), b"def") testcommon(b"%b", fb, b"123") + testcommon(b"%b", memoryview(b"abc"), b"abc") # # %s is an alias for %b -- should only be used for Py2/3 code testcommon(b"%s", b"abc", b"abc") testcommon(b"%s", bytearray(b"def"), b"def") testcommon(b"%s", fb, b"123") + testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') testcommon(b"%a", 3.14, b"3.14") @@ -377,9 +379,11 @@ def test_exc(formatstr, args, exception, excmsg): test_exc(b"%c", 3.14, TypeError, "%c requires an integer in range(256) or a single byte") test_exc(b"%b", "Xc", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") test_exc(b"%s", "Wd", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 708ed25b526b74..25730029ae76f1 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -361,6 +361,20 @@ def test_backslashes_in_string_part(self): self.assertEqual(f'2\x203', '2 3') self.assertEqual(f'\x203', ' 3') + with self.assertWarns(DeprecationWarning): # invalid escape sequence + value = eval(r"f'\{6*7}'") + self.assertEqual(value, '\\42') + self.assertEqual(f'\\{6*7}', '\\42') + self.assertEqual(fr'\{6*7}', '\\42') + + AMPERSAND = 'spam' + # Get the right unicode character (&), or pick up local variable + # depending on the number of backslashes. + self.assertEqual(f'\N{AMPERSAND}', '&') + self.assertEqual(f'\\N{AMPERSAND}', '\\Nspam') + self.assertEqual(fr'\N{AMPERSAND}', '\\Nspam') + self.assertEqual(f'\\\N{AMPERSAND}', '\\&') + def test_misformed_unicode_character_name(self): # These test are needed because unicode names are parsed # differently inside f-strings. diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 824549b80342ed..cd4664cec08745 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -89,6 +89,15 @@ def func(a=10, b=20): p(b=7) self.assertEqual(d, {'a':3}) + def test_kwargs_copy(self): + # Issue #29532: Altering a kwarg dictionary passed to a constructor + # should not affect a partial object after creation + d = {'a': 3} + p = self.partial(capture, **d) + self.assertEqual(p(), ((), {'a': 3})) + d['a'] = 5 + self.assertEqual(p(), ((), {'a': 3})) + def test_arg_combinations(self): # exercise special code paths for zero args in either partial # object or the caller @@ -393,6 +402,32 @@ def test_attributes_unwritable(self): else: self.fail('partial object allowed __dict__ to be deleted') + def test_manually_adding_non_string_keyword(self): + p = self.partial(capture) + # Adding a non-string/unicode keyword to partial kwargs + p.keywords[1234] = 'value' + r = repr(p) + self.assertIn('1234', r) + self.assertIn("'value'", r) + with self.assertRaises(TypeError): + p() + + def test_keystr_replaces_value(self): + p = self.partial(capture) + + class MutatesYourDict(object): + def __str__(self): + p.keywords[self] = ['sth2'] + return 'astr' + + # Raplacing the value during key formatting should keep the original + # value alive (at least long enough). + p.keywords[MutatesYourDict()] = ['sth'] + r = repr(p) + self.assertIn('astr', r) + self.assertIn("['sth']", r) + + class TestPartialPy(TestPartial, unittest.TestCase): partial = py_functools.partial diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 60f1d92846744d..b7554d698c9d6c 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -3,13 +3,14 @@ # The code for testing gdb was adapted from similar work in Unladen Swallow's # Lib/test/test_jit_gdb.py +import locale import os import re import subprocess import sys import sysconfig +import textwrap import unittest -import locale # Is this Python configured to support threads? try: @@ -845,7 +846,24 @@ def test_pycfunction(self): breakpoint='time_gmtime', cmds_after_breakpoint=['py-bt-full'], ) - self.assertIn('#0 " % clsname) + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b): + try: + repr(b) # Should not crash + except RuntimeError: + pass + def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush # and that flush() is called before file closed. @@ -2424,6 +2450,16 @@ def test_repr(self): t.buffer.detach() repr(t) # Should not raise an exception + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw) + with support.swap_attr(raw, 'name', t): + try: + repr(t) # Should not crash + except RuntimeError: + pass + def test_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) @@ -3478,6 +3514,7 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.readinto1, bytearray(1024)) self.assertRaises(ValueError, f.readline) self.assertRaises(ValueError, f.readlines) + self.assertRaises(ValueError, f.readlines, 1) self.assertRaises(ValueError, f.seek, 0) self.assertRaises(ValueError, f.tell) self.assertRaises(ValueError, f.truncate) @@ -3683,6 +3720,7 @@ def check_daemon_threads_shutdown_deadlock(self, stream_name): import sys import time import threading + from test.support import SuppressCrashReport file = sys.{stream_name} @@ -3691,6 +3729,10 @@ def run(): file.write('.') file.flush() + crash = SuppressCrashReport() + crash.__enter__() + # don't call __exit__(): the crash occurs at Python shutdown + thread = threading.Thread(target=run) thread.daemon = True thread.start() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index ea1f57caade12b..c431f0dc6e1d7f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1976,6 +1976,14 @@ def gen2(x): self.assertRaises(AssertionError, list, cycle(gen1())) self.assertEqual(hist, [0,1]) + def test_long_chain_of_empty_iterables(self): + # Make sure itertools.chain doesn't run into recursion limits when + # dealing with long chains of empty iterables. Even with a high + # number this would probably only fail in Py_DEBUG mode. + it = chain.from_iterable(() for unused in range(10000000)) + with self.assertRaises(StopIteration): + next(it) + class SubclassWithKwargsTest(unittest.TestCase): def test_keywords_in_subclass(self): # count is not subclassable... diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 15f373664e1278..9d93f931ca3958 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -2,7 +2,7 @@ import sys import textwrap import unittest -import subprocess +from subprocess import Popen, PIPE from test import support from test.support.script_helper import assert_python_ok @@ -61,12 +61,11 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - with subprocess.Popen( - (sys.executable, '-m', 'json.tool'), - stdin=subprocess.PIPE, stdout=subprocess.PIPE) as proc: + args = sys.executable, '-m', 'json.tool' + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.data.encode()) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, None) + self.assertEqual(err, b'') def _create_infile(self): infile = support.TESTFN diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 9dedc098ba5aca..474affdaebcd35 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1663,7 +1663,7 @@ def setUp(self): server.ready.wait() hcls = logging.handlers.SysLogHandler if isinstance(server.server_address, tuple): - self.sl_hdlr = hcls(('localhost', server.port)) + self.sl_hdlr = hcls((server.server_address[0], server.port)) else: self.sl_hdlr = hcls(server.server_address) self.log_output = '' @@ -1723,6 +1723,24 @@ def tearDown(self): SysLogHandlerTest.tearDown(self) support.unlink(self.address) +@unittest.skipUnless(support.IPV6_ENABLED, + 'IPv6 support required for this test.') +@unittest.skipUnless(threading, 'Threading required for this test.') +class IPv6SysLogHandlerTest(SysLogHandlerTest): + + """Test for SysLogHandler with IPv6 host.""" + + server_class = TestUDPServer + address = ('::1', 0) + + def setUp(self): + self.server_class.address_family = socket.AF_INET6 + super(IPv6SysLogHandlerTest, self).setUp() + + def tearDown(self): + self.server_class.address_family = socket.AF_INET + super(IPv6SysLogHandlerTest, self).tearDown() + @unittest.skipUnless(threading, 'Threading required for this test.') class HTTPHandlerTest(BaseTest): """Test for HTTPHandler.""" @@ -3163,6 +3181,7 @@ def setup_and_log(log_queue, ident): handler.close() @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_queue_queue(self, mock_handle): for i in range(self.repeat): log_queue = queue.Queue() @@ -3172,10 +3191,13 @@ def test_handle_called_with_queue_queue(self, mock_handle): @support.requires_multiprocessing_queue @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_mp_queue(self, mock_handle): for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) + log_queue.close() + log_queue.join_thread() self.assertEqual(mock_handle.call_count, 5 * self.repeat, 'correct number of handled log messages') @@ -3188,6 +3210,7 @@ def get_all_from_queue(log_queue): return [] @support.requires_multiprocessing_queue + @support.reap_threads def test_no_messages_in_queue_after_stop(self): """ Five messages are logged then the QueueListener is stopped. This @@ -3200,6 +3223,9 @@ def test_no_messages_in_queue_after_stop(self): self.setup_and_log(queue, '%s_%s' %(self.id(), i)) # time.sleep(1) items = list(self.get_all_from_queue(queue)) + queue.close() + queue.join_thread() + expected = [[], [logging.handlers.QueueListener._sentinel]] self.assertIn(items, expected, 'Found unexpected messages in queue: %s' % ( @@ -4370,7 +4396,7 @@ def test_main(): QueueHandlerTest, ShutdownTest, ModuleLevelMiscTest, BasicConfigTest, LoggerAdapterTest, LoggerTest, SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest, LastResortTest, LogRecordTest, - ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, + ExceptionTest, SysLogHandlerTest, IPv6SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, MiscTestCase diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index c46ded11866589..3e84f3429ee854 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -286,7 +286,12 @@ class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase): def setUpClass(cls): support.requires("network") with support.transient_internet(cls.NNTP_HOST): - cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) + try: + cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, + usenetrc=False) + except EOFError: + raise unittest.SkipTest(f"{cls} got EOF error on connecting " + f"to {cls.NNTP_HOST!r}") @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index d6e6f7157782ec..70cabb28598218 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -1,4 +1,6 @@ +import copy import parser +import pickle import unittest import operator import struct @@ -424,6 +426,52 @@ def test_junk(self): # not even remotely valid: self.check_bad_tree((1, 2, 3), "") + def test_illegal_terminal(self): + tree = \ + (257, + (269, + (270, + (271, + (277, + (1,))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too small items in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, b'pass'))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-string second item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', '0', 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-integer third item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', 0, 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too many items in terminal node") + def test_illegal_yield_1(self): # Illegal yield statement: def f(): return 1; yield 1 tree = \ @@ -628,6 +676,24 @@ def test_missing_import_source(self): (4, ''), (0, '')) self.check_bad_tree(tree, "from import fred") + def test_illegal_encoding(self): + # Illegal encoding declaration + tree = \ + (339, + (257, (0, ''))) + self.check_bad_tree(tree, "missed encoding") + tree = \ + (339, + (257, (0, '')), + b'iso-8859-1') + self.check_bad_tree(tree, "non-string encoding") + tree = \ + (339, + (257, (0, '')), + '\udcff') + with self.assertRaises(UnicodeEncodeError): + parser.sequence2st(tree) + class CompileTestCase(unittest.TestCase): @@ -772,6 +838,21 @@ def test_comparisons(self): self.assertRaises(TypeError, operator.lt, st1, 1815) self.assertRaises(TypeError, operator.gt, b'waterloo', st2) + def test_copy_pickle(self): + sts = [ + parser.expr('2 + 3'), + parser.suite('x = 2; y = x + 3'), + parser.expr('list(x**3 for x in range(20))') + ] + for st in sts: + st_copy = copy.copy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + st_copy = copy.deepcopy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + st_copy = pickle.loads(pickle.dumps(st, proto)) + self.assertEqual(st_copy.totuple(), st.totuple()) + check_sizeof = support.check_sizeof @support.cpython_only diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 88a93e5802c1cb..846f721e8d9abb 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -8,6 +8,7 @@ import stat import tempfile import unittest +from unittest import mock from test import support android_not_root = support.android_not_root @@ -1776,6 +1777,11 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + @only_nt # XXX: not sure how to test this on POSIX def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': @@ -1811,6 +1817,35 @@ def test_mkdir_no_parents_file(self): p.mkdir(exist_ok=True) self.assertEqual(cm.exception.errno, errno.EEXIST) + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(BASE, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + os.mkdir(path, mode) # from another process + concurrently_created.add(path) + os.mkdir(path, mode) # our real call + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + @with_symlinks def test_symlink_to(self): P = self.cls(BASE) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5f875eff6c7ede..ea6752b61d8f77 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -67,12 +67,12 @@ def test_processor(self): def setUp(self): self.save_version = sys.version - self.save_mercurial = sys._mercurial + self.save_git = sys._git self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys._mercurial = self.save_mercurial + sys._git = self.save_git sys.platform = self.save_platform def test_sys_version(self): @@ -102,7 +102,7 @@ def test_sys_version(self): ('CPython', '2.4.3', '', '', 'truncation', '', 'GCC')), ): # branch and revision are not "parsed", but fetched - # from sys._mercurial. Ignore them + # from sys._git. Ignore them (name, version, branch, revision, buildno, builddate, compiler) \ = platform._sys_version(input) self.assertEqual( @@ -149,10 +149,10 @@ def test_sys_version(self): sys_versions.items(): sys.version = version_tag if subversion is None: - if hasattr(sys, "_mercurial"): - del sys._mercurial + if hasattr(sys, "_git"): + del sys._git else: - sys._mercurial = subversion + sys._git = subversion if sys_platform is not None: sys.platform = sys_platform self.assertEqual(platform.python_implementation(), info[0]) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 5b6a4f06baaf41..03adc0a72294ab 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -427,6 +427,7 @@ def test_setstate_first_arg(self): self.assertRaises(ValueError, self.gen.setstate, (1, None, None)) def test_setstate_middle_arg(self): + start_state = self.gen.getstate() # Wrong type, s/b tuple self.assertRaises(TypeError, self.gen.setstate, (2, None, None)) # Wrong length, s/b 625 @@ -440,6 +441,10 @@ def test_setstate_middle_arg(self): self.gen.setstate((2, (1,)*624+(625,), None)) with self.assertRaises((ValueError, OverflowError)): self.gen.setstate((2, (1,)*624+(-1,), None)) + # Failed calls to setstate() should not have changed the state. + bits100 = self.gen.getrandbits(100) + self.gen.setstate(start_state) + self.assertEqual(self.gen.getrandbits(100), bits100) # Little trick to make "tuple(x % (2**32) for x in internalstate)" # raise ValueError. I cannot think of a simple way to achieve this, so diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 9e11e518f61e1f..679759ec6d676a 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -99,20 +99,24 @@ def test_large_operands(self): x = range(10**20+10, 10**20, 3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20, 10**20+10, -3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20+10, 10**20, -3) self.assertEqual(len(x), 4) self.assertEqual(len(list(x)), 4) + self.assertTrue(x) # Now test range() with longs - self.assertEqual(list(range(-2**100)), []) - self.assertEqual(list(range(0, -2**100)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) + for x in [range(-2**100), + range(0, -2**100), + range(0, 2**100, -1)]: + self.assertEqual(list(x), []) + self.assertFalse(x) a = int(10 * sys.maxsize) b = int(100 * sys.maxsize) @@ -153,6 +157,7 @@ def _range_len(x): step = x[1] - x[0] length = 1 + ((x[-1] - x[0]) // step) return length + a = -sys.maxsize b = sys.maxsize expected_len = b - a @@ -160,6 +165,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -177,6 +183,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -195,6 +202,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -213,6 +221,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index b945cf094e99c1..e88d0b3dcf2a78 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1303,32 +1303,43 @@ def test_inline_flags(self): upper_char = '\u1ea0' # Latin Capital Letter A with Dot Below lower_char = '\u1ea1' # Latin Small Letter A with Dot Below - p = re.compile(upper_char, re.I | re.U) - q = p.match(lower_char) + p = re.compile('.' + upper_char, re.I | re.S) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile(lower_char, re.I | re.U) - q = p.match(upper_char) + p = re.compile('.' + lower_char, re.I | re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?i)' + upper_char, re.U) - q = p.match(lower_char) + p = re.compile('(?i).' + upper_char, re.S) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?i)' + lower_char, re.U) - q = p.match(upper_char) + p = re.compile('(?i).' + lower_char, re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?iu)' + upper_char) - q = p.match(lower_char) + p = re.compile('(?is).' + upper_char) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?iu)' + lower_char) - q = p.match(upper_char) + p = re.compile('(?is).' + lower_char) + q = p.match('\n' + upper_char) self.assertTrue(q) - self.assertTrue(re.match('(?ixu) ' + upper_char, lower_char)) - self.assertTrue(re.match('(?ixu) ' + lower_char, upper_char)) + p = re.compile('(?s)(?i).' + upper_char) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('(?s)(?i).' + lower_char) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + self.assertTrue(re.match('(?ix) ' + upper_char, lower_char)) + self.assertTrue(re.match('(?ix) ' + lower_char, upper_char)) + self.assertTrue(re.match(' (?i) ' + upper_char, lower_char, re.X)) + self.assertTrue(re.match('(?x) (?i) ' + upper_char, lower_char)) + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char, re.X)) p = upper_char + '(?i)' with self.assertWarns(DeprecationWarning) as warns: @@ -1337,6 +1348,7 @@ def test_inline_flags(self): str(warns.warnings[0].message), 'Flags not at the start of the expression %s' % p ) + self.assertEqual(warns.warnings[0].filename, __file__) p = upper_char + '(?i)%s' % ('.?' * 100) with self.assertWarns(DeprecationWarning) as warns: @@ -1345,6 +1357,36 @@ def test_inline_flags(self): str(warns.warnings[0].message), 'Flags not at the start of the expression %s (truncated)' % p[:20] ) + self.assertEqual(warns.warnings[0].filename, __file__) + + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?i) ' + upper_char + ' (?x)', lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('^(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('$|(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match('(?:(?i)' + upper_char + ')', lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.fullmatch('(^)?(?(1)(?i)' + upper_char + ')', + lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.fullmatch('($)?(?(1)|(?i)' + upper_char + ')', + lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + def test_dollar_matches_twice(self): "$ matches the end of string, and just before the terminating \n" diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 2411895d9d12b1..2eb62905ffa882 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -4,6 +4,7 @@ from xml.sax import make_parser, ContentHandler, \ SAXException, SAXReaderNotAvailable, SAXParseException import unittest +from unittest import mock try: make_parser() except SAXReaderNotAvailable: @@ -175,12 +176,8 @@ def test_parse_bytes(self): with self.assertRaises(SAXException): self.check_parse(BytesIO(xml_bytes(self.data, 'iso-8859-1', None))) make_xml_file(self.data, 'iso-8859-1', None) - with support.check_warnings(('unclosed file', ResourceWarning)): - # XXX Failed parser leaks an opened file. - with self.assertRaises(SAXException): - self.check_parse(TESTFN) - # Collect leaked file. - gc.collect() + with self.assertRaises(SAXException): + self.check_parse(TESTFN) with open(TESTFN, 'rb') as f: with self.assertRaises(SAXException): self.check_parse(f) @@ -194,6 +191,21 @@ def test_parse_InputSource(self): input.setEncoding('iso-8859-1') self.check_parse(input) + def test_parse_close_source(self): + builtin_open = open + fileobj = None + + def mock_open(*args): + nonlocal fileobj + fileobj = builtin_open(*args) + return fileobj + + with mock.patch('xml.sax.saxutils.open', side_effect=mock_open): + make_xml_file(self.data, 'iso-8859-1', None) + with self.assertRaises(SAXException): + self.check_parse(TESTFN) + self.assertTrue(fileobj.closed) + def check_parseString(self, s): from xml.sax import parseString result = StringIO() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0620b242766057..4029617aa1d9c1 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -27,14 +27,27 @@ import site -if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): - # need to add user site directory for tests - try: - os.makedirs(site.USER_SITE) - site.addsitedir(site.USER_SITE) - except PermissionError as exc: - raise unittest.SkipTest('unable to create user site directory (%r): %s' - % (site.USER_SITE, exc)) + +OLD_SYS_PATH = None + + +def setUpModule(): + global OLD_SYS_PATH + OLD_SYS_PATH = sys.path[:] + + if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): + # need to add user site directory for tests + try: + os.makedirs(site.USER_SITE) + # modify sys.path: will be restored by tearDownModule() + site.addsitedir(site.USER_SITE) + except PermissionError as exc: + raise unittest.SkipTest('unable to create user site directory (%r): %s' + % (site.USER_SITE, exc)) + + +def tearDownModule(): + sys.path[:] = OLD_SYS_PATH class HelperFunctionsTests(unittest.TestCase): @@ -501,15 +514,15 @@ def _create_underpth_exe(self, lines): print(line, file=f) return exe_file except: - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) raise @classmethod def _cleanup_underpth_exe(self, exe_file): _pth_file = os.path.splitext(exe_file)[0] + '._pth' - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) @classmethod def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 97dc3cd76b24a5..80dfc405c71809 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -803,11 +803,6 @@ def testHostnameRes(self): self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) def test_host_resolution(self): - for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', - '1:1:1:1:1:1:1:1:1']: - self.assertRaises(OSError, socket.gethostbyname, addr) - self.assertRaises(OSError, socket.gethostbyaddr, addr) - for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: self.assertEqual(socket.gethostbyname(addr), addr) @@ -816,6 +811,21 @@ def test_host_resolution(self): for host in [support.HOST]: self.assertIn(host, socket.gethostbyaddr(host)[2]) + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): @@ -896,6 +906,7 @@ def testNtoH(self): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) + @support.cpython_only def testNtoHErrors(self): good_values = [ 1, 2, 3, 1, 2, 3 ] bad_values = [ -1, -2, -3, -1, -2, -3 ] @@ -4649,6 +4660,10 @@ def bind(self, sock, path): else: raise + def testUnbound(self): + # Issue #30205 + self.assertIn(self.sock.getsockname(), ('', None)) + def testStrAddr(self): # Test binding to and retrieving a normal string pathname. path = os.path.abspath(support.TESTFN) @@ -5469,7 +5484,7 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 3) # see test_aes_cbc + @support.requires_linux_version(4, 9) # see issue29324 def test_aead_aes_gcm(self): key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') @@ -5492,8 +5507,7 @@ def test_aead_aes_gcm(self): op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen, flags=socket.MSG_MORE) op.sendall(assoc, socket.MSG_MORE) - op.sendall(plain, socket.MSG_MORE) - op.sendall(b'\x00' * taglen) + op.sendall(plain) res = op.recv(assoclen + len(plain) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5501,7 +5515,7 @@ def test_aead_aes_gcm(self): # now with msg op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen) res = op.recv(assoclen + len(plain) + taglen) @@ -5512,7 +5526,7 @@ def test_aead_aes_gcm(self): pack_uint32 = struct.Struct('I').pack op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg( [msg], ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], @@ -5520,7 +5534,7 @@ def test_aead_aes_gcm(self): [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], ) ) - res = op.recv(len(msg)) + res = op.recv(len(msg) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5530,8 +5544,8 @@ def test_aead_aes_gcm(self): msg = assoc + expected_ct + expected_tag op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, assoclen=assoclen) - res = op.recv(len(msg)) - self.assertEqual(plain, res[assoclen:-taglen]) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) @support.requires_linux_version(4, 3) # see test_aes_cbc def test_drbg_pr_sha256(self): diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index d203cddbdfb5ee..85c59a618ce1e9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2065,7 +2065,7 @@ class AsyncoreEchoServer(threading.Thread): class EchoServer (asyncore.dispatcher): - class ConnectionHandler (asyncore.dispatcher_with_send): + class ConnectionHandler(asyncore.dispatcher_with_send): def __init__(self, conn, certfile): self.socket = test_wrap_socket(conn, server_side=True, @@ -2156,6 +2156,8 @@ def __exit__(self, *args): self.join() if support.verbose: sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) def start (self, flag=None): self.flag = flag diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e63f9f254cd2b2..d8d6d82713cded 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -347,6 +347,16 @@ def test_cwd(self): temp_dir = self._normalize_cwd(temp_dir) self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + def test_cwd_with_pathlike(self): + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + + class _PathLikeObj: + def __fspath__(self): + return temp_dir + + self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): # Check that Popen looks for args[0] relative to cwd if args[0] @@ -2421,7 +2431,7 @@ def test_fork_exec(self): with self.assertRaises(TypeError): _posixsubprocess.fork_exec( args, exe_list, - True, [], cwd, env_list, + True, (), cwd, env_list, -1, -1, -1, -1, 1, 2, 3, 4, True, True, func) @@ -2433,6 +2443,16 @@ def test_fork_exec(self): def test_fork_exec_sorted_fd_sanity_check(self): # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. import _posixsubprocess + class BadInt: + first = True + def __init__(self, value): + self.value = value + def __int__(self): + if self.first: + self.first = False + return self.value + raise ValueError + gc_enabled = gc.isenabled() try: gc.enable() @@ -2443,6 +2463,7 @@ def test_fork_exec_sorted_fd_sanity_check(self): (18, 23, 42, 2**63), # Out of range. (5, 4), # Not sorted. (6, 7, 7, 8), # Duplicate. + (BadInt(1), BadInt(2)), ): with self.assertRaises( ValueError, diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e83a4d6426a8b2..ddd65092312a3b 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -282,17 +282,34 @@ def test_python_is_optimized(self): def test_swap_attr(self): class Obj: - x = 1 + pass obj = Obj() - with support.swap_attr(obj, "x", 5): + obj.x = 1 + with support.swap_attr(obj, "x", 5) as x: self.assertEqual(obj.x, 5) + self.assertEqual(x, 1) self.assertEqual(obj.x, 1) + with support.swap_attr(obj, "y", 5) as y: + self.assertEqual(obj.y, 5) + self.assertIsNone(y) + self.assertFalse(hasattr(obj, 'y')) + with support.swap_attr(obj, "y", 5): + del obj.y + self.assertFalse(hasattr(obj, 'y')) def test_swap_item(self): - D = {"item":1} - with support.swap_item(D, "item", 5): - self.assertEqual(D["item"], 5) - self.assertEqual(D["item"], 1) + D = {"x":1} + with support.swap_item(D, "x", 5) as x: + self.assertEqual(D["x"], 5) + self.assertEqual(x, 1) + self.assertEqual(D["x"], 1) + with support.swap_item(D, "y", 5) as y: + self.assertEqual(D["y"], 5) + self.assertIsNone(y) + self.assertNotIn("y", D) + with support.swap_item(D, "y", 5): + del D["y"] + self.assertNotIn("y", D) class RefClass: attribute1 = None diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index df9ebd40859e21..e151f493d4e36c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -913,13 +913,15 @@ def inner(): return inner check(get_cell().__closure__[0], size('P')) # code - check(get_cell().__code__, size('6i13P')) - check(get_cell.__code__, size('6i13P')) + def check_code_size(a, expected_size): + self.assertGreaterEqual(sys.getsizeof(a), expected_size) + check_code_size(get_cell().__code__, size('6i13P')) + check_code_size(get_cell.__code__, size('6i13P')) def get_cell2(x): def inner(): return x return inner - check(get_cell2.__code__, size('6i13P') + 1) + check_code_size(get_cell2.__code__, size('6i13P') + calcsize('n')) # complex check(complex(0,1), size('2d')) # method_descriptor (descriptor object) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 619cbc03b28abc..fc79055421161c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4,6 +4,7 @@ from hashlib import md5 from contextlib import contextmanager from random import Random +import pathlib import unittest import unittest.mock @@ -440,6 +441,22 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) + def test_pathlike_name(self): + tarname = pathlib.Path(self.tarname) + with tarfile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with self.taropen(tarname) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with tarfile.TarFile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + if self.suffix == '': + with tarfile.TarFile(tarname, mode='r') as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + def test_illegal_mode_arg(self): with open(tmpname, 'wb'): pass @@ -582,6 +599,26 @@ def test_extract_directory(self): finally: support.rmtree(DIR) + def test_extractall_pathlike_name(self): + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + for tarinfo in directories: + path = DIR / tarinfo.name + self.assertEqual(os.path.getmtime(path), tarinfo.mtime) + + def test_extract_pathlike_name(self): + dirtype = "ustar/dirtype" + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + tarinfo = tar.getmember(dirtype) + tar.extract(tarinfo, path=DIR) + extracted = DIR / dirtype + self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) + def test_init_close_fobj(self): # Issue #7341: Close the internal file object in the TarFile # constructor in case of an error. For the test we rely on @@ -1092,6 +1129,17 @@ def test_directory_size(self): finally: support.rmdir(path) + def test_gettarinfo_pathlike_name(self): + with tarfile.open(tmpname, self.mode) as tar: + path = pathlib.Path(TEMPDIR) / "file" + with open(path, "wb") as fobj: + fobj.write(b"aaa") + tarinfo = tar.gettarinfo(path) + tarinfo2 = tar.gettarinfo(os.fspath(path)) + self.assertIsInstance(tarinfo.name, str) + self.assertEqual(tarinfo.name, tarinfo2.name) + self.assertEqual(tarinfo.size, 3) + @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1528,6 +1576,34 @@ def test_create_existing_taropen(self): self.assertEqual(len(names), 1) self.assertIn("spameggs42", names[0]) + def test_create_pathlike_name(self): + with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + def test_create_taropen_pathlike_name(self): + with self.taropen(pathlib.Path(tmpname), "x") as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + class GzipCreateTest(GzipTest, CreateTest): pass diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 51df1ecd7d18e6..d0cf04b0cb67ca 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -273,13 +273,12 @@ def raise_OSError(*args, **kwargs): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) - open = io.open def bad_writer(*args, **kwargs): - fp = open(*args, **kwargs) + fp = orig_open(*args, **kwargs) fp.write = raise_OSError return fp - with support.swap_attr(io, "open", bad_writer): + with support.swap_attr(io, "open", bad_writer) as orig_open: # test again with failing write() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index ef3059b68674b4..3909b75ccd4647 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -20,6 +20,7 @@ def verbose_print(arg): with _print_mutex: print(arg) + class BasicThreadTest(unittest.TestCase): def setUp(self): @@ -31,6 +32,9 @@ def setUp(self): self.running = 0 self.next_ident = 0 + key = support.threading_setup() + self.addCleanup(support.threading_cleanup, *key) + class ThreadRunningTests(BasicThreadTest): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 2c2914fd6d8969..0db028864d203a 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -170,6 +170,9 @@ def f(mutex): mutex.acquire() self.assertIn(tid, threading._active) self.assertIsInstance(threading._active[tid], threading._DummyThread) + #Issue 29376 + self.assertTrue(threading._active[tid].is_alive()) + self.assertRegex(repr(threading._active[tid]), '_DummyThread') del threading._active[tid] # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) @@ -467,13 +470,15 @@ def test_is_alive_after_fork(self): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - self.addCleanup(t.join) pid = os.fork() if pid == 0: - os._exit(1 if t.is_alive() else 0) + os._exit(11 if t.is_alive() else 10) else: + t.join() + pid, status = os.waitpid(pid, 0) - self.assertEqual(0, status) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(10, os.WEXITSTATUS(status)) def test_main_thread(self): main = threading.main_thread() diff --git a/Lib/test/test_tools/test_reindent.py b/Lib/test/test_tools/test_reindent.py index d7c20e1e5c7206..34df0c5d511904 100644 --- a/Lib/test/test_tools/test_reindent.py +++ b/Lib/test/test_tools/test_reindent.py @@ -7,6 +7,7 @@ import os import unittest from test.support.script_helper import assert_python_ok +from test.support import findfile from test.test_tools import scriptsdir, skip_if_missing @@ -23,6 +24,12 @@ def test_help(self): self.assertEqual(out, b'') self.assertGreater(err, b'') + def test_reindent_file_with_bad_encoding(self): + bad_coding_path = findfile('bad_coding.py') + rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path) + self.assertEqual(out, b'') + self.assertNotEqual(err, b'') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 790ab7ee606d60..742259b4396b98 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -865,6 +865,7 @@ def test_sys_xoptions_invalid(self): b'number of frames', stderr) + @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): # Issue #21639: Check that PyMem_Malloc(0) with tracemalloc enabled # does not crash. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fce6b5aaffdf93..b3cabda394497e 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6,7 +6,7 @@ from unittest import TestCase, main, skipUnless, SkipTest from copy import copy, deepcopy -from typing import Any +from typing import Any, NoReturn from typing import TypeVar, AnyStr from typing import T, KT, VT # Not in __all__. from typing import Union, Optional @@ -102,10 +102,6 @@ def test_cannot_instantiate(self): with self.assertRaises(TypeError): type(Any)() - def test_cannot_subscript(self): - with self.assertRaises(TypeError): - Any[int] - def test_any_works_with_alias(self): # These expressions must simply not fail. typing.Match[Any] @@ -113,6 +109,40 @@ def test_any_works_with_alias(self): typing.IO[Any] +class NoReturnTests(BaseTestCase): + + def test_noreturn_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, NoReturn) + + def test_noreturn_subclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(Employee, NoReturn) + with self.assertRaises(TypeError): + issubclass(NoReturn, Employee) + + def test_repr(self): + self.assertEqual(repr(NoReturn), 'typing.NoReturn') + + def test_not_generic(self): + with self.assertRaises(TypeError): + NoReturn[int] + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class A(NoReturn): + pass + with self.assertRaises(TypeError): + class A(type(NoReturn)): + pass + + def test_cannot_instantiate(self): + with self.assertRaises(TypeError): + NoReturn() + with self.assertRaises(TypeError): + type(NoReturn)() + + class TypeVarTests(BaseTestCase): def test_basic_plain(self): @@ -190,6 +220,10 @@ def test_bound_errors(self): with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVar('T', covariant=True, contravariant=True) + class UnionTests(BaseTestCase): @@ -254,6 +288,11 @@ def test_repr(self): self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) u = Union[int, Employee] self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + T = TypeVar('T') + u = Union[T, int][int] + self.assertEqual(repr(u), repr(int)) + u = Union[List[int], int] + self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') def test_cannot_subclass(self): with self.assertRaises(TypeError): @@ -304,6 +343,15 @@ def test_union_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, Union[int, str]) + def test_no_eval_union(self): + u = Union[int, str] + def f(x: u): ... + self.assertIs(get_type_hints(f)['x'], u) + + def test_function_repr_union(self): + def fun() -> int: ... + self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + def test_union_str_pattern(self): # Shouldn't crash; see http://bugs.python.org/issue25390 A = Union[str, Pattern] @@ -401,6 +449,8 @@ def test_callable_wrong_forms(self): Callable[[()], int] with self.assertRaises(TypeError): Callable[[int, 1], 2] + with self.assertRaises(TypeError): + Callable[int] def test_callable_instance_works(self): def f(): @@ -546,15 +596,27 @@ def test_basics(self): Y[str] with self.assertRaises(TypeError): Y[str, str] + self.assertIsSubclass(SimpleMapping[str, int], SimpleMapping) def test_generic_errors(self): T = TypeVar('T') + S = TypeVar('S') with self.assertRaises(TypeError): Generic[T]() + with self.assertRaises(TypeError): + Generic[T][T] + with self.assertRaises(TypeError): + Generic[T][S] with self.assertRaises(TypeError): isinstance([], List[int]) with self.assertRaises(TypeError): issubclass(list, List[int]) + with self.assertRaises(TypeError): + class NewGeneric(Generic): ... + with self.assertRaises(TypeError): + class MyGeneric(Generic[T], Generic[S]): ... + with self.assertRaises(TypeError): + class MyGeneric(List[T], Generic[S]): ... def test_init(self): T = TypeVar('T') @@ -642,6 +704,41 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_subscripted_generics_as_proxies(self): + T = TypeVar('T') + class C(Generic[T]): + x = 'def' + self.assertEqual(C[int].x, 'def') + self.assertEqual(C[C[int]].x, 'def') + C[C[int]].x = 'changed' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[str].x, 'changed') + C[List[str]].z = 'new' + self.assertEqual(C.z, 'new') + self.assertEqual(C[Tuple[int]].z, 'new') + + self.assertEqual(C().x, 'changed') + self.assertEqual(C[Tuple[str]]().z, 'new') + + class D(C[T]): + pass + self.assertEqual(D[int].x, 'changed') + self.assertEqual(D.z, 'new') + D.z = 'from derived z' + D[int].x = 'from derived x' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[int].z, 'new') + self.assertEqual(D.x, 'from derived x') + self.assertEqual(D[str].z, 'from derived z') + + def test_abc_registry_kept(self): + T = TypeVar('T') + class C(Generic[T]): ... + C.register(int) + self.assertIsInstance(1, C) + C[int] + self.assertIsInstance(1, C) + def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass self.assertNotIsInstance({}, MyMapping) @@ -738,6 +835,53 @@ def test_subscript_meta(self): self.assertEqual(Union[T, int][GenericMeta], Union[GenericMeta, int]) self.assertEqual(Callable[..., GenericMeta].__args__, (Ellipsis, GenericMeta)) + def test_generic_hashes(self): + try: + from test import mod_generics_cache + except ImportError: # for Python 3.4 and previous versions + import mod_generics_cache + class A(Generic[T]): + ... + + class B(Generic[T]): + class A(Generic[T]): + ... + + self.assertEqual(A, A) + self.assertEqual(mod_generics_cache.A[str], mod_generics_cache.A[str]) + self.assertEqual(B.A, B.A) + self.assertEqual(mod_generics_cache.B.A[B.A[str]], + mod_generics_cache.B.A[B.A[str]]) + + self.assertNotEqual(A, B.A) + self.assertNotEqual(A, mod_generics_cache.A) + self.assertNotEqual(A, mod_generics_cache.B.A) + self.assertNotEqual(B.A, mod_generics_cache.A) + self.assertNotEqual(B.A, mod_generics_cache.B.A) + + self.assertNotEqual(A[str], B.A[str]) + self.assertNotEqual(A[List[Any]], B.A[List[Any]]) + self.assertNotEqual(A[str], mod_generics_cache.A[str]) + self.assertNotEqual(A[str], mod_generics_cache.B.A[str]) + self.assertNotEqual(B.A[int], mod_generics_cache.A[int]) + self.assertNotEqual(B.A[List[Any]], mod_generics_cache.B.A[List[Any]]) + + self.assertNotEqual(Tuple[A[str]], Tuple[B.A[str]]) + self.assertNotEqual(Tuple[A[List[Any]]], Tuple[B.A[List[Any]]]) + self.assertNotEqual(Union[str, A[str]], Union[str, mod_generics_cache.A[str]]) + self.assertNotEqual(Union[A[str], A[str]], + Union[A[str], mod_generics_cache.A[str]]) + self.assertNotEqual(typing.FrozenSet[A[str]], + typing.FrozenSet[mod_generics_cache.B.A[str]]) + + if sys.version_info[:2] > (3, 2): + self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) + self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) + .endswith('mod_generics_cache.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) + .endswith('mod_generics_cache.B.A[str]]')) + def test_extended_generic_rules_eq(self): T = TypeVar('T') U = TypeVar('U') @@ -835,6 +979,8 @@ def test_fail_with_bare_generic(self): Tuple[Generic[T]] with self.assertRaises(TypeError): List[typing._Protocol] + with self.assertRaises(TypeError): + isinstance(1, Generic) def test_type_erasure_special(self): T = TypeVar('T') @@ -853,6 +999,11 @@ class MyDict(typing.Dict[T, T]): ... class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) self.assertIs(MyDef[int]().__orig_class__, MyDef[int]) + # ChainMap was added in 3.3 + if sys.version_info >= (3, 3): + class MyChain(typing.ChainMap[str, T]): ... + self.assertIs(MyChain[int]().__class__, MyChain) + self.assertIs(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) @@ -1203,6 +1354,19 @@ def test_forwardref_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, fr) + def test_forwardref_subclass_type_error(self): + fr = typing._ForwardRef('int') + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forward_equality(self): + fr = typing._ForwardRef('int') + self.assertEqual(fr, typing._ForwardRef('int')) + self.assertNotEqual(List['int'], List[int]) + + def test_forward_repr(self): + self.assertEqual(repr(List['int']), "typing.List[_ForwardRef('int')]") + def test_union_forward(self): def foo(a: Union['T']): @@ -1285,6 +1449,15 @@ def foo(a: 'whatevers') -> {}: ith = get_type_hints(C().foo) self.assertEqual(ith, {}) + def test_no_type_check_no_bases(self): + class C: + def meth(self, x: int): ... + @no_type_check + class D(C): + c = C + # verify that @no_type_check never affects bases + self.assertEqual(get_type_hints(C.meth), {'x': int}) + def test_meta_no_type_check(self): @no_type_check_decorator @@ -1401,11 +1574,16 @@ class A: class B(A): x: ClassVar[Optional['B']] = None y: int + b: int class CSub(B): z: ClassVar['CSub'] = B() class G(Generic[T]): lst: ClassVar[List[T]] = [] +class NoneAndForward: + parent: 'NoneAndForward' + meaning: None + class CoolEmployee(NamedTuple): name: str cool: int @@ -1419,10 +1597,13 @@ class XMeth(NamedTuple): def double(self): return 2 * self.x -class XMethBad(NamedTuple): +class XRepr(NamedTuple): x: int - def _fields(self): - return 'no chance for this' + y: int = 1 + def __str__(self): + return f'{self.x} -> {self.y}' + def __add__(self, other): + return 0 """ if PY36: @@ -1431,7 +1612,7 @@ def _fields(self): # fake names for the sake of static analysis ann_module = ann_module2 = ann_module3 = None A = B = CSub = G = CoolEmployee = CoolEmployeeWithDefault = object - XMeth = XMethBad = object + XMeth = XRepr = NoneAndForward = object gth = get_type_hints @@ -1466,6 +1647,8 @@ def test_get_type_hints_classes(self): {'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.S), {'x': str, 'y': str}) self.assertEqual(gth(ann_module.foo), {'x': int}) + self.assertEqual(gth(NoneAndForward, globals()), + {'parent': NoneAndForward, 'meaning': type(None)}) @skipUnless(PY36, 'Python 3.6 required') def test_respect_no_type_check(self): @@ -1482,17 +1665,22 @@ def meth(x: int): ... class Der(ABase): ... self.assertEqual(gth(ABase.meth), {'x': int}) - def test_get_type_hints_for_builins(self): + def test_get_type_hints_for_builtins(self): # Should not fail for built-in classes and functions. self.assertEqual(gth(int), {}) self.assertEqual(gth(type), {}) self.assertEqual(gth(dir), {}) self.assertEqual(gth(len), {}) + self.assertEqual(gth(object.__str__), {}) + self.assertEqual(gth(object().__str__), {}) + self.assertEqual(gth(str.join), {}) def test_previous_behavior(self): def testf(x, y): ... testf.__annotations__['x'] = 'int' self.assertEqual(gth(testf), {'x': int}) + def testg(x: None): ... + self.assertEqual(gth(testg), {'x': type(None)}) def test_get_type_hints_for_object_with_annotations(self): class A: ... @@ -1506,9 +1694,10 @@ def test_get_type_hints_ClassVar(self): self.assertEqual(gth(ann_module2.CV, ann_module2.__dict__), {'var': typing.ClassVar[ann_module2.CV]}) self.assertEqual(gth(B, globals()), - {'y': int, 'x': ClassVar[Optional[B]]}) + {'y': int, 'x': ClassVar[Optional[B]], 'b': int}) self.assertEqual(gth(CSub, globals()), - {'z': ClassVar[CSub], 'y': int, 'x': ClassVar[Optional[B]]}) + {'z': ClassVar[CSub], 'y': int, 'b': int, + 'x': ClassVar[Optional[B]]}) self.assertEqual(gth(G), {'lst': ClassVar[List[T]]}) @@ -1628,6 +1817,11 @@ def test_list(self): def test_deque(self): self.assertIsSubclass(collections.deque, typing.Deque) + class MyDeque(typing.Deque[int]): ... + self.assertIsInstance(MyDeque(), collections.deque) + + def test_counter(self): + self.assertIsSubclass(collections.Counter, typing.Counter) def test_set(self): self.assertIsSubclass(set, typing.Set) @@ -1680,13 +1874,10 @@ class MyDict(typing.Dict[str, int]): self.assertIsSubclass(MyDict, dict) self.assertNotIsSubclass(dict, MyDict) - def test_no_defaultdict_instantiation(self): - with self.assertRaises(TypeError): - typing.DefaultDict() - with self.assertRaises(TypeError): - typing.DefaultDict[KT, VT]() - with self.assertRaises(TypeError): - typing.DefaultDict[str, int]() + def test_defaultdict_instantiation(self): + self.assertIs(type(typing.DefaultDict()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[KT, VT]()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[str, int]()), collections.defaultdict) def test_defaultdict_subclass(self): @@ -1699,13 +1890,49 @@ class MyDefDict(typing.DefaultDict[str, int]): self.assertIsSubclass(MyDefDict, collections.defaultdict) self.assertNotIsSubclass(collections.defaultdict, MyDefDict) - def test_no_deque_instantiation(self): - with self.assertRaises(TypeError): - typing.Deque() - with self.assertRaises(TypeError): - typing.Deque[T]() - with self.assertRaises(TypeError): - typing.Deque[int]() + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_instantiation(self): + self.assertIs(type(typing.ChainMap()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[KT, VT]()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[str, int]()), collections.ChainMap) + class CM(typing.ChainMap[KT, VT]): ... + self.assertIs(type(CM[int, str]()), CM) + + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_subclass(self): + + class MyChainMap(typing.ChainMap[str, int]): + pass + + cm = MyChainMap() + self.assertIsInstance(cm, MyChainMap) + + self.assertIsSubclass(MyChainMap, collections.ChainMap) + self.assertNotIsSubclass(collections.ChainMap, MyChainMap) + + def test_deque_instantiation(self): + self.assertIs(type(typing.Deque()), collections.deque) + self.assertIs(type(typing.Deque[T]()), collections.deque) + self.assertIs(type(typing.Deque[int]()), collections.deque) + class D(typing.Deque[T]): ... + self.assertIs(type(D[int]()), D) + + def test_counter_instantiation(self): + self.assertIs(type(typing.Counter()), collections.Counter) + self.assertIs(type(typing.Counter[T]()), collections.Counter) + self.assertIs(type(typing.Counter[int]()), collections.Counter) + class C(typing.Counter[T]): ... + self.assertIs(type(C[int]()), C) + + def test_counter_subclass_instantiation(self): + + class MyCounter(typing.Counter[int]): + pass + + d = MyCounter() + self.assertIsInstance(d, MyCounter) + self.assertIsInstance(d, typing.Counter) + self.assertIsInstance(d, collections.Counter) def test_no_set_instantiation(self): with self.assertRaises(TypeError): @@ -2018,6 +2245,14 @@ def test_basics(self): collections.OrderedDict([('name', str), ('id', int)])) self.assertIs(Emp._field_types, Emp.__annotations__) + def test_namedtuple_pyversion(self): + if sys.version_info[:2] < (3, 6): + with self.assertRaises(TypeError): + NamedTuple('Name', one=int, other=str) + with self.assertRaises(TypeError): + class NotYet(NamedTuple): + whatever = 0 + @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) @@ -2055,10 +2290,26 @@ class NonDefaultAfterDefault(NamedTuple): @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage_with_methods(self): - self.assertEquals(XMeth(1).double(), 2) - self.assertEquals(XMeth(42).x, XMeth(42)[0]) - self.assertEquals(XMethBad(1)._fields, ('x',)) - self.assertEquals(XMethBad(1).__annotations__, {'x': int}) + self.assertEqual(XMeth(1).double(), 2) + self.assertEqual(XMeth(42).x, XMeth(42)[0]) + self.assertEqual(str(XRepr(42)), '42 -> 1') + self.assertEqual(XRepr(1, 2) + XRepr(3), 0) + + with self.assertRaises(AttributeError): + exec(""" +class XMethBad(NamedTuple): + x: int + def _fields(self): + return 'no chance for this' +""") + + with self.assertRaises(AttributeError): + exec(""" +class XMethBad2(NamedTuple): + x: int + def _source(self): + return 'no chance for this as well' +""") @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): @@ -2138,6 +2389,12 @@ def test_basics(self): Pattern[Union[str, bytes]] Match[Union[bytes, str]] + def test_alias_equality(self): + self.assertEqual(Pattern[str], Pattern[str]) + self.assertNotEqual(Pattern[str], Pattern[bytes]) + self.assertNotEqual(Pattern[str], Match[str]) + self.assertNotEqual(Pattern[str], str) + def test_errors(self): with self.assertRaises(TypeError): # Doesn't fit AnyStr. @@ -2152,6 +2409,9 @@ def test_errors(self): with self.assertRaises(TypeError): # We don't support isinstance(). isinstance(42, Pattern[str]) + with self.assertRaises(TypeError): + # We don't support issubclass(). + issubclass(Pattern[bytes], Pattern[str]) def test_repr(self): self.assertEqual(repr(Pattern), 'Pattern[~AnyStr]') @@ -2198,6 +2458,9 @@ def test_all(self): self.assertNotIn('sys', a) # Check that Text is defined. self.assertIn('Text', a) + # Check previously missing classes. + self.assertIn('SupportsBytes', a) + self.assertIn('SupportsComplex', a) if __name__ == '__main__': diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 86ebd45e7b031d..2844bc5540c38b 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1448,6 +1448,15 @@ def test_formatting_huge_precision(self): with self.assertRaises(ValueError): result = format_string % 2.34 + def test_issue28598_strsubclass_rhs(self): + # A subclass of str with an __rmod__ method should be able to hook + # into the % operator + class SubclassedStr(str): + def __rmod__(self, other): + return 'Success, self.__rmod__({!r}) was called'.format(other) + self.assertEqual('lhs %% %r' % SubclassedStr('rhs'), + "Success, self.__rmod__('lhs %% %r') was called") + @support.cpython_only def test_formatting_huge_precision_c_limits(self): from _testcapi import INT_MAX diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 5084486e5ab479..fa3757cc94be52 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -206,6 +206,7 @@ def test_iter(self): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + class ProxyTests(unittest.TestCase): def setUp(self): @@ -259,6 +260,7 @@ def test_proxy_bypass_environment_host_match(self): self.assertFalse(bypass('newdomain.com')) # no port self.assertFalse(bypass('newdomain.com:1235')) # wrong port + class ProxyTests_withOrderedEnv(unittest.TestCase): def setUp(self): @@ -294,6 +296,7 @@ def test_getproxies_environment_prefer_lowercase(self): proxies = urllib.request.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) + class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): """Test urlopen() opening a fake http connection.""" @@ -432,7 +435,6 @@ def test_ftp_cache_pruning(self): finally: self.unfakeftp() - def test_userpass_inurl(self): self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") try: @@ -476,6 +478,7 @@ def test_cafile_and_context(self): "https://localhost", cafile="/nonexistent/path", context=context ) + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -549,6 +552,7 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 34329f87162a62..876fcd4199fb92 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -141,44 +141,55 @@ def test_password_manager(self): mgr = urllib.request.HTTPPasswordMgr() add = mgr.add_password find_user_pass = mgr.find_user_password + add("Some Realm", "http://example.com/", "joe", "password") add("Some Realm", "http://example.com/ni", "ni", "ni") - add("c", "http://example.com/foo", "foo", "ni") - add("c", "http://example.com/bar", "bar", "nini") - add("b", "http://example.com/", "first", "blah") - add("b", "http://example.com/", "second", "spam") - add("a", "http://example.com", "1", "a") add("Some Realm", "http://c.example.com:3128", "3", "c") add("Some Realm", "d.example.com", "4", "d") add("Some Realm", "e.example.com:3128", "5", "e") + # For the same realm, password set the highest path is the winner. self.assertEqual(find_user_pass("Some Realm", "example.com"), ('joe', 'password')) - - #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), - # ('ni', 'ni')) - + self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), + ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com"), ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam"), - ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam/spam"), - ('joe', 'password')) + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam"), + ('joe', 'password')) + + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam/spam"), + ('joe', 'password')) + + # You can have different passwords for different paths. + + add("c", "http://example.com/foo", "foo", "ni") + add("c", "http://example.com/bar", "bar", "nini") + self.assertEqual(find_user_pass("c", "http://example.com/foo"), ('foo', 'ni')) + self.assertEqual(find_user_pass("c", "http://example.com/bar"), ('bar', 'nini')) + + # For the same path, newer password should be considered. + + add("b", "http://example.com/", "first", "blah") + add("b", "http://example.com/", "second", "spam") + self.assertEqual(find_user_pass("b", "http://example.com/"), ('second', 'spam')) # No special relationship between a.example.com and example.com: + add("a", "http://example.com", "1", "a") self.assertEqual(find_user_pass("a", "http://example.com/"), ('1', 'a')) + self.assertEqual(find_user_pass("a", "http://a.example.com/"), (None, None)) @@ -830,7 +841,6 @@ def test_file(self): for url, ftp in [ ("file://ftp.example.com//foo.txt", False), ("file://ftp.example.com///foo.txt", False), -# XXXX bug: fails with OSError, should be URLError ("file://ftp.example.com/foo.txt", False), ("file://somehost//foo/something.txt", False), ("file://localhost//foo/something.txt", False), @@ -838,8 +848,7 @@ def test_file(self): req = Request(url) try: h.file_open(req) - # XXXX remove OSError when bug fixed - except (urllib.error.URLError, OSError): + except urllib.error.URLError: self.assertFalse(ftp) else: self.assertIs(o.req, req) @@ -1414,7 +1423,6 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") - # TODO: This should be only for OSX @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { @@ -1690,7 +1698,6 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") - class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 949716c2b567e0..4103b6c07505d5 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -11,6 +11,7 @@ support.requires('network') + class URLTimeoutTest(unittest.TestCase): # XXX this test doesn't seem to test anything useful. @@ -25,11 +26,11 @@ def tearDown(self): def testURLread(self): with support.transient_internet("www.example.com"): f = urllib.request.urlopen("http://www.example.com/") - x = f.read() + f.read() class urlopenNetworkTests(unittest.TestCase): - """Tests urllib.reqest.urlopen using the network. + """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a good job overall of some of the basic interface features. There are no @@ -188,6 +189,7 @@ def test_data_header(self): def test_reporthook(self): records = [] + def recording_reporthook(blocks, block_size, total_size): records.append((blocks, block_size, total_size)) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 99c5c033e30d20..3c89ed928ff51f 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -6,8 +6,8 @@ RFC3986_BASE = 'http://a/b/c/d;p?q' SIMPLE_BASE = 'http://a/b/c/d' -# A list of test cases. Each test case is a two-tuple that contains -# a string with the query and a dictionary with the expected result. +# Each parse_qsl testcase is a two-tuple that contains +# a string with the query and a list with the expected result. parse_qsl_test_cases = [ ("", []), @@ -42,6 +42,9 @@ (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), ] +# Each parse_qs testcase is a two-tuple that contains +# a string with the query and a dictionary with the expected result. + parse_qs_test_cases = [ ("", {}), ("&", {}), @@ -290,7 +293,6 @@ def test_RFC2368(self): def test_RFC2396(self): # cases from RFC 2396 - self.checkJoin(RFC2396_BASE, 'g:h', 'g:h') self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g') self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g') @@ -333,9 +335,7 @@ def test_RFC2396(self): # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g') # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g') - def test_RFC3986(self): - # Test cases from RFC3986 self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x') self.checkJoin(RFC3986_BASE, 'g:h','g:h') @@ -363,7 +363,7 @@ def test_RFC3986(self): self.checkJoin(RFC3986_BASE, '../../g','http://a/g') self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g') - #Abnormal Examples + # Abnormal Examples # The 'abnormal scenarios' are incompatible with RFC2986 parsing # Tests are here for reference. diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c0144d1cb8febf..dbdad23a7423f2 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1879,6 +1879,118 @@ def test_recursive_repr(self): with self.assertRaises(RuntimeError): repr(e) # Should not crash + def test_element_get_text(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem.text + except NameError: + pass + + b = ET.TreeBuilder() + b.start('tag', {}) + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('tag') + + elem = b.close() + self.assertEqual(elem.text, 'ABCDEFGHIJKL') + + def test_element_get_tail(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem[0].tail + except NameError: + pass + + b = ET.TreeBuilder() + b.start('root', {}) + b.start('tag', {}) + b.end('tag') + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('root') + + elem = b.close() + self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') + + def test_element_iter(self): + # Issue #27863 + state = { + 'tag': 'tag', + '_children': [None], # non-Element + 'attrib': 'attr', + 'tail': 'tail', + 'text': 'text', + } + + e = ET.Element('tag') + try: + e.__setstate__(state) + except AttributeError: + e.__dict__ = state + + it = e.iter() + self.assertIs(next(it), e) + self.assertRaises(AttributeError, next, it) + + def test_subscr(self): + # Issue #27863 + class X: + def __index__(self): + del e[:] + return 1 + + e = ET.Element('elem') + e.append(ET.Element('child')) + e[:X()] # shouldn't crash + + e.append(ET.Element('child')) + e[0:10:X()] # shouldn't crash + + def test_ass_subscr(self): + # Issue #27863 + class X: + def __index__(self): + e[:] = [] + return 1 + + e = ET.Element('elem') + for _ in range(10): + e.insert(0, ET.Element('child')) + + e[0:10:X()] = [] # shouldn't crash + + def test_treebuilder_start(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.start, 'tag2', {}) + del b + gc_collect() + + def test_treebuilder_end(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.end, 'tag') + del b + gc_collect() + + class MutatingElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index df9c79e3df0689..30025e388ddf2e 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -343,6 +343,94 @@ def run_server(): self.assertEqual(p.method(), 5) self.assertEqual(p.method(), 5) + +class SimpleXMLRPCDispatcherTestCase(unittest.TestCase): + class DispatchExc(Exception): + """Raised inside the dispatched functions when checking for + chained exceptions""" + + def test_call_registered_func(self): + """Calls explicitly registered function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + def dispatched_func(*params): + raise self.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(dispatched_func) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_instance_func(self): + """Calls a registered instance attribute as a function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + class DispatchedClass: + def dispatched_func(self, *params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(DispatchedClass()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_dispatch_func(self): + """Calls the registered instance's `_dispatch` function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_method = 'method' + exp_params = 1, 2, 3 + + class TestInstance: + def _dispatch(self, method, params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc( + method, params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(TestInstance()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch(exp_method, exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_method, exp_params)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_registered_func_is_none(self): + """Calls explicitly registered function which is None""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(None, name='method') + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_instance_has_no_func(self): + """Attempts to call nonexistent function on a registered instance""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(object()) + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_cannot_locate_func(self): + """Calls a function that the dispatcher cannot locate""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + class HelperTestCase(unittest.TestCase): def test_escape(self): self.assertEqual(xmlrpclib.escape("a&b"), "a&b") @@ -1312,7 +1400,7 @@ def test_main(): KeepaliveServerTestCase1, KeepaliveServerTestCase2, GzipServerTestCase, GzipUtilTestCase, MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, - CGIHandlerTestCase) + CGIHandlerTestCase, SimpleXMLRPCDispatcherTestCase) if __name__ == "__main__": diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 0a43b20e2bcf0d..d09ad96fc8baaa 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2,6 +2,7 @@ import io import os import importlib.util +import pathlib import posixpath import time import struct @@ -13,7 +14,7 @@ from random import randint, random, getrandbits from test.support import script_helper -from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, +from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, temp_cwd, requires_zlib, requires_bz2, requires_lzma, captured_stdout, check_warnings) @@ -148,6 +149,12 @@ def test_open(self): for f in get_files(self): self.zip_open_test(f, self.compression) + def test_open_with_pathlike(self): + path = pathlib.Path(TESTFN2) + self.zip_open_test(path, self.compression) + with zipfile.ZipFile(path, "r", self.compression) as zipfp: + self.assertIsInstance(zipfp.filename, str) + def zip_random_open_test(self, f, compression): self.make_test_archive(f, compression) @@ -727,6 +734,48 @@ class LzmaTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, compression = zipfile.ZIP_LZMA +class AbstractWriterTests: + + def tearDown(self): + unlink(TESTFN2) + + def test_close_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + w.close() + self.assertTrue(w.closed) + self.assertEqual(zipf.read('test'), data) + + def test_write_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + self.assertRaises(ValueError, w.write, b'') + self.assertEqual(zipf.read('test'), data) + +class StoredWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib +class DeflateWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2 +class Bzip2WriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma +class LzmaWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + + class PyZipFileTests(unittest.TestCase): def assertCompiledIn(self, name, namelist): if name + 'o' not in namelist: @@ -906,22 +955,56 @@ def test_write_pyfile_bad_syntax(self): finally: rmtree(TESTFN2) + def test_write_pathlike(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w") as fp: + fp.write("print(42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + finally: + rmtree(TESTFN2) + class ExtractTests(unittest.TestCase): - def test_extract(self): + + def make_test_file(self): with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: for fpath, fdata in SMALL_TEST_DATA: zipfp.writestr(fpath, fdata) + def test_extract(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.normpath(correctfile) + + self.assertEqual(writtenfile, correctfile) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + def _test_extract_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath) + writtenfile = zipfp.extract(fpath, target) # make sure it was written to the right place - correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.join(target, fpath) correctfile = os.path.normpath(correctfile) - - self.assertEqual(writtenfile, correctfile) + self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) # make sure correct data is in correct file with open(writtenfile, "rb") as f: @@ -929,26 +1012,50 @@ def test_extract(self): unlink(writtenfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_with_target(self): + with temp_dir() as extdir: + self._test_extract_with_target(extdir) + + def test_extract_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_with_target(pathlib.Path(extdir)) def test_extract_all(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall() + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(os.getcwd(), fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + def _test_extract_all_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall() + zipfp.extractall(target) for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(os.getcwd(), fpath) + outfile = os.path.join(target, fpath) with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) unlink(outfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_all_with_target(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(extdir) + + def test_extract_all_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(pathlib.Path(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1188,6 +1295,8 @@ def test_is_zip_erroneous_file(self): with open(TESTFN, "w") as fp: fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a path-like object + self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2033,6 +2142,26 @@ def test_from_file(self): zi = zipfile.ZipInfo.from_file(__file__) self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_pathlike(self): + zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_bytes(self): + zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_fileno(self): + with open(__file__, 'rb') as f: + zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_dir(self): dirpath = os.path.dirname(os.path.abspath(__file__)) diff --git a/Lib/threading.py b/Lib/threading.py index 4829ff426e0bfd..95978d310a2f3a 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1217,6 +1217,10 @@ def __init__(self): def _stop(self): pass + def is_alive(self): + assert not self._is_stopped and self._started.is_set() + return True + def join(self, timeout=None): assert False, "cannot join a dummy thread" diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100644 new mode 100755 index 2770efa35a0160..8eea766b4c344b --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -208,7 +208,7 @@ def repeat(self, repeat=default_repeat, number=default_number): return r def autorange(self, callback=None): - """Return the number of loops so that total time >= 0.2. + """Return the number of loops and time taken so that total time >= 0.2. Calls the timeit method with *number* set to successive powers of ten (10, 100, 1000, ...) up to a maximum of one billion, until diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index ee2415da72a01e..deea791831ed66 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1004,7 +1004,7 @@ def winfo_ismapped(self): return self.tk.getint( self.tk.call('winfo', 'ismapped', self._w)) def winfo_manager(self): - """Return the window mananger name for this widget.""" + """Return the window manager name for this widget.""" return self.tk.call('winfo', 'manager', self._w) def winfo_name(self): """Return the name of this widget.""" @@ -1679,7 +1679,7 @@ def image_names(self): return self.tk.splitlist(self.tk.call('image', 'names')) def image_types(self): - """Return a list of all available image types (e.g. phote bitmap).""" + """Return a list of all available image types (e.g. photo bitmap).""" return self.tk.splitlist(self.tk.call('image', 'types')) @@ -1818,7 +1818,7 @@ def wm_focusmodel(self, model=None): return self.tk.call('wm', 'focusmodel', self._w, model) focusmodel = wm_focusmodel def wm_forget(self, window): # new in Tk 8.5 - """The window will be unmappend from the screen and will no longer + """The window will be unmapped from the screen and will no longer be managed by wm. toplevel windows will be treated like frame windows once they are no longer managed by wm, however, the menu option configuration will be remembered and the menus will return @@ -2527,7 +2527,7 @@ def find_closest(self, x, y, halo=None, start=None): """Return item which is closest to pixel at X, Y. If several match take the top-most. All items closer than HALO are considered overlapping (all are - closests). If START is specified the next below this tag is taken.""" + closest). If START is specified the next below this tag is taken.""" return self.find('closest', x, y, halo, start) def find_enclosed(self, x1, y1, x2, y2): """Return all items in rectangle defined @@ -2587,7 +2587,7 @@ def postscript(self, cnf={}, **kw): """Print the contents of the canvas to a postscript file. Valid options: colormap, colormode, file, fontmap, height, pageanchor, pageheight, pagewidth, pagex, pagey, - rotate, witdh, x, y.""" + rotate, width, x, y.""" return self.tk.call((self._w, 'postscript') + self._options(cnf, kw)) def tag_raise(self, *args): @@ -3522,7 +3522,7 @@ def height(self): return self.tk.getint( self.tk.call('image', 'height', self.name)) def type(self): - """Return the type of the imgage, e.g. "photo" or "bitmap".""" + """Return the type of the image, e.g. "photo" or "bitmap".""" return self.tk.call('image', 'type', self.name) def width(self): """Return the width of the image.""" diff --git a/Lib/typing.py b/Lib/typing.py index c9e341753769e0..645bc6f8ae0edd 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -10,6 +10,12 @@ import collections.abc as collections_abc except ImportError: import collections as collections_abc # Fallback for PY3.2. +try: + from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType +except ImportError: + WrapperDescriptorType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) # Please keep __all__ alphabetized within each category. @@ -57,11 +63,14 @@ # Structural checks, a.k.a. protocols. 'Reversible', 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', 'SupportsFloat', 'SupportsInt', 'SupportsRound', # Concrete collection types. + 'Counter', 'Deque', 'Dict', 'DefaultDict', @@ -413,6 +422,31 @@ def __subclasscheck__(self, cls): Any = _Any(_root=True) +class _NoReturn(_FinalTypingBase, _root=True): + """Special type indicating functions that never return. + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """ + + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError("NoReturn cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("NoReturn cannot be used with issubclass().") + + +NoReturn = _NoReturn(_root=True) + + class TypeVar(_TypingBase, _root=True): """Type variable. @@ -849,19 +883,6 @@ def _next_in_mro(cls): return next_in_mro -def _valid_for_check(cls): - """An internal helper to prohibit isinstance([1], List[str]) etc.""" - if cls is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % cls) - if ( - cls.__origin__ is not None and - sys._getframe(3).f_globals['__name__'] not in ['abc', 'functools'] - ): - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -872,7 +893,6 @@ def _make_subclasshook(cls): # Registered classes need not be checked here because # cls and its extra share the same _abc_registry. def __extrahook__(subclass): - _valid_for_check(cls) res = cls.__extra__.__subclasshook__(subclass) if res is not NotImplemented: return res @@ -887,7 +907,6 @@ def __extrahook__(subclass): else: # For non-ABC extras we'll just call issubclass(). def __extrahook__(subclass): - _valid_for_check(cls) if cls.__extra__ and issubclass(subclass, cls.__extra__): return True return NotImplemented @@ -974,6 +993,7 @@ def __new__(cls, name, bases, namespace, # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) self = super().__new__(cls, name, bases, namespace, _root=True) self.__parameters__ = tvars @@ -982,8 +1002,6 @@ def __new__(cls, name, bases, namespace, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) if args else None - self.__origin__ = origin - self.__extra__ = extra # Speed hack (https://github.com/python/typing/issues/196). self.__next_in_mro__ = _next_in_mro(self) # Preserve base classes on subclassing (__bases__ are type erased now). @@ -994,20 +1012,56 @@ def __new__(cls, name, bases, namespace, # with issubclass() and isinstance() in the same way as their # collections.abc counterparts (e.g., isinstance([], Iterable)). if ( - # allow overriding '__subclasshook__' not in namespace and extra or - hasattr(self.__subclasshook__, '__name__') and - self.__subclasshook__.__name__ == '__extrahook__' + # allow overriding + getattr(self.__subclasshook__, '__name__', '') == '__extrahook__' ): self.__subclasshook__ = _make_subclasshook(self) if isinstance(extra, abc.ABCMeta): self._abc_registry = extra._abc_registry + self._abc_cache = extra._abc_cache + elif origin is not None: + self._abc_registry = origin._abc_registry + self._abc_cache = origin._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ - self.__tree_hash__ = hash(self._subs_tree()) if origin else hash((self.__name__,)) + self.__tree_hash__ = (hash(self._subs_tree()) if origin else + super(GenericMeta, self).__hash__()) return self + # _abc_negative_cache and _abc_negative_cache_version + # realised as descriptors, since GenClass[t1, t2, ...] always + # share subclass info with GenClass. + # This is an important memory optimization. + @property + def _abc_negative_cache(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache + return _gorg(self)._abc_generic_negative_cache + + @_abc_negative_cache.setter + def _abc_negative_cache(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + else: + self._abc_generic_negative_cache = value + + @property + def _abc_negative_cache_version(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache_version + return _gorg(self)._abc_generic_negative_cache_version + + @_abc_negative_cache_version.setter + def _abc_negative_cache_version(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + else: + self._abc_generic_negative_cache_version = value + def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: _get_type_vars(self.__parameters__, tvars) @@ -1095,8 +1149,10 @@ def __getitem__(self, params): _check_generic(self, params) tvars = _type_vars(params) args = params + + prepend = (self,) if self.__origin__ is None else () return self.__class__(self.__name__, - self.__bases__, + prepend + self.__bases__, _no_slots_copy(self.__dict__), tvars=tvars, args=args, @@ -1104,6 +1160,17 @@ def __getitem__(self, params): extra=self.__extra__, orig_bases=self.__orig_bases__) + def __subclasscheck__(self, cls): + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if self is Generic: + raise TypeError("Class %r cannot be used with class " + "or instance checks" % self) + return super().__subclasscheck__(cls) + def __instancecheck__(self, instance): # Since we extend ABC.__subclasscheck__ and # ABC.__instancecheck__ inlines the cache checking done by the @@ -1118,6 +1185,16 @@ def __copy__(self): self.__parameters__, self.__args__, self.__origin__, self.__extra__, self.__orig_bases__) + def __setattr__(self, attr, value): + # We consider all the subscripted genrics as proxies for original class + if ( + attr.startswith('__') and attr.endswith('__') or + attr.startswith('_abc_') + ): + super(GenericMeta, self).__setattr__(attr, value) + else: + super(GenericMeta, _gorg(self)).__setattr__(attr, value) + # Prevent checks for Generic to crash when defining Generic. Generic = None @@ -1398,6 +1475,11 @@ def _get_defaults(func): return res +_allowed_types = (types.FunctionType, types.BuiltinFunctionType, + types.MethodType, types.ModuleType, + WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) + + def get_type_hints(obj, globalns=None, localns=None): """Return type hints for an object. @@ -1452,12 +1534,7 @@ def get_type_hints(obj, globalns=None, localns=None): hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. - if ( - isinstance(obj, types.FunctionType) or - isinstance(obj, types.BuiltinFunctionType) or - isinstance(obj, types.MethodType) or - isinstance(obj, types.ModuleType) - ): + if isinstance(obj, _allowed_types): return {} else: raise TypeError('{!r} is not a module, class, method, ' @@ -1824,8 +1901,7 @@ class Deque(collections.deque, MutableSequence[T], extra=collections.deque): def __new__(cls, *args, **kwds): if _geqv(cls, Deque): - raise TypeError("Type Deque cannot be instantiated; " - "use deque() instead") + return collections.deque(*args, **kwds) return _generic_new(collections.deque, cls, *args, **kwds) @@ -1894,11 +1970,35 @@ class DefaultDict(collections.defaultdict, MutableMapping[KT, VT], def __new__(cls, *args, **kwds): if _geqv(cls, DefaultDict): - raise TypeError("Type DefaultDict cannot be instantiated; " - "use collections.defaultdict() instead") + return collections.defaultdict(*args, **kwds) return _generic_new(collections.defaultdict, cls, *args, **kwds) +class Counter(collections.Counter, Dict[T, int], extra=collections.Counter): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, Counter): + return collections.Counter(*args, **kwds) + return _generic_new(collections.Counter, cls, *args, **kwds) + + +if hasattr(collections, 'ChainMap'): + # ChainMap only exists in 3.3+ + __all__.append('ChainMap') + + class ChainMap(collections.ChainMap, MutableMapping[KT, VT], + extra=collections.ChainMap): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, ChainMap): + return collections.ChainMap(*args, **kwds) + return _generic_new(collections.ChainMap, cls, *args, **kwds) + + # Determine what base class to use for Generator. if hasattr(collections_abc, 'Generator'): # Sufficiently recent versions of 3.5 have a Generator ABC. @@ -1975,6 +2075,13 @@ def _make_nmtuple(name, types): _PY36 = sys.version_info[:2] >= (3, 6) +# attributes prohibited to set in NamedTuple class syntax +_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', + '_fields', '_field_defaults', '_field_types', + '_make', '_replace', '_asdict', '_source') + +_special = ('__module__', '__name__', '__qualname__', '__annotations__') + class NamedTupleMeta(type): @@ -2002,7 +2109,9 @@ def __new__(cls, typename, bases, ns): nm_tpl._field_defaults = defaults_dict # update from user namespace without overriding special namedtuple attributes for key in ns: - if not hasattr(nm_tpl, key): + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: setattr(nm_tpl, key, ns[key]) return nm_tpl diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 09fefe11649bd3..807604f08dfd14 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -14,6 +14,7 @@ %(prog)s test_module - run tests from test_module %(prog)s module.TestClass - run tests from module.TestClass %(prog)s module.Class.test_method - run specified test method + %(prog)s path/to/test_file.py - run tests from test_file.py """ MODULE_EXAMPLES = """\ diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index b6b38362341f16..5f97728de28c59 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -815,8 +815,8 @@ def _error_message(): def assert_called_once_with(_mock_self, *args, **kwargs): - """assert that the mock was called exactly once and with the specified - arguments.""" + """assert that the mock was called exactly once and that that call was + with the specified arguments.""" self = _mock_self if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times." % diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py index d5f9e7c1d69204..d2202a7b4132e9 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/unittest/test/testmock/testhelpers.py @@ -307,18 +307,9 @@ def test_two_args_call(self): self.assertEqual(args, other_args) def test_call_with_name(self): - self.assertEqual( - 'foo', - _Call((), 'foo')[0], - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), ), )[0] - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), {'hello': 'world'}), )[0] - ) + self.assertEqual(_Call((), 'foo')[0], 'foo') + self.assertEqual(_Call((('bar', 'barz'),),)[0], '') + self.assertEqual(_Call((('bar', 'barz'), {'hello': 'world'}),)[0], '') class SpecSignatureTest(unittest.TestCase): diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d16188b8..851515bc307d62 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,10 +16,6 @@ __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a08d7e77..888247b8e90f56 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -38,29 +38,37 @@ "DefragResult", "ParseResult", "SplitResult", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] -# A classification of schemes ('' means apply by default) -uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', +# A classification of schemes. +# The empty string classifies URLs with no scheme specified, +# being the default value returned by “urlsplit” and “urlparse”. + +uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', '', 'sftp', + 'prospero', 'rtsp', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] -uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', + +uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', + 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', 'ws', 'wss'] -uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', + +uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', '', 'sftp', 'tel'] + 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) + non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] -uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] -uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', + +uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', + 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + +uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', - 'file', 'prospero', ''] + 'file', 'prospero'] # Characters valid in scheme names scheme_chars = ('abcdefghijklmnopqrstuvwxyz' @@ -612,6 +620,7 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. @@ -633,6 +642,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, @@ -644,28 +655,29 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index b6690c3d0e4bf5..96921440edea11 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1488,7 +1488,6 @@ def open_local_file(self, req): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') @@ -1658,14 +1657,10 @@ def pathname2url(pathname): of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index 98b5ca953983c7..421358fa5bc7f0 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -105,9 +105,16 @@ def parse(self, source): source = saxutils.prepare_input_source(source) self._source = source - self.reset() - self._cont_handler.setDocumentLocator(ExpatLocator(self)) - xmlreader.IncrementalParser.parse(self, source) + try: + self.reset() + self._cont_handler.setDocumentLocator(ExpatLocator(self)) + xmlreader.IncrementalParser.parse(self, source) + except: + # bpo-30264: Close the source on error to not leak resources: + # xml.sax.parse() doesn't give access to the underlying parser + # to the caller + self._close_source() + raise def prepareParser(self, source): if source.getSystemId() is not None: @@ -213,6 +220,17 @@ def feed(self, data, isFinal = 0): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def _close_source(self): + source = self._source + try: + file = source.getCharacterStream() + if file is not None: + file.close() + finally: + file = source.getByteStream() + if file is not None: + file.close() + def close(self): if (self._entity_stack or self._parser is None or isinstance(self._parser, _ClosedParser)): @@ -232,14 +250,7 @@ def close(self): parser.ErrorColumnNumber = self._parser.ErrorColumnNumber parser.ErrorLineNumber = self._parser.ErrorLineNumber self._parser = parser - try: - file = self._source.getCharacterStream() - if file is not None: - file.close() - finally: - file = self._source.getByteStream() - if file is not None: - file.close() + self._close_source() def _reset_cont_handler(self): self._parser.ProcessingInstructionHandler = \ diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 849bfddd84816f..6faa2d6f8fa319 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -386,31 +386,36 @@ def _dispatch(self, method, params): not be called. """ - func = None try: - # check to see if a matching function has been registered + # call the matching registered function func = self.funcs[method] except KeyError: - if self.instance is not None: - # check for a _dispatch method - if hasattr(self.instance, '_dispatch'): - return self.instance._dispatch(method, params) - else: - # call instance method directly - try: - func = resolve_dotted_attribute( - self.instance, - method, - self.allow_dotted_names - ) - except AttributeError: - pass - - if func is not None: - return func(*params) + pass else: + if func is not None: + return func(*params) raise Exception('method "%s" is not supported' % method) + if self.instance is not None: + if hasattr(self.instance, '_dispatch'): + # call the `_dispatch` method on the instance + return self.instance._dispatch(method, params) + + # call the instance's method directly + try: + func = resolve_dotted_attribute( + self.instance, + method, + self.allow_dotted_names + ) + except AttributeError: + pass + else: + if func is not None: + return func(*params) + + raise Exception('method "%s" is not supported' % method) + class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): """Simple XML-RPC request handler class. diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 7f2b43ce1cc848..9164f8ab086a16 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -479,6 +479,8 @@ def from_file(cls, filename, arcname=None): this will be the same as filename, but without a drive letter and with leading path separators removed). """ + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) @@ -979,6 +981,8 @@ def writable(self): return True def write(self, data): + if self.closed: + raise ValueError('I/O operation on closed file.') nbytes = len(data) self._file_size += nbytes self._crc = crc32(data, self._crc) @@ -989,6 +993,8 @@ def write(self, data): return nbytes def close(self): + if self.closed: + return super().close() # Flush any data from the compressor, and update header info if self._compressor: @@ -1070,6 +1076,8 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): self._comment = b'' # Check if we were passed a file-like object + if isinstance(file, os.PathLike): + file = os.fspath(file) if isinstance(file, str): # No, it's a filename self._filePassed = 0 @@ -1102,7 +1110,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self._start_disk = 0 try: self.start_dir = self.fp.tell() except (AttributeError, OSError): @@ -1128,7 +1135,7 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self.start_dir = self._start_disk = self.fp.tell() + self.start_dir = self.fp.tell() else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1172,18 +1179,17 @@ def _RealGetContents(self): offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment - # self._start_disk: Position of the start of ZIP archive - # It is zero, unless ZIP was concatenated to another file - self._start_disk = endrec[_ECD_LOCATION] - size_cd - offset_cd + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd if endrec[_ECD_SIGNATURE] == stringEndArchive64: # If Zip64 extension structures are present, account for them - self._start_disk -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) if self.debug > 2: - inferred = self._start_disk + offset_cd - print("given, inferred, offset", offset_cd, inferred, self._start_disk) + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + self._start_disk + self.start_dir = offset_cd + concat fp.seek(self.start_dir, 0) data = fp.read(size_cd) fp = io.BytesIO(data) @@ -1223,7 +1229,7 @@ def _RealGetContents(self): t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) x._decodeExtra() - x.header_offset = x.header_offset + self._start_disk + x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x @@ -1470,11 +1476,10 @@ def extract(self, member, path=None, pwd=None): as possible. `member' may be a filename or a ZipInfo object. You can specify a different directory using `path'. """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - if path is None: path = os.getcwd() + else: + path = os.fspath(path) return self._extract_member(member, path, pwd) @@ -1487,8 +1492,13 @@ def extractall(self, path=None, members=None, pwd=None): if members is None: members = self.namelist() + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + for zipinfo in members: - self.extract(zipinfo, path, pwd) + self._extract_member(zipinfo, path, pwd) @classmethod def _sanitize_windows_name(cls, arcname, pathsep): @@ -1509,6 +1519,9 @@ def _extract_member(self, member, targetpath, pwd): """Extract the ZipInfo object 'member' to a physical file on the path targetpath. """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + # build the destination pathname, replacing # forward slashes to platform specific separators. arcname = member.filename.replace('/', os.path.sep) @@ -1687,10 +1700,11 @@ def _write_end_record(self): file_size = zinfo.file_size compress_size = zinfo.compress_size - header_offset = zinfo.header_offset - self._start_disk - if header_offset > ZIP64_LIMIT: - extra.append(header_offset) + if zinfo.header_offset > ZIP64_LIMIT: + extra.append(zinfo.header_offset) header_offset = 0xffffffff + else: + header_offset = zinfo.header_offset extra_data = zinfo.extra min_version = 0 @@ -1737,7 +1751,7 @@ def _write_end_record(self): # Write end-of-zip-archive record centDirCount = len(self.filelist) centDirSize = pos2 - self.start_dir - centDirOffset = self.start_dir - self._start_disk + centDirOffset = self.start_dir requires_zip64 = None if centDirCount > ZIP_FILECOUNT_LIMIT: requires_zip64 = "Files count" @@ -1801,6 +1815,7 @@ def writepy(self, pathname, basename="", filterfunc=None): If filterfunc(pathname) is given, it is called with every argument. When it is False, the file or directory is skipped. """ + pathname = os.fspath(pathname) if filterfunc and not filterfunc(pathname): if self.debug: label = 'path' if os.path.isdir(pathname) else 'file' diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 8dfd092d08b512..7b4376f42b88f3 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -13,7 +13,7 @@ Python 2.6. In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed version of hg and a third-party version of +requires an installed version of git and a third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to @@ -23,7 +23,7 @@ 32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates), hg, and for the documentation +(2.5 is needed for Python parser updates), git, and for the documentation build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). Usage: see USAGE variable in the script. @@ -213,9 +213,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2j", - url="https://www.openssl.org/source/openssl-1.0.2j.tar.gz", - checksum='96322138f0b69e61b7212bc53d5e912b', + name="OpenSSL 1.0.2k", + url="https://www.openssl.org/source/openssl-1.0.2k.tar.gz", + checksum='f965fc0bf01bf882b31314b61391ae65', patches=[ "openssl_sdk_makedepend.patch", ], @@ -635,9 +635,9 @@ def checkEnvironment(): base_path = base_path + ':' + OLD_DEVELOPER_TOOLS os.environ['PATH'] = base_path print("Setting default PATH: %s"%(os.environ['PATH'])) - # Ensure ws have access to hg and to sphinx-build. + # Ensure ws have access to git and to sphinx-build. # You may have to create links in /usr/bin for them. - runCommand('hg --version') + runCommand('git --version') runCommand('sphinx-build --version') def parseOptions(args=None): @@ -1142,9 +1142,6 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - print("Running make touch") - runCommand("make touch") - print("Running make") runCommand("make") diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch index e22d67e4e45143..0caac0a64c1ec6 100644 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ b/Mac/BuildScript/openssl_sdk_makedepend.patch @@ -1,6 +1,6 @@ # HG changeset patch # -# using openssl 1.0.2j +# using openssl 1.0.2k # # - support building with an OS X SDK diff --git a/Makefile.pre.in b/Makefile.pre.in index 93aae9124726bd..82e830727ef849 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,9 +41,9 @@ RANLIB= @RANLIB@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ -HGVERSION= @HGVERSION@ -HGTAG= @HGTAG@ -HGBRANCH= @HGBRANCH@ +GITVERSION= @GITVERSION@ +GITTAG= @GITTAG@ +GITBRANCH= @GITBRANCH@ PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ @@ -107,6 +107,8 @@ ARFLAGS= @ARFLAGS@ CFLAGSFORSHARED=@CFLAGSFORSHARED@ # C flags used for building the interpreter object files PY_CORE_CFLAGS= $(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED) -DPy_BUILD_CORE +# Strict or non-strict aliasing flags used to compile dtoa.c, see above +CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Machine-dependent subdirectories @@ -227,7 +229,7 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -PYTHON_FOR_GEN=@PYTHON_FOR_GEN@ +PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ @@ -271,11 +273,6 @@ IO_OBJS= \ Modules/_io/stringio.o ########################################################################## -# Grammar -GRAMMAR_H= Include/graminit.h -GRAMMAR_C= Python/graminit.c -GRAMMAR_INPUT= $(srcdir)/Grammar/Grammar - LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ @@ -314,38 +311,9 @@ PARSER_HEADERS= \ PGENOBJS= $(POBJS) $(PGOBJS) -########################################################################## -# opcode.h generation -OPCODE_H_DIR= $(srcdir)/Include -OPCODE_H_SCRIPT= $(srcdir)/Tools/scripts/generate_opcode_h.py -OPCODE_H= $(OPCODE_H_DIR)/opcode.h -OPCODE_H_GEN= $(PYTHON_FOR_GEN) $(OPCODE_H_SCRIPT) $(srcdir)/Lib/opcode.py $(OPCODE_H) - -########################################################################## -# AST -AST_H_DIR= Include -AST_H= $(AST_H_DIR)/Python-ast.h -AST_C_DIR= Python -AST_C= $(AST_C_DIR)/Python-ast.c -AST_ASDL= $(srcdir)/Parser/Python.asdl - -ASDLGEN_FILES= $(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py -# Note that a build now requires Python to exist before the build starts. -# Use "hg touch" to fix up screwed up file mtimes in a checkout. -ASDLGEN= $(PYTHON_FOR_GEN) $(srcdir)/Parser/asdl_c.py - ########################################################################## # Python -OPCODETARGETS_H= \ - Python/opcode_targets.h - -OPCODETARGETGEN= \ - $(srcdir)/Python/makeopcodetargets.py - -OPCODETARGETGEN_FILES= \ - $(OPCODETARGETGEN) $(srcdir)/Lib/opcode.py - PYTHON_OBJS= \ Python/_warnings.o \ Python/Python-ast.o \ @@ -545,7 +513,8 @@ coverage-lcov: @echo "lcov report at $(COVERAGE_REPORT)/index.html" @echo -coverage-report: +# Force regeneration of parser and importlib +coverage-report: regen-grammar regen-importlib : # force rebuilding of parser and importlib @touch $(GRAMMAR_INPUT) @touch $(srcdir)/Lib/importlib/_bootstrap.py @@ -721,14 +690,24 @@ Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -Python/importlib_external.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap_external.py Programs/_freeze_importlib Python/marshal.c +.PHONY: regen-importlib +regen-importlib: Programs/_freeze_importlib + # Regenerate Python/importlib_external.h + # from Lib/importlib/_bootstrap_external.py using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap_external.py Python/importlib_external.h - -Python/importlib.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap.py Programs/_freeze_importlib Python/marshal.c + $(srcdir)/Lib/importlib/_bootstrap_external.py \ + $(srcdir)/Python/importlib_external.h + # Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py + # using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h + $(srcdir)/Lib/importlib/_bootstrap.py \ + $(srcdir)/Python/importlib.h + + +############################################################################ +# Regenerate all generated files +regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar regen-ast regen-importlib ############################################################################ # Special rules for object files @@ -740,9 +719,9 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ $(MODOBJS) \ $(srcdir)/Modules/getbuildinfo.c $(CC) -c $(PY_CORE_CFLAGS) \ - -DHGVERSION="\"`LC_ALL=C $(HGVERSION)`\"" \ - -DHGTAG="\"`LC_ALL=C $(HGTAG)`\"" \ - -DHGBRANCH="\"`LC_ALL=C $(HGBRANCH)`\"" \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile @@ -787,15 +766,18 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(IO_OBJS): $(IO_H) -$(GRAMMAR_H): @GENERATED_COMMENT@ $(GRAMMAR_INPUT) $(PGEN) - @$(MKDIR_P) Include - $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) -$(GRAMMAR_C): @GENERATED_COMMENT@ $(GRAMMAR_H) - touch $(GRAMMAR_C) - $(PGEN): $(PGENOBJS) $(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN) +.PHONY: regen-grammar +regen-grammar: $(PGEN) + # Regenerate Include/graminit.h and Python/graminit.c + # from Grammar/Grammar using pgen + @$(MKDIR_P) Include + $(PGEN) $(srcdir)/Grammar/Grammar \ + $(srcdir)/Include/graminit.h \ + $(srcdir)/Python/graminit.c + Parser/grammar.o: $(srcdir)/Parser/grammar.c \ $(srcdir)/Include/token.h \ $(srcdir)/Include/grammar.h @@ -807,18 +789,28 @@ Parser/printgrammar.o: $(srcdir)/Parser/printgrammar.c Parser/pgenmain.o: $(srcdir)/Include/parsetok.h -$(AST_H): $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_H_DIR) - $(ASDLGEN) -h $(AST_H_DIR) $(AST_ASDL) - -$(AST_C): $(AST_H) $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_C_DIR) - $(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL) - -$(OPCODE_H): $(srcdir)/Lib/opcode.py $(OPCODE_H_SCRIPT) - $(OPCODE_H_GEN) - -Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H) +.PHONY=regen-ast +regen-ast: + # Regenerate Include/Python-ast.h using Parser/asdl_c.py -h + $(MKDIR_P) $(srcdir)/Include + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -h $(srcdir)/Include \ + $(srcdir)/Parser/Python.asdl + # Regenerate Python/Python-ast.c using Parser/asdl_c.py -c + $(MKDIR_P) $(srcdir)/Python + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -c $(srcdir)/Python \ + $(srcdir)/Parser/Python.asdl + +.PHONY: regen-opcode +regen-opcode: + # Regenerate Include/opcode.h from Lib/opcode.py + # using Tools/scripts/generate_opcode_h.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \ + $(srcdir)/Lib/opcode.py \ + $(srcdir)/Include/opcode.h + +Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -868,17 +860,22 @@ Objects/odictobject.o: $(srcdir)/Objects/dict-common.h Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h $(srcdir)/Objects/dict-common.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h -$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES) - $(PYTHON_FOR_GEN) $(OPCODETARGETGEN) $(OPCODETARGETS_H) +.PHONY: regen-opcode-targets +regen-opcode-targets: + # Regenerate Python/opcode_targets.h from Lib/opcode.py + # using Python/makeopcodetargets.py + $(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \ + $(srcdir)/Python/opcode_targets.h -Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h +Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h -Python/frozen.o: Python/importlib.h Python/importlib_external.h +Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to # follow our naming conventions. dtrace(1) uses the output filename to generate # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d + $(MKDIR_P) Include $(DTRACE) $(DFLAGS) -o $@ -h -s $< : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp @@ -888,8 +885,14 @@ Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc -Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py - $(PYTHON_FOR_GEN) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h Objects/typeslots.inc + +.PHONY: regen-typeslots +regen-typeslots: + # Regenerate Objects/typeslots.inc from Include/typeslotsh + # using Objects/typeslots.py + $(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \ + < $(srcdir)/Include/typeslots.h \ + $(srcdir)/Objects/typeslots.inc ############################################################################ # Header files @@ -942,7 +945,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/node.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ - $(OPCODE_H) \ + $(srcdir)/Include/opcode.h \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ @@ -985,7 +988,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/weakrefobject.h \ pyconfig.h \ $(PARSER_HEADERS) \ - $(AST_H) \ + $(srcdir)/Include/Python-ast.h \ $(DTRACE_HEADERS) $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -1000,7 +1003,7 @@ TESTTIMEOUT= 1200 # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -test: all platform +test: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. @@ -1010,7 +1013,7 @@ test: all platform # the bytecode read from a .pyc file had the bug, sometimes the directly # generated bytecode. This is sometimes a very shy bug needing a lot of # sample data. -testall: all platform +testall: @DEF_MAKE_RULE@ platform -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f @@ -1019,7 +1022,7 @@ testall: all platform # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. -testuniversal: all platform +testuniversal: @DEF_MAKE_RULE@ platform if [ `arch` != 'i386' ];then \ echo "This can only be used on OSX/i386" ;\ exit 1 ;\ @@ -1042,7 +1045,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multiprocessing_forkserver \ test_mailbox test_socket test_poll \ test_select test_zipfile test_concurrent_futures -quicktest: all platform +quicktest: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(QUICKTESTOPTS) @@ -1379,7 +1382,7 @@ LIBPL= @LIBPL@ # pkgconfig directory LIBPC= $(LIBDIR)/pkgconfig -libainstall: all python-config +libainstall: @DEF_MAKE_RULE@ python-config @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ do \ if test ! -d $(DESTDIR)$$i; then \ @@ -1535,6 +1538,13 @@ config.status: $(srcdir)/configure .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +# bpo-30104: dtoa.c uses union to cast double to unsigned long[2]. clang 4.0 +# with -O2 or higher and strict aliasing miscompiles the ratio() function +# causing rounding issues. Compile dtoa.c using -fno-strict-aliasing on clang. +# https://bugs.llvm.org//show_bug.cgi?id=31928 +Python/dtoa.o: Python/dtoa.c + $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_ALIASING) -o $@ $< + # Run reindent on the library reindent: ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib @@ -1545,9 +1555,12 @@ recheck: $(SHELL) config.status --recheck $(SHELL) config.status -# Rebuild the configure script from configure.ac; also rebuild pyconfig.h.in +# Regenerate configure and pyconfig.h.in +.PHONY: autoconf autoconf: + # Regenerate the configure script from configure.ac using autoconf (cd $(srcdir); autoconf -Wall) + # Regenerate pyconfig.h.in from configure.ac using autoheader (cd $(srcdir); autoheader -Wall) # Create a tags file for vi @@ -1564,14 +1577,6 @@ TAGS:: etags Include/*.h; \ for i in $(SRCDIRS); do etags -a $$i/*.[ch]; done -# This fixes up the mtimes of checked-in generated files, assuming that they -# only *appear* to be outdated because of checkout order. -# This is run while preparing a source release tarball, and can be run manually -# to avoid bootstrap issues. -touch: - cd $(srcdir); \ - hg --config extensions.touch=Tools/hg/hgtouch.py touch -v - # Sanitation targets -- clean leaves libraries, executables and tags # files, which clobber removes as well pycremoval: @@ -1626,7 +1631,8 @@ distclean: clobber done -rm -f core Makefile Makefile.pre config.status \ Modules/Setup Modules/Setup.local Modules/Setup.config \ - Modules/ld_so_aix Modules/python.exp Misc/python.pc + Modules/ld_so_aix Modules/python.exp Misc/python.pc \ + Misc/python-config.sh -rm -f python*-gdb.py # Issue #28258: set LC_ALL to avoid issues with Estonian locale. # Expansion is performed here by shell (spawned by make) itself before @@ -1639,7 +1645,7 @@ distclean: clobber -exec rm -f {} ';' # Check for smelly exported symbols (not starting with Py/_Py) -smelly: all +smelly: @DEF_MAKE_RULE@ nm -p $(LIBRARY) | \ sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ @@ -1676,7 +1682,7 @@ funny: -o -print # Perform some verification checks on any modified files. -patchcheck: all +patchcheck: @DEF_MAKE_RULE@ $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py # Dependencies @@ -1689,7 +1695,7 @@ Python/thread.o: @THREADHEADERS@ .PHONY: maninstall libinstall inclinstall libainstall sharedinstall .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools -.PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean +.PHONY: frameworkaltinstallunixtools recheck clean clobber distclean .PHONY: smelly funny patchcheck touch altmaninstall commoninstall .PHONY: gdbhooks diff --git a/Misc/ACKS b/Misc/ACKS index a6b3df12ff230c..7fc5fecc8a4605 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -269,6 +269,7 @@ Albert Chin-A-Young Adal Chiriliuc Matt Chisholm Lita Cho +Sayan Chowdhury Anders Chrigström Tom Christiansen Renee Chu @@ -347,6 +348,7 @@ A. Jesse Jiryu Davis Merlijn van Deen John DeGood Ned Deily +Jim DeLaHunt Vincent Delft Arnaud Delobelle Konrad Delong @@ -540,6 +542,7 @@ Tim Graham Kim Gräsman Nathaniel Gray Eddy De Greef +Duane Griffin Grant Griffin Andrea Griffini Duncan Grisby @@ -549,6 +552,8 @@ Eric Groo Daniel Andrade Groppe Dag Gruneau Filip Gruszczyński +Andrii Grynenko +Grzegorz Grzywacz Thomas Guettler Yuyang Guo Anuj Gupta @@ -768,11 +773,13 @@ Lawrence Kesteloot Vivek Khera Dhiru Kholia Akshit Khurana +Sanyam Khurana Mads Kiilerich Jason Killen Jan Kim Taek Joo Kim Sam Kimbrel +Tomohiko Kinebuchi James King W. Trevor King Paul Kippes @@ -956,6 +963,7 @@ David Marek Doug Marien Sven Marnach Alex Martelli +Dennis Mårtensson Anthony Martin Owen Martin Sidney San Martín @@ -1056,6 +1064,7 @@ R. David Murray Matti Mäki Jörg Müller Kaushik N +Dong-hee Na Dale Nagata John Nagle Takahiro Nakayama @@ -1101,6 +1110,7 @@ Milan Oberkirch Pascal Oberndoerfer Jeffrey Ollie Adam Olsen +Bryan Olson Grant Olson Koray Oner Piet van Oostrum @@ -1109,6 +1119,7 @@ Jason Orendorff Douglas Orr William Orr Michele Orrù +Tomáš Orsava Oleg Oshmyan Denis S. Otkidach Peter Otten @@ -1369,6 +1380,7 @@ Federico Schwindt Barry Scott Steven Scott Nick Seidenman +Michael Seifert Žiga Seilnacht Yury Selivanov Fred Sells @@ -1469,6 +1481,7 @@ Daniel Stokes Michael Stone Serhiy Storchaka Ken Stox +Charalampos Stratakis Dan Stromberg Donald Stufft Daniel Stutzbach @@ -1706,6 +1719,7 @@ Artur Zaprzala Mike Zarnstorff Yury V. Zaytsev Siebren van der Zee +Christophe Zeitouny Nickolai Zeldovich Yuxiao Zeng Uwe Zessin diff --git a/Misc/NEWS b/Misc/NEWS index d79ff2450defaa..1656e07f6eb1df 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,7 +2,7 @@ Python News +++++++++++ -What's New in Python 3.6.1 release candidate 1? +What's New in Python 3.6.2 release candidate 1? =============================================== *Release date: XXXX-XX-XX* @@ -10,6 +10,277 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29104: Fixed parsing backslashes in f-strings. + +- bpo-27945: Fixed various segfaults with dict when input collections are + mutated during searching, inserting or comparing. Based on patches by + Duane Griffin and Tim Mitchell. + +- bpo-25794: Fixed type.__setattr__() and type.__delattr__() for + non-interned attribute names. Based on patch by Eryk Sun. + +- bpo-12414: sys.getsizeof() on a code object now returns the sizes + which includes the code struct and sizes of objects which it references. + Patch by Dong-hee Na. + +- bpo-29949: Fix memory usage regression of set and frozenset object. + +- bpo-29935: Fixed error messages in the index() method of tuple, list and deque + when pass indices of wrong type. + +- bpo-29859: Show correct error messages when any of the pthread_* calls in + thread_pthread.h fails. + +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + +- bpo-28856: Fix an oversight that %b format for bytes should support objects + follow the buffer protocol. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +Library +------- + +- bpo-30378: Fix the problem that logging.handlers.SysLogHandler cannot + handle IPv6 addresses. + +- bpo-29960: Preserve generator state when _random.Random.setstate() + raises an exception. Patch by Bryan Olson. + +- bpo-30414: multiprocessing.Queue._feed background running + thread do not break from main loop on exception. + +- bpo-30003: Fix handling escape characters in HZ codec. Based on patch + by Ma Lin. + +- bpo-30301: Fix AttributeError when using SimpleQueue.empty() under + *spawn* and *forkserver* start methods. + +- bpo-30329: imaplib and poplib now catch the Windows socket WSAEINVAL error + (code 10022) on shutdown(SHUT_RDWR): An invalid operation was attempted. + This error occurs sometimes on SSL connections. + +- bpo-30375: Warnings emitted when compile a regular expression now always + point to the line in the user code. Previously they could point into inners + of the re module if emitted from inside of groups or conditionals. + +- bpo-30048: Fixed ``Task.cancel()`` can be ignored when the task is + running coroutine and the coroutine returned without any more ``await``. + +- bpo-30298: Weaken the condition of deprecation warnings for inline modifiers. + Now allowed several subsequential inline modifiers at the start of the + pattern (e.g. ``'(?i)(?s)...'``). In verbose mode whitespaces and comments + now are allowed before and between inline modifiers (e.g. + ``'(?x) (?i) (?s)...'``). + +- bpo-29990: Fix range checking in GB18030 decoder. Original patch by Ma Lin. + +- Revert bpo-26293 for zipfile breakage. See also bpo-29094. + +- bpo-30243: Removed the __init__ methods of _json's scanner and encoder. + Misusing them could cause memory leaks or crashes. Now scanner and encoder + objects are completely initialized in the __new__ methods. + +- bpo-30185: Avoid KeyboardInterrupt tracebacks in forkserver helper process + when Ctrl-C is received. + +- bpo-28556: Various updates to typing module: add typing.NoReturn type, use + WrapperDescriptorType, minor bug-fixes. Original PRs by + Jim Fasarakis-Hilliard and Ivan Levkivskyi. + +- bpo-30205: Fix getsockname() for unbound AF_UNIX sockets on Linux. + +- bpo-30070: Fixed leaks and crashes in errors handling in the parser module. + +- bpo-30061: Fixed crashes in IOBase methods __next__() and readlines() when + readline() or __next__() respectively return non-sizeable object. + Fixed possible other errors caused by not checking results of PyObject_Size(), + PySequence_Size(), or PyMapping_Size(). + +- bpo-30017: Allowed calling the close() method of the zip entry writer object + multiple times. Writing to a closed writer now always produces a ValueError. + +- bpo-30068: _io._IOBase.readlines will check if it's closed first when + hint is present. + +- bpo-29694: Fixed race condition in pathlib mkdir with flags + parents=True. Patch by Armin Rigo. + +- bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in + contextlib.contextmanager. Patch by Siddharth Velankar. + +- bpo-29998: Pickling and copying ImportError now preserves name and path + attributes. + +- bpo-29953: Fixed memory leaks in the replace() method of datetime and time + objects when pass out of bound fold argument. + +- bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering + long runs of empty iterables. + +- bpo-27863: Fixed multiple crashes in ElementTree caused by race conditions + and wrong types. + +- bpo-28699: Fixed a bug in pools in multiprocessing.pool that raising an + exception at the very first of an iterable may swallow the exception or + make the program hang. Patch by Davin Potts and Xiang Zhang. + +- bpo-25803: Avoid incorrect errors raised by Path.mkdir(exist_ok=True) + when the OS gives priority to errors such as EACCES over EEXIST. + +- bpo-29861: Release references to tasks, their arguments and their results + as soon as they are finished in multiprocessing.Pool. + +- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. + Patch by Christophe Zeitouny. + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +C API +----- + +- Issue #27867: Function PySlice_GetIndicesEx() no longer replaced with a macro + if Py_LIMITED_API is not set. + + +Build +----- + +- bpo-29941: Add ``--with-assertions`` configure flag to explicitly enable + C ``assert()`` checks. Defaults to off. ``--with-pydebug`` implies + ``--with-assertions``. + +- bpo-28787: Fix out-of-tree builds of Python when configured with + ``--with--dtrace``. + +- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, + ``make install`` and some other make targets when configured with + ``--enable-optimizations``. + +- bpo-23404: Don't regenerate generated files based on file modification time + anymore: the action is now explicit. Replace ``make touch`` with + ``make regen-all``. + +- bpo-29643: Fix ``--enable-optimization`` didn't work. + +Documentation +------------- + +- Issue #30052: the link targets for :func:`bytes` and + :func:`bytearray` are now their respective type definitions, rather + than the corresponding builtin function entries. Use :ref:`bytes ` + and :ref:`bytearray ` to reference the latter. + + In order to ensure this and future cross-reference updates are applied + automatically, the daily documentation builds now disable the default + output caching features in Sphinx. + +- bpo-26985: Add missing info of code object in inspect documentation. + +Tools/Demos +----------- + +- Issue #29367: python-gdb.py now supports also ``method-wrapper`` + (``wrapperobject``) objects. + +Tests +----- + +* bpo-30357: test_thread: setUp() now uses support.threading_setup() and + support.threading_cleanup() to wait until threads complete to avoid + random side effects on following tests. Initial patch written by Grzegorz + Grzywacz. + +- bpo-30197: Enhanced functions swap_attr() and swap_item() in the + test.support module. They now work when delete replaced attribute or item + inside the with statement. The old value of the attribute or item (or None + if it doesn't exist) now will be assigned to the target of the "as" clause, + if there is one. + + +What's New in Python 3.6.1? +=========================== + +*Release date: 2017-03-21* + +Core and Builtins +----------------- + +- bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a + regression by revealing an inconsistency in how sys.path is initialized when + executing ``__main__`` from a zipfile, directory, or other import location. + The interpreter now consistently avoids ever adding the import location's + parent directory to ``sys.path``, and ensures no other ``sys.path`` entries + are inadvertently modified when inserting the import location named on the + command line. + +Build +----- + +- bpo-27593: fix format of git information used in sys.version + +- Fix incompatible comment in python.h + + +What's New in Python 3.6.1 release candidate 1 +============================================== + +*Release date: 2017-03-04* + +Core and Builtins +----------------- + +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + +- Issue #28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + +- bpo-29438: Fixed use-after-free problem in key sharing dict. + - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - Issue #29337: Fixed possible BytesWarning when compare the code objects. @@ -57,6 +328,39 @@ Extension Modules Library ------- +- bpo-29623: Allow use of path-like object as a single argument in + ConfigParser.read(). Patch by David Ellis. + +- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback + implemented in C. + +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes + before all pipes are closed. + +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Łukasz Langa. + - Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python 3.6.0: check minimum and maximum years. @@ -126,7 +430,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #29055: Neaten-up empty population error on random.choice() by suppressing the upstream exception. @@ -154,6 +458,8 @@ Library Windows ------- +- bpo-29579: Removes readme.txt from the installer + - Issue #29326: Ignores blank lines in ._pth files (Patch by Alexey Izbyshev) - Issue #28164: Correctly handle special console filenames (patch by Eryk Sun) @@ -186,6 +492,11 @@ C API Documentation ------------- +- bpo-28929: Link the documentation to its source file on GitHub. + +- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to + aiosmtpd, a third-party asyncio-based replacement. + - Issue #26355: Add canonical header link on each page to corresponding major version of the documentation. Patch by Matthias Bussonnier. @@ -194,6 +505,15 @@ Documentation Tests ----- +- bpo-28087: Skip test_asyncore and test_eintr poll failures on macOS. + Skip some tests of select.poll when running on macOS due to unresolved + issues with the underlying system poll function on some macOS versions. + +- Issue #29571: to match the behaviour of the ``re.LOCALE`` flag, + test_re.test_locale_flag now uses ``locale.getpreferredencoding(False)`` to + determine the candidate encoding for the test regex (allowing it to correctly + skip the test when the default locale encoding is a multi-byte encoding) + - Issue #28950: Disallow -j0 to be combined with -T/-l in regrtest command line arguments. @@ -206,6 +526,12 @@ Tests Build ----- +- bpo-27593: sys.version and the platform module python_build(), + python_branch(), and python_revision() functions now use + git information rather than hg when building from a repo. + +- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. + - Issue #26851: Set Android compilation and link flags. - Issue #28768: Fix implicit declaration of function _setmode. Patch by @@ -510,7 +836,7 @@ Library non-None value is passed to it.send(val). - Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). + for readability. - Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin a workaround to Tix library bug. @@ -872,7 +1198,7 @@ Core and Builtins alpha releases, where backslashes are allowed anywhere in an f-string. Also, require that expressions inside f-strings be enclosed within literal braces, and not escapes like - f'\x7b"hi"\x7d'. + ``f'\x7b"hi"\x7d'``. - Issue #28046: Remove platform-specific directories from sys.path. @@ -1091,7 +1417,7 @@ Library - Issue #24277: The new email API is no longer provisional, and the docs have been reorganized and rewritten to emphasize the new API. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -1110,9 +1436,9 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27331: The email.mime classes now all accept an optional policy keyword. +- Issue #27331: The email.mime classes now all accept an optional policy keyword. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #16113: Add SHA-3 and SHAKE support to hashlib module. @@ -2220,7 +2546,7 @@ Core and Builtins in ``def f(): 1.0``. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used @@ -2778,7 +3104,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -2914,7 +3240,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -3549,7 +3875,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #28871: Fixed a crash when deallocate deep ElementTree. @@ -3672,7 +3998,7 @@ Library - Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). -- Issue #19003:m email.generator now replaces only \r and/or \n line +- Issue #19003:m email.generator now replaces only ``\r`` and/or ``\n`` line endings, per the RFC, instead of all unicode line endings. - Issue #28019: itertools.count() no longer rounds non-integer step in range @@ -3695,7 +4021,7 @@ Library - Issue #27445: Don't pass str(_charset) to MIMEText.set_payload(). Patch by Claude Paroz. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -3710,7 +4036,7 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #27691: Fix ssl module's parsing of GEN_RID subject alternative name fields in X.509 certs. @@ -4161,7 +4487,7 @@ Core and Builtins cookie names. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #27138: Fix the doc comment for FileFinder.find_spec(). @@ -4541,7 +4867,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -4951,7 +5277,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -6494,7 +6820,7 @@ Library - Issue #23521: Corrected pure python implementation of timedelta division. - * Eliminated OverflowError from timedelta * float for some floats; + * Eliminated OverflowError from ``timedelta * float`` for some floats; * Corrected rounding in timedlta true division. - Issue #21619: Popen objects no longer leave a zombie after exit in the with @@ -7293,7 +7619,7 @@ Library character instead of truncating it. Based on patch by Victor Stinner. - Issue #13968: The glob module now supports recursive search in - subdirectories using the "**" pattern. + subdirectories using the ``**`` pattern. - Issue #21951: Fixed a crash in Tkinter on AIX when called Tcl command with empty string or tuple argument. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fff90468a5927c..75327fa3014be9 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -521,7 +521,7 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) return NULL; } - for (i = 0; i < len; i++) { + for (i = 0; i < PyList_GET_SIZE(self->fut_callbacks); i++) { int ret; PyObject *item = PyList_GET_ITEM(self->fut_callbacks, i); @@ -1413,7 +1413,7 @@ TaskObj_get_fut_waiter(TaskObj *task) @classmethod _asyncio.Task.current_task - loop: 'O' = NULL + loop: 'O' = None Return the currently running task in an event loop or None. @@ -1424,12 +1424,12 @@ None is returned when called not in the context of a Task. static PyObject * _asyncio_Task_current_task_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=99fbe7332c516e03 input=cd784537f02cf833]*/ +/*[clinic end generated code: output=99fbe7332c516e03 input=a0d6cdf2e3b243e1]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } @@ -1500,7 +1500,7 @@ task_all_tasks(PyObject *loop) @classmethod _asyncio.Task.all_tasks - loop: 'O' = NULL + loop: 'O' = None Return a set of all tasks for an event loop. @@ -1509,12 +1509,12 @@ By default all tasks for the current event loop are returned. static PyObject * _asyncio_Task_all_tasks_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=11f9b20749ccca5d input=cd64aa5f88bd5c49]*/ +/*[clinic end generated code: output=11f9b20749ccca5d input=c6f5b53bd487488f]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } @@ -1984,6 +1984,16 @@ task_step_impl(TaskObj *task, PyObject *exc) if (_PyGen_FetchStopIterationValue(&o) == 0) { /* The error is StopIteration and that means that the underlying coroutine has resolved */ + if (task->task_must_cancel) { + // Task is cancelled right before coro stops. + Py_DECREF(o); + task->task_must_cancel = 0; + et = asyncio_CancelledError; + Py_INCREF(et); + ev = NULL; + tb = NULL; + goto set_exception; + } PyObject *res = future_set_result((FutureObj*)task, o); Py_DECREF(o); if (res == NULL) { @@ -2001,6 +2011,8 @@ task_step_impl(TaskObj *task, PyObject *exc) /* Some other exception; pop it and call Task.set_exception() */ PyErr_Fetch(&et, &ev, &tb); + +set_exception: assert(et); if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) { PyErr_NormalizeException(&et, &ev, &tb); diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e6111c64e77de2..30157701d70b47 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1051,8 +1051,8 @@ deque_index(dequeobject *deque, PyObject *args) int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(deque); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index df3aedec6eeab5..12234e254e1a7a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4273,11 +4273,10 @@ Array_subscript(PyObject *myself, PyObject *item) PyObject *np; Py_ssize_t start, stop, step, slicelen, cur, i; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for array object instances */ @@ -4414,11 +4413,10 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, otherlen, i, cur; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); if ((step < 0 && start < stop) || (step > 0 && start > stop)) stop = start; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 629ddf66bc4212..6119ecdaf90dc6 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -44,6 +44,19 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) func(in); } +/* + * See issue 29565. Update a structure passed by value; + * the caller should not see any change. + */ + +EXPORT(void) +_testfunc_large_struct_update_value(Test in) +{ + ((volatile Test *)&in)->first = 0x0badf00d; + ((volatile Test *)&in)->second = 0x0badf00d; + ((volatile Test *)&in)->third = 0x0badf00d; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/libffi_msvc/ffi.c b/Modules/_ctypes/libffi_msvc/ffi.c index 1d82929f530220..91a27dce3f2575 100644 --- a/Modules/_ctypes/libffi_msvc/ffi.c +++ b/Modules/_ctypes/libffi_msvc/ffi.c @@ -239,6 +239,16 @@ ffi_call(/*@dependent@*/ ffi_cif *cif, break; #else case FFI_SYSV: + /* If a single argument takes more than 8 bytes, + then a copy is passed by reference. */ + for (unsigned i = 0; i < cif->nargs; i++) { + size_t z = cif->arg_types[i]->size; + if (z > 8) { + void *temp = alloca(z); + memcpy(temp, avalue[i], z); + avalue[i] = temp; + } + } /*@-usedef@*/ return ffi_call_AMD64(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, fn); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 3bf2ca7ce15392..d88d06e78220ec 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -2082,7 +2082,7 @@ static PyGetSetDef PyCursesWindow_getsets[] = { PyTypeObject PyCursesWindow_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_curses.curses window", /*tp_name*/ + "_curses.window", /*tp_name*/ sizeof(PyCursesWindowObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c784d0f4a90999..c2ad9a203e9652 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3926,16 +3926,16 @@ time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) time_kws, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = time_new(Py_TYPE(self), tuple, NULL); if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } TIME_SET_FOLD(clone, fold); } Py_DECREF(tuple); @@ -5019,17 +5019,16 @@ datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) &y, &m, &d, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } DATE_SET_FOLD(clone, fold); } Py_DECREF(tuple); diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2cda98e61127d6..bef702ebe69c93 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -131,7 +131,7 @@ elementtree_free(void *m) LOCAL(PyObject*) list_join(PyObject* list) { - /* join list elements (destroying the list in the process) */ + /* join list elements */ PyObject* joiner; PyObject* result; @@ -140,8 +140,6 @@ list_join(PyObject* list) return NULL; result = PyUnicode_Join(joiner, list); Py_DECREF(joiner); - if (result) - Py_DECREF(list); return result; } @@ -508,15 +506,17 @@ element_get_text(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->text; + PyObject *res = self->text; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->text = res; + self->text = tmp; + Py_DECREF(res); + res = tmp; } } @@ -528,15 +528,17 @@ element_get_tail(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->tail; + PyObject *res = self->tail; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->tail = res; + self->tail = tmp; + Py_DECREF(res); + res = tmp; } } @@ -1710,11 +1712,11 @@ element_subscr(PyObject* self_, PyObject* item) if (!self->extra) return PyList_New(0); - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return PyList_New(0); @@ -1766,11 +1768,11 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) return -1; } - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (value == NULL) { /* Delete slice */ @@ -2147,6 +2149,12 @@ elementiter_next(ElementIterObject *it) continue; } + if (!PyObject_TypeCheck(extra->children[child_index], &Element_Type)) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute 'iter'", + Py_TYPE(extra->children[child_index])->tp_name); + return NULL; + } elem = (ElementObject *)extra->children[child_index]; item->child_index++; Py_INCREF(elem); @@ -2396,40 +2404,51 @@ treebuilder_dealloc(TreeBuilderObject *self) /* helpers for handling of arbitrary element-like objects */ static int -treebuilder_set_element_text_or_tail(PyObject *element, PyObject *data, +treebuilder_set_element_text_or_tail(PyObject *element, PyObject **data, PyObject **dest, _Py_Identifier *name) { if (Element_CheckExact(element)) { - Py_DECREF(JOIN_OBJ(*dest)); - *dest = JOIN_SET(data, PyList_CheckExact(data)); + PyObject *tmp = JOIN_OBJ(*dest); + *dest = JOIN_SET(*data, PyList_CheckExact(*data)); + *data = NULL; + Py_DECREF(tmp); return 0; } else { - PyObject *joined = list_join(data); + PyObject *joined = list_join(*data); int r; if (joined == NULL) return -1; r = _PyObject_SetAttrId(element, name, joined); Py_DECREF(joined); - return r; + if (r < 0) + return -1; + Py_CLEAR(*data); + return 0; } } -/* These two functions steal a reference to data */ -static int -treebuilder_set_element_text(PyObject *element, PyObject *data) +LOCAL(int) +treebuilder_flush_data(TreeBuilderObject* self) { - _Py_IDENTIFIER(text); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->text, &PyId_text); -} + PyObject *element = self->last; -static int -treebuilder_set_element_tail(PyObject *element, PyObject *data) -{ - _Py_IDENTIFIER(tail); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->tail, &PyId_tail); + if (!self->data) { + return 0; + } + + if (self->this == element) { + _Py_IDENTIFIER(text); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->text, &PyId_text); + } + else { + _Py_IDENTIFIER(tail); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->tail, &PyId_tail); + } } static int @@ -2479,16 +2498,8 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, PyObject* this; elementtreestate *st = ET_STATE_GLOBAL; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } - else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (!self->element_factory || self->element_factory == Py_None) { @@ -2591,15 +2602,8 @@ treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) { PyObject* item; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (self->index == 0) { diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index f785a7260e6a2e..1bcf16a7e00f53 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -88,10 +88,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else { + else if (Py_REFCNT(kw) == 1) { Py_INCREF(kw); pto->kw = kw; } + else { + pto->kw = PyDict_Copy(kw); + } } else { pto->kw = PyDict_Copy(pkw); @@ -247,8 +250,11 @@ partial_repr(partialobject *pto) /* Pack keyword arguments */ assert (PyDict_Check(pto->kw)); for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { - Py_SETREF(arglist, PyUnicode_FromFormat("%U, %U=%R", arglist, + /* Prevent key.__str__ from deleting the value. */ + Py_INCREF(value); + Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); + Py_DECREF(value); if (arglist == NULL) goto done; } diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index cbe7425eaefa65..efc7d05b7d01b9 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1416,8 +1416,18 @@ buffered_repr(buffered *self) res = PyUnicode_FromFormat("<%s>", Py_TYPE(self)->tp_name); } else { - res = PyUnicode_FromFormat("<%s name=%R>", - Py_TYPE(self)->tp_name, nameobj); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat("<%s name=%R>", + Py_TYPE(self)->tp_name, nameobj); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 7f3bcab9625ef5..833ea8e7b50894 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1082,9 +1082,19 @@ fileio_repr(fileio *self) self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { - res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat( + "<_io.FileIO name=%R mode='%s' closefd=%s>", + nameobj, mode_string(self), self->closefd ? "True" : "False"); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 472ef3b97cd38e..c8642040ae426a 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -625,7 +625,8 @@ iobase_iternext(PyObject *self) if (line == NULL) return NULL; - if (PyObject_Size(line) == 0) { + if (PyObject_Size(line) <= 0) { + /* Error or empty */ Py_DECREF(line); return NULL; } @@ -650,7 +651,7 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) /*[clinic end generated code: output=2f50421677fa3dea input=1961c4a95e96e661]*/ { Py_ssize_t length = 0; - PyObject *result; + PyObject *result, *it = NULL; result = PyList_New(0); if (result == NULL) @@ -664,19 +665,23 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) PyObject *ret = _PyObject_CallMethodId(result, &PyId_extend, "O", self); if (ret == NULL) { - Py_DECREF(result); - return NULL; + goto error; } Py_DECREF(ret); return result; } + it = PyObject_GetIter(self); + if (it == NULL) { + goto error; + } + while (1) { - PyObject *line = PyIter_Next(self); + Py_ssize_t line_length; + PyObject *line = PyIter_Next(it); if (line == NULL) { if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + goto error; } else break; /* StopIteration raised */ @@ -684,16 +689,25 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) if (PyList_Append(result, line) < 0) { Py_DECREF(line); - Py_DECREF(result); - return NULL; + goto error; } - length += PyObject_Size(line); + line_length = PyObject_Size(line); Py_DECREF(line); - - if (length > hint) + if (line_length < 0) { + goto error; + } + if (line_length > hint - length) break; + length += line_length; } + + Py_DECREF(it); return result; + + error: + Py_XDECREF(it); + Py_DECREF(result); + return NULL; } /*[clinic input] diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4df55626d4bc23..bc8d11efa5258c 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2483,6 +2483,7 @@ static PyObject * textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; + int status; CHECK_INITIALIZED(self); @@ -2490,6 +2491,15 @@ textiowrapper_repr(textio *self) if (res == NULL) return NULL; + status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } + goto error; + } nameobj = _PyObject_GetAttrId((PyObject *) self, &PyId_name); if (nameobj == NULL) { if (PyErr_ExceptionMatches(PyExc_Exception)) @@ -2504,7 +2514,7 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } modeobj = _PyObject_GetAttrId((PyObject *) self, &PyId_mode); if (modeobj == NULL) { @@ -2520,14 +2530,21 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } s = PyUnicode_FromFormat("%U encoding=%R>", res, self->encoding); Py_DECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return s; -error: + + error: Py_XDECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return NULL; } diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 1d169e2764ed27..cbe21422bb4653 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1017,7 +1017,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); if (wlen) { res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL); - if (n < wlen) { + if (res && n < wlen) { /* Wrote fewer characters than expected, which means our * len value may be wrong. So recalculate it from the * characters that were written. As this could potentially diff --git a/Modules/_json.c b/Modules/_json.c index faa213491b6387..a84b0851090779 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -89,16 +89,12 @@ static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds); static void scanner_dealloc(PyObject *self); static int scanner_clear(PyObject *self); static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds); static void encoder_dealloc(PyObject *self); static int @@ -1203,38 +1199,21 @@ static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyScannerObject *s; - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->strict = NULL; - s->object_hook = NULL; - s->object_pairs_hook = NULL; - s->parse_float = NULL; - s->parse_int = NULL; - s->parse_constant = NULL; - } - return (PyObject *)s; -} - -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ PyObject *ctx; static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return -1; + return NULL; - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; + s = (PyScannerObject *)type->tp_alloc(type, 0); + if (s == NULL) { + return NULL; } + s->memo = PyDict_New(); + if (s->memo == NULL) + goto bail; + /* All of these will fail "gracefully" so we don't need to verify them */ s->strict = PyObject_GetAttrString(ctx, "strict"); if (s->strict == NULL) @@ -1255,16 +1234,11 @@ scanner_init(PyObject *self, PyObject *args, PyObject *kwds) if (s->parse_constant == NULL) goto bail; - return 0; + return (PyObject *)s; bail: - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->object_pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - return -1; + Py_DECREF(s); + return NULL; } PyDoc_STRVAR(scanner_doc, "JSON scanner object"); @@ -1306,7 +1280,7 @@ PyTypeObject PyScannerType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - scanner_init, /* tp_init */ + 0, /* tp_init */ 0,/* PyType_GenericAlloc, */ /* tp_alloc */ scanner_new, /* tp_new */ 0,/* PyObject_GC_Del, */ /* tp_free */ @@ -1315,25 +1289,6 @@ PyTypeObject PyScannerType = { static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyEncoderObject *s; - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->markers = NULL; - s->defaultfn = NULL; - s->encoder = NULL; - s->indent = NULL; - s->key_separator = NULL; - s->item_separator = NULL; - s->sort_keys = NULL; - s->skipkeys = NULL; - } - return (PyObject *)s; -} - -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* initialize Encoder object */ static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; @@ -1341,22 +1296,23 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) PyObject *item_separator, *sort_keys, *skipkeys; int allow_nan; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUOOp:make_encoder", kwlist, &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) - return -1; + return NULL; if (markers != Py_None && !PyDict_Check(markers)) { PyErr_Format(PyExc_TypeError, "make_encoder() argument 1 must be dict or None, " "not %.200s", Py_TYPE(markers)->tp_name); - return -1; + return NULL; } + s = (PyEncoderObject *)type->tp_alloc(type, 0); + if (s == NULL) + return NULL; + s->markers = markers; s->defaultfn = defaultfn; s->encoder = encoder; @@ -1383,7 +1339,7 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) Py_INCREF(s->item_separator); Py_INCREF(s->sort_keys); Py_INCREF(s->skipkeys); - return 0; + return (PyObject *)s; } static PyObject * @@ -1914,7 +1870,7 @@ PyTypeObject PyEncoderType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - encoder_init, /* tp_init */ + 0, /* tp_init */ 0, /* tp_alloc */ encoder_new, /* tp_new */ 0, /* tp_free */ @@ -1957,10 +1913,8 @@ PyInit__json(void) PyObject *m = PyModule_Create(&jsonmodule); if (!m) return NULL; - PyScannerType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyScannerType) < 0) goto fail; - PyEncoderType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyEncoderType) < 0) goto fail; Py_INCREF((PyObject*)&PyScannerType); diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 5007a39bc2443d..d1434d59f818b4 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -111,13 +111,17 @@ _is_fdescfs_mounted_on_dev_fd(void) static int _sanity_check_python_fd_sequence(PyObject *fd_sequence) { - Py_ssize_t seq_idx, seq_len = PySequence_Length(fd_sequence); + Py_ssize_t seq_idx; long prev_fd = -1; - for (seq_idx = 0; seq_idx < seq_len; ++seq_idx) { - PyObject* py_fd = PySequence_Fast_GET_ITEM(fd_sequence, seq_idx); - long iter_fd = PyLong_AsLong(py_fd); + for (seq_idx = 0; seq_idx < PyTuple_GET_SIZE(fd_sequence); ++seq_idx) { + PyObject* py_fd = PyTuple_GET_ITEM(fd_sequence, seq_idx); + long iter_fd; + if (!PyLong_Check(py_fd)) { + return 1; + } + iter_fd = PyLong_AsLong(py_fd); if (iter_fd < 0 || iter_fd <= prev_fd || iter_fd > INT_MAX) { - /* Negative, overflow, not a Long, unsorted, too big for a fd. */ + /* Negative, overflow, unsorted, too big for a fd. */ return 1; } prev_fd = iter_fd; @@ -132,13 +136,12 @@ _is_fd_in_sorted_fd_sequence(int fd, PyObject *fd_sequence) { /* Binary search. */ Py_ssize_t search_min = 0; - Py_ssize_t search_max = PySequence_Length(fd_sequence) - 1; + Py_ssize_t search_max = PyTuple_GET_SIZE(fd_sequence) - 1; if (search_max < 0) return 0; do { long middle = (search_min + search_max) / 2; - long middle_fd = PyLong_AsLong( - PySequence_Fast_GET_ITEM(fd_sequence, middle)); + long middle_fd = PyLong_AsLong(PyTuple_GET_ITEM(fd_sequence, middle)); if (fd == middle_fd) return 1; if (fd > middle_fd) @@ -154,9 +157,9 @@ make_inheritable(PyObject *py_fds_to_keep, int errpipe_write) { Py_ssize_t i, len; - len = PySequence_Length(py_fds_to_keep); + len = PyTuple_GET_SIZE(py_fds_to_keep); for (i = 0; i < len; ++i) { - PyObject* fdobj = PySequence_Fast_GET_ITEM(py_fds_to_keep, i); + PyObject* fdobj = PyTuple_GET_ITEM(py_fds_to_keep, i); long fd = PyLong_AsLong(fdobj); assert(!PyErr_Occurred()); assert(0 <= fd && fd <= INT_MAX); @@ -213,14 +216,13 @@ static void _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) { long end_fd = safe_get_max_fd(); - Py_ssize_t num_fds_to_keep = PySequence_Length(py_fds_to_keep); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); Py_ssize_t keep_seq_idx; int fd_num; /* As py_fds_to_keep is sorted we can loop through the list closing * fds inbetween any in the keep list falling within our range. */ for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { - PyObject* py_keep_fd = PySequence_Fast_GET_ITEM(py_fds_to_keep, - keep_seq_idx); + PyObject* py_keep_fd = PyTuple_GET_ITEM(py_fds_to_keep, keep_seq_idx); int keep_fd = PyLong_AsLong(py_keep_fd); if (keep_fd < start_fd) continue; @@ -306,7 +308,7 @@ _close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep) /* Close all open file descriptors from start_fd and higher. - * Do not close any in the sorted py_fds_to_keep list. + * Do not close any in the sorted py_fds_to_keep tuple. * * This function violates the strict use of async signal safe functions. :( * It calls opendir(), readdir() and closedir(). Of these, the one most @@ -562,8 +564,9 @@ subprocess_fork_exec(PyObject* self, PyObject *args) #endif if (!PyArg_ParseTuple( - args, "OOpOOOiiiiiiiiiiO:fork_exec", - &process_args, &executable_list, &close_fds, &py_fds_to_keep, + args, "OOpO!OOiiiiiiiiiiO:fork_exec", + &process_args, &executable_list, + &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, &p2cread, &p2cwrite, &c2pread, &c2pwrite, &errread, &errwrite, &errpipe_read, &errpipe_write, @@ -574,10 +577,6 @@ subprocess_fork_exec(PyObject* self, PyObject *args) PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3"); return NULL; } - if (PySequence_Length(py_fds_to_keep) < 0) { - PyErr_SetString(PyExc_ValueError, "cannot get length of fds_to_keep"); - return NULL; - } if (_sanity_check_python_fd_sequence(py_fds_to_keep)) { PyErr_SetString(PyExc_ValueError, "bad value(s) in fds_to_keep"); return NULL; @@ -631,6 +630,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) goto cleanup; for (arg_num = 0; arg_num < num_args; ++arg_num) { PyObject *borrowed_arg, *converted_arg; + if (PySequence_Fast_GET_SIZE(fast_args) != num_args) { + PyErr_SetString(PyExc_RuntimeError, "args changed during iteration"); + goto cleanup; + } borrowed_arg = PySequence_Fast_GET_ITEM(fast_args, arg_num); if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) goto cleanup; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0d3282db8f2810..d006aebf927c38 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -2,7 +2,7 @@ /* ------------------------------------------------------------------ The code in this module was based on a download from: - http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html It was modified in 2002 by Raymond Hettinger as follows: @@ -60,8 +60,8 @@ Any feedback is very welcome. - http://www.math.keio.ac.jp/matumoto/emt.html - email: matumoto@math.keio.ac.jp + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html + email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) */ /* ---------------------------------------------------------------*/ @@ -348,6 +348,7 @@ random_setstate(RandomObject *self, PyObject *state) int i; unsigned long element; long index; + uint32_t new_state[N]; if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, @@ -364,7 +365,7 @@ random_setstate(RandomObject *self, PyObject *state) element = PyLong_AsUnsignedLong(PyTuple_GET_ITEM(state, i)); if (element == (unsigned long)-1 && PyErr_Occurred()) return NULL; - self->state[i] = (uint32_t)element; + new_state[i] = (uint32_t)element; } index = PyLong_AsLong(PyTuple_GET_ITEM(state, i)); @@ -375,6 +376,8 @@ random_setstate(RandomObject *self, PyObject *state) return NULL; } self->index = (int)index; + for (i = 0; i < N; i++) + self->state[i] = new_state[i]; Py_INCREF(Py_None); return Py_None; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 39f7a6508c2aef..8341fb8480172c 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -511,10 +511,9 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); - /* For backwards compatibility reasons, do not start a transaction if a - DDL statement is encountered. If anybody wants transactional DDL, - they can issue a BEGIN statement manually. */ - if (self->connection->begin_statement && !sqlite3_stmt_readonly(self->statement->st) && !self->statement->is_ddl) { + /* We start a transaction implicitly before a DML statement. + SELECT is the only exception. See #9924. */ + if (self->connection->begin_statement && self->statement->is_dml) { if (sqlite3_get_autocommit(self->connection->db)) { result = _pysqlite_connection_begin(self->connection); if (!result) { @@ -609,7 +608,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* } } - if (!sqlite3_stmt_readonly(self->statement->st)) { + if (self->statement->is_dml) { self->rowcount += (long)sqlite3_changes(self->connection->db); } else { self->rowcount= -1L; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 0df661b9c7ad31..087375be9b63d8 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -73,8 +73,9 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con Py_INCREF(sql); self->sql = sql; - /* determine if the statement is a DDL statement */ - self->is_ddl = 0; + /* Determine if the statement is a DML statement. + SELECT is the only exception. See #9924. */ + self->is_dml = 0; for (p = sql_cstr; *p != 0; p++) { switch (*p) { case ' ': @@ -84,9 +85,10 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con continue; } - self->is_ddl = (PyOS_strnicmp(p, "create ", 7) == 0) - || (PyOS_strnicmp(p, "drop ", 5) == 0) - || (PyOS_strnicmp(p, "reindex ", 8) == 0); + self->is_dml = (PyOS_strnicmp(p, "insert ", 7) == 0) + || (PyOS_strnicmp(p, "update ", 7) == 0) + || (PyOS_strnicmp(p, "delete ", 7) == 0) + || (PyOS_strnicmp(p, "replace ", 8) == 0); break; } diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h index 6eef16857f7b85..8db10f6649ce80 100644 --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -38,7 +38,7 @@ typedef struct sqlite3_stmt* st; PyObject* sql; int in_use; - int is_ddl; + int is_dml; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Statement; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b1988570604db1..2a2c18fe2f7309 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1210,10 +1210,6 @@ _get_crl_dp(X509 *certificate) { int i, j; PyObject *lst, *res = NULL; -#if OPENSSL_VERSION_NUMBER >= 0x10001000L - /* Calls x509v3_cache_extensions and sets up crldp */ - X509_check_ca(certificate); -#endif dps = X509_get_ext_d2i(certificate, NID_crl_distribution_points, NULL, NULL); if (dps == NULL) @@ -1258,9 +1254,7 @@ _get_crl_dp(X509 *certificate) { done: Py_XDECREF(lst); -#if OPENSSL_VERSION_NUMBER < 0x10001000L - sk_DIST_POINT_free(dps); -#endif + CRL_DIST_POINTS_free(dps); return res; } @@ -2729,12 +2723,12 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) #endif -#ifndef OPENSSL_NO_ECDH +#if !defined(OPENSSL_NO_ECDH) && !defined(OPENSSL_VERSION_1_1) /* Allow automatic ECDH curve selection (on OpenSSL 1.0.2+), or use prime256v1 by default. This is Apache mod_ssl's initialization policy, so we should be safe. OpenSSL 1.1 has it enabled by default. */ -#if defined(SSL_CTX_set_ecdh_auto) && !defined(OPENSSL_VERSION_1_1) +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(self->ctx, 1); #else { diff --git a/Modules/_struct.c b/Modules/_struct.c index 796d1682f094f1..2635af9db69597 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -539,7 +539,7 @@ np_ubyte(char *p, PyObject *v, const formatdef *f) "ubyte format requires 0 <= number <= 255"); return -1; } - *p = (char)x; + *(unsigned char *)p = (unsigned char)x; return 0; } @@ -868,6 +868,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -880,7 +881,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffL); x >>= 8; } while (i > 0); return 0; @@ -891,6 +892,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -901,7 +903,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffUL); x >>= 8; } while (i > 0); return 0; @@ -1087,6 +1089,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -1099,7 +1102,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffL); x >>= 8; } while (--i > 0); return 0; @@ -1110,6 +1113,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -1120,7 +1124,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffUL); x >>= 8; } while (--i > 0); return 0; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 4e1ce6851a5eb6..6b8ab34d931cd3 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1715,10 +1715,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { @@ -1935,9 +1935,10 @@ slice_indices(PyObject *self, PyObject *args) "first argument must be a slice object"); return NULL; } - if (PySlice_GetIndicesEx(key, len, &s[0], &s[1], &s[2], &s[3]) < 0) { + if (PySlice_Unpack(key, &s[0], &s[1], &s[2]) < 0) { return NULL; } + s[3] = PySlice_AdjustIndices(len, &s[0], &s[1], s[2]); ret = PyTuple_New(4); if (ret == NULL) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f09205f63cc5ab..c76eefab4e80cf 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1571,7 +1571,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) { PyObject *sub_args; PyObject *sub_kwargs; - char *sub_format; + const char *sub_format; PyObject *sub_keywords; Py_ssize_t i, size; @@ -1584,7 +1584,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) double buffers[8][4]; /* double ensures alignment where necessary */ - if (!PyArg_ParseTuple(args, "OOyO:parse_tuple_and_keywords", + if (!PyArg_ParseTuple(args, "OOsO:parse_tuple_and_keywords", &sub_args, &sub_kwargs, &sub_format, &sub_keywords)) return NULL; @@ -3812,7 +3812,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - seconds = PyLong_FromLong((long long)tv.tv_sec); + seconds = PyLong_FromLongLong(tv.tv_sec); if (seconds == NULL) return NULL; return Py_BuildValue("Nl", seconds, tv.tv_usec); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 91d4f0172c3c50..248f4582c60fb4 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -722,17 +722,22 @@ getenvironment(PyObject* environment) return NULL; } - envsize = PyMapping_Length(environment); - keys = PyMapping_Keys(environment); values = PyMapping_Values(environment); if (!keys || !values) goto error; + envsize = PySequence_Fast_GET_SIZE(keys); + if (PySequence_Fast_GET_SIZE(values) != envsize) { + PyErr_SetString(PyExc_RuntimeError, + "environment changed size during iteration"); + goto error; + } + totalsize = 1; /* trailing null character */ for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (! PyUnicode_Check(key) || ! PyUnicode_Check(value)) { PyErr_SetString(PyExc_TypeError, @@ -760,8 +765,8 @@ getenvironment(PyObject* environment) end = buffer + totalsize; for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (!PyUnicode_AsUCS4(key, p, end - p, 0)) goto error; p += PyUnicode_GET_LENGTH(key); diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2caa8ee5a8e892..64e0f172fd7d82 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -331,35 +331,51 @@ II_getitem(arrayobject *ap, Py_ssize_t i) (unsigned long) ((unsigned int *)ap->ob_item)[i]); } +static PyObject * +get_int_unless_float(PyObject *v) +{ + if (PyFloat_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "array item must be integer"); + return NULL; + } + return (PyObject *)_PyLong_FromNbInt(v); +} + static int II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned int is less than minimum"); - return -1; + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long)y; - + return -1; } if (x > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, - "unsigned int is greater than maximum"); + "unsigned int is greater than maximum"); + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -390,31 +406,28 @@ static int LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) - return -1; - } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is less than minimum"); + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; } - x = (unsigned long)y; - + do_decref = 1; } - if (x > ULONG_MAX) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is greater than maximum"); + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -446,25 +459,28 @@ static int QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLongLong(v); - if (x == (unsigned long long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long long y; - if (!PyArg_Parse(v, "L;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long long is less than minimum"); - return -1; + x = PyLong_AsUnsignedLongLong(v); + if (x == (unsigned long long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long long)y; + return -1; } - if (i >= 0) ((unsigned long long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -2281,10 +2297,11 @@ array_subscr(arrayobject* self, PyObject* item) arrayobject* ar; int itemsize = self->ob_descr->itemsize; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return newarrayobject(&Arraytype, 0, self->ob_descr); @@ -2352,11 +2369,11 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) return (*self->ob_descr->setitem)(self, i, value); } else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx(item, - Py_SIZE(self), &start, &stop, - &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); } else { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index 1a070f2f393219..1fcc220b8db0f4 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -279,7 +279,9 @@ DECODER(gb18030) REQUIRE_INBUF(4); c3 = INBYTE3; c4 = INBYTE4; - if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39) + if (c < 0x81 || c > 0xFE || + c3 < 0x81 || c3 > 0xFE || + c4 < 0x30 || c4 > 0x39) return 1; c -= 0x81; c2 -= 0x30; c3 -= 0x81; c4 -= 0x30; @@ -348,15 +350,17 @@ ENCODER(hz) DBCHAR code; if (c < 0x80) { - if (state->i == 0) { - WRITEBYTE1((unsigned char)c); - NEXT(1, 1); - } - else { - WRITEBYTE3('~', '}', (unsigned char)c); - NEXT(1, 3); + if (state->i) { + WRITEBYTE2('~', '}'); + NEXT_OUT(2); state->i = 0; } + WRITEBYTE1((unsigned char)c); + NEXT(1, 1); + if (c == '~') { + WRITEBYTE1('~'); + NEXT_OUT(1); + } continue; } @@ -407,17 +411,14 @@ DECODER(hz) unsigned char c2 = INBYTE2; REQUIRE_INBUF(2); - if (c2 == '~') { + if (c2 == '~' && state->i == 0) OUTCHAR('~'); - NEXT_IN(2); - continue; - } else if (c2 == '{' && state->i == 0) state->i = 1; /* set GB */ + else if (c2 == '\n' && state->i == 0) + ; /* line-continuation */ else if (c2 == '}' && state->i == 1) state->i = 0; /* set ASCII */ - else if (c2 == '\n') - ; /* line-continuation */ else return 1; NEXT_IN(2); diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index d1da189ddd3bef..d6efc77d20c883 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1670,6 +1670,9 @@ _multibytecodec_MultibyteStreamWriter_writelines(MultibyteStreamWriterObject *se if (r == -1) return NULL; } + /* PySequence_Length() can fail */ + if (PyErr_Occurred()) + return NULL; Py_RETURN_NONE; } diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 052d252331f6a7..41f11f4cac5494 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -278,7 +278,7 @@ _asyncio_Task_current_task(PyTypeObject *type, PyObject **args, Py_ssize_t nargs PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:current_task", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -310,7 +310,7 @@ _asyncio_Task_all_tasks(PyTypeObject *type, PyObject **args, Py_ssize_t nargs, P PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:all_tasks", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -517,4 +517,4 @@ _asyncio_Task__wakeup(TaskObj *self, PyObject **args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=8f036321bb083066 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=40ca6c9da517da73 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c1e4fb7d17760..61fc4908b0fe9b 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -55,6 +55,9 @@ static struct { int fd; int all_threads; PyInterpreterState *interp; +#ifdef MS_WINDOWS + void *exc_handler; +#endif } fatal_error = {0, NULL, -1, 0}; #ifdef FAULTHANDLER_LATER @@ -124,6 +127,7 @@ static const size_t faulthandler_nsignals = \ #ifdef HAVE_SIGALTSTACK static stack_t stack; +static stack_t old_stack; #endif @@ -461,7 +465,8 @@ faulthandler_enable(void) } #ifdef MS_WINDOWS - AddVectoredExceptionHandler(1, faulthandler_exc_handler); + assert(fatal_error.exc_handler == NULL); + fatal_error.exc_handler = AddVectoredExceptionHandler(1, faulthandler_exc_handler); #endif return 0; } @@ -513,7 +518,12 @@ faulthandler_disable(void) faulthandler_disable_fatal_handler(handler); } } - +#ifdef MS_WINDOWS + if (fatal_error.exc_handler != NULL) { + RemoveVectoredExceptionHandler(fatal_error.exc_handler); + fatal_error.exc_handler = NULL; + } +#endif Py_CLEAR(fatal_error.file); } @@ -1310,7 +1320,7 @@ int _PyFaulthandler_Init(void) stack.ss_size = SIGSTKSZ; stack.ss_sp = PyMem_Malloc(stack.ss_size); if (stack.ss_sp != NULL) { - err = sigaltstack(&stack, NULL); + err = sigaltstack(&stack, &old_stack); if (err) { PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; @@ -1366,6 +1376,20 @@ void _PyFaulthandler_Fini(void) faulthandler_disable(); #ifdef HAVE_SIGALTSTACK if (stack.ss_sp != NULL) { + /* Fetch the current alt stack */ + stack_t current_stack; + if (sigaltstack(NULL, ¤t_stack) == 0) { + if (current_stack.ss_sp == stack.ss_sp) { + /* The current alt stack is the one that we installed. + It is safe to restore the old stack that we found when + we installed ours */ + sigaltstack(&old_stack, NULL); + } else { + /* Someone switched to a different alt stack and didn't + restore ours when they were done (if they're done). + There's not much we can do in this unlikely case */ + } + } PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; } diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 0971a64fccbadc..5f941a26e1d54c 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -21,47 +21,47 @@ #endif /* XXX Only unix build process has been tested */ -#ifndef HGVERSION -#define HGVERSION "" +#ifndef GITVERSION +#define GITVERSION "" #endif -#ifndef HGTAG -#define HGTAG "" +#ifndef GITTAG +#define GITTAG "" #endif -#ifndef HGBRANCH -#define HGBRANCH "" +#ifndef GITBRANCH +#define GITBRANCH "" #endif const char * Py_GetBuildInfo(void) { - static char buildinfo[50 + sizeof(HGVERSION) + - ((sizeof(HGTAG) > sizeof(HGBRANCH)) ? - sizeof(HGTAG) : sizeof(HGBRANCH))]; - const char *revision = _Py_hgversion(); + static char buildinfo[50 + sizeof(GITVERSION) + + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? + sizeof(GITTAG) : sizeof(GITBRANCH))]; + const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; - const char *hgid = _Py_hgidentifier(); - if (!(*hgid)) - hgid = "default"; + const char *gitid = _Py_gitidentifier(); + if (!(*gitid)) + gitid = "default"; PyOS_snprintf(buildinfo, sizeof(buildinfo), - "%s%s%s, %.20s, %.9s", hgid, sep, revision, + "%s%s%s, %.20s, %.9s", gitid, sep, revision, DATE, TIME); return buildinfo; } const char * -_Py_hgversion(void) +_Py_gitversion(void) { - return HGVERSION; + return GITVERSION; } const char * -_Py_hgidentifier(void) +_Py_gitidentifier(void) { - const char *hgtag, *hgid; - hgtag = HGTAG; - if ((*hgtag) && strcmp(hgtag, "tip") != 0) - hgid = hgtag; + const char *gittag, *gitid; + gittag = GITTAG; + if ((*gittag) && strcmp(gittag, "undefined") != 0) + gitid = gittag; else - hgid = HGBRANCH; - return hgid; + gitid = GITBRANCH; + return gitid; } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6bf04cbee31c35..8e22ec9c4b7cc1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1864,33 +1864,37 @@ chain_next(chainobject *lz) { PyObject *item; - if (lz->source == NULL) - return NULL; /* already stopped */ - - if (lz->active == NULL) { - PyObject *iterable = PyIter_Next(lz->source); - if (iterable == NULL) { - Py_CLEAR(lz->source); - return NULL; /* no more input sources */ - } - lz->active = PyObject_GetIter(iterable); - Py_DECREF(iterable); + /* lz->source is the iterator of iterables. If it's NULL, we've already + * consumed them all. lz->active is the current iterator. If it's NULL, + * we should grab a new one from lz->source. */ + while (lz->source != NULL) { if (lz->active == NULL) { - Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + PyObject *iterable = PyIter_Next(lz->source); + if (iterable == NULL) { + Py_CLEAR(lz->source); + return NULL; /* no more input sources */ + } + lz->active = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (lz->active == NULL) { + Py_CLEAR(lz->source); + return NULL; /* input not iterable */ + } } + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); + if (item != NULL) + return item; + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else + return NULL; /* input raised an exception */ + } + /* lz->active is consumed, try with the next iterable. */ + Py_CLEAR(lz->active); } - item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); - if (item != NULL) - return item; - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - return NULL; /* input raised an exception */ - } - Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + /* Everything had been consumed already. */ + return NULL; } static PyObject * diff --git a/Modules/main.c b/Modules/main.c index 2e6a60b1673f88..475a2fdc36db9b 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,7 +53,7 @@ static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : debug output from parser; also PYTHONDEBUG=x\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ @@ -225,55 +225,60 @@ static int RunModule(wchar_t *modname, int set_argv0) return 0; } -static int -RunMainFromImporter(wchar_t *filename) +static PyObject * +AsImportPathEntry(wchar_t *filename) { - PyObject *argv0 = NULL, *importer, *sys_path, *sys_path0; - int sts; + PyObject *sys_path0 = NULL, *importer; - argv0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (argv0 == NULL) + sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + if (sys_path0 == NULL) goto error; - importer = PyImport_GetImporter(argv0); + importer = PyImport_GetImporter(sys_path0); if (importer == NULL) goto error; if (importer == Py_None) { - Py_DECREF(argv0); + Py_DECREF(sys_path0); Py_DECREF(importer); - return -1; + return NULL; } Py_DECREF(importer); + return sys_path0; + +error: + Py_XDECREF(sys_path0); + PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + PyErr_Print(); + PyErr_Clear(); + return NULL; +} + + +static int +RunMainFromImporter(PyObject *sys_path0) +{ + PyObject *sys_path; + int sts; - /* argv0 is usable as an import source, so put it in sys.path[0] - and import __main__ */ + /* Assume sys_path0 has already been checked by AsImportPathEntry, + * so put it in sys.path[0] and import __main__ */ sys_path = PySys_GetObject("path"); if (sys_path == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); goto error; } - sys_path0 = PyList_GetItem(sys_path, 0); - sts = 0; - if (!sys_path0) { - PyErr_Clear(); - sts = PyList_Append(sys_path, argv0); - } else if (PyObject_IsTrue(sys_path0)) { - sts = PyList_Insert(sys_path, 0, argv0); - } else { - sts = PyList_SetItem(sys_path, 0, argv0); - } + sts = PyList_Insert(sys_path, 0, sys_path0); if (sts) { - argv0 = NULL; + sys_path0 = NULL; goto error; } - Py_INCREF(argv0); sts = RunModule(L"__main__", 0); return sts != 0; error: - Py_XDECREF(argv0); + Py_XDECREF(sys_path0); PyErr_Print(); return 1; } @@ -358,6 +363,7 @@ Py_Main(int argc, wchar_t **argv) int saw_unbuffered_flag = 0; char *opt; PyCompilerFlags cf; + PyObject *main_importer_path = NULL; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -714,7 +720,17 @@ Py_Main(int argc, wchar_t **argv) argv[_PyOS_optind] = L"-m"; } - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + if (filename != NULL) { + main_importer_path = AsImportPathEntry(filename); + } + + if (main_importer_path != NULL) { + /* Let RunMainFromImporter adjust sys.path[0] later */ + PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + } else { + /* Use config settings to decide whether or not to update sys.path[0] */ + PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + } if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && isatty(fileno(stdin)) && @@ -744,11 +760,11 @@ Py_Main(int argc, wchar_t **argv) sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (main_importer_path != NULL) { + sts = RunMainFromImporter(main_importer_path); } - if (sts==-1 && filename!=NULL) { + if (sts==-1 && filename != NULL) { fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5f1615ff82992c..426b7cacebb573 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -806,10 +806,10 @@ mmap_subscript(mmap_object *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->size, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); @@ -932,11 +932,10 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) Py_ssize_t start, stop, step, slicelen; Py_buffer vbuf; - if (PySlice_GetIndicesEx(item, - self->size, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap object doesn't support slice deletion"); diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index b2566951d06de1..a4443350ef1cdc 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -698,7 +698,7 @@ validate_node(node *tree) short a_label = dfa_state->s_arc[arc].a_lbl; assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels); if (_PyParser_Grammar.g_ll.ll_label[a_label].lb_type == ch_type) { - /* The child is acceptable; if non-terminal, validate it recursively. */ + /* The child is acceptable; if non-terminal, validate it recursively. */ if (ISNONTERMINAL(ch_type) && !validate_node(ch)) return 0; @@ -776,32 +776,35 @@ parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) */ tree = build_node_tree(tuple); if (tree != 0) { - node *validation_root = tree; + node *validation_root = NULL; int tree_type = 0; switch (TYPE(tree)) { case eval_input: /* Might be an eval form. */ tree_type = PyST_EXPR; + validation_root = tree; break; case encoding_decl: /* This looks like an encoding_decl so far. */ - if (NCH(tree) != 1) + if (NCH(tree) == 1) { + tree_type = PyST_SUITE; + validation_root = CHILD(tree, 0); + } + else { err_string("Error Parsing encoding_decl"); - validation_root = CHILD(tree, 0); - /* Fall through */ + } + break; case file_input: /* This looks like an exec form so far. */ - tree_type = PyST_SUITE; + validation_root = tree; break; default: /* This is a fragment, at best. */ - PyNode_Free(tree); err_string("parse tree does not use a valid start symbol"); - return (0); } - if (validate_node(validation_root)) + if (validation_root != NULL && validate_node(validation_root)) st = parser_newstobject(tree, tree_type); else PyNode_Free(tree); @@ -831,6 +834,9 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_ssize_t i; int err; + if (len < 0) { + return NULL; + } for (i = 1; i < len; ++i) { /* elem must always be a sequence, however simple */ PyObject* elem = PySequence_GetItem(tuple, i); @@ -851,7 +857,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if (type == -1 && PyErr_Occurred()) { Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } } Py_DECREF(temp); @@ -863,7 +869,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) PyErr_SetObject(parser_error, err); Py_XDECREF(err); Py_XDECREF(elem); - return (0); + return NULL; } if (ISTERMINAL(type)) { Py_ssize_t len = PyObject_Size(elem); @@ -872,11 +878,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if ((len != 2) && (len != 3)) { err_string("terminal nodes must have 2 or 3 entries"); - return 0; + Py_DECREF(elem); + return NULL; } temp = PySequence_GetItem(elem, 1); - if (temp == NULL) - return 0; + if (temp == NULL) { + Py_DECREF(elem); + return NULL; + } if (!PyUnicode_Check(temp)) { PyErr_Format(parser_error, "second item in terminal node must be a string," @@ -884,46 +893,49 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_TYPE(temp)->tp_name); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } if (len == 3) { PyObject *o = PySequence_GetItem(elem, 2); - if (o != NULL) { - if (PyLong_Check(o)) { - int num = _PyLong_AsInt(o); - if (num == -1 && PyErr_Occurred()) { - Py_DECREF(o); - Py_DECREF(temp); - Py_DECREF(elem); - return 0; - } - *line_num = num; - } - else { - PyErr_Format(parser_error, - "third item in terminal node must be an" - " integer, found %s", - Py_TYPE(temp)->tp_name); + if (o == NULL) { + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + if (PyLong_Check(o)) { + int num = _PyLong_AsInt(o); + if (num == -1 && PyErr_Occurred()) { Py_DECREF(o); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } + *line_num = num; + } + else { + PyErr_Format(parser_error, + "third item in terminal node must be an" + " integer, found %s", + Py_TYPE(temp)->tp_name); Py_DECREF(o); + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; } + Py_DECREF(o); } temp_str = PyUnicode_AsUTF8AndSize(temp, &len); if (temp_str == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); - return 0; + Py_DECREF(elem); + return NULL; } strn = (char *)PyObject_MALLOC(len + 1); if (strn == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); + Py_DECREF(elem); PyErr_NoMemory(); - return 0; + return NULL; } (void) memcpy(strn, temp_str, len + 1); Py_DECREF(temp); @@ -933,20 +945,21 @@ build_node_children(PyObject *tuple, node *root, int *line_num) * It has to be one or the other; this is an error. * Raise an exception. */ - PyObject *err = Py_BuildValue("os", elem, "unknown node type."); + PyObject *err = Py_BuildValue("Os", elem, "unknown node type."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } err = PyNode_AddChild(root, type, strn, *line_num, 0); if (err == E_NOMEM) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); - return (node *) PyErr_NoMemory(); + PyErr_NoMemory(); + return NULL; } if (err == E_OVERFLOW) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); PyErr_SetString(PyExc_ValueError, "unsupported number of child nodes"); @@ -957,14 +970,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) node* new_child = CHILD(root, i - 1); if (new_child != build_node_children(elem, new_child, line_num)) { - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } } else if (type == NEWLINE) { /* It's true: we increment the */ ++(*line_num); /* line number *after* the newline! */ } - Py_XDECREF(elem); + Py_DECREF(elem); } return root; } @@ -999,10 +1012,23 @@ build_node_tree(PyObject *tuple) if (num == encoding_decl) { encoding = PySequence_GetItem(tuple, 2); + if (encoding == NULL) { + PyErr_SetString(parser_error, "missed encoding"); + return NULL; + } + if (!PyUnicode_Check(encoding)) { + PyErr_Format(parser_error, + "encoding must be a string, found %.200s", + Py_TYPE(encoding)->tp_name); + Py_DECREF(encoding); + return NULL; + } /* tuple isn't borrowed anymore here, need to DECREF */ tuple = PySequence_GetSlice(tuple, 0, 2); - if (tuple == NULL) + if (tuple == NULL) { + Py_DECREF(encoding); return NULL; + } } res = PyNode_New(num); if (res != NULL) { @@ -1015,31 +1041,33 @@ build_node_tree(PyObject *tuple) const char *temp; temp = PyUnicode_AsUTF8AndSize(encoding, &len); if (temp == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); return NULL; } res->n_str = (char *)PyObject_MALLOC(len + 1); if (res->n_str == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); PyErr_NoMemory(); return NULL; } (void) memcpy(res->n_str, temp, len + 1); - Py_DECREF(encoding); - Py_DECREF(tuple); } } + if (encoding != NULL) { + Py_DECREF(encoding); + Py_DECREF(tuple); + } } else { /* The tuple is illegal -- if the number is neither TERMINAL nor * NONTERMINAL, we can't use it. Not sure the implementation * allows this condition, but the API doesn't preclude it. */ - PyObject *err = Py_BuildValue("os", tuple, + PyObject *err = Py_BuildValue("Os", tuple, "Illegal component tuple."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); @@ -1074,7 +1102,6 @@ parser__pickler(PyObject *self, PyObject *args) result = Py_BuildValue("O(O)", pickle_constructor, tuple); Py_DECREF(tuple); } - Py_DECREF(empty_dict); Py_DECREF(newargs); } finally: diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8f8ba255ec48f6..f4bbc8931b39fa 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1932,11 +1932,13 @@ _pystat_fromstructstat(STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); -#ifdef HAVE_LARGEFILE_SUPPORT +#if defined(HAVE_LARGEFILE_SUPPORT) || defined(MS_WINDOWS) + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(st->st_ino)); PyStructSequence_SET_ITEM(v, 1, - PyLong_FromLongLong((long long)st->st_ino)); + PyLong_FromUnsignedLongLong(st->st_ino)); #else - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long)st->st_ino)); + Py_BUILD_ASSERT(sizeof(unsigned long) >= sizeof(st->st_ino)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLong(st->st_ino)); #endif #ifdef MS_WINDOWS PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); @@ -6648,7 +6650,7 @@ static PyObject * os_setgroups(PyObject *module, PyObject *groups) /*[clinic end generated code: output=3fcb32aad58c5ecd input=fa742ca3daf85a7e]*/ { - int i, len; + Py_ssize_t i, len; gid_t grouplist[MAX_GROUPS]; if (!PySequence_Check(groups)) { @@ -6656,6 +6658,9 @@ os_setgroups(PyObject *module, PyObject *groups) return NULL; } len = PySequence_Size(groups); + if (len < 0) { + return NULL; + } if (len > MAX_GROUPS) { PyErr_SetString(PyExc_ValueError, "too many groups"); return NULL; @@ -7884,9 +7889,9 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) static Py_ssize_t -iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, int cnt, int type) +iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type) { - int i, j; + Py_ssize_t i, j; Py_ssize_t blen, total = 0; *iov = PyMem_New(struct iovec, cnt); @@ -7963,8 +7968,7 @@ static Py_ssize_t os_readv_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=792da062d3fcebdb input=e679eb5dbfa0357d]*/ { - int cnt; - Py_ssize_t n; + Py_ssize_t cnt, n; int async_err = 0; struct iovec *iov; Py_buffer *buf; @@ -7976,6 +7980,8 @@ os_readv_impl(PyObject *module, int fd, PyObject *buffers) } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) return -1; @@ -8114,15 +8120,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() headers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.hdr_cnt = PySequence_Size(headers); - if (sf.hdr_cnt > 0 && - (i = iov_setup(&(sf.headers), &hbuf, - headers, sf.hdr_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(headers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() header is too large"); return NULL; + } + if (i > 0) { + sf.hdr_cnt = (int)i; + i = iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } if (trailers != NULL) { @@ -8131,15 +8146,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() trailers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.trl_cnt = PySequence_Size(trailers); - if (sf.trl_cnt > 0 && - (i = iov_setup(&(sf.trailers), &tbuf, - trailers, sf.trl_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(trailers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() trailer is too large"); return NULL; + } + if (i > 0) { + sf.trl_cnt = (int)i; + i = iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } @@ -8409,7 +8433,7 @@ static Py_ssize_t os_writev_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=56565cfac3aac15b input=5b8d17fe4189d2fe]*/ { - int cnt; + Py_ssize_t cnt; Py_ssize_t result; int async_err = 0; struct iovec *iov; @@ -8421,6 +8445,8 @@ os_writev_impl(PyObject *module, int fd, PyObject *buffers) return -1; } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0) { return -1; @@ -11156,7 +11182,7 @@ typedef struct { PyObject *lstat; #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; - __int64 win32_file_index; + uint64_t win32_file_index; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -11419,7 +11445,8 @@ DirEntry_inode(DirEntry *self) self->win32_file_index = stat.st_ino; self->got_file_index = 1; } - return PyLong_FromLongLong((long long)self->win32_file_index); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->win32_file_index)); + return PyLong_FromUnsignedLongLong(self->win32_file_index); #else /* POSIX */ #ifdef HAVE_LARGEFILE_SUPPORT return PyLong_FromLongLong((long long)self->d_ino); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b3259d57aa114c..47f70a27f15846 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1191,7 +1191,7 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec Py_DECREF(self); return NULL; } -#if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT) +#if XML_COMBINED_VERSION >= 20100 || defined(XML_HAS_SET_HASH_SALT) /* This feature was added upstream in libexpat 2.1.0. Our expat copy * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT * to indicate that we can still use it. */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f4edc062fd6539..42aec59ca7f6e5 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -288,36 +288,6 @@ if_indextoname(index) -- return the corresponding interface name\n\ #include #endif -#ifdef HAVE_SOCKADDR_ALG -#include -#ifndef AF_ALG -#define AF_ALG 38 -#endif -#ifndef SOL_ALG -#define SOL_ALG 279 -#endif - -/* Linux 3.19 */ -#ifndef ALG_SET_AEAD_ASSOCLEN -#define ALG_SET_AEAD_ASSOCLEN 4 -#endif -#ifndef ALG_SET_AEAD_AUTHSIZE -#define ALG_SET_AEAD_AUTHSIZE 5 -#endif -/* Linux 4.8 */ -#ifndef ALG_SET_PUBKEY -#define ALG_SET_PUBKEY 6 -#endif - -#ifndef ALG_OP_SIGN -#define ALG_OP_SIGN 2 -#endif -#ifndef ALG_OP_VERIFY -#define ALG_OP_VERIFY 3 -#endif - -#endif /* HAVE_SOCKADDR_ALG */ - /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" @@ -1242,9 +1212,9 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto) { struct sockaddr_un *a = (struct sockaddr_un *) addr; #ifdef __linux__ - if (a->sun_path[0] == 0) { /* Linux abstract namespace */ - addrlen -= offsetof(struct sockaddr_un, sun_path); - return PyBytes_FromStringAndSize(a->sun_path, addrlen); + size_t linuxaddrlen = addrlen - offsetof(struct sockaddr_un, sun_path); + if (linuxaddrlen > 0 && a->sun_path[0] == 0) { /* Linux abstract namespace */ + return PyBytes_FromStringAndSize(a->sun_path, linuxaddrlen); } else #endif /* linux */ diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 3cce927e0b35f0..03f982b91083f3 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -98,6 +98,37 @@ typedef int socklen_t; #include #endif +#ifdef HAVE_SOCKADDR_ALG +#include +#ifndef AF_ALG +#define AF_ALG 38 +#endif +#ifndef SOL_ALG +#define SOL_ALG 279 +#endif + +/* Linux 3.19 */ +#ifndef ALG_SET_AEAD_ASSOCLEN +#define ALG_SET_AEAD_ASSOCLEN 4 +#endif +#ifndef ALG_SET_AEAD_AUTHSIZE +#define ALG_SET_AEAD_AUTHSIZE 5 +#endif +/* Linux 4.8 */ +#ifndef ALG_SET_PUBKEY +#define ALG_SET_PUBKEY 6 +#endif + +#ifndef ALG_OP_SIGN +#define ALG_OP_SIGN 2 +#endif +#ifndef ALG_OP_VERIFY +#define ALG_OP_VERIFY 3 +#endif + +#endif /* HAVE_SOCKADDR_ALG */ + + #ifndef Py__SOCKET_H #define Py__SOCKET_H #ifdef __cplusplus @@ -159,6 +190,9 @@ typedef union sock_addr { #ifdef HAVE_SYS_KERN_CONTROL_H struct sockaddr_ctl ctl; #endif +#ifdef HAVE_SOCKADDR_ALG + struct sockaddr_alg alg; +#endif } sock_addr_t; /* The object holding a socket. It holds some extra information, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ebd44ad525bd81..328b84f3338ea3 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -279,7 +279,7 @@ static PyTypeObject StructTimeType; static PyObject * tmtotuple(struct tm *p #ifndef HAVE_STRUCT_TM_TM_ZONE - , const char *zone, int gmtoff + , const char *zone, time_t gmtoff #endif ) { @@ -305,7 +305,7 @@ tmtotuple(struct tm *p #else PyStructSequence_SET_ITEM(v, 9, PyUnicode_DecodeLocale(zone, "surrogateescape")); - SET(10, gmtoff); + PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ #undef SET if (PyErr_Occurred()) { @@ -397,7 +397,7 @@ time_localtime(PyObject *self, PyObject *args) { struct tm local = buf; char zone[100]; - int gmtoff; + time_t gmtoff; strftime(zone, sizeof(zone), "%Z", &buf); gmtoff = timegm(&buf) - when; return tmtotuple(&local, zone, gmtoff); diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 59046aaae41f51..cccc033d2d9fab 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -1102,7 +1102,7 @@ get_decompress_func(void) _Py_IDENTIFIER(decompress); if (importing_zlib != 0) - /* Someone has a zlib.py[co] in their Zip file; + /* Someone has a zlib.pyc in their Zip file; let's avoid a stack overflow. */ return NULL; importing_zlib = 1; @@ -1261,7 +1261,7 @@ eq_mtime(time_t t1, time_t t2) return d <= 1; } -/* Given the contents of a .py[co] file in a buffer, unmarshal the data +/* Given the contents of a .pyc file in a buffer, unmarshal the data and return the code object. Return None if it the magic word doesn't match (we do this instead of raising an exception as we fall back to .py if available and we don't want to mask other errors). @@ -1403,7 +1403,7 @@ get_mtime_of_source(ZipImporter *self, PyObject *path) PyObject *toc_entry, *stripped; time_t mtime; - /* strip 'c' or 'o' from *.py[co] */ + /* strip 'c' from *.pyc */ if (PyUnicode_READY(path) == -1) return (time_t)-1; stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a8d69802508736..a9c8ca6f1f35a8 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -254,7 +254,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -400,11 +400,11 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) } else if (PySlice_Check(index)) { Py_ssize_t start, stop, step, slicelength, cur, i; - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), + &start, &stop, step); if (slicelength <= 0) return PyByteArray_FromStringAndSize("", 0); @@ -630,11 +630,11 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu } } else if (PySlice_Check(index)) { - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), &start, + &stop, step); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index d5c4fe6346fc53..625e242d563d89 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -542,7 +542,11 @@ _Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_index__doc__, "B.index(sub[, start[, end]]) -> int\n\ \n\ -Like B.find() but raise ValueError when the subsection is not found."); +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the subsection is not found."); PyObject * _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) @@ -579,7 +583,11 @@ _Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_rindex__doc__, "B.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like B.rfind() but raise ValueError when the subsection is not found."); +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raise ValueError when the subsection is not found."); PyObject * _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) @@ -811,4 +819,3 @@ PyDoc_STRVAR_shared(_Py_zfill__doc__, "\n" "Pad a numeric string B with zeros on the left, to fill a field\n" "of the specified width. B is never truncated."); - diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 5d484409600852..4c55294d9d8c8a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -528,6 +528,8 @@ byte_converter(PyObject *arg, char *p) return 0; } +static PyObject *_PyBytes_FromBuffer(PyObject *x); + static PyObject * format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) { @@ -564,8 +566,19 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) *plen = PyBytes_GET_SIZE(result); return result; } + /* does it support buffer protocol? */ + if (PyObject_CheckBuffer(v)) { + /* maybe we can avoid making a copy of the buffer object here? */ + result = _PyBytes_FromBuffer(v); + if (result == NULL) + return NULL; + *pbuf = PyBytes_AS_STRING(result); + *plen = PyBytes_GET_SIZE(result); + return result; + } PyErr_Format(PyExc_TypeError, - "%%b requires bytes, or an object that implements __bytes__, not '%.100s'", + "%%b requires a bytes-like object, " + "or an object that implements __bytes__, not '%.100s'", Py_TYPE(v)->tp_name); return NULL; } @@ -619,11 +632,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, Py_ssize_t len; char *pos; - pos = strchr(fmt + 1, '%'); + pos = (char *)memchr(fmt + 1, '%', fmtcnt); if (pos != NULL) len = pos - fmt; else - len = format_len - (fmt - format); + len = fmtcnt + 1; assert(len != 0); memcpy(res, fmt, len); @@ -1425,7 +1438,7 @@ bytes_concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -1670,11 +1683,11 @@ bytes_subscript(PyBytesObject* self, PyObject* item) char* result_buf; PyObject* result; - if (PySlice_GetIndicesEx(item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyBytes_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyBytes_FromStringAndSize("", 0); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d8a675f9fbfc6..22c4f856cd83e6 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -446,11 +446,15 @@ code_dealloc(PyCodeObject *co) static PyObject * code_sizeof(PyCodeObject *co, void *unused) { - Py_ssize_t res; + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co)); + _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; - res = _PyObject_SIZE(Py_TYPE(co)); if (co->co_cell2arg != NULL && co->co_cellvars != NULL) - res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(unsigned char); + res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(Py_ssize_t); + + if (co_extra != NULL) + res += co_extra->ce_size * sizeof(co_extra->ce_extras[0]); + return PyLong_FromSsize_t(res); } @@ -856,16 +860,15 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra *) o->co_extra; if (co_extra == NULL) { - o->co_extra = (_PyCodeObjectExtra*) PyMem_Malloc( - sizeof(_PyCodeObjectExtra)); - if (o->co_extra == NULL) { + co_extra = PyMem_Malloc(sizeof(_PyCodeObjectExtra)); + if (co_extra == NULL) { return -1; } - co_extra = (_PyCodeObjectExtra *) o->co_extra; co_extra->ce_extras = PyMem_Malloc( tstate->co_extra_user_count * sizeof(void*)); if (co_extra->ce_extras == NULL) { + PyMem_Free(co_extra); return -1; } @@ -874,20 +877,25 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { co_extra->ce_extras[i] = NULL; } + + o->co_extra = co_extra; } else if (co_extra->ce_size <= index) { - co_extra->ce_extras = PyMem_Realloc( + void** ce_extras = PyMem_Realloc( co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*)); - if (co_extra->ce_extras == NULL) { + if (ce_extras == NULL) { return -1; } - co_extra->ce_size = tstate->co_extra_user_count; - - for (Py_ssize_t i = co_extra->ce_size; i < co_extra->ce_size; i++) { - co_extra->ce_extras[i] = NULL; + for (Py_ssize_t i = co_extra->ce_size; + i < tstate->co_extra_user_count; + i++) { + ce_extras[i] = NULL; } + + co_extra->ce_extras = ce_extras; + co_extra->ce_size = tstate->co_extra_user_count; } co_extra->ce_extras[index] = extra; diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 31e12784cc34f9..cfaba688c684f6 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1025,11 +1025,11 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } cr.real = PyFloat_AsDouble(tmp); - cr.imag = 0.0; /* Shut up compiler warning */ + cr.imag = 0.0; Py_DECREF(tmp); } if (i == NULL) { - ci.real = 0.0; + ci.real = cr.imag; } else if (PyComplex_Check(i)) { ci = ((PyComplexObject*)i)->cval; @@ -1051,7 +1051,7 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ci_is_complex) { cr.real -= ci.imag; } - if (cr_is_complex) { + if (cr_is_complex && i != NULL) { ci.real += cr.imag; } return complex_subtype_from_doubles(type, cr.real, ci.real); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7b403bcecc5da..b0f583a067b4e1 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -436,7 +436,7 @@ static PyObject *empty_values[1] = { NULL }; /* #define DEBUG_PYDICT */ -#ifdef Py_DEBUG +#ifndef NDEBUG static int _PyDict_CheckConsistency(PyDictObject *mp) { @@ -676,7 +676,8 @@ Christian Tismer. lookdict() is general-purpose, and may return DKIX_ERROR if (and only if) a comparison raises an exception. lookdict_unicode() below is specialized to string keys, comparison of which can -never raise an exception; that function can never return DKIX_ERROR. +never raise an exception; that function can never return DKIX_ERROR when key +is string. Otherwise, it falls back to lookdict(). lookdict_unicode_nodummy is further specialized for string keys that cannot be the value. For both, when the key isn't found a DKIX_EMPTY is returned. hashpos returns @@ -1114,18 +1115,18 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) PyDictKeyEntry *ep, *ep0; Py_ssize_t hashpos, ix; + Py_INCREF(key); + Py_INCREF(value); if (mp->ma_values != NULL && !PyUnicode_CheckExact(key)) { if (insertion_resize(mp) < 0) - return -1; + goto Fail; } ix = mp->ma_keys->dk_lookup(mp, key, hash, &value_addr, &hashpos); - if (ix == DKIX_ERROR) { - return -1; - } + if (ix == DKIX_ERROR) + goto Fail; assert(PyUnicode_CheckExact(key) || mp->ma_keys->dk_lookup == lookdict); - Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); /* When insertion order is different from shared key, we can't share @@ -1134,10 +1135,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) if (_PyDict_HasSplitTable(mp) && ((ix >= 0 && *value_addr == NULL && mp->ma_used != ix) || (ix == DKIX_EMPTY && mp->ma_used != mp->ma_keys->dk_nentries))) { - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); ix = DKIX_EMPTY; } @@ -1146,16 +1145,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) /* Insert into new slot. */ if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); } ep0 = DK_ENTRIES(mp->ma_keys); ep = &ep0[mp->ma_keys->dk_nentries]; dk_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - Py_INCREF(key); ep->me_key = key; ep->me_hash = hash; if (mp->ma_values) { @@ -1183,6 +1179,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) assert(_PyDict_CheckConsistency(mp)); Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ + Py_DECREF(key); return 0; } @@ -1193,7 +1190,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) mp->ma_used++; mp->ma_version_tag = DICT_NEXT_VERSION(); assert(_PyDict_CheckConsistency(mp)); + Py_DECREF(key); return 0; + +Fail: + Py_DECREF(value); + Py_DECREF(key); + return -1; } /* @@ -1928,7 +1931,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, ESTIMATE_SIZE(Py_SIZE(iterable)))) { + if (dictresize(mp, ESTIMATE_SIZE(((PyDictObject *)iterable)->ma_used))) { Py_DECREF(d); return NULL; } @@ -2431,11 +2434,18 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) /* Update/merge with this (key, value) pair. */ key = PySequence_Fast_GET_ITEM(fast, 0); value = PySequence_Fast_GET_ITEM(fast, 1); + Py_INCREF(key); + Py_INCREF(value); if (override || PyDict_GetItem(d, key) == NULL) { int status = PyDict_SetItem(d, key, value); - if (status < 0) + if (status < 0) { + Py_DECREF(key); + Py_DECREF(value); goto Fail; + } } + Py_DECREF(key); + Py_DECREF(value); Py_DECREF(fast); Py_DECREF(item); } @@ -2736,14 +2746,15 @@ dict_equal(PyDictObject *a, PyDictObject *b) bval = NULL; else bval = *vaddr; - Py_DECREF(key); if (bval == NULL) { + Py_DECREF(key); Py_DECREF(aval); if (PyErr_Occurred()) return -1; return 0; } cmp = PyObject_RichCompareBool(aval, bval, Py_EQ); + Py_DECREF(key); Py_DECREF(aval); if (cmp <= 0) /* error or not equal */ return cmp; @@ -3632,7 +3643,7 @@ PyTypeObject PyDictIterValue_Type = { static PyObject * dictiter_iternextitem(dictiterobject *di) { - PyObject *key, *value, *result = di->di_result; + PyObject *key, *value, *result; Py_ssize_t i, n; PyDictObject *d = di->di_dict; @@ -3673,20 +3684,25 @@ dictiter_iternextitem(dictiterobject *di) } di->di_pos = i+1; di->len--; - if (result->ob_refcnt == 1) { + Py_INCREF(key); + Py_INCREF(value); + result = di->di_result; + if (Py_REFCNT(result) == 1) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ Py_INCREF(result); - Py_DECREF(PyTuple_GET_ITEM(result, 0)); - Py_DECREF(PyTuple_GET_ITEM(result, 1)); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); } else { result = PyTuple_New(2); if (result == NULL) return NULL; + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ } - Py_INCREF(key); - Py_INCREF(value); - PyTuple_SET_ITEM(result, 0, key); /* steals reference */ - PyTuple_SET_ITEM(result, 1, value); /* steals reference */ return result; fail: @@ -4179,6 +4195,7 @@ dictitems_iter(_PyDictViewObject *dv) static int dictitems_contains(_PyDictViewObject *dv, PyObject *obj) { + int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) return 0; @@ -4192,7 +4209,10 @@ dictitems_contains(_PyDictViewObject *dv, PyObject *obj) return -1; return 0; } - return PyObject_RichCompareBool(value, found, Py_EQ); + Py_INCREF(found); + result = PyObject_RichCompareBool(value, found, Py_EQ); + Py_DECREF(found); + return result; } static PySequenceMethods dictitems_as_sequence = { @@ -4376,15 +4396,19 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, } if (value == NULL) { res = PyDict_DelItem(dict, key); - if (cached != ((PyDictObject *)dict)->ma_keys) { + // Since key sharing dict doesn't allow deletion, PyDict_DelItem() + // always converts dict to combined form. + if ((cached = CACHED_KEYS(tp)) != NULL) { CACHED_KEYS(tp) = NULL; DK_DECREF(cached); } } else { - int was_shared = cached == ((PyDictObject *)dict)->ma_keys; + int was_shared = (cached == ((PyDictObject *)dict)->ma_keys); res = PyDict_SetItem(dict, key, value); - if (was_shared && cached != ((PyDictObject *)dict)->ma_keys) { + if (was_shared && + (cached = CACHED_KEYS(tp)) != NULL && + cached != ((PyDictObject *)dict)->ma_keys) { /* PyDict_SetItem() may call dictresize and convert split table * into combined table. In such case, convert it to split * table again and update type's shared key only when this is diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f63f06a145bcb1..d158b9768d45f5 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -686,6 +686,53 @@ ImportError_str(PyImportErrorObject *self) } } +static PyObject * +ImportError_getstate(PyImportErrorObject *self) +{ + PyObject *dict = ((PyBaseExceptionObject *)self)->dict; + if (self->name || self->path) { + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(path); + dict = dict ? PyDict_Copy(dict) : PyDict_New(); + if (dict == NULL) + return NULL; + if (self->name && _PyDict_SetItemId(dict, &PyId_name, self->name) < 0) { + Py_DECREF(dict); + return NULL; + } + if (self->path && _PyDict_SetItemId(dict, &PyId_path, self->path) < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } + else if (dict) { + Py_INCREF(dict); + return dict; + } + else { + Py_RETURN_NONE; + } +} + +/* Pickling support */ +static PyObject * +ImportError_reduce(PyImportErrorObject *self) +{ + PyObject *res; + PyObject *args; + PyObject *state = ImportError_getstate(self); + if (state == NULL) + return NULL; + args = ((PyBaseExceptionObject *)self)->args; + if (state == Py_None) + res = PyTuple_Pack(2, Py_TYPE(self), args); + else + res = PyTuple_Pack(3, Py_TYPE(self), args, state); + Py_DECREF(state); + return res; +} + static PyMemberDef ImportError_members[] = { {"msg", T_OBJECT, offsetof(PyImportErrorObject, msg), 0, PyDoc_STR("exception message")}, @@ -697,6 +744,7 @@ static PyMemberDef ImportError_members[] = { }; static PyMethodDef ImportError_methods[] = { + {"__reduce__", (PyCFunction)ImportError_reduce, METH_NOARGS}, {NULL} }; diff --git a/Objects/genobject.c b/Objects/genobject.c index 2680ab0e129da4..1c29e296afe2a8 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -575,8 +575,7 @@ _PyGen_SetStopIterationValue(PyObject *value) PyObject *e; if (value == NULL || - (!PyTuple_Check(value) && - !PyObject_TypeCheck(value, (PyTypeObject *) PyExc_StopIteration))) + (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) { /* Delay exception instantiation if we can */ PyErr_SetObject(PyExc_StopIteration, value); diff --git a/Objects/listobject.c b/Objects/listobject.c index dcd7b5efe5b0fe..547bdf0b95f1ee 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2153,8 +2153,8 @@ listindex(PyListObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -2420,10 +2420,11 @@ list_subscript(PyListObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return PyList_New(0); @@ -2469,10 +2470,11 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (step == 1) return list_ass_slice(self, start, stop, value); diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 515375772e6317..3dab2b98661695 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,17 +1,18 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes -disguised as a Python string. It is used to map bytecode offsets to source code -line #s for tracebacks and to identify line number boundaries for line tracing. +disguised as a Python bytes object. It is used to map bytecode offsets to +source code line #s for tracebacks and to identify line number boundaries for +line tracing. The array is conceptually a compressed list of (bytecode offset increment, line number increment) pairs. The details are important and delicate, best illustrated by example: byte code offset source code line number - 0 1 - 6 2 - 50 7 + 0 1 + 6 2 + 50 7 350 207 361 208 @@ -24,7 +25,8 @@ look like: The above doesn't really work, but it's a start. An unsigned byte (byte code offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the -above example contains two such values. So we make two tweaks: +above example contains two such values. (Note that before 3.6, line number +was also encoded by an unsigned byte.) So we make two tweaks: (a) there's a deep assumption that byte code offsets increase monotonically, and @@ -52,7 +54,7 @@ the example above, assemble_lnotab in compile.c should not (as was actually done until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 128, 0, 72. + 255, 0, 45, 127, 0, 73. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c @@ -83,30 +85,34 @@ Consider this code: 1: def f(a): 2: while a: -3: print 1, +3: print(1) 4: break 5: else: -6: print 2, +6: print(2) which compiles to this: - 2 0 SETUP_LOOP 19 (to 22) - >> 3 LOAD_FAST 0 (a) - 6 POP_JUMP_IF_FALSE 17 + 2 0 SETUP_LOOP 26 (to 28) + >> 2 LOAD_FAST 0 (a) + 4 POP_JUMP_IF_FALSE 18 - 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 3 6 LOAD_GLOBAL 0 (print) + 8 LOAD_CONST 1 (1) + 10 CALL_FUNCTION 1 + 12 POP_TOP - 4 13 BREAK_LOOP - 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + 4 14 BREAK_LOOP + 16 JUMP_ABSOLUTE 2 + >> 18 POP_BLOCK - 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM - >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 6 20 LOAD_GLOBAL 0 (print) + 22 LOAD_CONST 2 (2) + 24 CALL_FUNCTION 1 + 26 POP_TOP + >> 28 LOAD_CONST 0 (None) + 30 RETURN_VALUE -If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 +If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 18 and the co_lnotab will claim that execution has moved to line 4, which is wrong. In this case, we could instead associate the POP_BLOCK with line 5, but that would break jumps around loops without else clauses. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b1798a2073d108..e1ac7281783f38 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2285,10 +2285,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index a1142f3b09ad9e..32e7ecbe1e0436 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1227,7 +1227,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) _Py_AllocatedBlocks++; - assert(nelem <= PY_SSIZE_T_MAX / elsize); + assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; #ifdef WITH_VALGRIND diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 8449fc7a24d394..8f5fc434bd4955 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -668,6 +668,16 @@ static PyMappingMethods range_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; +static int +range_bool(rangeobject* self) +{ + return PyObject_IsTrue(self->length); +} + +static PyNumberMethods range_as_number = { + .nb_bool = (inquiry)range_bool, +}; + static PyObject * range_iter(PyObject *seq); static PyObject * range_reverse(PyObject *seq); @@ -707,7 +717,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)range_repr, /* tp_repr */ - 0, /* tp_as_number */ + &range_as_number, /* tp_as_number */ &range_as_sequence, /* tp_as_sequence */ &range_as_mapping, /* tp_as_mapping */ (hashfunc)range_hash, /* tp_hash */ diff --git a/Objects/setobject.c b/Objects/setobject.c index fdb9d3600d9fad..c1bc1e1234799f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -236,7 +236,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); found_active: Py_DECREF(key); @@ -304,7 +304,6 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry small_copy[PySet_MINSIZE]; assert(minused >= 0); - minused = (minused > 50000) ? minused * 2 : minused * 4; /* Find the smallest table size > minused. */ /* XXX speed-up with intrinsics */ @@ -646,8 +645,8 @@ set_merge(PySetObject *so, PyObject *otherset) * that there will be no (or few) overlapping keys. */ if ((so->fill + other->used)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + other->used) != 0) - return -1; + if (set_table_resize(so, (so->used + other->used)*2) != 0) + return -1; } so_entry = so->table; other_entry = other->table; @@ -990,7 +989,7 @@ set_update_internal(PySetObject *so, PyObject *other) if (dictsize < 0) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + dictsize) != 0) + if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { @@ -1511,7 +1510,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* If more than 1/4th are dummies, then resize them away. */ if ((size_t)(so->fill - so->used) <= (size_t)so->mask / 4) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } static PyObject * @@ -1551,20 +1550,26 @@ set_difference(PySetObject *so, PyObject *other) PyObject *key; Py_hash_t hash; setentry *entry; - Py_ssize_t pos = 0; + Py_ssize_t pos = 0, other_size; int rv; if (PySet_GET_SIZE(so) == 0) { return set_copy(so); } - if (!PyAnySet_Check(other) && !PyDict_CheckExact(other)) { + if (PyAnySet_Check(other)) { + other_size = PySet_GET_SIZE(other); + } + else if (PyDict_CheckExact(other)) { + other_size = PyDict_Size(other); + } + else { return set_copy_and_difference(so, other); } /* If len(so) much more than len(other), it's more efficient to simply copy * so and then iterate other looking for common elements. */ - if ((PySet_GET_SIZE(so) >> 2) > PyObject_Size(other)) { + if ((PySet_GET_SIZE(so) >> 2) > other_size) { return set_copy_and_difference(so, other); } diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 6a690213d14518..4e4b32d32441a8 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -197,6 +197,8 @@ PySlice_Unpack(PyObject *_r, PySliceObject *r = (PySliceObject*)_r; /* this is harder to get right than you might think */ + Py_BUILD_ASSERT(PY_SSIZE_T_MIN + 1 <= -PY_SSIZE_T_MAX); + if (r->step == Py_None) { *step = 1; } @@ -217,14 +219,14 @@ PySlice_Unpack(PyObject *_r, } if (r->start == Py_None) { - *start = *step < 0 ? PY_SSIZE_T_MAX-1 : 0;; + *start = *step < 0 ? PY_SSIZE_T_MAX : 0; } else { if (!_PyEval_SliceIndex(r->start, start)) return -1; } if (r->stop == Py_None) { - *stop = *step < 0 ? -PY_SSIZE_T_MAX : PY_SSIZE_T_MAX; + *stop = *step < 0 ? PY_SSIZE_T_MIN : PY_SSIZE_T_MAX; } else { if (!_PyEval_SliceIndex(r->stop, stop)) return -1; @@ -258,7 +260,7 @@ PySlice_AdjustIndices(Py_ssize_t length, *stop = (step < 0) ? -1 : 0; } } - else if (*stop >= length) { + else if (*stop >= length) { *stop = (step < 0) ? length - 1 : length; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c0ff499e72c7dd..0dada74dc7c986 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -522,8 +522,8 @@ tupleindex(PyTupleObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -720,11 +720,11 @@ tuplesubscript(PyTupleObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, - PyTuple_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyTuple_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyTuple_New(0); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 329261b037320a..271b93575c6dcf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3075,6 +3075,7 @@ type_getattro(PyTypeObject *type, PyObject *name) static int type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) { + int res; if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { PyErr_Format( PyExc_TypeError, @@ -3082,9 +3083,35 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) type->tp_name); return -1; } - if (PyObject_GenericSetAttr((PyObject *)type, name, value) < 0) - return -1; - return update_slot(type, name); + if (PyUnicode_Check(name)) { + if (PyUnicode_CheckExact(name)) { + if (PyUnicode_READY(name) == -1) + return -1; + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + PyUnicode_InternInPlace(&name); + if (!PyUnicode_CHECK_INTERNED(name)) { + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; + } + } + else { + /* Will fail in _PyObject_GenericSetAttrWithDict. */ + Py_INCREF(name); + } + res = PyObject_GenericSetAttr((PyObject *)type, name, value); + if (res == 0) { + res = update_slot(type, name); + } + Py_DECREF(name); + return res; } extern void @@ -6929,7 +6956,7 @@ init_slotdefs(void) /* Slots must be ordered by their offset in the PyHeapTypeObject. */ assert(!p[1].name || p->offset <= p[1].offset); p->name_strobj = PyUnicode_InternFromString(p->name); - if (!p->name_strobj) + if (!p->name_strobj || !PyUnicode_CHECK_INTERNED(p->name_strobj)) Py_FatalError("Out of memory interning slotdef names"); } slotdefs_initialized = 1; @@ -6954,6 +6981,9 @@ update_slot(PyTypeObject *type, PyObject *name) slotdef **pp; int offset; + assert(PyUnicode_CheckExact(name)); + assert(PyUnicode_CHECK_INTERNED(name)); + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. This could possibly be unified with the update_subclasses() recursion below, but carefully: @@ -6964,7 +6994,6 @@ update_slot(PyTypeObject *type, PyObject *name) init_slotdefs(); pp = ptrs; for (p = slotdefs; p->name; p++) { - /* XXX assume name is interned! */ if (p->name_strobj == name) *pp++ = p; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a5ae454b495dee..4e0c663e338a52 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3902,6 +3902,7 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) PyObject *output = NULL; if (arg == NULL) { Py_DECREF(*(PyObject**)addr); + *(PyObject**)addr = NULL; return 1; } @@ -11701,7 +11702,11 @@ unicode_hash(PyObject *self) PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ \n\ -Like S.find() but raise ValueError when the substring is not found."); +Return the lowest index in S where substring sub is found, \n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_index(PyObject *self, PyObject *args) @@ -12758,7 +12763,11 @@ unicode_rfind(PyObject *self, PyObject *args) PyDoc_STRVAR(rindex__doc__, "S.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like S.rfind() but raise ValueError when the substring is not found."); +Return the highest index in S where substring sub is found,\n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_rindex(PyObject *self, PyObject *args) @@ -13915,10 +13924,11 @@ unicode_subscript(PyObject* self, PyObject* item) int src_kind, dest_kind; Py_UCS4 ch, max_char, kind_limit; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self), + &start, &stop, step); if (slicelength <= 0) { _Py_RETURN_UNICODE_EMPTY(); diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ab6b23525552b8..9ca386da2563cb 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -24,6 +24,8 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) { self->hash = -1; self->wr_object = ob; + self->wr_prev = NULL; + self->wr_next = NULL; Py_XINCREF(callback); self->wr_callback = callback; } diff --git a/PC/getpathp.c b/PC/getpathp.c index 1eeebfe9c19451..e7be704a9a78dd 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -185,7 +185,7 @@ exists(wchar_t *filename) may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ +ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ { size_t n; @@ -196,7 +196,7 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/ n = wcsnlen_s(filename, MAXPATHLEN+1); if (n < MAXPATHLEN) { int exist = 0; - filename[n] = Py_OptimizeFlag ? L'o' : L'c'; + filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); if (!update_filename) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 70ab340905e4c9..f7f2858d7d9d99 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -105,9 +105,9 @@ if "%platf%"=="x64" ( ) ) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if exist "%HG%" set HGProperty=/p:HG="%HG%" -if not exist "%HG%" echo Cannot find Mercurial on PATH & set HGProperty= +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" +if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment call "%dir%env.bat" %vs_platf% >nul @@ -145,7 +145,7 @@ msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ - /p:UseTestMarker=%UseTestMarker% %HGProperty%^ + /p:UseTestMarker=%UseTestMarker% %GITProperty%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index a5185becee5a96..3a2656e961ef7f 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2j +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k set libraries=%libraries% sqlite-3.14.2.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 @@ -66,7 +66,7 @@ for %%e in (%libraries%) do ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - svn export %SVNROOT%%%e + svn export -q %SVNROOT%%%e ) ) diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 0e65811ae70731..580879930a83d0 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -96,6 +96,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsol EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj", "{12728250-16EC-4DC6-94D7-E21DD88947F8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -728,6 +730,38 @@ Global {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.ActiveCfg = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.Build.0 = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.ActiveCfg = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.Build.0 = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.ActiveCfg = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.Build.0 = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.ActiveCfg = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/python.props b/PCbuild/python.props index dde94f733a13ae..d6bfd0877e09df 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -45,7 +45,7 @@ $(ExternalsDir)sqlite-3.14.2.0\ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.2.2\ - $(ExternalsDir)openssl-1.0.2j\ + $(ExternalsDir)openssl-1.0.2k\ $(opensslDir)include32 $(opensslDir)include64 $(ExternalsDir)\nasm-2.11.06\ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 6b23d8ecacce3b..6ea184877f99cd 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -407,24 +407,24 @@ - hg - <_HG>$(HG) - <_HG Condition="$(HG.Contains(` `))">"$(HG)" + git + <_GIT>$(GIT) + <_GIT Condition="$(GIT.Contains(` `))">"$(GIT)" - + - - - + + + - $([System.IO.File]::ReadAllText('$(IntDir)hgbranch.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgversion.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgtag.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gittag.txt').Trim()) - + - HGVERSION="$(HgVersion)";HGTAG="$(HgTag)";HGBRANCH="$(HgBranch)";%(PreprocessorDefinitions) + GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c04ba4e46b9894..8924b3fcfa2e80 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -169,7 +169,7 @@ _lzma Homepage: http://tukaani.org/xz/ _ssl - Python wrapper for version 1.0.2j of the OpenSSL secure sockets + Python wrapper for version 1.0.2k of the OpenSSL secure sockets library, which is built by ssl.vcxproj Homepage: http://www.openssl.org/ diff --git a/PCbuild/rmpyc.py b/PCbuild/rmpyc.py index a1e75bb7ae5b4b..0b58f687729f7e 100644 --- a/PCbuild/rmpyc.py +++ b/PCbuild/rmpyc.py @@ -1,25 +1,19 @@ -# Remove all the .pyc and .pyo files under ../Lib. +# Remove all the .pyc files under ../Lib. def deltree(root): import os from os.path import join - npyc = npyo = 0 + npyc = 0 for root, dirs, files in os.walk(root): for name in files: - delete = False - if name.endswith('.pyc'): - delete = True + # to be thorough + if name.endswith(('.pyc', '.pyo')): npyc += 1 - elif name.endswith('.pyo'): - delete = True - npyo += 1 - - if delete: os.remove(join(root, name)) - return npyc, npyo + return npyc -npyc, npyo = deltree("../Lib") -print(npyc, ".pyc deleted,", npyo, ".pyo deleted") +npyc = deltree("../Lib") +print(npyc, ".pyc deleted") diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 35826727f3fa27..e73ac0457646e9 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -4,8 +4,8 @@ rem Usage: rt [-d] [-O] [-q] [-x64] regrtest_args rem -d Run Debug build (python_d.exe). Else release build. rem -O Run python.exe or python_d.exe (see -d) with -O. rem -q "quick" -- normally the tests are run twice, the first time -rem after deleting all the .py[co] files reachable from Lib/. -rem -q runs the tests just once, and without deleting .py[co] files. +rem after deleting all the .pyc files reachable from Lib/. +rem -q runs the tests just once, and without deleting .pyc files. rem -x64 Run the 64-bit build of python (or python_d if -d was specified) rem from the 'amd64' dir instead of the 32-bit build in this dir. rem All leading instances of these switches are shifted off, and @@ -45,7 +45,7 @@ set exe=%prefix%python%suffix%.exe set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% if defined qmode goto Qmode -echo Deleting .pyc/.pyo files ... +echo Deleting .pyc files ... "%exe%" "%pcbuild%rmpyc.py" echo Cleaning _pth files ... @@ -55,7 +55,7 @@ echo on %cmd% @echo off -echo About to run again without deleting .pyc/.pyo first: +echo About to run again without deleting .pyc first: pause :Qmode diff --git a/Python/ast.c b/Python/ast.c index 217ea14bf310be..ed05a1e53bdb5e 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4170,9 +4170,11 @@ decode_unicode_with_escapes(struct compiling *c, const node *n, const char *s, while (s < end) { if (*s == '\\') { *p++ = *s++; - if (*s & 0x80) { + if (s >= end || *s & 0x80) { strcpy(p, "u005c"); p += 5; + if (s >= end) + break; } } if (*s & 0x80) { /* XXX inefficient */ @@ -4325,30 +4327,37 @@ fstring_find_literal(const char **str, const char *end, int raw, brace (which isn't part of a unicode name escape such as "\N{EULER CONSTANT}"), or the end of the string. */ - const char *literal_start = *str; - const char *literal_end; - int in_named_escape = 0; + const char *s = *str; + const char *literal_start = s; int result = 0; assert(*literal == NULL); - for (; *str < end; (*str)++) { - char ch = **str; - if (!in_named_escape && ch == '{' && (*str)-literal_start >= 2 && - *(*str-2) == '\\' && *(*str-1) == 'N') { - in_named_escape = 1; - } else if (in_named_escape && ch == '}') { - in_named_escape = 0; - } else if (ch == '{' || ch == '}') { + while (s < end) { + char ch = *s++; + if (!raw && ch == '\\' && s < end) { + ch = *s++; + if (ch == 'N') { + if (s < end && *s++ == '{') { + while (s < end && *s++ != '}') { + } + continue; + } + break; + } + if (ch == '{' && warn_invalid_escape_sequence(c, n, ch) < 0) { + return -1; + } + } + if (ch == '{' || ch == '}') { /* Check for doubled braces, but only at the top level. If we checked at every level, then f'{0:{3}}' would fail with the two closing braces. */ if (recurse_lvl == 0) { - if (*str+1 < end && *(*str+1) == ch) { + if (s < end && *s == ch) { /* We're going to tell the caller that the literal ends here, but that they should continue scanning. But also skip over the second brace when we resume scanning. */ - literal_end = *str+1; - *str += 2; + *str = s + 1; result = 1; goto done; } @@ -4356,6 +4365,7 @@ fstring_find_literal(const char **str, const char *end, int raw, /* Where a single '{' is the start of a new expression, a single '}' is not allowed. */ if (ch == '}') { + *str = s - 1; ast_error(c, n, "f-string: single '}' is not allowed"); return -1; } @@ -4363,21 +4373,22 @@ fstring_find_literal(const char **str, const char *end, int raw, /* We're either at a '{', which means we're starting another expression; or a '}', which means we're at the end of this f-string (for a nested format_spec). */ + s--; break; } } - literal_end = *str; - assert(*str <= end); - assert(*str == end || **str == '{' || **str == '}'); + *str = s; + assert(s <= end); + assert(s == end || *s == '{' || *s == '}'); done: - if (literal_start != literal_end) { + if (literal_start != s) { if (raw) *literal = PyUnicode_DecodeUTF8Stateful(literal_start, - literal_end-literal_start, + s - literal_start, NULL, NULL); else *literal = decode_unicode_with_escapes(c, n, literal_start, - literal_end-literal_start); + s - literal_start); if (!*literal) return -1; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ef5a34cae900b6..597e26ec69a95a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1927,12 +1927,15 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *result; size_t len; + /* stdin is a text stream, so it must have an encoding. */ stdin_encoding = _PyObject_GetAttrId(fin, &PyId_encoding); stdin_errors = _PyObject_GetAttrId(fin, &PyId_errors); - if (!stdin_encoding || !stdin_errors) - /* stdin is a text stream, so it must have an - encoding. */ + if (!stdin_encoding || !stdin_errors || + !PyUnicode_Check(stdin_encoding) || + !PyUnicode_Check(stdin_errors)) { + tty = 0; goto _readline_errors; + } stdin_encoding_str = PyUnicode_AsUTF8(stdin_encoding); stdin_errors_str = PyUnicode_AsUTF8(stdin_errors); if (!stdin_encoding_str || !stdin_errors_str) @@ -1948,8 +1951,12 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *stringpo; stdout_encoding = _PyObject_GetAttrId(fout, &PyId_encoding); stdout_errors = _PyObject_GetAttrId(fout, &PyId_errors); - if (!stdout_encoding || !stdout_errors) + if (!stdout_encoding || !stdout_errors || + !PyUnicode_Check(stdout_encoding) || + !PyUnicode_Check(stdout_errors)) { + tty = 0; goto _readline_errors; + } stdout_encoding_str = PyUnicode_AsUTF8(stdout_encoding); stdout_errors_str = PyUnicode_AsUTF8(stdout_errors); if (!stdout_encoding_str || !stdout_errors_str) @@ -2003,13 +2010,17 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_XDECREF(po); PyMem_FREE(s); return result; + _readline_errors: Py_XDECREF(stdin_encoding); Py_XDECREF(stdout_encoding); Py_XDECREF(stdin_errors); Py_XDECREF(stdout_errors); Py_XDECREF(po); - return NULL; + if (tty) + return NULL; + + PyErr_Clear(); } /* Fallback if we're not interactive */ diff --git a/Python/ceval.c b/Python/ceval.c index d5172b9631f43d..5dc0444a1acf5c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1409,9 +1409,15 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(BINARY_MODULO) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *res = PyUnicode_CheckExact(dividend) ? - PyUnicode_Format(dividend, divisor) : - PyNumber_Remainder(dividend, divisor); + PyObject *res; + if (PyUnicode_CheckExact(dividend) && ( + !PyUnicode_Check(divisor) || PyUnicode_CheckExact(divisor))) { + // fast path; string formatting, but not if the RHS is a str subclass + // (see issue28598) + res = PyUnicode_Format(dividend, divisor); + } else { + res = PyNumber_Remainder(dividend, divisor); + } Py_DECREF(divisor); Py_DECREF(dividend); SET_TOP(res); @@ -1898,13 +1904,13 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(iter); if (awaitable == NULL) { - SET_TOP(NULL); - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __aiter__: %.100s", Py_TYPE(iter)->tp_name); + SET_TOP(NULL); Py_DECREF(iter); goto error; } else { @@ -1963,7 +1969,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", @@ -2851,13 +2857,16 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(IMPORT_STAR) { PyObject *from = POP(), *locals; int err; - if (PyFrame_FastToLocalsWithError(f) < 0) + if (PyFrame_FastToLocalsWithError(f) < 0) { + Py_DECREF(from); goto error; + } locals = f->f_locals; if (locals == NULL) { PyErr_SetString(PyExc_SystemError, "no locals found during 'import *'"); + Py_DECREF(from); goto error; } err = import_all_from(locals, from); @@ -4690,11 +4699,7 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) assert(!PyErr_Occurred()); #endif - if (args == NULL) { - return _PyObject_FastCallDict(func, NULL, 0, kwargs); - } - - if (!PyTuple_Check(args)) { + if (args != NULL && !PyTuple_Check(args)) { PyErr_SetString(PyExc_TypeError, "argument list must be a tuple"); return NULL; @@ -4706,7 +4711,12 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) return NULL; } - return PyObject_Call(func, args, kwargs); + if (args == NULL) { + return _PyObject_FastCallDict(func, NULL, 0, kwargs); + } + else { + return PyObject_Call(func, args, kwargs); + } } const char * @@ -5061,17 +5071,13 @@ do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict) /* Extract a slice index from a PyLong or an object with the nb_index slot defined, and store in *pi. Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, - and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. + and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN. Return 0 on error, 1 on success. */ -/* Note: If v is NULL, return success without storing into *pi. This - is because_PyEval_SliceIndex() is called by apply_slice(), which can be - called by the SLICE opcode with v and/or w equal to NULL. -*/ int _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) { - if (v != NULL) { + if (v != Py_None) { Py_ssize_t x; if (PyIndex_Check(v)) { x = PyNumber_AsSsize_t(v, NULL); @@ -5089,6 +5095,26 @@ _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) return 1; } +int +_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) +{ + Py_ssize_t x; + if (PyIndex_Check(v)) { + x = PyNumber_AsSsize_t(v, NULL); + if (x == -1 && PyErr_Occurred()) + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + return 0; + } + *pi = x; + return 1; +} + + #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" diff --git a/Python/compile.c b/Python/compile.c index 0e1607585245b9..6255ec7d47f5a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1043,7 +1043,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case CALL_FUNCTION_KW: return -oparg-1; case CALL_FUNCTION_EX: - return - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0); + return -1 - ((oparg & 0x01) != 0); case MAKE_FUNCTION: return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); diff --git a/Python/fileutils.c b/Python/fileutils.c index e84d66e99a473d..f3764e4b3c5a2d 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -583,7 +583,7 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); result->st_nlink = info->nNumberOfLinks; - result->st_ino = (((__int64)info->nFileIndexHigh)<<32) + info->nFileIndexLow; + result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; if (reparse_tag == IO_REPARSE_TAG_SYMLINK) { /* first clear the S_IFMT bits */ result->st_mode ^= (result->st_mode & S_IFMT); @@ -653,7 +653,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) _Py_attribute_data_to_stat(&info, 0, status); /* specific to fstat() */ - status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow; return 0; #else return fstat(fd, status); diff --git a/Python/importlib.h b/Python/importlib.h index 195fbfd13b6227..4607c40de60e7c 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -392,7 +392,7 @@ const unsigned char _Py_M__importlib[] = { 111,118,101,100,197,0,0,0,115,2,0,0,0,0,8,114, 58,0,0,0,114,33,0,0,0,41,1,218,9,118,101,114, 98,111,115,105,116,121,99,1,0,0,0,1,0,0,0,3, - 0,0,0,5,0,0,0,71,0,0,0,115,54,0,0,0, + 0,0,0,4,0,0,0,71,0,0,0,115,54,0,0,0, 116,0,106,1,106,2,124,1,107,5,114,50,124,0,106,3, 100,6,131,1,115,30,100,3,124,0,23,0,125,0,116,4, 124,0,106,5,124,2,142,0,116,0,106,6,100,4,141,2, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 2f40978fb78e13..886aa68009ec03 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -578,7 +578,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,17,95,102,105,110,100,95,109,111,100,117,108,101,95,115, 104,105,109,157,1,0,0,115,10,0,0,0,0,10,14,1, 16,1,4,1,22,1,114,127,0,0,0,99,4,0,0,0, - 0,0,0,0,11,0,0,0,22,0,0,0,67,0,0,0, + 0,0,0,0,11,0,0,0,19,0,0,0,67,0,0,0, 115,136,1,0,0,105,0,125,4,124,2,100,1,107,9,114, 22,124,2,124,4,100,2,60,0,110,4,100,3,125,2,124, 3,100,1,107,9,114,42,124,3,124,4,100,4,60,0,124, diff --git a/Python/marshal.c b/Python/marshal.c index 87a4b240a41c87..7b12ab7510dcfb 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -549,7 +549,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_lnotab, p); } else if (PyObject_CheckBuffer(v)) { - /* Write unknown bytes-like objects as a byte string */ + /* Write unknown bytes-like objects as a bytes object */ Py_buffer view; if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { w_byte(TYPE_UNKNOWN, p); @@ -1086,7 +1086,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (bytes object size out of range)"); break; } v = PyBytes_FromStringAndSize((char *)NULL, n); @@ -1110,7 +1110,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } goto _read_ascii; @@ -1150,7 +1150,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } if (n != 0) { @@ -1612,7 +1612,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) if (wf.ptr - base > PY_SSIZE_T_MAX) { Py_DECREF(wf.str); PyErr_SetString(PyExc_OverflowError, - "too much marshal data for a string"); + "too much marshal data for a bytes object"); return NULL; } if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) @@ -1658,8 +1658,7 @@ PyDoc_STRVAR(dump_doc, "dump(value, file[, version])\n\ \n\ Write the value on the open file. The value must be a supported type.\n\ -The file must be an open file object such as sys.stdout or returned by\n\ -open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\ +The file must be a writeable binary file.\n\ \n\ If the value has (or contains an object that has) an unsupported type, a\n\ ValueError exception is raised - but garbage data will also be written\n\ @@ -1715,8 +1714,7 @@ PyDoc_STRVAR(load_doc, Read one value from the open file and return it. If no valid value is\n\ read (e.g. because the data has a different Python version's\n\ incompatible marshal format), raise EOFError, ValueError or TypeError.\n\ -The file must be an open file object opened in binary mode ('rb' or\n\ -'r+b').\n\ +The file must be a readable binary file.\n\ \n\ Note: If an object containing an unsupported type was marshalled with\n\ dump(), load() will substitute None for the unmarshallable type."); @@ -1735,7 +1733,7 @@ marshal_dumps(PyObject *self, PyObject *args) PyDoc_STRVAR(dumps_doc, "dumps(value[, version])\n\ \n\ -Return the string that would be written to a file by dump(value, file).\n\ +Return the bytes object that would be written to a file by dump(value, file).\n\ The value must be a supported type. Raise a ValueError exception if\n\ value has (or contains an object that has) an unsupported type.\n\ \n\ @@ -1771,8 +1769,8 @@ marshal_loads(PyObject *self, PyObject *args) PyDoc_STRVAR(loads_doc, "loads(bytes)\n\ \n\ -Convert the bytes object to a value. If no valid value is found, raise\n\ -EOFError, ValueError or TypeError. Extra characters in the input are\n\ +Convert the bytes-like object to a value. If no valid value is found,\n\ +raise EOFError, ValueError or TypeError. Extra bytes in the input are\n\ ignored."); static PyMethodDef marshal_methods[] = { @@ -1810,8 +1808,8 @@ Functions:\n\ \n\ dump() -- write value to a file\n\ load() -- read value from a file\n\ -dumps() -- write value to a string\n\ -loads() -- read value from a string"); +dumps() -- marshal value as a bytes object\n\ +loads() -- read value from a bytes-like object"); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a4f7f823bc695d..640271fd20b709 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -88,7 +88,7 @@ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ -int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ +int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ @@ -1044,6 +1044,14 @@ initsite(void) static int is_valid_fd(int fd) { +#ifdef __APPLE__ + /* bpo-30225: On macOS Tiger, when stdout is redirected to a pipe + and the other side of the pipe is closed, dup(1) succeed, whereas + fstat(1, &st) fails with EBADF. Prefer fstat() over dup() to detect + such error. */ + struct stat st; + return (fstat(fd, &st) == 0); +#else int fd2; if (fd < 0) return 0; @@ -1056,6 +1064,7 @@ is_valid_fd(int fd) close(fd2); _Py_END_SUPPRESS_IPH return fd2 >= 0; +#endif } /* returns Py_None if the fd is not valid */ diff --git a/Python/pystate.c b/Python/pystate.c index 65c244e6f73617..ccb0092c42b3a4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -743,6 +743,10 @@ _PyGILState_Fini(void) void _PyGILState_Reinit(void) { +#ifdef WITH_THREAD + head_mutex = NULL; + HEAD_INIT(); +#endif PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_delete_key(autoTLSkey); if ((autoTLSkey = PyThread_create_key()) == -1) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 52034ff7fb5ddf..99e6b5ea018ea1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1942,9 +1942,9 @@ _PySys_Init(void) PyUnicode_FromString(Py_GetVersion())); SET_SYS_FROM_STRING("hexversion", PyLong_FromLong(PY_VERSION_HEX)); - SET_SYS_FROM_STRING("_mercurial", - Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), - _Py_hgversion())); + SET_SYS_FROM_STRING("_git", + Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(), + _Py_gitversion())); SET_SYS_FROM_STRING("dont_write_bytecode", PyBool_FromLong(Py_DontWriteBytecodeFlag)); SET_SYS_FROM_STRING("api_version", diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 27e0dc84bcb52e..ba7393f03de6ae 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -143,6 +143,8 @@ typedef struct { } pthread_lock; #define CHECK_STATUS(name) if (status != 0) { perror(name); error = 1; } +#define CHECK_STATUS_PTHREAD(name) if (status != 0) { fprintf(stderr, \ + "%s: %s\n", name, strerror(status)); error = 1; } /* * Initialization. @@ -417,7 +419,7 @@ PyThread_allocate_lock(void) status = pthread_mutex_init(&lock->mut, pthread_mutexattr_default); - CHECK_STATUS("pthread_mutex_init"); + CHECK_STATUS_PTHREAD("pthread_mutex_init"); /* Mark the pthread mutex underlying a Python mutex as pure happens-before. We can't simply mark the Python-level mutex as a mutex because it can be @@ -427,7 +429,7 @@ PyThread_allocate_lock(void) status = pthread_cond_init(&lock->lock_released, pthread_condattr_default); - CHECK_STATUS("pthread_cond_init"); + CHECK_STATUS_PTHREAD("pthread_cond_init"); if (error) { PyMem_RawFree((void *)lock); @@ -452,10 +454,10 @@ PyThread_free_lock(PyThread_type_lock lock) * and must have the cond destroyed first. */ status = pthread_cond_destroy( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_destroy"); + CHECK_STATUS_PTHREAD("pthread_cond_destroy"); status = pthread_mutex_destroy( &thelock->mut ); - CHECK_STATUS("pthread_mutex_destroy"); + CHECK_STATUS_PTHREAD("pthread_mutex_destroy"); PyMem_RawFree((void *)thelock); } @@ -472,7 +474,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, lock, microseconds, intr_flag)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); if (thelock->locked == 0) { success = PY_LOCK_ACQUIRED; @@ -494,13 +496,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, &thelock->mut, &ts); if (status == ETIMEDOUT) break; - CHECK_STATUS("pthread_cond_timed_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); } else { status = pthread_cond_wait( &thelock->lock_released, &thelock->mut); - CHECK_STATUS("pthread_cond_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_wait"); } if (intr_flag && status == 0 && thelock->locked) { @@ -518,7 +520,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, } if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); if (error) success = PY_LOCK_FAILURE; dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", @@ -536,16 +538,16 @@ PyThread_release_lock(PyThread_type_lock lock) dprintf(("PyThread_release_lock(%p) called\n", lock)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[3]"); thelock->locked = 0; /* wake up someone (anyone, if any) waiting on the lock */ status = pthread_cond_signal( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_signal"); + CHECK_STATUS_PTHREAD("pthread_cond_signal"); status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[3]"); } #endif /* USE_SEMAPHORES */ diff --git a/README b/README deleted file mode 100644 index 4fc97e6013063e..00000000000000 --- a/README +++ /dev/null @@ -1,234 +0,0 @@ -This is Python version 3.6.1 release candidate 1 -================================================ - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights -reserved. - -Python 3.x is a new version of the language, which is incompatible with the -2.x line of releases. The language is mostly the same, but many details, -especially how built-in objects like dictionaries and strings work, -have changed considerably, and a lot of deprecated features have finally -been removed. - - -Build Instructions ------------------- - -On Unix, Linux, BSD, OSX, and Cygwin: - - ./configure - make - make test - sudo make install - -This will install Python as python3. - -You can pass many options to the configure script; run "./configure --help" to -find out more. On OSX and Cygwin, the executable is called python.exe; -elsewhere it's just python. - -On Mac OS X, if you have configured Python with --enable-framework, you should -use "make frameworkinstall" to do the installation. Note that this installs the -Python executable in a place that is not normally on your PATH, you may want to -set up a symlink in /usr/local/bin. - -On Windows, see PCbuild/readme.txt. - -If you wish, you can create a subdirectory and invoke configure from there. -For example: - - mkdir debug - cd debug - ../configure --with-pydebug - make - make test - -(This will fail if you *also* built at the top-level directory. -You should do a "make clean" at the toplevel first.) - -To get an optimized build of Python, "configure --enable-optimizations" before -you run make. This sets the default make targets up to enable Profile Guided -Optimization (PGO) and may be used to auto-enable Link Time Optimization (LTO) -on some platforms. For more details, see the sections bellow. - - -Profile Guided Optimization ---------------------------- - -PGO takes advantage of recent versions of the GCC or Clang compilers. -If ran, "make profile-opt" will do several steps. - -First, the entire Python directory is cleaned of temporary files that -may have resulted in a previous compilation. - -Then, an instrumented version of the interpreter is built, using suitable -compiler flags for each flavour. Note that this is just an intermediary -step and the binary resulted after this step is not good for real life -workloads, as it has profiling instructions embedded inside. - -After this instrumented version of the interpreter is built, the Makefile -will automatically run a training workload. This is necessary in order to -profile the interpreter execution. Note also that any output, both stdout -and stderr, that may appear at this step is suppressed. - -Finally, the last step is to rebuild the interpreter, using the information -collected in the previous one. The end result will be a Python binary -that is optimized and suitable for distribution or production installation. - - -Link Time Optimization ----------------------- - -Enabled via configure's --with-lto flag. LTO takes advantages of recent -compiler toolchains ability to optimize across the otherwise arbitrary .o file -boundary when building final executables or shared libraries for additional -performance gains. - - -What's New ----------- - -We have a comprehensive overview of the changes in the "What's New in -Python 3.6" document, found at - - https://docs.python.org/3.6/whatsnew/3.6.html - -For a more detailed change log, read Misc/NEWS (though this file, too, -is incomplete, and also doesn't list anything merged in from the 2.7 -release under development). - -If you want to install multiple versions of Python see the section below -entitled "Installing multiple versions". - - -Documentation -------------- - -Documentation for Python 3.6 is online, updated daily: - - https://docs.python.org/3.6/ - -It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version -is primarily for documentation authors, translators, and people with special -formatting requirements. - -If you would like to contribute to the development of Python, relevant -documentation is available at: - - https://docs.python.org/devguide/ - -For information about building Python's documentation, refer to Doc/README.txt. - - -Converting From Python 2.x to 3.x ---------------------------------- - -Python starting with 2.6 contains features to help locating code that needs to -be changed, such as optional warnings when deprecated features are used, and -backported versions of certain key Python 3.x features. - -A source-to-source translation tool, "2to3", can take care of the mundane task -of converting large amounts of source code. It is not a complete solution but -is complemented by the deprecation warnings in 2.6. See -https://docs.python.org/3.6/library/2to3.html for more information. - - -Testing -------- - -To test the interpreter, type "make test" in the top-level directory. -The test set produces some output. You can generally ignore the messages -about skipped tests due to optional features which can't be imported. -If a message is printed about a failed test or a traceback or core dump -is produced, something is wrong. - -By default, tests are prevented from overusing resources like disk space and -memory. To enable these tests, run "make testall". - -IMPORTANT: If the tests fail and you decide to mail a bug report, *don't* -include the output of "make test". It is useless. Run the failing test -manually, as follows: - - ./python -m test -v test_whatever - -(substituting the top of the source tree for '.' if you built in a different -directory). This runs the test in verbose mode. - - -Installing multiple versions ----------------------------- - -On Unix and Mac systems if you intend to install multiple versions of Python -using the same installation prefix (--prefix argument to the configure script) -you must take care that your primary python executable is not overwritten by the -installation of a different version. All files and directories installed using -"make altinstall" contain the major and minor version and can thus live -side-by-side. "make install" also creates ${prefix}/bin/python3 which refers to -${prefix}/bin/pythonX.Y. If you intend to install multiple versions using the -same prefix you must decide which version (if any) is your "primary" version. -Install that version using "make install". Install all other versions using -"make altinstall". - -For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the -primary version, you would execute "make install" in your 3.6 build directory -and "make altinstall" in the others. - - -Issue Tracker and Mailing List ------------------------------- - -We're soliciting bug reports about all aspects of the language. Fixes are also -welcome, preferably in unified diff format. Please use the issue tracker: - - https://bugs.python.org/ - -If you're not sure whether you're dealing with a bug or a feature, use the -mailing list: - - python-dev@python.org - -To subscribe to the list, use the mailman form: - - https://mail.python.org/mailman/listinfo/python-dev/ - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or python-ideas mailing lists for initial feedback. A Python -Enhancement Proposal (PEP) may be submitted if your idea gains ground. All -current PEPs, as well as guidelines for submitting a new PEP, are listed at -https://www.python.org/dev/peps/. - - -Release Schedule ----------------- - -See PEP 494 for release details: https://www.python.org/dev/peps/pep-0494/ - - -Copyright and License Information ---------------------------------- - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. - -Copyright (c) 2000 BeOpen.com. All rights reserved. - -Copyright (c) 1995-2001 Corporation for National Research Initiatives. All -rights reserved. - -Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. - -See the file "LICENSE" for information on the history of this software, -terms & conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. - -This Python distribution contains *no* GNU General Public License (GPL) code, -so it may be used in proprietary projects. There are interfaces to some GNU -code but these are entirely optional. - -All trademarks referenced herein are property of their respective holders. - diff --git a/README.rst b/README.rst new file mode 100644 index 00000000000000..b0ce13599ad653 --- /dev/null +++ b/README.rst @@ -0,0 +1,251 @@ +This is Python version 3.6.1+ +============================= + +.. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 + :alt: CPython build status on Travis CI + :target: https://travis-ci.org/python/cpython + +.. image:: https://ci.appveyor.com/api/projects/status/4mew1a93xdkbf5ua/branch/3.6?svg=true + :alt: CPython build status on Appveyor + :target: https://ci.appveyor.com/project/python/cpython/branch/3.6 + +.. image:: https://codecov.io/gh/python/cpython/branch/3.6/graph/badge.svg + :alt: CPython code coverage on Codecov + :target: https://codecov.io/gh/python/cpython + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights +reserved. + +See the end of this file for further copyright and license information. + +General Information +------------------- + +- Website: https://www.python.org +- Source code: https://github.com/python/cpython +- Issue tracker: https://bugs.python.org +- Documentation: https://docs.python.org +- Developer's Guide: https://docs.python.org/devguide/ + +Contributing to CPython +----------------------- + +For more complete instructions on contributing to CPython development, +see the `Developer Guide`_. + +.. _Developer Guide: https://docs.python.org/devguide/ + +Using Python +------------ + +Installable Python kits, and information about using Python, are available at +`python.org`_. + +.. _python.org: https://www.python.org/ + + +Build Instructions +------------------ + +On Unix, Linux, BSD, macOS, and Cygwin:: + + ./configure + make + make test + sudo make install + +This will install Python as python3. + +You can pass many options to the configure script; run ``./configure --help`` +to find out more. On macOS and Cygwin, the executable is called ``python.exe``; +elsewhere it's just ``python``. + +On macOS, if you have configured Python with ``--enable-framework``, you +should use ``make frameworkinstall`` to do the installation. Note that this +installs the Python executable in a place that is not normally on your PATH, +you may want to set up a symlink in ``/usr/local/bin``. + +On Windows, see `PCbuild/readme.txt +`_. + +If you wish, you can create a subdirectory and invoke configure from there. +For example:: + + mkdir debug + cd debug + ../configure --with-pydebug + make + make test + +(This will fail if you *also* built at the top-level directory. You should do +a ``make clean`` at the toplevel first.) + +To get an optimized build of Python, ``configure --enable-optimizations`` +before you run ``make``. This sets the default make targets up to enable +Profile Guided Optimization (PGO) and may be used to auto-enable Link Time +Optimization (LTO) on some platforms. For more details, see the sections +below. + + +Profile Guided Optimization +--------------------------- + +PGO takes advantage of recent versions of the GCC or Clang compilers. If ran, +``make profile-opt`` will do several steps. + +First, the entire Python directory is cleaned of temporary files that may have +resulted in a previous compilation. + +Then, an instrumented version of the interpreter is built, using suitable +compiler flags for each flavour. Note that this is just an intermediary step +and the binary resulted after this step is not good for real life workloads, as +it has profiling instructions embedded inside. + +After this instrumented version of the interpreter is built, the Makefile will +automatically run a training workload. This is necessary in order to profile +the interpreter execution. Note also that any output, both stdout and stderr, +that may appear at this step is suppressed. + +Finally, the last step is to rebuild the interpreter, using the information +collected in the previous one. The end result will be a Python binary that is +optimized and suitable for distribution or production installation. + + +Link Time Optimization +---------------------- + +Enabled via configure's ``--with-lto`` flag. LTO takes advantage of the +ability of recent compiler toolchains to optimize across the otherwise +arbitrary ``.o`` file boundary when building final executables or shared +libraries for additional performance gains. + + +What's New +---------- + +We have a comprehensive overview of the changes in the `What's New in Python +3.6 `_ document. For a more +detailed change log, read `Misc/NEWS +`_, but a full +accounting of changes can only be gleaned from the `commit history +`_. + +If you want to install multiple versions of Python see the section below +entitled "Installing multiple versions". + + +Documentation +------------- + +`Documentation for Python 3.6 `_ is online, +updated daily. + +It can also be downloaded in many formats for faster access. The documentation +is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is primarily for documentation authors, translators, and people with special +formatting requirements. + +For information about building Python's documentation, refer to `Doc/README.rst +`_. + + +Converting From Python 2.x to 3.x +--------------------------------- + +Significant backward incompatible changes were made for the release of Python +3.0, which may cause programs written for Python 2 to fail when run with Python +3. For more information about porting your code from Python 2 to Python 3, see +the `Porting HOWTO `_. + + +Testing +------- + +To test the interpreter, type ``make test`` in the top-level directory. The +test set produces some output. You can generally ignore the messages about +skipped tests due to optional features which can't be imported. If a message +is printed about a failed test or a traceback or core dump is produced, +something is wrong. + +By default, tests are prevented from overusing resources like disk space and +memory. To enable these tests, run ``make testall``. + +If any tests fail, you can re-run the failing test(s) in verbose mode:: + + make test TESTOPTS="-v test_that_failed" + +If the failure persists and appears to be a problem with Python rather than +your environment, you can `file a bug report `_ and +include relevant output from that command to show the issue. + + +Installing multiple versions +---------------------------- + +On Unix and Mac systems if you intend to install multiple versions of Python +using the same installation prefix (``--prefix`` argument to the configure +script) you must take care that your primary python executable is not +overwritten by the installation of a different version. All files and +directories installed using ``make altinstall`` contain the major and minor +version and can thus live side-by-side. ``make install`` also creates +``${prefix}/bin/python3`` which refers to ``${prefix}/bin/pythonX.Y``. If you +intend to install multiple versions using the same prefix you must decide which +version (if any) is your "primary" version. Install that version using ``make +install``. Install all other versions using ``make altinstall``. + +For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the +primary version, you would execute ``make install`` in your 3.6 build directory +and ``make altinstall`` in the others. + + +Issue Tracker and Mailing List +------------------------------ + +Bug reports are welcome! You can use the `issue tracker +`_ to report bugs, and/or submit pull requests `on +GitHub `_. + +You can also follow development discussion on the `python-dev mailing list +`_. + + +Proposals for enhancement +------------------------- + +If you have a proposal to change Python, you may want to send an email to the +comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A +Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. +All current PEPs, as well as guidelines for submitting a new PEP, are listed at +`python.org/dev/peps/ `_. + +.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ + + +Release Schedule +---------------- + +See :pep:`494` for Python 3.6 release details. + + +Copyright and License Information +--------------------------------- + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. + +Copyright (c) 2000 BeOpen.com. All rights reserved. + +Copyright (c) 1995-2001 Corporation for National Research Initiatives. All +rights reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. + +See the file "LICENSE" for information on the history of this software, terms & +conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. + +This Python distribution contains *no* GNU General Public License (GPL) code, +so it may be used in proprietary projects. There are interfaces to some GNU +code but these are entirely optional. + +All trademarks referenced herein are property of their respective holders. diff --git a/Tools/README b/Tools/README index 0d961de23d6cb8..edbf4fb83321ba 100644 --- a/Tools/README +++ b/Tools/README @@ -45,4 +45,4 @@ unittestgui A Tkinter based GUI test runner for unittest, with test discovery. -(*) A generic benchmark suite is maintained separately at http://hg.python.org/benchmarks/ +(*) A generic benchmark suite is maintained separately at https://github.com/python/performance diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index cc1afbe16d01d1..31ae8117c785ff 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -362,6 +362,7 @@ def subclass_from_type(cls, t): 'set' : PySetObjectPtr, 'frozenset' : PySetObjectPtr, 'builtin_function_or_method' : PyCFunctionObjectPtr, + 'method-wrapper': wrapperobject, } if tp_name in name_map: return name_map[tp_name] @@ -714,7 +715,7 @@ def _get_entries(self, keys): try: # <= Python 3.5 return keys['dk_entries'], dk_size - except gdb.error: + except RuntimeError: # >= Python 3.6 pass @@ -1330,6 +1331,39 @@ def write_repr(self, out, visited): out.write(quote) +class wrapperobject(PyObjectPtr): + _typename = 'wrapperobject' + + def safe_name(self): + try: + name = self.field('descr')['d_base']['name'].string() + return repr(name) + except (NullPyObjectPtr, RuntimeError): + return '' + + def safe_tp_name(self): + try: + return self.field('self')['ob_type']['tp_name'].string() + except (NullPyObjectPtr, RuntimeError): + return '' + + def safe_self_addresss(self): + try: + address = long(self.field('self')) + return '%#x' % address + except (NullPyObjectPtr, RuntimeError): + return '' + + def proxyval(self, visited): + name = self.safe_name() + tp_name = self.safe_tp_name() + self_address = self.safe_self_addresss() + return ("" + % (name, tp_name, self_address)) + + def write_repr(self, out, visited): + proxy = self.proxyval(visited) + out.write(proxy) def int_from_int(gdbval): @@ -1364,11 +1398,13 @@ def to_string (self): def pretty_printer_lookup(gdbval): type = gdbval.type.unqualified() - if type.code == gdb.TYPE_CODE_PTR: - type = type.target().unqualified() - t = str(type) - if t in ("PyObject", "PyFrameObject", "PyUnicodeObject"): - return PyObjectPtrPrinter(gdbval) + if type.code != gdb.TYPE_CODE_PTR: + return None + + type = type.target().unqualified() + t = str(type) + if t in ("PyObject", "PyFrameObject", "PyUnicodeObject", "wrapperobject"): + return PyObjectPtrPrinter(gdbval) """ During development, I've been manually invoking the code in this way: @@ -1497,11 +1533,8 @@ def is_other_python_frame(self): return 'Garbage-collecting' # Detect invocations of PyCFunction instances: - older = self.older() - if not older: - return False - - caller = older._gdbframe.name() + frame = self._gdbframe + caller = frame.name() if not caller: return False @@ -1513,18 +1546,25 @@ def is_other_python_frame(self): # "self" is the (PyObject*) of the 'self' try: # Use the prettyprinter for the func: - func = older._gdbframe.read_var('func') + func = frame.read_var('func') return str(func) except RuntimeError: return 'PyCFunction invocation (unable to read "func")' elif caller == '_PyCFunction_FastCallDict': try: - func = older._gdbframe.read_var('func_obj') + func = frame.read_var('func_obj') return str(func) except RuntimeError: return 'PyCFunction invocation (unable to read "func_obj")' + if caller == 'wrapper_call': + try: + func = frame.read_var('wp') + return str(func) + except RuntimeError: + return '' + # This frame isn't worth reporting: return False diff --git a/Tools/hg/hgtouch.py b/Tools/hg/hgtouch.py deleted file mode 100644 index fbca469ba94c25..00000000000000 --- a/Tools/hg/hgtouch.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Bring time stamps of generated checked-in files into the right order - -A versioned configuration file .hgtouch specifies generated files, in the -syntax of make rules. - - output: input1 input2 - -In addition to the dependency syntax, #-comments are supported. -""" -import errno -import os -import time - -def parse_config(repo): - try: - fp = repo.wfile(".hgtouch") - except IOError, e: - if e.errno != errno.ENOENT: - raise - return {} - result = {} - with fp: - for line in fp: - # strip comments - line = line.split('#')[0].strip() - if ':' not in line: - continue - outputs, inputs = line.split(':', 1) - outputs = outputs.split() - inputs = inputs.split() - for o in outputs: - try: - result[o].extend(inputs) - except KeyError: - result[o] = inputs - return result - -def check_rule(ui, repo, modified, basedir, output, inputs): - """Verify that the output is newer than any of the inputs. - Return (status, stamp), where status is True if the update succeeded, - and stamp is the newest time stamp assigned to any file (might be in - the future). - - If basedir is nonempty, it gives a directory in which the tree is to - be checked. - """ - f_output = repo.wjoin(os.path.join(basedir, output)) - try: - o_time = os.stat(f_output).st_mtime - except OSError: - ui.warn("Generated file %s does not exist\n" % output) - return False, 0 - youngest = 0 # youngest dependency - backdate = None - backdate_source = None - for i in inputs: - f_i = repo.wjoin(os.path.join(basedir, i)) - try: - i_time = os.stat(f_i).st_mtime - except OSError: - ui.warn(".hgtouch input file %s does not exist\n" % i) - return False, 0 - if i in modified: - # input is modified. Need to backdate at least to i_time - if backdate is None or backdate > i_time: - backdate = i_time - backdate_source = i - continue - youngest = max(i_time, youngest) - if backdate is not None: - ui.warn("Input %s for file %s locally modified\n" % (backdate_source, output)) - # set to 1s before oldest modified input - backdate -= 1 - os.utime(f_output, (backdate, backdate)) - return False, 0 - if youngest >= o_time: - ui.note("Touching %s\n" % output) - youngest += 1 - os.utime(f_output, (youngest, youngest)) - return True, youngest - else: - # Nothing to update - return True, 0 - -def do_touch(ui, repo, basedir): - if basedir: - if not os.path.isdir(repo.wjoin(basedir)): - ui.warn("Abort: basedir %r does not exist\n" % basedir) - return - modified = [] - else: - modified = repo.status()[0] - dependencies = parse_config(repo) - success = True - tstamp = 0 # newest time stamp assigned - # try processing all rules in topological order - hold_back = {} - while dependencies: - output, inputs = dependencies.popitem() - # check whether any of the inputs is generated - for i in inputs: - if i in dependencies: - hold_back[output] = inputs - continue - _success, _tstamp = check_rule(ui, repo, modified, basedir, output, inputs) - success = success and _success - tstamp = max(tstamp, _tstamp) - # put back held back rules - dependencies.update(hold_back) - hold_back = {} - now = time.time() - if tstamp > now: - # wait until real time has passed the newest time stamp, to - # avoid having files dated in the future - time.sleep(tstamp-now) - if hold_back: - ui.warn("Cyclic dependency involving %s\n" % (' '.join(hold_back.keys()))) - return False - return success - -def touch(ui, repo, basedir): - "touch generated files that are older than their sources after an update." - do_touch(ui, repo, basedir) - -cmdtable = { - "touch": (touch, - [('b', 'basedir', '', 'base dir of the tree to apply touching')], - "hg touch [-b BASEDIR]") -} diff --git a/Tools/importbench/README b/Tools/importbench/README index 81a5544a383b4c..6ba386c2b608d5 100644 --- a/Tools/importbench/README +++ b/Tools/importbench/README @@ -3,4 +3,4 @@ Importbench is a set of micro-benchmarks for various import scenarios. It should not be used as an overall benchmark of import performance, but rather an easy way to measure impact of possible code changes. For a real-world benchmark of import, use the normal_startup benchmark from -hg.python.org/benchmarks. +https://github.com/python/performance diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 4659a32b838cc1..81a3f86cab36b0 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -65,8 +65,8 @@ if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if not exist "%HG%" echo Cannot find Mercurial on PATH && exit /B 1 +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" diff --git a/Tools/msi/exe/exe_files.wxs b/Tools/msi/exe/exe_files.wxs index 01385874fa90b7..e675c21c8975ef 100644 --- a/Tools/msi/exe/exe_files.wxs +++ b/Tools/msi/exe/exe_files.wxs @@ -8,9 +8,6 @@ - - - diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index f78e6ffa28fb72..b3588b7a0bae90 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -17,15 +17,12 @@ rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) - set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - + diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 4e319ce4c98ccb..670836be8c1c8b 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -9,6 +9,8 @@ set USER= set TARGET= set DRYRUN=false set NOGPG= +set PURGE_OPTION=/p:Purge=true +set NOTEST= :CheckOpts if "%1" EQU "-h" goto Help @@ -19,7 +21,11 @@ if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts -if "%1" EQU "--no-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && godo CheckOpts +if "%1" EQU "--skip-test" (set NOTEST=true) && shift && godo CheckOpts +if "%1" EQU "-T" (set NOTEST=true) && shift && godo CheckOpts +if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 if not defined PLINK where plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" if not defined PLINK where /R "%ProgramFiles(x86)%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" @@ -35,7 +41,7 @@ echo Found pscp.exe at %PSCP% if defined NOGPG ( set GPG= - echo Skipping GPG signature generation because of --no-gpg + echo Skipping GPG signature generation because of --skip-gpg ) else ( if not defined GPG where gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" if not defined GPG where /R "%PCBUILD%..\externals\windows-installer" gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" @@ -45,8 +51,12 @@ if defined NOGPG ( call "%PCBUILD%env.bat" > nul 2> nul pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +if not defined NOTEST ( + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 +) msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd @@ -55,9 +65,12 @@ exit /B 0 :Help echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] echo. -echo --host (-o) Specify the upload host (required) -echo --user (-u) Specify the user on the host (required) -echo --target (-t) Specify the target directory on the host -echo --dry-run Display commands and filenames without executing them -echo -h Display this help information +echo --host (-o) Specify the upload host (required) +echo --user (-u) Specify the user on the host (required) +echo --target (-t) Specify the target directory on the host +echo --dry-run Display commands and filenames without executing them +echo --skip-gpg Does not generate GPG signatures before uploading +echo --skip-purge Does not perform CDN purge after uploading +echo --skip-test (-T) Does not perform post-upload tests +echo -h Display this help information echo. diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 0d472dea5428a0..75840f2f851ecd 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -8,7 +8,9 @@ $(TARGET) /srv/www.python.org/ftp/python true + true false + false @@ -64,7 +66,37 @@ echo. echo." /> - + + + + + + + + + + $(TEMP)\%(Filename)_source + $(TEMP)\%(Filename)_source\%(Filename)%(Extension) + $(TEMP)\%(Filename)_layout + $(OutputPath)\%(Filename)_layoutlog + $(OutputPath)\%(Filename)_layoutlog\%(Filename).log + + + + + + + + + + + + + + + + diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index d7e932cee54d3c..464ef0456af47a 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -34,9 +34,8 @@ $(NugetArguments) -Version "$(NuspecVersion)" $(NugetArguments) -NoPackageAnalysis -NonInteractive - setlocal -set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) @@ -45,8 +44,7 @@ set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140. - + diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 58b081a9c53ff5..33a9fead879325 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -12,7 +12,6 @@ SRCDIR = sysconfig.get_config_var('srcdir') - def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -46,27 +45,76 @@ def mq_patches_applied(): return st.returncode == 0 and bstdout +def get_git_branch(): + """Get the symbolic name for the current git branch""" + cmd = "git rev-parse --abbrev-ref HEAD".split() + try: + return subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return None + + +def get_git_upstream_remote(): + """Get the remote name to use for upstream branches + + Uses "upstream" if it exists, "origin" otherwise + """ + cmd = "git remote get-url upstream".split() + try: + subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return "origin" + return "upstream" + + +@status("Getting base branch for PR", + info=lambda x: x if x is not None else "not a PR branch") +def get_base_branch(): + if not os.path.exists(os.path.join(SRCDIR, '.git')): + # Not a git checkout, so there's no base branch + return None + version = sys.version_info + if version.releaselevel == 'alpha': + base_branch = "master" + else: + base_branch = "{0.major}.{0.minor}".format(version) + this_branch = get_git_branch() + if this_branch is None or this_branch == base_branch: + # Not on a git PR branch, so there's no base branch + return None + upstream_remote = get_git_upstream_remote() + return upstream_remote + "/" + base_branch + + @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) -def changed_files(): +def changed_files(base_branch=None): """Get the list of changed or added files from Mercurial or git.""" if os.path.isdir(os.path.join(SRCDIR, '.hg')): + if base_branch is not None: + sys.exit('need a git checkout to check PR status') cmd = 'hg status --added --modified --no-status' if mq_patches_applied(): cmd += ' --rev qparent' with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] - elif os.path.isdir(os.path.join(SRCDIR, '.git')): - cmd = 'git status --porcelain' + elif os.path.exists(os.path.join(SRCDIR, '.git')): + # We just use an existence check here as: + # directory = normal git checkout/clone + # file = git worktree directory + if base_branch: + cmd = 'git diff --name-status ' + base_branch + else: + cmd = 'git status --porcelain' filenames = [] with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: for line in st.stdout: line = line.decode().rstrip() - status = set(line[:2]) + status_text, filename = line.split(maxsplit=1) + status = set(status_text) # modified, added or unmerged files if not status.intersection('MAU'): continue - filename = line[3:] if ' -> ' in filename: # file is renamed filename = filename.split(' -> ', 2)[1].strip() @@ -165,7 +213,8 @@ def regenerated_pyconfig_h_in(file_paths): return "not needed" def main(): - file_paths = changed_files() + base_branch = get_base_branch() + file_paths = changed_files(base_branch) python_files = [fn for fn in file_paths if fn.endswith('.py')] c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] doc_files = [fn for fn in file_paths if fn.startswith('Doc') and diff --git a/Tools/scripts/reindent.py b/Tools/scripts/reindent.py index 18424dea1437ac..f6dadaac5a5206 100755 --- a/Tools/scripts/reindent.py +++ b/Tools/scripts/reindent.py @@ -118,7 +118,11 @@ def check(file): if verbose: print("checking", file, "...", end=' ') with open(file, 'rb') as f: - encoding, _ = tokenize.detect_encoding(f.readline) + try: + encoding, _ = tokenize.detect_encoding(f.readline) + except SyntaxError as se: + errprint("%s: SyntaxError: %s" % (file, str(se))) + return try: with open(file, encoding=encoding) as f: r = Reindenter(f) diff --git a/aclocal.m4 b/aclocal.m4 index 9a9cc557281571..2a745e57466cae 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -13,7 +13,7 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -dnl serial 11 (pkg-config-0.29) +dnl serial 11 (pkg-config-0.29.1) dnl dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson @@ -55,7 +55,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29]) +[m4_define([PKG_MACROS_VERSION], [0.29.1]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ diff --git a/configure b/configure index 1500cea4a8292b..09a94624c1b2b4 100755 --- a/configure +++ b/configure @@ -668,6 +668,7 @@ OTHER_LIBTOOL_OPT UNIVERSAL_ARCH_FLAGS CFLAGS_NODIST BASECFLAGS +CFLAGS_ALIASING OPT LLVM_PROF_FOUND target_os @@ -749,9 +750,8 @@ UNIVERSALSDK CONFIG_ARGS SOVERSION VERSION -GENERATED_COMMENT PYTHON_FOR_BUILD -PYTHON_FOR_GEN +PYTHON_FOR_REGEN host_os host_vendor host_cpu @@ -760,10 +760,10 @@ build_os build_vendor build_cpu build -HAS_HG -HGBRANCH -HGTAG -HGVERSION +HAS_GIT +GITBRANCH +GITTAG +GITVERSION BASECPPFLAGS target_alias host_alias @@ -784,6 +784,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -817,6 +818,7 @@ with_suffix enable_shared enable_profiling with_pydebug +with_assertions enable_optimizations with_lto with_hash_algorithm @@ -894,6 +896,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1146,6 +1149,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1283,7 +1295,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1436,6 +1448,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1501,6 +1514,7 @@ Optional Packages: compiler --with-suffix=.exe set executable suffix --with-pydebug build with Py_DEBUG defined + --with-assertions build with C assertions enabled --with-lto Enable Link Time Optimization in PGO builds. Disabled by default. --with-hash-algorithm=[fnv|siphash24] @@ -2698,17 +2712,17 @@ fi -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git then -# Extract the first word of "hg", so it can be a program name with args. -set dummy hg; ac_word=$2 +# Extract the first word of "git", so it can be a program name with args. +set dummy git; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_HAS_HG+:} false; then : +if ${ac_cv_prog_HAS_GIT+:} false; then : $as_echo_n "(cached) " >&6 else - if test -n "$HAS_HG"; then - ac_cv_prog_HAS_HG="$HAS_HG" # Let the user override the test. + if test -n "$HAS_GIT"; then + ac_cv_prog_HAS_GIT="$HAS_GIT" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH @@ -2717,7 +2731,7 @@ do test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_HAS_HG="found" + ac_cv_prog_HAS_GIT="found" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -2725,13 +2739,13 @@ done done IFS=$as_save_IFS - test -z "$ac_cv_prog_HAS_HG" && ac_cv_prog_HAS_HG="not-found" + test -z "$ac_cv_prog_HAS_GIT" && ac_cv_prog_HAS_GIT="not-found" fi fi -HAS_HG=$ac_cv_prog_HAS_HG -if test -n "$HAS_HG"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_HG" >&5 -$as_echo "$HAS_HG" >&6; } +HAS_GIT=$ac_cv_prog_HAS_GIT +if test -n "$HAS_GIT"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_GIT" >&5 +$as_echo "$HAS_GIT" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } @@ -2739,17 +2753,17 @@ fi else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi @@ -2868,11 +2882,11 @@ do set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_PYTHON_FOR_GEN+:} false; then : +if ${ac_cv_prog_PYTHON_FOR_REGEN+:} false; then : $as_echo_n "(cached) " >&6 else - if test -n "$PYTHON_FOR_GEN"; then - ac_cv_prog_PYTHON_FOR_GEN="$PYTHON_FOR_GEN" # Let the user override the test. + if test -n "$PYTHON_FOR_REGEN"; then + ac_cv_prog_PYTHON_FOR_REGEN="$PYTHON_FOR_REGEN" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH @@ -2881,7 +2895,7 @@ do test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_PYTHON_FOR_GEN="$ac_prog" + ac_cv_prog_PYTHON_FOR_REGEN="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -2891,25 +2905,20 @@ IFS=$as_save_IFS fi fi -PYTHON_FOR_GEN=$ac_cv_prog_PYTHON_FOR_GEN -if test -n "$PYTHON_FOR_GEN"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_GEN" >&5 -$as_echo "$PYTHON_FOR_GEN" >&6; } +PYTHON_FOR_REGEN=$ac_cv_prog_PYTHON_FOR_REGEN +if test -n "$PYTHON_FOR_REGEN"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_REGEN" >&5 +$as_echo "$PYTHON_FOR_REGEN" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi - test -n "$PYTHON_FOR_GEN" && break + test -n "$PYTHON_FOR_REGEN" && break done -test -n "$PYTHON_FOR_GEN" || PYTHON_FOR_GEN="not-found" +test -n "$PYTHON_FOR_REGEN" || PYTHON_FOR_REGEN="python3" -if test "$PYTHON_FOR_GEN" = not-found; then - PYTHON_FOR_GEN='@echo "Cannot generate $@, python not found !" && \ - echo "To skip re-generation of $@ run or ." && \ - echo "Otherwise, set python in PATH and run configure or run ." && false &&' -fi if test "$cross_compiling" = yes; then @@ -2930,18 +2939,14 @@ $as_echo_n "checking for python interpreter for cross build... " >&6; } $as_echo "$interp" >&6; } PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp fi - # Used to comment out stuff for rebuilding generated files - GENERATED_COMMENT='#' elif test "$cross_compiling" = maybe; then as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5 else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' - GENERATED_COMMENT='' fi - if test "$prefix" != "/"; then prefix=`echo "$prefix" | sed -e 's/\/$//g'` fi @@ -6512,6 +6517,33 @@ $as_echo "no" >&6; } fi +# Check for --with-assertions. Py_DEBUG implies assertions, but also changes +# the ABI. This allows enabling assertions without changing the ABI. +assertions='false' +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-assertions" >&5 +$as_echo_n "checking for --with-assertions... " >&6; } + +# Check whether --with-assertions was given. +if test "${with_assertions+set}" = set; then : + withval=$with_assertions; +if test "$withval" != no +then + assertions='true' +fi +fi + +if test "$assertions" = 'true'; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +elif test "$Py_DEBUG" = 'true'; then + assertions='true' + { $as_echo "$as_me:${as_lineno-$LINENO}: result: implied by --with-pydebug" >&5 +$as_echo "implied by --with-pydebug" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + # Enable optimization flags @@ -6521,7 +6553,7 @@ $as_echo_n "checking for --enable-optimizations... " >&6; } # Check whether --enable-optimizations was given. if test "${enable_optimizations+set}" = set; then : enableval=$enable_optimizations; -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 @@ -6839,6 +6871,7 @@ esac # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line + if test "${OPT-unset}" = "unset" then case $GCC in @@ -6851,30 +6884,49 @@ then WRAP="-fwrapv" fi - # Clang also needs -fwrapv case $CC in - *clang*) WRAP="-fwrapv" - ;; + *clang*) + cc_is_clang=1 + ;; + *) + if $CC --version 2>&1 | grep -q clang + then + cc_is_clang=1 + else + cc_is_clang= + fi esac + if test -n "${cc_is_clang}" + then + # Clang also needs -fwrapv + WRAP="-fwrapv" + # bpo-30104: disable strict aliasing to compile correctly dtoa.c, + # see Makefile.pre.in for more information + CFLAGS_ALIASING="-fno-strict-aliasing" + fi + case $ac_cv_prog_cc_g in yes) if test "$Py_DEBUG" = 'true' ; then # Optimization messes up debuggers, so turn it off for # debug builds. if "$CC" -v --help 2>/dev/null |grep -- -Og > /dev/null; then - OPT="-g -Og -Wall $STRICT_PROTO" + OPT="-g -Og -Wall" else - OPT="-g -O0 -Wall $STRICT_PROTO" + OPT="-g -O0 -Wall" fi else - OPT="-g $WRAP -O3 -Wall $STRICT_PROTO" + OPT="-g $WRAP -O3 -Wall" fi ;; *) - OPT="-O3 -Wall $STRICT_PROTO" + OPT="-O3 -Wall" ;; esac + + OPT="$OPT $STRICT_PROTO" + case $ac_sys_system in SCO_SV*) OPT="$OPT -m486 -DSCO5" ;; @@ -7401,7 +7453,7 @@ case "$CC" in ;; esac -if test "$Py_DEBUG" = 'true'; then +if test "$assertions" = 'true'; then : else OPT="-DNDEBUG $OPT" @@ -9697,8 +9749,11 @@ esac # check for systems that require aligned memory access { $as_echo "$as_me:${as_lineno-$LINENO}: checking aligned memory access is required" >&5 $as_echo_n "checking aligned memory access is required... " >&6; } -if test "$cross_compiling" = yes; then : - aligned_required=yes +if ${ac_cv_aligned_required+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : + ac_cv_aligned_required=yes else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9715,26 +9770,26 @@ int main() return 1; return 0; } - _ACEOF if ac_fn_c_try_run "$LINENO"; then : - aligned_required=no + ac_cv_aligned_required=no else - aligned_required=yes + ac_cv_aligned_required=yes fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi -if test "$aligned_required" = yes ; then +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_aligned_required" >&5 +$as_echo "$ac_cv_aligned_required" >&6; } +if test "$ac_cv_aligned_required" = yes ; then $as_echo "#define HAVE_ALIGNED_REQUIRED 1" >>confdefs.h fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $aligned_required" >&5 -$as_echo "$aligned_required" >&6; } - # str, bytes and memoryview hash algorithm diff --git a/configure.ac b/configure.ac index 64f12f28790a7d..146ae22a632a59 100644 --- a/configure.ac +++ b/configure.ac @@ -25,25 +25,25 @@ else BASECPPFLAGS="" fi -AC_SUBST(HGVERSION) -AC_SUBST(HGTAG) -AC_SUBST(HGBRANCH) +AC_SUBST(GITVERSION) +AC_SUBST(GITTAG) +AC_SUBST(GITBRANCH) -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git then -AC_CHECK_PROG(HAS_HG, hg, found, not-found) +AC_CHECK_PROG(HAS_GIT, git, found, not-found) else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi AC_CONFIG_SRCDIR([Include/object.h]) @@ -56,13 +56,8 @@ AC_SUBST(host) # pybuilddir.txt will be created by --generate-posix-vars in the Makefile rm -f pybuilddir.txt -AC_CHECK_PROGS(PYTHON_FOR_GEN, python$PACKAGE_VERSION python3 python, not-found) -if test "$PYTHON_FOR_GEN" = not-found; then - PYTHON_FOR_GEN='@echo "Cannot generate $@, python not found !" && \ - echo "To skip re-generation of $@ run or ." && \ - echo "Otherwise, set python in PATH and run configure or run ." && false &&' -fi -AC_SUBST(PYTHON_FOR_GEN) +AC_CHECK_PROGS(PYTHON_FOR_REGEN, python$PACKAGE_VERSION python3 python, python3) +AC_SUBST(PYTHON_FOR_REGEN) if test "$cross_compiling" = yes; then AC_MSG_CHECKING([for python interpreter for cross build]) @@ -80,16 +75,12 @@ if test "$cross_compiling" = yes; then AC_MSG_RESULT($interp) PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp fi - # Used to comment out stuff for rebuilding generated files - GENERATED_COMMENT='#' elif test "$cross_compiling" = maybe; then AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH]) else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' - GENERATED_COMMENT='' fi AC_SUBST(PYTHON_FOR_BUILD) -AC_SUBST(GENERATED_COMMENT) dnl Ensure that if prefix is specified, it does not end in a slash. If dnl it does, we get path names containing '//' which is both ugly and @@ -1267,6 +1258,27 @@ else AC_MSG_RESULT(no); Py_DEBUG='false' fi], [AC_MSG_RESULT(no)]) +# Check for --with-assertions. Py_DEBUG implies assertions, but also changes +# the ABI. This allows enabling assertions without changing the ABI. +assertions='false' +AC_MSG_CHECKING(for --with-assertions) +AC_ARG_WITH(assertions, + AC_HELP_STRING([--with-assertions], [build with C assertions enabled]), +[ +if test "$withval" != no +then + assertions='true' +fi], +[]) +if test "$assertions" = 'true'; then + AC_MSG_RESULT(yes) +elif test "$Py_DEBUG" = 'true'; then + assertions='true' + AC_MSG_RESULT(implied by --with-pydebug) +else + AC_MSG_RESULT(no) +fi + # Enable optimization flags AC_SUBST(DEF_MAKE_ALL_RULE) AC_SUBST(DEF_MAKE_RULE) @@ -1274,7 +1286,7 @@ Py_OPT='false' AC_MSG_CHECKING(for --enable-optimizations) AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive optimizations (PGO, etc). Disabled by default.]), [ -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' AC_MSG_RESULT(yes); @@ -1447,6 +1459,7 @@ esac # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line AC_SUBST(OPT) +AC_SUBST(CFLAGS_ALIASING) if test "${OPT-unset}" = "unset" then case $GCC in @@ -1459,30 +1472,49 @@ then WRAP="-fwrapv" fi - # Clang also needs -fwrapv case $CC in - *clang*) WRAP="-fwrapv" - ;; + *clang*) + cc_is_clang=1 + ;; + *) + if $CC --version 2>&1 | grep -q clang + then + cc_is_clang=1 + else + cc_is_clang= + fi esac + if test -n "${cc_is_clang}" + then + # Clang also needs -fwrapv + WRAP="-fwrapv" + # bpo-30104: disable strict aliasing to compile correctly dtoa.c, + # see Makefile.pre.in for more information + CFLAGS_ALIASING="-fno-strict-aliasing" + fi + case $ac_cv_prog_cc_g in yes) if test "$Py_DEBUG" = 'true' ; then # Optimization messes up debuggers, so turn it off for # debug builds. if "$CC" -v --help 2>/dev/null |grep -- -Og > /dev/null; then - OPT="-g -Og -Wall $STRICT_PROTO" + OPT="-g -Og -Wall" else - OPT="-g -O0 -Wall $STRICT_PROTO" + OPT="-g -O0 -Wall" fi else - OPT="-g $WRAP -O3 -Wall $STRICT_PROTO" + OPT="-g $WRAP -O3 -Wall" fi ;; *) - OPT="-O3 -Wall $STRICT_PROTO" + OPT="-O3 -Wall" ;; esac + + OPT="$OPT $STRICT_PROTO" + case $ac_sys_system in SCO_SV*) OPT="$OPT -m486 -DSCO5" ;; @@ -1844,7 +1876,7 @@ case "$CC" in ;; esac -if test "$Py_DEBUG" = 'true'; then +if test "$assertions" = 'true'; then : else OPT="-DNDEBUG $OPT" @@ -2696,7 +2728,8 @@ esac # check for systems that require aligned memory access AC_MSG_CHECKING(aligned memory access is required) -AC_TRY_RUN([ +AC_CACHE_VAL(ac_cv_aligned_required, +[AC_RUN_IFELSE([AC_LANG_SOURCE([[ int main() { char s[16]; @@ -2708,18 +2741,16 @@ int main() if (*p1 == *p2) return 1; return 0; -} - ], - [aligned_required=no], - [aligned_required=yes], - [aligned_required=yes]) - -if test "$aligned_required" = yes ; then +}]])], +[ac_cv_aligned_required=no], +[ac_cv_aligned_required=yes], +[ac_cv_aligned_required=yes]) +]) +AC_MSG_RESULT($ac_cv_aligned_required) +if test "$ac_cv_aligned_required" = yes ; then AC_DEFINE([HAVE_ALIGNED_REQUIRED], [1], [Define if aligned memory access is required]) fi -AC_MSG_RESULT($aligned_required) - # str, bytes and memoryview hash algorithm AH_TEMPLATE(Py_HASH_ALGORITHM,