diff --git a/.gitattributes b/.gitattributes
index 0a998a5369fff9..4a487c3c2a14e5 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -37,3 +37,20 @@ Lib/test/test_importlib/data01/* -text
*.proj text eol=crlf
PCbuild/readme.txt text eol=crlf
PC/readme.txt text eol=crlf
+
+# Generated files
+# https://github.com/github/linguist#generated-code
+Include/graminit.h linguist-generated=true
+Python/graminit.h linguist-generated=true
+Modules/clinic/*.h linguist-generated=true
+Objects/clinic/*.h linguist-generated=true
+PC/clinic/*.h linguist-generated=true
+Python/clinic/*.h linguist-generated=true
+Python/importlib.h linguist-generated=true
+Python/importlib_external.h linguist-generated=true
+Include/Python-ast.h linguist-generated=true
+Python/Python-ast.c linguist-generated=true
+Include/opcode.h linguist-generated=true
+Python/opcode_targets.h linguist-generated=true
+Objects/typeslots.inc linguist-generated=true
+Modules/unicodedata_db.h linguist-generated=true
diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst
index d559bd5e189448..7f912e87084190 100644
--- a/.github/CONTRIBUTING.rst
+++ b/.github/CONTRIBUTING.rst
@@ -8,6 +8,10 @@ Build Status
+ `Stable buildbots `_
+- 3.7
+
+ + `Stable buildbots `_
+
- 3.6
+ `Stable buildbots `_
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 0f238d0558cd00..55e4168747e10d 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,9 +1,11 @@
!!! If this is a backport PR (PR made against branches other than `master`),
please ensure that the PR title is in the following format:
+
```
[X.Y]
(GH-NNNN)
```
-Where: [X.Y] is the branch name, e.g. [3.6].
+
+Where: [X.Y] is the branch name, e.g. [3.7].
GH-NNNN refers to the PR number from `master`.
diff --git a/.github/appveyor.yml b/.github/appveyor.yml
index b7d40787318985..6662732326a3b8 100644
--- a/.github/appveyor.yml
+++ b/.github/appveyor.yml
@@ -7,12 +7,32 @@ branches:
- buildbot-custom
cache:
- externals -> PCbuild\*
+before_build:
+ - ps: |+
+ if ($env:APPVEYOR_RE_BUILD) {
+ echo 'Doing full build due to re-build request.'
+ } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) {
+ echo 'Not a PR, doing full build.'
+ } else {
+ git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH
+ $mergebase = git merge-base HEAD FETCH_HEAD
+ $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)'
+ If (!$changes) {
+ echo 'Only docs were updated, stopping build process.'
+ Exit-AppveyorBuild
+ } else {
+ echo 'Doing full build due to non-doc changes in these files:'
+ echo $changes
+ }
+ }
+
+
build_script:
-- cmd: PCbuild\build.bat -e
-- cmd: PCbuild\win32\python.exe -m test.pythoninfo
+ - cmd: PCbuild\build.bat -e
+ - cmd: PCbuild\win32\python.exe -m test.pythoninfo
test_script:
-- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 --fail-env-changed -j0
+ - cmd: PCbuild\rt.bat -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0
environment:
HOST_PYTHON: C:\Python36\python.exe
image:
-- Visual Studio 2017
+ - Visual Studio 2017
diff --git a/.gitignore b/.gitignore
index 05fb6cba0875a1..58f8bf72f2b9bd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -113,3 +113,4 @@ Tools/ssl/amd64
Tools/ssl/win32
.vs/
.vscode/
+gmon.out
diff --git a/.hgeol b/.hgeol
deleted file mode 100644
index eb19a6c88d28d0..00000000000000
--- a/.hgeol
+++ /dev/null
@@ -1,58 +0,0 @@
-[patterns]
-
-# Non human-editable files are binary
-
-**.dsp = BIN
-**.dsw = BIN
-**.mk = BIN
-**.sln = BIN
-**.vcproj = BIN
-**.vsprops = BIN
-
-**.aif = BIN
-**.aifc = BIN
-**.aiff = BIN
-**.au = BIN
-**.bmp = BIN
-**.db = BIN
-**.exe = BIN
-**.icns = BIN
-**.gif = BIN
-**.ico = BIN
-**.info = BIN
-**.jpg = BIN
-**.pck = BIN
-**.png = BIN
-**.psd = BIN
-**.tar = BIN
-**.wav = BIN
-**.whl = BIN
-**.xar = BIN
-**.zip = BIN
-
-Lib/test/cjkencodings/* = BIN
-Lib/test/decimaltestdata/*.decTest = BIN
-Lib/test/sndhdrdata/sndhdr.* = BIN
-Lib/test/test_email/data/msg_26.txt = BIN
-Lib/test/xmltestdata/* = BIN
-
-Lib/venv/scripts/nt/* = BIN
-
-Lib/test/coding20731.py = BIN
-
-# Windows only zlib upstream file
-Modules/zlib/zlib.map = CRLF
-
-# Windows batch files work best with CRLF, there can be subtle problems with LF
-**.bat = CRLF
-
-# The Windows readme is likely to be read in Notepad, so make it readable
-PCbuild/readme.txt = CRLF
-
-# All other files (which presumably are human-editable) are "native".
-# This must be the last rule!
-
-** = native
-
-[repository]
-native = LF
diff --git a/.hgignore b/.hgignore
deleted file mode 100644
index 68c607f2e8d420..00000000000000
--- a/.hgignore
+++ /dev/null
@@ -1,106 +0,0 @@
-.gdb_history
-.purify
-.svn/
-^.idea/
-^.vscode/
-.DS_Store
-Makefile$
-Makefile.pre$
-TAGS$
-autom4te.cache$
-^build/
-^Doc/build/
-^Doc/venv/
-buildno$
-config.cache
-config.log
-config.status
-config.status.lineno
-db_home
-platform$
-pyconfig.h$
-python$
-python.bat$
-python.exe$
-python-config$
-python-config.py$
-reflog.txt$
-tags$
-Misc/python.pc
-Misc/python-config.sh$
-Modules/Setup$
-Modules/Setup.config
-Modules/Setup.local
-Modules/config.c
-Modules/ld_so_aix$
-^lcov-report/
-^core
-^python-gdb.py
-^python.exe-gdb.py
-^pybuilddir.txt
-
-syntax: glob
-libpython*.a
-libpython*.so*
-libpython*.dylib
-libpython*.dll
-*.swp
-*.o
-*.pyc
-*.pyo
-*.pyd
-*.cover
-*~
-*.gc??
-*.profclang?
-*.profraw
-*.dyn
-Include/pydtrace_probes.h
-Lib/distutils/command/*.pdb
-Lib/lib2to3/*.pickle
-Lib/test/data/*
-Misc/*.wpu
-PC/python_nt*.h
-PC/pythonnt_rc*.h
-PC/*/*.exe
-PC/*/*.exp
-PC/*/*.lib
-PC/*/*.bsc
-PC/*/*.dll
-PC/*/*.pdb
-PC/*/*.user
-PC/*/*.ncb
-PC/*/*.suo
-PC/*/Win32-temp-*
-PC/*/x64-temp-*
-PC/*/amd64
-PCbuild/*.user
-PCbuild/*.suo
-PCbuild/*.*sdf
-PCbuild/*-pgi
-PCbuild/*-pgo
-PCbuild/.vs
-PCbuild/amd64
-PCbuild/obj
-PCbuild/win32
-Tools/unicode/build/
-Tools/unicode/MAPPINGS/
-BuildLog.htm
-__pycache__
-Parser/pgen{,.exe}
-Programs/_freeze_importlib{,.exe}
-Programs/_testembed{,.exe}
-.coverage
-coverage/
-externals/
-htmlcov/
-*.gcda
-*.gcno
-*.gcov
-ipch/
-coverage.info
-Tools/msi/obj
-Tools/ssl/amd64
-Tools/ssl/win32
-.vs/
-.vscode/
diff --git a/.hgtags b/.hgtags
deleted file mode 100644
index 8f51c2ced49aed..00000000000000
--- a/.hgtags
+++ /dev/null
@@ -1,182 +0,0 @@
-64cc5439e10a6fdf984effaf0141e94fa4cc1004 v0.9.8
-78a7ed6953025e7ecdde9585099b01a6ae40b76a v0.9.9
-b15b8cc9b8d10e0352a0b8b7e8d51fa309db6df3 v1.0.1
-0326b5d61445ee3a8d3de28119f9652cb72d2e3f v1.0.2
-832615ec07646e310c85316b8ba6bc9b17ad3547 v1.1
-9895475d18c7b5f32adaf78f71886ae041e4d10c v1.1.1
-16eb4c51ee97169046340998e850a63c65225b0a v1.2b1
-b45c688756d04fb84d4a0d518fc3d7e3cb25fa8d v1.2b2
-9e82daf7605bad7976a9abc997cb5e0abe434078 v1.2b3
-065e31cf5862e27521cf5725b003aed211f091b2 v1.2b4
-e72257e655454d569468da8b1189e0ec336f3536 v1.2
-e63d83f8275853aaaa3d1972cb86564505e65583 v1.3b1
-7d743c865a9aa6bde8b603e32e0542031bba3c33 v1.3
-4fc85c82cc222554ae6b9c0b87776ed5f2b70c6e v1.4b1
-129f1299d4e97e884bbbbdd00baf101d178973e6 v1.4b2
-44a82ac654a4175569deed8e8a94b0cc8edee08d v1.4b3
-db49494c93dc73de06d5721c74eab533a947a92c v1.4
-062aed8a4ce2c91c81b80e29f02faff1cf5a761b v1.5a1
-c9498ac988372575cf7028b86395b900c9b0a840 v1.5a2
-dc5c968ec992aab3d40a7189df0c443d1c7a1a68 v1.5a3
-746654a0af680c7d9b814b210a026eb91bec9533 v1.5a4
-8ff58b5730f06be08fbbdc2bf592226f7a736201 v1.5b1
-eb78658d819fb0af09a8e6f9bedcb670805ed5f6 v1.5b2
-84461011a1a0ab402e352f06748f29fb5b5559e5 v1.5
-44aba4d26b01fbae0403efe654f9fd0347606732 v1.5.1
-fed63ccbe6dc3ac663bfe97a2f7006b1b28568f9 v1.5.2a1
-21d71f2e27248a0f4e393d0fc321ecf9b89321d2 v1.5.2a2
-f08c7a2a56f80741f5f192fd0ebe0b0967a203cf v1.5.2b1
-8fe7ec4b4fc1518fcac89e6bf674fbbce16150a9 v1.5.2b2
-39fb0dcc83dc375c1565ba65dbce0ed59b1359c9 v1.5.2c1
-61c91c7f101bab3149adfcd5646ae40e048de712 v1.5.2
-605eb9326ffe1fd1e43f40e2338d6652ab449fdf v1.6a1
-011bee8fd9f7f4da457ec71596484fb0882c0614 v1.6a2
-35c4fc1414a59888614b9be784a25f233ba67984 v2.0b1
-55bba197d4870cdae62aeca00e20240a756b84f8 v2.0b2
-e276329cce036a5f9e9d3451256dca5984e543dc v2.0c1
-2fa4e35083e02342ca014bf5bfba46aecb816c31 v2.0
-b60831eeab5a06dd3c5e8297a99e39297aa8794b v2.1a1
-b382f1f07ec6b2c95551658b30c6139eeb32077a v2.1a2
-d0c830db5e68edd4aaa3401216e610c9ff145826 v2.1b1
-b59a536ae1ef3774fd85c17f623e8926b7b6c095 v2.1b2
-d611276e9ad53b5d32d1e8065e1d811c32f7d96f v2.1c1
-ff065e674af6c9ab895bd9eff7d9e9039a376c7d v2.1c2
-020e95d8180d7943fe54701e1db0a7d7d87e2b1e v2.1
-08796a137f1ada2462f7a3177306df5f67a767e1 v2.2a3
-d054c29647f90bccb8345bd779bca1eecf2dd7f2 v2.3c1
-fce5c9e9abc722394cb2e909b7e2a39080d4448e v2.3c2
-92ca658fd420095b6284c9ce6e9082a80285ec9c v2.4a1
-055fc6955f3c6522bfeb7ed4c671c97d5baaaac2 v2.4a2
-186b72550e53533ef6175f6411f932c1298193d7 v2.4a3
-53cff04283855adf88ed0c0fd3698827ca843637 v2.4b1
-7e387a9dcc79954a77695adef8b593da35be1214 v2.4b2
-ff80d8bbef6e13426c8a85d7f9d837b8f8f89834 v2.4c1
-f31e18d313c7a4fc66914b2d27e130a0f72c0b69 v2.4
-cd3f783cd08a16781e236c0b9cb5717d1d995fa9 v3.0a1
-65e82140e281bf26f2e22eda05a7f9956c420f8b v3.0a2
-df15827f34881b9af0936350813ced5c123c8230 v3.0a3
-15f773f7300e372c56a21d59fe49ca26955a6477 v3.0a4
-e35935475153377d6727d64e6c52f72c3b84015b v3.0a5
-a335c4d643b1cfe14197a9ef195c9b2804f608fc v3.0b1
-16ec4bb14a68ea428acf09ebf0c92981da2646f3 v3.0b2
-509e30a7968e01be329ec121540b3e755fc4e566 v3.0b3
-507ede9c7f7f475dfafbd4a52c22d767d10a2bc0 v3.0rc1
-8fae465a39627b590385462e6095eb63af45240a v3.0rc2
-e83a60c69d53f5551a306e77a6d38e9b11485496 v3.0rc3
-bc1ce368986e45b1faf96f93995df46bcd75e7b8 v3.1a1
-ee430e5075db2adf8124e6b94916a89ca41d3171 v3.1a2
-b63020797f9678adaf4d2c3e26574a9eef2ef028 v3.1b1
-4353fd0843cb31b356adc50f93d220e2e7255ef2 v3.1rc1
-0b87e438e1b53e3f812cad963a7fdb65d198ba2f v3.1rc2
-a69a031ac1402dede8b1ef80096436bca6d371f3 v3.1
-35efb1054ec6ceca72017a587263cb6a9257340b v3.1.1rc1
-8b9c0f573ab29c41c6c5fdcca82a1fe0ff5355af v3.1.1
-149b8b87514d10416b598884db5f74651f625b38 v3.1.2rc1
-960efa327c5d9c18df995437b0ac550cb89c9f85 v3.1.2
-d18e9d71f369d8211f6ac87252c6d3211f9bd09f v3.1.3rc1
-a4f75773c0060cee38b0bb651a7aba6f56b0e996 v3.1.3
-32fcb9e94985cb19ce37ba9543f091c0dbe9d7dd v3.1.4rc1
-c918ec9f3a76d6afedfbb5d455004de880443a3d v3.1.4
-ee26aca3219cf4bb0b93352e83edcc9cb28c7802 v3.1.5rc1
-75db2bc69fc9a3e4801e94e3e19801cb096208d8 v3.1.5rc2
-7395330e495ec3316862ca1f6ce0aaf7bdf6785b v3.1.5
-b37b7834757492d009b99cf0ca4d42d2153d7fac v3.2a1
-56d4373cecb73c8b45126ba7b045b3c7b3f94b0b v3.2a2
-da012d9a2c23d144e399d2e01a55b8a83ad94573 v3.2a3
-d92a5b850f5e56808bedc01723906ed64c5e6e2e v3.2a4
-b635cea94195780c8716e236479af319bcc26253 v3.2b1
-e3af5f3a7904c0d5343ec9633ea66e7acfd23a66 v3.2b2
-865d5b24bf28ca41b536befc326407c03e74a4d5 v3.2rc1
-acf3e24dd0d0dfd1e20c907d696d3da965a8f56f v3.2rc2
-18c1f52896501c7ee13b038454a39acb45a87979 v3.2rc3
-a222a015e28d8ae9af3899258dc6c15c3d40add0 v3.2
-8ffac2337a3323323d02153ac919fd1483176652 v3.2.1b1
-cfa9364997c7f2e67b9cbb45c3a5fa3bba4e4999 v3.2.1rc1
-5df549718fb4841ff521fe051f6b54f290fad5d8 v3.2.1rc2
-ac1f7e5c05104d557d5acd922e95625ba5d1fe10 v3.2.1
-c860feaa348d663e598986894ee4680480577e15 v3.2.2rc1
-137e45f15c0bd262c9ad4c032d97425bc0589456 v3.2.2
-7085403daf439adb3f9e70ef13f6bedb1c447376 v3.2.3rc1
-428f05cb7277e1d42bb9dd8d1af6b6270ebc6112 v3.2.3rc2
-3d0686d90f55a78f96d9403da2c52dc2411419d0 v3.2.3
-b2cb7bc1edb8493c0a78f9331eae3e8fba6a881d v3.2.4rc1
-1e10bdeabe3de02f038a63c001911561ac1d13a7 v3.2.4
-cef745775b6583446572cffad704100983db2bea v3.2.5
-51382a5598ec96119cb84594572901c9c964dc3c v3.2.6rc1
-0bd5f4f14de965ca8e44c6e3965fee106176cfc4 v3.2.6
-f1a9a6505731714f0e157453ff850e3b71615c45 v3.3.0a1
-2f69db52d6de306cdaef0a0cc00cc823fb350b01 v3.3.0a2
-0b53b70a40a00013505eb35e3660057b62be77be v3.3.0a3
-7c51388a3aa7ce76a8541bbbdfc05d2d259a162c v3.3.0a4
-e15c554cd43eb23bc0a528a4e8741da9bbec9607 v3.3.0b1
-4972a8f1b2aa3d7cdd64dc96aa7fa112fe1ea343 v3.3.0b2
-8bb5c7bc46ba43804480f3e328e1fa956672c885 v3.3.0rc1
-88a0792e8ba3e4916b24c7e7a522c277d326d66e v3.3.0rc2
-c191d21cefafb3832c45570e84854e309aa62eaa v3.3.0rc3
-bd8afb90ebf28ba4edc901d4a235f75e7bbc79fd v3.3.0
-92c2cfb924055ce68c4f78f836dcfe688437ceb8 v3.3.1rc1
-d9893d13c6289aa03d33559ec67f97dcbf5c9e3c v3.3.1
-d047928ae3f6314a13b6137051315453d0ae89b6 v3.3.2
-fd53c500f8b80f54f3ecedec9da2e8c7e52a6888 v3.3.3rc1
-d32442c0e60dfbd71234e807d3d1dedd227495a9 v3.3.3rc2
-c3896275c0f61b2510a6c7e6c458a750359a91b8 v3.3.3
-fa92f5f940c6c0d839d7f0611e4b717606504a3c v3.3.4rc1
-7ff62415e4263c432c8acf6e424224209211eadb v3.3.4
-9ec811df548ed154a9bf9815383a916d6df31b98 v3.3.5rc1
-ca5635efe090f78806188ac2758f9948596aa8b2 v3.3.5rc2
-62cf4e77f78564714e7ea3d4bf1479ca1fbd0758 v3.3.5
-51317c9786f54267975abf2e9c502e6aaaa4a249 v3.3.6rc1
-971fec30da1fc5bf2b9fb28e09812a5127014211 v3.3.6
-46535f65e7f3bcdcf176f36d34bc1fed719ffd2b v3.4.0a1
-9265a2168e2cb2a84785d8717792acc661e6b692 v3.4.0a2
-dd9cdf90a5073510877e9dd5112f8e6cf20d5e89 v3.4.0a3
-e245b0d7209bb6d0e19316e1e2af1aa9c2139104 v3.4.0a4
-3405dc9a6afaa0a06dd1f6f182ec5c998dce6f5f v3.4.0b1
-ba32913eb13ec545a46dd0ce18035b6c416f0d78 v3.4.0b2
-a97ce3ecc96af79bd2e1ac66ce48d9138e0ca749 v3.4.0b3
-5e088cea8660677969113741c1313d570d977e02 v3.4.0rc1
-a300712ed38c9a242b736c44e806caea25a6dc05 v3.4.0rc2
-8a81cdab3e9d521daaef989fade94b16455fc3b8 v3.4.0rc3
-04f714765c13824c3bc2835d7b008908862e083a v3.4.0
-c67a19e11a7191baf30f313bf55e2e0b6c6f574e v3.4.1rc1
-c0e311e010fcb5bae8d87ca22051cd0845ea0ca0 v3.4.1
-8711a09513848cfc48c689d983495ee64f4668ca v3.4.2rc1
-ab2c023a9432f16652e89c404bbc84aa91bf55af v3.4.2
-69dd528ca6255a66c37cc5cf680e8357d108b036 v3.4.3rc1
-b4cbecbc0781e89a309d03b60a1f75f8499250e6 v3.4.3
-04f3f725896c6961212c3a12e8ac25be6958f4fa v3.4.4rc1
-737efcadf5a678b184e0fa431aae11276bf06648 v3.4.4
-3631bb4a2490292ebf81d3e947ae36da145da564 v3.4.5rc1
-619b61e505d0e2ccc8516b366e4ddd1971b46a6f v3.4.5
-e199a272ccdac5a8c073d4690f60c13e0b6d86b0 v3.4.6rc1
-b662f47769213f23325144b80782c05764d0f053 v3.4.6
-5d4b6a57d5fd7564bf73f3db0e46fe5eeb00bcd8 v3.5.0a1
-0337bd7ebcb6559d69679bc7025059ad1ce4f432 v3.5.0a2
-82656e28b5e5c4ae48d8dd8b5f0d7968908a82b6 v3.5.0a3
-413e0e0004f4f954331cb8122aa55fe208984955 v3.5.0a4
-071fefbb5e3db770c6c19fba9994699f121b1cea v3.5.0b1
-7a088af5615bf04024e9912068f4bd8f43ed3917 v3.5.0b2
-0035fcd9b9243ae52c2e830204fd9c1f7d528534 v3.5.0b3
-c0d64105463581f85d0e368e8d6e59b7fd8f12b1 v3.5.0b4
-1a58b1227501e046eee13d90f113417b60843301 v3.5.0rc1
-cc15d736d860303b9da90d43cd32db39bab048df v3.5.0rc2
-66ed52375df802f9d0a34480daaa8ce79fc41313 v3.5.0rc3
-2d033fedfa7f1e325fd14ccdaa9cb42155da206f v3.5.0rc4
-374f501f4567b7595f2ad7798aa09afa2456bb28 v3.5.0
-948ef16a69513ba1ff15c9d7d0b012b949df4c80 v3.5.1rc1
-37a07cee5969e6d3672583187a73cf636ff28e1b v3.5.1
-68feec6488b26327a85a634605dd28eca4daa5f1 v3.5.2rc1
-4def2a2901a5618ea45bcc8f2a1411ef33af18ad v3.5.2
-de530d7f21c0398bb2a2b67716e0638e5fadf727 v3.5.3rc1
-1880cb95a742cd001c67677de5c4efeab169416c v3.5.3
-5896da372fb044e38595fb74495de1e1e7c8fb3c v3.6.0a1
-37889342355223e2fc1438de3dc7ffcd625c60f7 v3.6.0a2
-f3edf13dc339b8942ae6b309771ab197dd8ce6fa v3.6.0a3
-017cf260936b444788c9b671d195b7bfd83dbd25 v3.6.0a4
-5b0ca4ed5e2f0669d76ece7ef975c544580f12b4 v3.6.0b1
-b9fadc7d1c3f9c3c77f32f35afbe1a1cc38070e6 v3.6.0b2
-8345e066c0ed713c3e510cbc8fafc1c38d6d306b v3.6.0b3
-18496abdb3d5c2730a659b747a89261b2219fecf v3.6.0b4
-29a273eee9a523ee178f6a66c4ac9d317c8fc84f v3.6.0rc1
-800a67f7806de45a7abd5273359e704bf147c079 v3.6.0rc2
-41df79263a11f2429d1dd0cfe12553de3dcb5508 v3.6.0
diff --git a/.travis.yml b/.travis.yml
index d7387e5f9831b2..8b1219636677a5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,17 +5,18 @@ group: beta
# To cache doc-building dependencies and C compiler output.
cache:
- - pip
- - ccache
- - directories:
- - $HOME/multissl
+ - pip
+ - ccache
+ - directories:
+ - $HOME/multissl
env:
global:
- - OPENSSL=1.1.0g
+ - OPENSSL=1.1.0h
- OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
- PATH="${OPENSSL_DIR}/bin:$PATH"
- - CFLAGS="-I${OPENSSL_DIR}/include"
+ # Use -O3 because we don't use debugger on Travis-CI
+ - CFLAGS="-I${OPENSSL_DIR}/include -O3"
- LDFLAGS="-L${OPENSSL_DIR}/lib"
# Set rpath with env var instead of -Wl,-rpath linker flag
# OpenSSL ignores LDFLAGS when linking bin/openssl
@@ -25,6 +26,7 @@ branches:
only:
- master
- /^\d\.\d$/
+ - buildbot-custom
matrix:
fast_finish: true
@@ -38,6 +40,10 @@ matrix:
# compiler here and the other to run the coverage build. Clang is preferred
# in this instance for its better error messages.
env: TESTING=cpython
+ addons:
+ apt:
+ packages:
+ - xvfb
- os: linux
language: python
# Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs.
@@ -54,72 +60,79 @@ matrix:
language: c
compiler: gcc
env: OPTIONAL=true
+ addons:
+ apt:
+ packages:
+ - xvfb
before_script:
- - |
- if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)'
- then
- echo "Only docs were updated, stopping build process."
- exit
- fi
- python3 Tools/ssl/multissltests.py --steps=library \
- --base-directory ${HOME}/multissl \
- --openssl ${OPENSSL} >/dev/null
- openssl version
- ./configure
- make -s -j4
- # Need a venv that can parse covered code.
- ./python -m venv venv
- ./venv/bin/python -m pip install -U coverage
- ./venv/bin/python -m test.pythoninfo
+ - ./configure
+ - make -s -j4
+ # Need a venv that can parse covered code.
+ - ./python -m venv venv
+ - ./venv/bin/python -m pip install -U coverage
+ - ./venv/bin/python -m test.pythoninfo
script:
# Skip tests that re-run the entire test suite.
- - ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
+ - xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
after_script: # Probably should be after_success once test suite updated to run under coverage.py.
# Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
- source ./venv/bin/activate
- bash <(curl -s https://codecov.io/bash)
-# Travis provides only 2 cores, so don't overdo the parallelism and waste memory.
-before_script:
+
+before_install:
+ - set -e
- |
- set -e
- if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
- files_changed=$(git diff --name-only $TRAVIS_COMMIT_RANGE)
- else
- # Pull requests are slightly complicated because merging the PR commit without
- # rebasing causes it to retain its old commit date. Meaning in history if any
- # commits have been made on master that post-date it, they will be accidentally
- # included in the diff if we use the TRAVIS_COMMIT_RANGE variable.
- files_changed=$(git diff --name-only HEAD $(git merge-base HEAD $TRAVIS_BRANCH))
+ # Check short-circuit conditions
+ if [ "${TESTING}" != "docs" ]
+ then
+ if [ "$TRAVIS_PULL_REQUEST" = "false" ]
+ then
+ echo "Not a PR, doing full build."
+ else
+ # Pull requests are slightly complicated because $TRAVIS_COMMIT_RANGE
+ # may include more changes than desired if the history is convoluted.
+ # Instead, explicitly fetch the base branch and compare against the
+ # merge-base commit.
+ git fetch -q origin +refs/heads/$TRAVIS_BRANCH
+ changes=$(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD))
+ echo "Files changed:"
+ echo "$changes"
+ if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)'
+ then
+ echo "Only docs were updated, stopping build process."
+ exit
+ fi
+ fi
fi
- # Prints changed files in this commit to help debug doc-only build issues.
- echo "Files changed: "
- echo $files_changed
-
- if ! echo $files_changed | grep -qvE '(\.rst$)|(^Doc)|(^Misc)'
+install:
+ - |
+ # Install OpenSSL as necessary
+ if [ "${TESTING}" != "docs" ]
then
- echo "Only docs were updated, stopping build process."
- exit
- fi
- if [ "${TESTING}" != "docs" ]; then
# clang complains about unused-parameter a lot, redirect stderr
python3 Tools/ssl/multissltests.py --steps=library \
--base-directory ${HOME}/multissl \
--openssl ${OPENSSL} >/dev/null 2>&1
fi
- openssl version
- ./configure --with-pydebug
- make -j4
- make -j4 regen-all clinic
- changes=`git status --porcelain`
+ - openssl version
+
+# Travis provides only 2 cores, so don't overdo the parallelism and waste memory.
+before_script:
+ - ./configure --with-pydebug
+ - make -j4 regen-all
+ - changes=`git status --porcelain`
+ - |
+ # Check for changes in regenerated files
if ! test -z "$changes"
then
echo "Generated files not up to date"
echo "$changes"
exit 1
fi
- make pythoninfo
+ - make -j4
+ - make pythoninfo
script:
# Using the built Python as patchcheck.py is built around the idea of using
@@ -127,10 +140,10 @@ script:
# should be compared against.
# Only run on Linux as the check only needs to be run once.
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi
- # `-r -w` implicitly provided through `make buildbottest`.
- - make buildbottest TESTOPTS="-j4 -uall,-cpu"
# Check that all symbols exported by libpython start with "Py" or "_Py"
- make smelly
+ # `-r -w` implicitly provided through `make buildbottest`.
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then XVFB_RUN=xvfb-run; fi; $XVFB_RUN make buildbottest TESTOPTS="-j4 -uall,-cpu"
notifications:
email: false
diff --git a/.vsts/docs-release.yml b/.vsts/docs-release.yml
new file mode 100644
index 00000000000000..e90428a42494e0
--- /dev/null
+++ b/.vsts/docs-release.yml
@@ -0,0 +1,43 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted Linux Preview
+
+#variables:
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
+ displayName: 'Install LaTeX'
+
+- task: UsePythonVersion@0
+ displayName: 'Use Python 3.6 or later'
+ inputs:
+ versionSpec: '>=3.6'
+
+- script: python -m pip install sphinx blurb python-docs-theme
+ displayName: 'Install build dependencies'
+
+- script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
+ workingDirectory: '$(build.sourcesDirectory)/Doc'
+ displayName: 'Build documentation'
+
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish build'
+ inputs:
+ PathToPublish: '$(build.sourcesDirectory)/Doc/build'
+ ArtifactName: build
+ publishLocation: Container
+
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish dist'
+ inputs:
+ PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
+ ArtifactName: dist
+ publishLocation: Container
diff --git a/.vsts/docs.yml b/.vsts/docs.yml
new file mode 100644
index 00000000000000..93a7282f770ab4
--- /dev/null
+++ b/.vsts/docs.yml
@@ -0,0 +1,58 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted Linux Preview
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ include:
+ - Doc/*
+
+#variables:
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: |
+ git fetch -q origin $(system.pullRequest.targetBranch)
+ if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "No docs were updated, stopping build process."
+ echo "##vso[task.setvariable variable=NoDocs]true"
+ exit
+ fi
+ displayName: Detect doc-only changes
+ condition: and(succeeded(), variables['system.pullRequest.targetBranch'])
+
+- task: UsePythonVersion@0
+ displayName: 'Use Python 3.6 or later'
+ inputs:
+ versionSpec: '>=3.6'
+ condition: and(succeeded(), ne(variables['NoDocs'], 'true'))
+
+- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme
+ displayName: 'Install build dependencies'
+ condition: and(succeeded(), ne(variables['NoDocs'], 'true'))
+
+- script: make check suspicious html PYTHON=python
+ workingDirectory: '$(build.sourcesDirectory)/Doc'
+ displayName: 'Build documentation'
+ condition: and(succeeded(), ne(variables['NoDocs'], 'true'))
+
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish build'
+ condition: and(and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')), ne(variables['NoDocs'], 'true'))
+ inputs:
+ PathToPublish: '$(build.sourcesDirectory)/Doc/build'
+ ArtifactName: build
+ publishLocation: Container
diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml
new file mode 100644
index 00000000000000..fc2c8ca2486e1b
--- /dev/null
+++ b/.vsts/linux-buildbot.yml
@@ -0,0 +1,72 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted Linux Preview
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ exclude:
+ - Doc/*
+ - Tools/*
+
+variables:
+ # Copy-pasted from linux-deps.yml until template support arrives
+ OPENSSL: 1.1.0g
+ OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)"
+
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+#- template: linux-deps.yml
+
+# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
+# For now, we copy/paste the steps
+- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
+ displayName: 'Update apt-get lists'
+
+- script: echo ##vso[task.prependpath]$(OPENSSL_DIR)
+ displayName: 'Add $(OPENSSL_DIR) to PATH'
+- script: >
+ sudo apt-get -yq install
+ build-essential
+ zlib1g-dev
+ libbz2-dev
+ liblzma-dev
+ libncurses5-dev
+ libreadline6-dev
+ libsqlite3-dev
+ libssl-dev
+ libgdbm-dev
+ tk-dev
+ lzma
+ lzma-dev
+ liblzma-dev
+ libffi-dev
+ uuid-dev
+ xvfb
+ displayName: 'Install dependencies'
+- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux
+ displayName: 'python multissltests.py'
+
+- script: ./configure --with-pydebug
+ displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+
+- script: make pythoninfo
+ displayName: 'Display build info'
+
+- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ displayName: 'Tests'
diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml
new file mode 100644
index 00000000000000..1112555ab93f9b
--- /dev/null
+++ b/.vsts/linux-coverage.yml
@@ -0,0 +1,98 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted Linux Preview
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ exclude:
+ - Doc/*
+ - Tools/*
+
+variables:
+ # Copy-pasted from linux-deps.yml until template support arrives
+ OPENSSL: 1.1.0g
+ OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)"
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: |
+ git fetch -q origin $(system.pullRequest.targetBranch)
+ if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "Only docs were updated, stopping build process."
+ echo "##vso[task.setvariable variable=DocOnly]true"
+ exit
+ fi
+ displayName: Detect doc-only changes
+ condition: and(succeeded(), variables['system.pullRequest.targetBranch'])
+
+#- template: linux-deps.yml
+
+# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
+# For now, we copy/paste the steps
+- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
+ displayName: 'Update apt-get lists'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: echo ##vso[task.prependpath]$(OPENSSL_DIR)
+ displayName: 'Add $(OPENSSL_DIR) to PATH'
+- script: >
+ sudo apt-get -yq install
+ build-essential
+ zlib1g-dev
+ libbz2-dev
+ liblzma-dev
+ libncurses5-dev
+ libreadline6-dev
+ libsqlite3-dev
+ libssl-dev
+ libgdbm-dev
+ tk-dev
+ lzma
+ lzma-dev
+ liblzma-dev
+ libffi-dev
+ uuid-dev
+ xvfb
+ displayName: 'Install dependencies'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux
+ displayName: 'python multissltests.py'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+
+- script: ./configure --with-pydebug
+ displayName: 'Configure CPython (debug)'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
+ displayName: 'Set up virtual environment'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: ./venv/bin/python -m test.pythoninfo
+ displayName: 'Display build info'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
+ displayName: 'Tests with coverage'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
+ displayName: 'Publish code coverage results'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
diff --git a/.vsts/linux-deps.yml b/.vsts/linux-deps.yml
new file mode 100644
index 00000000000000..83b0b5961721d0
--- /dev/null
+++ b/.vsts/linux-deps.yml
@@ -0,0 +1,37 @@
+# Note: this file is not currently used, but when template support comes to VSTS it
+# will be referenced from the other scripts..
+
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+parameters:
+ OPENSSL: 1.1.0g
+ OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)"
+
+steps:
+- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
+ displayName: 'Update apt-get lists'
+
+- script: echo ##vso[task.prependpath]$(OPENSSL_DIR)
+ displayName: 'Add $(OPENSSL_DIR) to PATH'
+- script: >
+ sudo apt-get -yq install
+ build-essential
+ zlib1g-dev
+ libbz2-dev
+ liblzma-dev
+ libncurses5-dev
+ libreadline6-dev
+ libsqlite3-dev
+ libssl-dev
+ libgdbm-dev
+ tk-dev
+ lzma
+ lzma-dev
+ liblzma-dev
+ libffi-dev
+ uuid-dev
+ xvfb
+ displayName: 'Install dependencies'
+- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux
+ displayName: 'python multissltests.py'
diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml
new file mode 100644
index 00000000000000..145ebb38016a61
--- /dev/null
+++ b/.vsts/linux-pr.yml
@@ -0,0 +1,98 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted Linux Preview
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ exclude:
+ - Doc/*
+ - Tools/*
+
+variables:
+ # Copy-pasted from linux-deps.yml until template support arrives
+ OPENSSL: 1.1.0g
+ OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)"
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: |
+ git fetch -q origin $(system.pullRequest.targetBranch)
+ if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "Only docs were updated, stopping build process."
+ echo "##vso[task.setvariable variable=DocOnly]true"
+ exit
+ fi
+ displayName: Detect doc-only changes
+ condition: and(succeeded(), variables['system.pullRequest.targetBranch'])
+
+#- template: linux-deps.yml
+
+# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
+# For now, we copy/paste the steps
+- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
+ displayName: 'Update apt-get lists'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: echo ##vso[task.prependpath]$(OPENSSL_DIR)
+ displayName: 'Add $(OPENSSL_DIR) to PATH'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: >
+ sudo apt-get -yq install
+ build-essential
+ zlib1g-dev
+ libbz2-dev
+ liblzma-dev
+ libncurses5-dev
+ libreadline6-dev
+ libsqlite3-dev
+ libssl-dev
+ libgdbm-dev
+ tk-dev
+ lzma
+ lzma-dev
+ liblzma-dev
+ libffi-dev
+ uuid-dev
+ xvfb
+ displayName: 'Install dependencies'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux
+ displayName: 'python multissltests.py'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+
+- script: ./configure --with-pydebug
+ displayName: 'Configure CPython (debug)'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: make pythoninfo
+ displayName: 'Display build info'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+# Run patchcheck and fail if anything is discovered
+- script: ./python Tools/scripts/patchcheck.py --travis true
+ displayName: 'Run patchcheck.py'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ displayName: 'Tests'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
diff --git a/.vsts/macos-buildbot.yml b/.vsts/macos-buildbot.yml
new file mode 100644
index 00000000000000..d9b2297283b0a8
--- /dev/null
+++ b/.vsts/macos-buildbot.yml
@@ -0,0 +1,37 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted macOS Preview
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ exclude:
+ - Doc/*
+ - Tools/*
+
+#variables:
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts
+ displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+
+- script: make pythoninfo
+ displayName: 'Display build info'
+
+- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ displayName: 'Tests'
diff --git a/.vsts/macos-pr.yml b/.vsts/macos-pr.yml
new file mode 100644
index 00000000000000..a3fd4487ed3a70
--- /dev/null
+++ b/.vsts/macos-pr.yml
@@ -0,0 +1,55 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted macOS Preview
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ exclude:
+ - Doc/*
+ - Tools/*
+
+#variables:
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: |
+ git fetch -q origin $(system.pullRequest.targetBranch)
+ changes = $(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD))
+ echo "Files changed:"
+ echo "$changes"
+ if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)'
+ then
+ echo "Only docs were updated, stopping build process."
+ echo "##vso[task.setvariable variable=DocOnly]true"
+ exit
+ fi
+ displayName: Detect doc-only changes
+ condition: and(succeeded(), variables['system.pullRequest.targetBranch'])
+
+- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts
+ displayName: 'Configure CPython (debug)'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: make pythoninfo
+ displayName: 'Display build info'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ displayName: 'Tests'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
diff --git a/.vsts/windows-buildbot.yml b/.vsts/windows-buildbot.yml
new file mode 100644
index 00000000000000..5ec4522796cea9
--- /dev/null
+++ b/.vsts/windows-buildbot.yml
@@ -0,0 +1,49 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted VS2017
+ parallel: 2
+ matrix:
+ amd64:
+ buildOpt: -p x64
+ outDirSuffix: amd64
+ win32:
+ buildOpt:
+ outDirSuffix: win32
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ exclude:
+ - Doc/*
+ - Tools/*
+
+variables:
+ # Relocate build outputs outside of source directory to make cleaning faster
+ Py_IntDir: $(Build.BinariesDirectory)\obj
+ # UNDONE: Do not build to a different directory because of broken tests
+ Py_OutDir: $(Build.SourcesDirectory)\PCbuild
+ EXTERNAL_DIR: $(Build.BinariesDirectory)\externals
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: PCbuild\build.bat -e $(buildOpt)
+ displayName: 'Build CPython'
+
+- script: python.bat -m test.pythoninfo
+ displayName: 'Display build info'
+
+- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+ displayName: 'Tests'
+ env:
+ PREFIX: $(Py_OutDir)\$(outDirSuffix)
diff --git a/.vsts/windows-pr.yml b/.vsts/windows-pr.yml
new file mode 100644
index 00000000000000..3dd5609a32e514
--- /dev/null
+++ b/.vsts/windows-pr.yml
@@ -0,0 +1,61 @@
+# Current docs for the syntax of this file are at:
+# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+ name: Hosted VS2017
+ parallel: 2
+ matrix:
+ amd64:
+ buildOpt: -p x64
+ outDirSuffix: amd64
+ win32:
+ buildOpt:
+ outDirSuffix: win32
+
+trigger:
+ branches:
+ include:
+ - master
+ - 3.7
+ - 3.6
+ paths:
+ exclude:
+ - Doc/*
+ - Tools/*
+
+variables:
+ # Relocate build outputs outside of source directory to make cleaning faster
+ Py_IntDir: $(Build.BinariesDirectory)\obj
+ # UNDONE: Do not build to a different directory because of broken tests
+ Py_OutDir: $(Build.SourcesDirectory)\PCbuild
+ EXTERNAL_DIR: $(Build.BinariesDirectory)\externals
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- powershell: |
+ git fetch -q origin $(System.PullRequest.TargetBranch)
+ if (-not (git diff --name-only HEAD (git merge-base HEAD FETCH_HEAD) | sls -NotMatch '(\.rst$)|(^Doc)|(^Misc)')) {
+ Write-Host 'Only docs were updated. Skipping build'
+ Write-Host '##vso[task.setvariable variable=DocOnly]true'
+ }
+ displayName: Detect doc-only changes
+ condition: and(succeeded(), variables['System.PullRequest.TargetBranch'])
+
+- script: PCbuild\build.bat -e $(buildOpt)
+ displayName: 'Build CPython'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: python.bat -m test.pythoninfo
+ displayName: 'Display build info'
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+ displayName: 'Tests'
+ env:
+ PREFIX: $(Py_OutDir)\$(outDirSuffix)
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
diff --git a/.github/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst
similarity index 100%
rename from .github/CODE_OF_CONDUCT.rst
rename to CODE_OF_CONDUCT.rst
diff --git a/Doc/README.rst b/Doc/README.rst
index a29d1f3a708a43..d7bcc5ba7919bb 100644
--- a/Doc/README.rst
+++ b/Doc/README.rst
@@ -33,7 +33,7 @@ To get started on UNIX, you can create a virtual environment with the command ::
make venv
That will install all the tools necessary to build the documentation. Assuming
-the virtual environment was created in the ``env`` directory (the default;
+the virtual environment was created in the ``venv`` directory (the default;
configurable with the VENVDIR variable), you can run the following command to
build the HTML output files::
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index e4b48e66bc2d1c..b41130ede416e3 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -151,19 +151,35 @@ which disallows mutable objects such as :class:`bytearray`.
Previously, :exc:`TypeError` was raised when embedded null code points
were encountered in the Python string.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``u#`` (:class:`str`) [const Py_UNICODE \*, int]
This variant on ``u`` stores into two C variables, the first one a pointer to a
Unicode data buffer, the second one its length. This variant allows
null code points.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``Z`` (:class:`str` or ``None``) [const Py_UNICODE \*]
Like ``u``, but the Python object may also be ``None``, in which case the
:c:type:`Py_UNICODE` pointer is set to *NULL*.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, int]
Like ``u#``, but the Python object may also be ``None``, in which case the
:c:type:`Py_UNICODE` pointer is set to *NULL*.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``U`` (:class:`str`) [PyObject \*]
Requires that the Python object is a Unicode object, without attempting
any conversion. Raises :exc:`TypeError` if the object is not a Unicode
@@ -552,12 +568,13 @@ Building values
``z#`` (:class:`str` or ``None``) [const char \*, int]
Same as ``s#``.
- ``u`` (:class:`str`) [const Py_UNICODE \*]
- Convert a null-terminated buffer of Unicode (UCS-2 or UCS-4) data to a Python
- Unicode object. If the Unicode buffer pointer is *NULL*, ``None`` is returned.
+ ``u`` (:class:`str`) [const wchar_t \*]
+ Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4)
+ data to a Python Unicode object. If the Unicode buffer pointer is *NULL*,
+ ``None`` is returned.
- ``u#`` (:class:`str`) [const Py_UNICODE \*, int]
- Convert a Unicode (UCS-2 or UCS-4) data buffer and its length to a Python
+ ``u#`` (:class:`str`) [const wchar_t \*, int]
+ Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python
Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored
and ``None`` is returned.
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 8c2de9691f3e1a..5a9a46fc67e899 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -473,6 +473,15 @@ Buffer-related functions
(*order* is ``'A'``). Return ``0`` otherwise.
+.. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order)
+
+ Copy *len* bytes from *src* to its contiguous representation in *buf*.
+ *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
+ ``0`` is returned on success, ``-1`` on error.
+
+ This function fails if *len* != *src->len*.
+
+
.. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order)
Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if
@@ -497,6 +506,3 @@ Buffer-related functions
If this function is used as part of a :ref:`getbufferproc `,
*exporter* MUST be set to the exporting object and *flags* must be passed
unmodified. Otherwise, *exporter* MUST be NULL.
-
-
-
diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst
index b8642d0aba9248..8eb6695e22de18 100644
--- a/Doc/c-api/capsule.rst
+++ b/Doc/c-api/capsule.rst
@@ -9,6 +9,8 @@ Capsules
Refer to :ref:`using-capsules` for more information on using these objects.
+.. versionadded:: 3.1
+
.. c:type:: PyCapsule
@@ -19,6 +21,7 @@ Refer to :ref:`using-capsules` for more information on using these objects.
regular import mechanism can be used to access C APIs defined in dynamically
loaded modules.
+
.. c:type:: PyCapsule_Destructor
The type of a destructor callback for a capsule. Defined as::
@@ -104,8 +107,8 @@ Refer to :ref:`using-capsules` for more information on using these objects.
import the module conventionally (using :c:func:`PyImport_ImportModule`).
Return the capsule's internal *pointer* on success. On failure, set an
- exception and return *NULL*. However, if :c:func:`PyCapsule_Import` failed to
- import the module, and *no_block* was true, no exception is set.
+ exception and return *NULL*.
+
.. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name)
@@ -122,18 +125,21 @@ Refer to :ref:`using-capsules` for more information on using these objects.
Return a nonzero value if the object is valid and matches the name passed in.
Return ``0`` otherwise. This function will not fail.
+
.. c:function:: int PyCapsule_SetContext(PyObject *capsule, void *context)
Set the context pointer inside *capsule* to *context*.
Return ``0`` on success. Return nonzero and set an exception on failure.
+
.. c:function:: int PyCapsule_SetDestructor(PyObject *capsule, PyCapsule_Destructor destructor)
Set the destructor inside *capsule* to *destructor*.
Return ``0`` on success. Return nonzero and set an exception on failure.
+
.. c:function:: int PyCapsule_SetName(PyObject *capsule, const char *name)
Set the name inside *capsule* to *name*. If non-*NULL*, the name must
@@ -142,6 +148,7 @@ Refer to :ref:`using-capsules` for more information on using these objects.
Return ``0`` on success. Return nonzero and set an exception on failure.
+
.. c:function:: int PyCapsule_SetPointer(PyObject *capsule, void *pointer)
Set the void pointer inside *capsule* to *pointer*. The pointer may not be
diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst
index 47dab81891d38f..9558a4a583cba5 100644
--- a/Doc/c-api/concrete.rst
+++ b/Doc/c-api/concrete.rst
@@ -113,5 +113,5 @@ Other Objects
capsule.rst
gen.rst
coro.rst
+ contextvars.rst
datetime.rst
-
diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst
new file mode 100644
index 00000000000000..4c33ba4f994471
--- /dev/null
+++ b/Doc/c-api/contextvars.rst
@@ -0,0 +1,125 @@
+.. highlightlang:: c
+
+.. _contextvarsobjects:
+
+Context Variables Objects
+-------------------------
+
+.. versionadded:: 3.7
+
+This section details the public C API for the :mod:`contextvars` module.
+
+.. c:type:: PyContext
+
+ The C structure used to represent a :class:`contextvars.Context`
+ object.
+
+.. c:type:: PyContextVar
+
+ The C structure used to represent a :class:`contextvars.ContextVar`
+ object.
+
+.. c:type:: PyContextToken
+
+ The C structure used to represent a :class:`contextvars.Token` object.
+
+.. c:var:: PyTypeObject PyContext_Type
+
+ The type object representing the *context* type.
+
+.. c:var:: PyTypeObject PyContextVar_Type
+
+ The type object representing the *context variable* type.
+
+.. c:var:: PyTypeObject PyContextToken_Type
+
+ The type object representing the *context variable token* type.
+
+
+Type-check macros:
+
+.. c:function:: int PyContext_CheckExact(PyObject *o)
+
+ Return true if *o* is of type :c:data:`PyContext_Type`. *o* must not be
+ *NULL*. This function always succeeds.
+
+.. c:function:: int PyContextVar_CheckExact(PyObject *o)
+
+ Return true if *o* is of type :c:data:`PyContextVar_Type`. *o* must not be
+ *NULL*. This function always succeeds.
+
+.. c:function:: int PyContextToken_CheckExact(PyObject *o)
+
+ Return true if *o* is of type :c:data:`PyContextToken_Type`.
+ *o* must not be *NULL*. This function always succeeds.
+
+
+Context object management functions:
+
+.. c:function:: PyContext *PyContext_New(void)
+
+ Create a new empty context object. Returns ``NULL`` if an error
+ has occurred.
+
+.. c:function:: PyContext *PyContext_Copy(PyContext *ctx)
+
+ Create a shallow copy of the passed *ctx* context object.
+ Returns ``NULL`` if an error has occurred.
+
+.. c:function:: PyContext *PyContext_CopyCurrent(void)
+
+ Create a shallow copy of the current thread context.
+ Returns ``NULL`` if an error has occurred.
+
+.. c:function:: int PyContext_Enter(PyContext *ctx)
+
+ Set *ctx* as the current context for the current thread.
+ Returns ``0`` on success, and ``-1`` on error.
+
+.. c:function:: int PyContext_Exit(PyContext *ctx)
+
+ Deactivate the *ctx* context and restore the previous context as the
+ current context for the current thread. Returns ``0`` on success,
+ and ``-1`` on error.
+
+.. c:function:: int PyContext_ClearFreeList()
+
+ Clear the context variable free list. Return the total number of
+ freed items. This function always succeeds.
+
+
+Context variable functions:
+
+.. c:function:: PyContextVar *PyContextVar_New(const char *name, PyObject *def)
+
+ Create a new ``ContextVar`` object. The *name* parameter is used
+ for introspection and debug purposes. The *def* parameter may optionally
+ specify the default value for the context variable. If an error has
+ occurred, this function returns ``NULL``.
+
+.. c:function:: int PyContextVar_Get(PyContextVar *var, PyObject *default_value, PyObject **value)
+
+ Get the value of a context variable. Returns ``-1`` if an error has
+ occurred during lookup, and ``0`` if no error occurred, whether or not
+ a value was found.
+
+ If the context variable was found, *value* will be a pointer to it.
+ If the context variable was *not* found, *value* will point to:
+
+ - *default_value*, if not ``NULL``;
+ - the default value of *var*, if not ``NULL``;
+ - ``NULL``
+
+ If the value was found, the function will create a new reference to it.
+
+.. c:function:: PyContextToken *PyContextVar_Set(PyContextVar *var, PyObject *value)
+
+ Set the value of *var* to *value* in the current context. Returns a
+ pointer to a :c:type:`PyContextToken` object, or ``NULL`` if an error
+ has occurred.
+
+.. c:function:: int PyContextVar_Reset(PyContextVar *var, PyContextToken *token)
+
+ Reset the state of the *var* context variable to that it was in before
+ :c:func:`PyContextVar_Set` that returned the *token* was called.
+ This function returns ``0`` on success and ``-1`` on error.
diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst
index f5e0d7ec9c79c3..7f54b6a9cff8c5 100644
--- a/Doc/c-api/gcsupport.rst
+++ b/Doc/c-api/gcsupport.rst
@@ -49,7 +49,7 @@ Constructors for container types must conform to two rules:
.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize)
Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the
- resized object or *NULL* on failure.
+ resized object or *NULL* on failure. *op* must not be tracked by the collector yet.
.. c:function:: void PyObject_GC_Track(PyObject *op)
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index bae49d5ba8125e..694b4669eea897 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -31,6 +31,9 @@ The following functions can be safely called before Python is initialized:
* :c:func:`Py_SetProgramName`
* :c:func:`Py_SetPythonHome`
* :c:func:`Py_SetStandardStreamEncoding`
+ * :c:func:`PySys_AddWarnOption`
+ * :c:func:`PySys_AddXOption`
+ * :c:func:`PySys_ResetWarnOptions`
* Informative functions:
@@ -273,8 +276,8 @@ Initializing and finalizing the interpreter
the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory
allocated by the Python interpreter. This is a no-op when called for a second
time (without calling :c:func:`Py_Initialize` again first). Normally the
- return value is 0. If there were errors during finalization
- (flushing buffered data), -1 is returned.
+ return value is ``0``. If there were errors during finalization
+ (flushing buffered data), ``-1`` is returned.
This function is provided for a number of reasons. An embedding application
might want to restart Python without having to restart the application itself.
@@ -1018,7 +1021,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
.. c:function:: PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp)
Return the interpreter's unique ID. If there was any error in doing
- so then -1 is returned and an error is set.
+ so then ``-1`` is returned and an error is set.
.. versionadded:: 3.7
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index 4f16b578eb5999..5b1f386fb7e58f 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -137,7 +137,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Raise :exc:`OverflowError` if the value of *obj* is out of range for a
:c:type:`long`.
- Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow)
@@ -151,7 +151,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception
occurs set *\*overflow* to ``0`` and return ``-1`` as usual.
- Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: long long PyLong_AsLongLong(PyObject *obj)
@@ -166,7 +166,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Raise :exc:`OverflowError` if the value of *obj* is out of range for a
:c:type:`long`.
- Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow)
@@ -180,7 +180,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other
exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual.
- Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. versionadded:: 3.2
@@ -197,7 +197,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Raise :exc:`OverflowError` if the value of *pylong* is out of range for a
:c:type:`Py_ssize_t`.
- Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong)
@@ -259,7 +259,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
If the value of *obj* is out of range for an :c:type:`unsigned long`,
return the reduction of that value modulo ``ULONG_MAX + 1``.
- Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj)
@@ -271,7 +271,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
If the value of *obj* is out of range for an :c:type:`unsigned long long`,
return the reduction of that value modulo ``PY_ULLONG_MAX + 1``.
- Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: double PyLong_AsDouble(PyObject *pylong)
@@ -282,7 +282,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Raise :exc:`OverflowError` if the value of *pylong* is out of range for a
:c:type:`double`.
- Returns -1.0 on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns ``-1.0`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong)
@@ -292,4 +292,4 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
is only assured to produce a usable :c:type:`void` pointer for values created
with :c:func:`PyLong_FromVoidPtr`.
- Returns NULL on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+ Returns *NULL* on error. Use :c:func:`PyErr_Occurred` to disambiguate.
diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst
index 308a9761f87ea9..b8eaadbd702c10 100644
--- a/Doc/c-api/mapping.rst
+++ b/Doc/c-api/mapping.rst
@@ -5,11 +5,17 @@
Mapping Protocol
================
+See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and
+:c:func:`PyObject_DelItem`.
+
.. c:function:: int PyMapping_Check(PyObject *o)
- Return ``1`` if the object provides mapping protocol, and ``0`` otherwise. This
- function always succeeds.
+ Return ``1`` if the object provides mapping protocol or supports slicing,
+ and ``0`` otherwise. Note that it returns ``1`` for Python classes with
+ a :meth:`__getitem__` method since in general case it is impossible to
+ determine what the type of keys it supports. This function always
+ succeeds.
.. c:function:: Py_ssize_t PyMapping_Size(PyObject *o)
@@ -17,35 +23,49 @@ Mapping Protocol
.. index:: builtin: len
- Returns the number of keys in object *o* on success, and ``-1`` on failure. For
- objects that do not provide mapping protocol, this is equivalent to the Python
- expression ``len(o)``.
+ Returns the number of keys in object *o* on success, and ``-1`` on failure.
+ This is equivalent to the Python expression ``len(o)``.
-.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key)
+.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key)
+
+ Return element of *o* corresponding to the string *key* or *NULL* on failure.
+ This is the equivalent of the Python expression ``o[key]``.
+ See also :c:func:`PyObject_GetItem`.
+
- Remove the mapping for object *key* from the object *o*. Return ``-1`` on
- failure. This is equivalent to the Python statement ``del o[key]``.
+.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v)
+
+ Map the string *key* to the value *v* in object *o*. Returns ``-1`` on
+ failure. This is the equivalent of the Python statement ``o[key] = v``.
+ See also :c:func:`PyObject_SetItem`.
.. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key)
- Remove the mapping for object *key* from the object *o*. Return ``-1`` on
- failure. This is equivalent to the Python statement ``del o[key]``.
+ Remove the mapping for the object *key* from the object *o*. Return ``-1``
+ on failure. This is equivalent to the Python statement ``del o[key]``.
+ This is an alias of :c:func:`PyObject_DelItem`.
-.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key)
+.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key)
- On success, return ``1`` if the mapping object has the key *key* and ``0``
- otherwise. This is equivalent to the Python expression ``key in o``.
- This function always succeeds.
+ Remove the mapping for the string *key* from the object *o*. Return ``-1``
+ on failure. This is equivalent to the Python statement ``del o[key]``.
.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key)
- Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This
- is equivalent to the Python expression ``key in o``. This function always
- succeeds.
+ Return ``1`` if the mapping object has the key *key* and ``0`` otherwise.
+ This is equivalent to the Python expression ``key in o``.
+ This function always succeeds.
+
+
+.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key)
+
+ Return ``1`` if the mapping object has the key *key* and ``0`` otherwise.
+ This is equivalent to the Python expression ``key in o``.
+ This function always succeeds.
.. c:function:: PyObject* PyMapping_Keys(PyObject *o)
@@ -73,15 +93,3 @@ Mapping Protocol
.. versionchanged:: 3.7
Previously, the function returned a list or a tuple.
-
-
-.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key)
-
- Return element of *o* corresponding to the object *key* or *NULL* on failure.
- This is the equivalent of the Python expression ``o[key]``.
-
-
-.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v)
-
- Map the object *key* to the value *v* in object *o*. Returns ``-1`` on failure.
- This is the equivalent of the Python statement ``o[key] = v``.
diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst
index c6d1d02a2fa510..17ec621610b5e3 100644
--- a/Doc/c-api/marshal.rst
+++ b/Doc/c-api/marshal.rst
@@ -40,12 +40,6 @@ unmarshalling. Version 2 uses a binary format for floating point numbers.
The following functions allow marshalled values to be read back in.
-XXX What about error detection? It appears that reading past the end of the
-file will always result in a negative numeric value (where that's relevant),
-but it's not clear that negative values won't be handled properly when there's
-no error. What's the right way to tell? Should only non-negative values be
-written using these routines?
-
.. c:function:: long PyMarshal_ReadLongFromFile(FILE *file)
@@ -53,7 +47,8 @@ written using these routines?
for reading. Only a 32-bit value can be read in using this function,
regardless of the native size of :c:type:`long`.
- On error, raise an exception and return ``-1``.
+ On error, sets the appropriate exception (:exc:`EOFError`) and returns
+ ``-1``.
.. c:function:: int PyMarshal_ReadShortFromFile(FILE *file)
@@ -62,7 +57,8 @@ written using these routines?
for reading. Only a 16-bit value can be read in using this function,
regardless of the native size of :c:type:`short`.
- On error, raise an exception and return ``-1``.
+ On error, sets the appropriate exception (:exc:`EOFError`) and returns
+ ``-1``.
.. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file)
@@ -70,8 +66,8 @@ written using these routines?
Return a Python object from the data stream in a :c:type:`FILE\*` opened for
reading.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
.. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file)
@@ -84,8 +80,8 @@ written using these routines?
file. Only use these variant if you are certain that you won't be reading
anything else from the file.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len)
@@ -93,6 +89,6 @@ written using these routines?
Return a Python object from the data stream in a byte buffer
containing *len* bytes pointed to by *data*.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
index 2af0c46d451f05..9f7b13c1472953 100644
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -35,7 +35,7 @@ operate within the bounds of the private heap.
It is important to understand that the management of the Python heap is
performed by the interpreter itself and that the user has no control over it,
-even if she regularly manipulates object pointers to memory blocks inside that
+even if they regularly manipulate object pointers to memory blocks inside that
heap. The allocation of heap space for Python objects and other internal
buffers is performed on demand by the Python memory manager through the Python/C
API functions listed in this document.
@@ -530,7 +530,7 @@ tracemalloc C API
Track an allocated memory block in the :mod:`tracemalloc` module.
- Return 0 on success, return ``-1`` on error (failed to allocate memory to
+ Return ``0`` on success, return ``-1`` on error (failed to allocate memory to
store the trace). Return ``-2`` if tracemalloc is disabled.
If memory block is already tracked, update the existing trace.
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 7efab28af724aa..017b656854a8cd 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel
.. c:member:: traverseproc m_traverse
A traversal function to call during GC traversal of the module object, or
- *NULL* if not needed.
+ *NULL* if not needed. This function may be called before module state
+ is allocated (:c:func:`PyModule_GetState()` may return `NULL`),
+ and before the :c:member:`Py_mod_exec` function is executed.
.. c:member:: inquiry m_clear
A clear function to call during GC clearing of the module object, or
- *NULL* if not needed.
+ *NULL* if not needed. This function may be called before module state
+ is allocated (:c:func:`PyModule_GetState()` may return `NULL`),
+ and before the :c:member:`Py_mod_exec` function is executed.
.. c:member:: freefunc m_free
A function to call during deallocation of the module object, or *NULL* if
- not needed.
+ not needed. This function may be called before module state
+ is allocated (:c:func:`PyModule_GetState()` may return `NULL`),
+ and before the :c:member:`Py_mod_exec` function is executed.
Single-phase initialization
...........................
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index 754dedc1c60370..f0b2005f2d12cc 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -379,8 +379,8 @@ Object Protocol
parameters must be non-*NULL*.
-.. c:function:: Py_ssize_t PyObject_Length(PyObject *o)
- Py_ssize_t PyObject_Size(PyObject *o)
+.. c:function:: Py_ssize_t PyObject_Size(PyObject *o)
+ Py_ssize_t PyObject_Length(PyObject *o)
.. index:: builtin: len
@@ -414,8 +414,8 @@ Object Protocol
.. c:function:: int PyObject_DelItem(PyObject *o, PyObject *key)
- Delete the mapping for *key* from *o*. Returns ``-1`` on failure. This is the
- equivalent of the Python statement ``del o[key]``.
+ Remove the mapping for the object *key* from the object *o*. Return ``-1``
+ on failure. This is equivalent to the Python statement ``del o[key]``.
.. c:function:: PyObject* PyObject_Dir(PyObject *o)
diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst
index f1825f079be474..6d22f35e22b1f2 100644
--- a/Doc/c-api/sequence.rst
+++ b/Doc/c-api/sequence.rst
@@ -9,7 +9,10 @@ Sequence Protocol
.. c:function:: int PySequence_Check(PyObject *o)
Return ``1`` if the object provides sequence protocol, and ``0`` otherwise.
- This function always succeeds.
+ Note that it returns ``1`` for Python classes with a :meth:`__getitem__`
+ method unless they are :class:`dict` subclasses since in general case it
+ is impossible to determine what the type of keys it supports. This
+ function always succeeds.
.. c:function:: Py_ssize_t PySequence_Size(PyObject *o)
@@ -17,9 +20,8 @@ Sequence Protocol
.. index:: builtin: len
- Returns the number of objects in sequence *o* on success, and ``-1`` on failure.
- For objects that do not provide sequence protocol, this is equivalent to the
- Python expression ``len(o)``.
+ Returns the number of objects in sequence *o* on success, and ``-1`` on
+ failure. This is equivalent to the Python expression ``len(o)``.
.. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2)
@@ -120,18 +122,27 @@ Sequence Protocol
.. index:: builtin: tuple
- Return a tuple object with the same contents as the arbitrary sequence *o* or
- *NULL* on failure. If *o* is a tuple, a new reference will be returned,
+ Return a tuple object with the same contents as the sequence or iterable *o*,
+ or *NULL* on failure. If *o* is a tuple, a new reference will be returned,
otherwise a tuple will be constructed with the appropriate contents. This is
equivalent to the Python expression ``tuple(o)``.
.. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m)
- Return the sequence *o* as a list, unless it is already a tuple or list, in
+ Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in
which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access
the members of the result. Returns *NULL* on failure. If the object is not
- a sequence, raises :exc:`TypeError` with *m* as the message text.
+ a sequence or iterable, raises :exc:`TypeError` with *m* as the message text.
+
+
+.. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o)
+
+ Returns the length of *o*, assuming that *o* was returned by
+ :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be
+ gotten by calling :c:func:`PySequence_Size` on *o*, but
+ :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list
+ or tuple.
.. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i)
@@ -156,12 +167,3 @@ Sequence Protocol
:c:func:`PySequence_GetItem` but without checking that
:c:func:`PySequence_Check` on *o* is true and without adjustment for negative
indices.
-
-
-.. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o)
-
- Returns the length of *o*, assuming that *o* was returned by
- :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be
- gotten by calling :c:func:`PySequence_Size` on *o*, but
- :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list
- or tuple.
diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst
index e4da96c493cd88..994509aa50f2ad 100644
--- a/Doc/c-api/sys.rst
+++ b/Doc/c-api/sys.rst
@@ -205,16 +205,24 @@ accessible to C code. They all work with the current interpreter thread's
.. c:function:: void PySys_ResetWarnOptions()
- Reset :data:`sys.warnoptions` to an empty list.
+ Reset :data:`sys.warnoptions` to an empty list. This function may be
+ called prior to :c:func:`Py_Initialize`.
.. c:function:: void PySys_AddWarnOption(const wchar_t *s)
- Append *s* to :data:`sys.warnoptions`.
+ Append *s* to :data:`sys.warnoptions`. This function must be called prior
+ to :c:func:`Py_Initialize` in order to affect the warnings filter list.
.. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode)
Append *unicode* to :data:`sys.warnoptions`.
+ Note: this function is not currently usable from outside the CPython
+ implementation, as it must be called prior to the implicit import of
+ :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be
+ called until enough of the runtime has been initialized to permit the
+ creation of Unicode objects.
+
.. c:function:: void PySys_SetPath(const wchar_t *path)
Set :data:`sys.path` to a list object of paths found in *path* which should
@@ -260,7 +268,8 @@ accessible to C code. They all work with the current interpreter thread's
.. c:function:: void PySys_AddXOption(const wchar_t *s)
Parse *s* as a set of :option:`-X` options and add them to the current
- options mapping as returned by :c:func:`PySys_GetXOptions`.
+ options mapping as returned by :c:func:`PySys_GetXOptions`. This function
+ may be called prior to :c:func:`Py_Initialize`.
.. versionadded:: 3.2
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 3bdf45ad9b61f7..6cbcc273c1f153 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -1167,21 +1167,24 @@ Mapping Object Structures
.. c:member:: lenfunc PyMappingMethods.mp_length
- This function is used by :c:func:`PyMapping_Length` and
+ This function is used by :c:func:`PyMapping_Size` and
:c:func:`PyObject_Size`, and has the same signature. This slot may be set to
*NULL* if the object has no defined length.
.. c:member:: binaryfunc PyMappingMethods.mp_subscript
- This function is used by :c:func:`PyObject_GetItem` and has the same
- signature. This slot must be filled for the :c:func:`PyMapping_Check`
- function to return ``1``, it can be *NULL* otherwise.
+ This function is used by :c:func:`PyObject_GetItem` and
+ :c:func:`PySequence_GetSlice`, and has the same signature as
+ :c:func:`!PyObject_GetItem`. This slot must be filled for the
+ :c:func:`PyMapping_Check` function to return ``1``, it can be *NULL*
+ otherwise.
.. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript
- This function is used by :c:func:`PyObject_SetItem` and
- :c:func:`PyObject_DelItem`. It has the same signature as
- :c:func:`PyObject_SetItem`, but *v* can also be set to *NULL* to delete
+ This function is used by :c:func:`PyObject_SetItem`,
+ :c:func:`PyObject_DelItem`, :c:func:`PyObject_SetSlice` and
+ :c:func:`PyObject_DelSlice`. It has the same signature as
+ :c:func:`!PyObject_SetItem`, but *v* can also be set to *NULL* to delete
an item. If this slot is *NULL*, the object does not support item
assignment and deletion.
@@ -1201,26 +1204,29 @@ Sequence Object Structures
.. c:member:: lenfunc PySequenceMethods.sq_length
- This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`,
- and has the same signature.
+ This function is used by :c:func:`PySequence_Size` and
+ :c:func:`PyObject_Size`, and has the same signature. It is also used for
+ handling negative indices via the :c:member:`~PySequenceMethods.sq_item`
+ and the :c:member:`~PySequenceMethods.sq_ass_item` slots.
.. c:member:: binaryfunc PySequenceMethods.sq_concat
This function is used by :c:func:`PySequence_Concat` and has the same
signature. It is also used by the ``+`` operator, after trying the numeric
- addition via the :c:member:`~PyTypeObject.tp_as_number.nb_add` slot.
+ addition via the :c:member:`~PyNumberMethods.nb_add` slot.
.. c:member:: ssizeargfunc PySequenceMethods.sq_repeat
This function is used by :c:func:`PySequence_Repeat` and has the same
signature. It is also used by the ``*`` operator, after trying numeric
- multiplication via the :c:member:`~PyTypeObject.tp_as_number.nb_multiply`
- slot.
+ multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot.
.. c:member:: ssizeargfunc PySequenceMethods.sq_item
This function is used by :c:func:`PySequence_GetItem` and has the same
- signature. This slot must be filled for the :c:func:`PySequence_Check`
+ signature. It is also used by :c:func:`PyObject_GetItem`, after trying
+ the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot.
+ This slot must be filled for the :c:func:`PySequence_Check`
function to return ``1``, it can be *NULL* otherwise.
Negative indexes are handled as follows: if the :attr:`sq_length` slot is
@@ -1231,28 +1237,36 @@ Sequence Object Structures
.. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item
This function is used by :c:func:`PySequence_SetItem` and has the same
- signature. This slot may be left to *NULL* if the object does not support
+ signature. It is also used by :c:func:`PyObject_SetItem` and
+ :c:func:`PyObject_DelItem`, after trying the item assignment and deletion
+ via the :c:member:`~PyMappingMethods.mp_ass_subscript` slot.
+ This slot may be left to *NULL* if the object does not support
item assignment and deletion.
.. c:member:: objobjproc PySequenceMethods.sq_contains
This function may be used by :c:func:`PySequence_Contains` and has the same
signature. This slot may be left to *NULL*, in this case
- :c:func:`PySequence_Contains` simply traverses the sequence until it finds a
- match.
+ :c:func:`!PySequence_Contains` simply traverses the sequence until it
+ finds a match.
.. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat
This function is used by :c:func:`PySequence_InPlaceConcat` and has the same
- signature. It should modify its first operand, and return it.
+ signature. It should modify its first operand, and return it. This slot
+ may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceConcat`
+ will fall back to :c:func:`PySequence_Concat`. It is also used by the
+ augmented assignment ``+=``, after trying numeric inplace addition
+ via the :c:member:`~PyNumberMethods.nb_inplace_add` slot.
.. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat
This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same
- signature. It should modify its first operand, and return it.
-
-.. XXX need to explain precedence between mapping and sequence
-.. XXX explains when to implement the sq_inplace_* slots
+ signature. It should modify its first operand, and return it. This slot
+ may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceRepeat`
+ will fall back to :c:func:`PySequence_Repeat`. It is also used by the
+ augmented assignment ``*=``, after trying numeric inplace multiplication
+ via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot.
.. _buffer-structs:
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index 6dc86fc5e54cd4..62cc93832783b7 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -152,6 +152,47 @@ PyComplex_ImagAsDouble:PyObject*:op:0:
PyComplex_RealAsDouble:double:::
PyComplex_RealAsDouble:PyObject*:op:0:
+PyContext_CheckExact:int:::
+PyContext_CheckExact:PyObject*:o:0:
+
+PyContext_ClearFreeList:int:::
+
+PyContext_Copy:PyObject*::+1:
+PyContext_Copy:PyObject*:ctx:0:
+
+PyContext_CopyCurrent:PyObject*::+1:
+
+PyContext_Enter:int:::
+PyContext_Enter:PyObject*:ctx:+1:
+
+PyContext_Exit:int:::
+PyContext_Exit:PyObject*:ctx:-1:
+
+PyContext_New:PyObject*::+1:
+
+PyContextToken_CheckExact:int:::
+PyContextToken_CheckExact:PyObject*:o:0:
+
+PyContextVar_CheckExact:int:::
+PyContextVar_CheckExact:PyObject*:o:0:
+
+PyContextVar_Get:int:::
+PyContextVar_Get:PyObject*:var:0:
+PyContextVar_Get:PyObject*:default_value:0:
+PyContextVar_Get:PyObject**:value:+1:???
+
+PyContextVar_New:PyObject*::+1:
+PyContextVar_New:const char*:name::
+PyContextVar_New:PyObject*:def:+1:
+
+PyContextVar_Set:PyObject*::+1:
+PyContextVar_Set:PyObject*:var:0:
+PyContextVar_Set:PyObject*:value:+1:
+
+PyContextVar_Reset:int:::
+PyContextVar_Reset:PyObject*:var:0:
+PyContextVar_Reset:PyObject*:token:-1:
+
PyDate_FromDate:PyObject*::+1:
PyDate_FromDate:int:year::
PyDate_FromDate:int:month::
@@ -541,6 +582,9 @@ PyImport_ExecCodeModuleEx:const char*:pathname::
PyImport_GetMagicNumber:long:::
+PyImport_GetModule:PyObject*::+1:
+PyImport_GetModule:PyObject*:name:0:
+
PyImport_GetModuleDict:PyObject*::0:
PyImport_Import:PyObject*::+1:
@@ -605,6 +649,9 @@ PyInterpreterState_Clear:PyInterpreterState*:interp::
PyInterpreterState_Delete:void:::
PyInterpreterState_Delete:PyInterpreterState*:interp::
+PyInterpreterState_GetID:int64_t:::
+PyInterpreterState_GetID:PyInterpreterState*:interp::
+
PyInterpreterState_New:PyInterpreterState*:::
PyIter_Check:int:o:0:
@@ -933,6 +980,14 @@ PyNumber_Xor:PyObject*:o2:0:
PyObject_AsFileDescriptor:int:::
PyObject_AsFileDescriptor:PyObject*:o:0:
+PyOS_AfterFork:void:::
+
+PyOS_AfterFork_Child:void:::
+
+PyOS_AfterFork_Parent:void:::
+
+PyOS_BeforeFork:void:::
+
PyOS_FSPath:PyObject*::+1:
PyOS_FSPath:PyObject*:path:0:
@@ -1252,6 +1307,12 @@ PySet_Pop:PyObject*:set:0:
PySet_Size:int:::
PySet_Size:PyObject*:anyset:0:
+PySlice_AdjustIndices:Py_ssize_t:::
+PySlice_AdjustIndices:Py_ssize_t:length::
+PySlice_AdjustIndices:Py_ssize_t*:start::
+PySlice_AdjustIndices:Py_ssize_t*:stop::
+PySlice_AdjustIndices:Py_ssize_t*:step::
+
PySlice_Check:int:::
PySlice_Check:PyObject*:ob:0:
@@ -1260,6 +1321,12 @@ PySlice_New:PyObject*:start:0:
PySlice_New:PyObject*:stop:0:
PySlice_New:PyObject*:step:0:
+PySlice_Unpack:int:::
+PySlice_Unpack:PyObject*:slice:0:
+PySlice_Unpack:Py_ssize_t*:start::
+PySlice_Unpack:Py_ssize_t*:stop::
+PySlice_Unpack:Py_ssize_t*:step::
+
PyString_AS_STRING:const char*:::
PyString_AS_STRING:PyObject*:string:0:
@@ -1383,12 +1450,42 @@ PyThreadState_New:PyInterpreterState*:interp::
PyThreadState_Swap:PyThreadState*:::
PyThreadState_Swap:PyThreadState*:tstate::
+PyThread_tss_alloc:Py_tss_t*:::
+
+PyThread_tss_create:int:::
+PyThread_tss_create:Py_tss_t*:key::
+
+PyThread_tss_delete:void:::
+PyThread_tss_delete:Py_tss_t*:key::
+
+PyThread_tss_free:void:::
+PyThread_tss_free:Py_tss_t*:key::
+
+PyThread_tss_get:void*:::
+PyThread_tss_get:Py_tss_t*:key::
+
+PyThread_tss_is_created:int:::
+PyThread_tss_is_created:Py_tss_t*:key::
+
+PyThread_tss_set:int:::
+PyThread_tss_set:Py_tss_t*:key::
+PyThread_tss_set:void*:value::
+
PyTime_FromTime:PyObject*::+1:
PyTime_FromTime:int:hour::
PyTime_FromTime:int:minute::
PyTime_FromTime:int:second::
PyTime_FromTime:int:usecond::
+PyTraceMalloc_Track:int:::
+PyTraceMalloc_Track:unsigned int:domain::
+PyTraceMalloc_Track:uintptr_t:ptr::
+PyTraceMalloc_Track:size_t:size::
+
+PyTraceMalloc_Untrack:int:::
+PyTraceMalloc_Untrack:unsigned int:domain::
+PyTraceMalloc_Untrack:uintptr_t:ptr::
+
PyTuple_Check:int:::
PyTuple_Check:PyObject*:p:0:
diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst
index aedbe712d3db33..5dd14b1f7a60b9 100644
--- a/Doc/distributing/index.rst
+++ b/Doc/distributing/index.rst
@@ -31,7 +31,7 @@ installing other Python projects, refer to the
Key terms
=========
-* the `Python Packaging Index `__ is a public
+* the `Python Packaging Index `__ is a public
repository of open source licensed packages made available for use by
other Python users
* the `Python Packaging Authority
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index 9fce46ad266962..3c89468220a4dc 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -78,7 +78,7 @@ setup script). Indirectly provides the :class:`distutils.dist.Distribution` and
| | be built | :class:`distutils.core.Extension` |
+--------------------+--------------------------------+-------------------------------------------------------------+
| *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPI |
- | | package | `_. |
+ | | package | `_. |
+--------------------+--------------------------------+-------------------------------------------------------------+
| *distclass* | the :class:`Distribution` | a subclass of |
| | class to use | :class:`distutils.core.Distribution` |
diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst
index f523a672340876..758bd141ac956b 100644
--- a/Doc/distutils/builtdist.rst
+++ b/Doc/distutils/builtdist.rst
@@ -21,7 +21,7 @@ specialty---writing code and creating source distributions---while an
intermediary species called *packagers* springs up to turn source distributions
into built distributions for as many platforms as there are packagers.
-Of course, the module developer could be his own packager; or the packager could
+Of course, the module developer could be their own packager; or the packager could
be a volunteer "out there" somewhere who has access to a platform which the
original developer does not; or it could be software periodically grabbing new
source distributions and turning them into built distributions for as many
diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst
index 21f1acdace5f19..cd10a7fdf31513 100644
--- a/Doc/distutils/configfile.rst
+++ b/Doc/distutils/configfile.rst
@@ -36,7 +36,9 @@ consequences:
* installers can override anything in :file:`setup.cfg` using the command-line
options to :file:`setup.py`
-The basic syntax of the configuration file is simple::
+The basic syntax of the configuration file is simple:
+
+.. code-block:: ini
[command]
option=value
@@ -51,9 +53,11 @@ option values can be split across multiple lines simply by indenting the
continuation lines.
You can find out the list of options supported by a particular command with the
-universal :option:`!--help` option, e.g. ::
+universal :option:`!--help` option, e.g.
+
+.. code-block:: shell-session
- > python setup.py --help build_ext
+ $ python setup.py --help build_ext
[...]
Options for 'build_ext' command:
--build-lib (-b) directory for compiled extension modules
@@ -75,14 +79,18 @@ For example, say you want your extensions to be built "in-place"---that is, you
have an extension :mod:`pkg.ext`, and you want the compiled extension file
(:file:`ext.so` on Unix, say) to be put in the same source directory as your
pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the
-:option:`!--inplace` option on the command-line to ensure this::
+:option:`!--inplace` option on the command-line to ensure this:
+
+.. code-block:: sh
python setup.py build_ext --inplace
But this requires that you always specify the :command:`build_ext` command
explicitly, and remember to provide :option:`!--inplace`. An easier way is to
"set and forget" this option, by encoding it in :file:`setup.cfg`, the
-configuration file for this distribution::
+configuration file for this distribution:
+
+.. code-block:: ini
[build_ext]
inplace=1
@@ -103,7 +111,9 @@ information comes from the setup script, and some is automatically generated by
the Distutils (such as the list of files installed). But some of it has to be
supplied as options to :command:`bdist_rpm`, which would be very tedious to do
on the command-line for every run. Hence, here is a snippet from the Distutils'
-own :file:`setup.cfg`::
+own :file:`setup.cfg`:
+
+.. code-block:: ini
[bdist_rpm]
release = 1
diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst
index 8f46bd74c5b01a..7721484fe73717 100644
--- a/Doc/distutils/introduction.rst
+++ b/Doc/distutils/introduction.rst
@@ -94,7 +94,7 @@ containing your setup script :file:`setup.py`, and your module :file:`foo.py`.
The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and
will unpack into a directory :file:`foo-1.0`.
-If an end-user wishes to install your :mod:`foo` module, all she has to do is
+If an end-user wishes to install your :mod:`foo` module, all they have to do is
download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and---from the
:file:`foo-1.0` directory---run ::
@@ -193,8 +193,8 @@ modules using the Distutils:
module distribution
a collection of Python modules distributed together as a single downloadable
resource and meant to be installed *en masse*. Examples of some well-known
- module distributions are NumPy, SciPy, PIL (the Python Imaging
- Library), or mxBase. (This would be called a *package*, except that term is
+ module distributions are NumPy, SciPy, Pillow,
+ or mxBase. (This would be called a *package*, except that term is
already taken in the Python context: a single module distribution may contain
zero, one, or many Python packages.)
diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst
index 44556e3df9c5b1..50cb74f2b6ca90 100644
--- a/Doc/distutils/packageindex.rst
+++ b/Doc/distutils/packageindex.rst
@@ -156,7 +156,9 @@ The :command:`register` and :command:`upload` commands both check for the
existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`.
If this file exists, the command uses the username, password, and repository
URL configured in the file. The format of a :file:`.pypirc` file is as
-follows::
+follows:
+
+.. code-block:: ini
[distutils]
index-servers =
@@ -179,7 +181,9 @@ Each section describing a repository defines three variables:
will be prompt to type it when needed.
If you want to define another server a new section can be created and
-listed in the *index-servers* variable::
+listed in the *index-servers* variable:
+
+.. code-block:: ini
[distutils]
index-servers =
@@ -246,4 +250,4 @@ without warnings does not guarantee that PyPI will convert the content
successfully.
-.. _Python Package Index (PyPI): https://pypi.python.org/pypi
+.. _Python Package Index (PyPI): https://pypi.org
diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst
index 952607a4073b25..1d96acbe98f696 100644
--- a/Doc/distutils/setupscript.rst
+++ b/Doc/distutils/setupscript.rst
@@ -625,7 +625,7 @@ Notes:
(7)
The valid classifiers are listed on
- `PyPI `_.
+ `PyPI `_.
(8)
To preserve backward compatibility, this field also accepts a string. If
diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst
index ab2f61614afc4e..7e4fc19db83b2a 100644
--- a/Doc/extending/embedding.rst
+++ b/Doc/extending/embedding.rst
@@ -323,7 +323,7 @@ options. In this case, the :mod:`sysconfig` module is a useful tool to
programmatically extract the configuration values that you will want to
combine together. For example:
-.. code-block:: python
+.. code-block:: pycon
>>> import sysconfig
>>> sysconfig.get_config_var('LIBS')
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index e02f7837b69ed2..82b689e064c1d5 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -43,7 +43,9 @@ Let's create an extension module called ``spam`` (the favorite food of Monty
Python fans...) and let's say we want to create a Python interface to the C
library function :c:func:`system` [#]_. This function takes a null-terminated
character string as argument and returns an integer. We want this function to
-be callable from Python as follows::
+be callable from Python as follows:
+
+.. code-block:: pycon
>>> import spam
>>> status = spam.system("ls -l")
@@ -439,7 +441,9 @@ part of the Python interpreter, you will have to change the configuration setup
and rebuild the interpreter. Luckily, this is very simple on Unix: just place
your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory
of an unpacked source distribution, add a line to the file
-:file:`Modules/Setup.local` describing your file::
+:file:`Modules/Setup.local` describing your file:
+
+.. code-block:: sh
spam spammodule.o
@@ -450,7 +454,9 @@ subdirectory, but then you must first rebuild :file:`Makefile` there by running
:file:`Setup` file.)
If your module requires additional libraries to link with, these can be listed
-on the line in the configuration file as well, for instance::
+on the line in the configuration file as well, for instance:
+
+.. code-block:: sh
spam spammodule.o -lX11
diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst
index 80594e357fd497..0994e3e8627dfa 100644
--- a/Doc/extending/index.rst
+++ b/Doc/extending/index.rst
@@ -26,9 +26,11 @@ Recommended third party tools
=============================
This guide only covers the basic tools for creating extensions provided
-as part of this version of CPython. Third party tools like Cython,
-``cffi``, SWIG and Numba offer both simpler and more sophisticated
-approaches to creating C and C++ extensions for Python.
+as part of this version of CPython. Third party tools like
+`Cython `_, `cffi `_,
+`SWIG `_ and `Numba `_
+offer both simpler and more sophisticated approaches to creating C and C++
+extensions for Python.
.. seealso::
@@ -52,6 +54,7 @@ C extensions.
:numbered:
extending.rst
+ newtypes_tutorial.rst
newtypes.rst
building.rst
windows.rst
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index 62fbdb87a53000..d0d2ec1f88207c 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -1,889 +1,11 @@
.. highlightlang:: c
-
-.. _defining-new-types:
-
-******************
-Defining New Types
-******************
-
-.. sectionauthor:: Michael Hudson
-.. sectionauthor:: Dave Kuhlman
-.. sectionauthor:: Jim Fulton
-
-
-As mentioned in the last chapter, Python allows the writer of an extension
-module to define new types that can be manipulated from Python code, much like
-strings and lists in core Python.
-
-This is not hard; the code for all extension types follows a pattern, but there
-are some details that you need to understand before you can get started.
-
-
-.. _dnt-basics:
-
-The Basics
-==========
-
-The Python runtime sees all Python objects as variables of type
-:c:type:`PyObject\*`, which serves as a "base type" for all Python objects.
-:c:type:`PyObject` itself only contains the refcount and a pointer to the
-object's "type object". This is where the action is; the type object determines
-which (C) functions get called when, for instance, an attribute gets looked
-up on an object or it is multiplied by another object. These C functions
-are called "type methods".
-
-So, if you want to define a new object type, you need to create a new type
-object.
-
-This sort of thing can only be explained by example, so here's a minimal, but
-complete, module that defines a new type:
-
-.. literalinclude:: ../includes/noddy.c
-
-
-Now that's quite a bit to take in at once, but hopefully bits will seem familiar
-from the last chapter.
-
-The first bit that will be new is::
-
- typedef struct {
- PyObject_HEAD
- } noddy_NoddyObject;
-
-This is what a Noddy object will contain---in this case, nothing more than what
-every Python object contains---a field called ``ob_base`` of type
-:c:type:`PyObject`. :c:type:`PyObject` in turn, contains an ``ob_refcnt``
-field and a pointer to a type object. These can be accessed using the macros
-:c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE` respectively. These are the fields
-the :c:macro:`PyObject_HEAD` macro brings in. The reason for the macro is to
-standardize the layout and to enable special debugging fields in debug builds.
-
-Note that there is no semicolon after the :c:macro:`PyObject_HEAD` macro;
-one is included in the macro definition. Be wary of adding one by
-accident; it's easy to do from habit, and your compiler might not complain,
-but someone else's probably will! (On Windows, MSVC is known to call this an
-error and refuse to compile the code.)
-
-For contrast, let's take a look at the corresponding definition for standard
-Python floats::
-
- typedef struct {
- PyObject_HEAD
- double ob_fval;
- } PyFloatObject;
-
-Moving on, we come to the crunch --- the type object. ::
-
- static PyTypeObject noddy_NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(noddy_NoddyObject), /* tp_basicsize */
- 0, /* tp_itemsize */
- 0, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_as_async */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT, /* tp_flags */
- "Noddy objects", /* tp_doc */
- };
-
-Now if you go and look up the definition of :c:type:`PyTypeObject` in
-:file:`object.h` you'll see that it has many more fields that the definition
-above. The remaining fields will be filled with zeros by the C compiler, and
-it's common practice to not specify them explicitly unless you need them.
-
-This is so important that we're going to pick the top of it apart still
-further::
-
- PyVarObject_HEAD_INIT(NULL, 0)
-
-This line is a bit of a wart; what we'd like to write is::
-
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
-
-as the type of a type object is "type", but this isn't strictly conforming C and
-some compilers complain. Fortunately, this member will be filled in for us by
-:c:func:`PyType_Ready`. ::
-
- "noddy.Noddy", /* tp_name */
-
-The name of our type. This will appear in the default textual representation of
-our objects and in some error messages, for example::
-
- >>> "" + noddy.new_noddy()
- Traceback (most recent call last):
- File "", line 1, in
- TypeError: cannot add type "noddy.Noddy" to string
-
-Note that the name is a dotted name that includes both the module name and the
-name of the type within the module. The module in this case is :mod:`noddy` and
-the type is :class:`Noddy`, so we set the type name to :class:`noddy.Noddy`.
-One side effect of using an undotted name is that the pydoc documentation tool
-will not list the new type in the module documentation. ::
-
- sizeof(noddy_NoddyObject), /* tp_basicsize */
-
-This is so that Python knows how much memory to allocate when you call
-:c:func:`PyObject_New`.
-
-.. note::
-
- If you want your type to be subclassable from Python, and your type has the same
- :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple
- inheritance. A Python subclass of your type will have to list your type first
- in its :attr:`~class.__bases__`, or else it will not be able to call your type's
- :meth:`__new__` method without getting an error. You can avoid this problem by
- ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its
- base type does. Most of the time, this will be true anyway, because either your
- base type will be :class:`object`, or else you will be adding data members to
- your base type, and therefore increasing its size.
-
-::
-
- 0, /* tp_itemsize */
-
-This has to do with variable length objects like lists and strings. Ignore this
-for now.
-
-Skipping a number of type methods that we don't provide, we set the class flags
-to :const:`Py_TPFLAGS_DEFAULT`. ::
-
- Py_TPFLAGS_DEFAULT, /* tp_flags */
-
-All types should include this constant in their flags. It enables all of the
-members defined until at least Python 3.3. If you need further members,
-you will need to OR the corresponding flags.
-
-We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. ::
-
- "Noddy objects", /* tp_doc */
-
-Now we get into the type methods, the things that make your objects different
-from the others. We aren't going to implement any of these in this version of
-the module. We'll expand this example later to have more interesting behavior.
-
-For now, all we want to be able to do is to create new :class:`Noddy` objects.
-To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` implementation.
-In this case, we can just use the default implementation provided by the API
-function :c:func:`PyType_GenericNew`. ::
-
- PyType_GenericNew, /* tp_new */
-
-All the other type methods are *NULL*, so we'll go over them later --- that's
-for a later section!
-
-Everything else in the file should be familiar, except for some code in
-:c:func:`PyInit_noddy`::
-
- if (PyType_Ready(&noddy_NoddyType) < 0)
- return;
-
-This initializes the :class:`Noddy` type, filing in a number of members,
-including :attr:`ob_type` that we initially set to *NULL*. ::
-
- PyModule_AddObject(m, "Noddy", (PyObject *)&noddy_NoddyType);
-
-This adds the type to the module dictionary. This allows us to create
-:class:`Noddy` instances by calling the :class:`Noddy` class::
-
- >>> import noddy
- >>> mynoddy = noddy.Noddy()
-
-That's it! All that remains is to build it; put the above code in a file called
-:file:`noddy.c` and ::
-
- from distutils.core import setup, Extension
- setup(name="noddy", version="1.0",
- ext_modules=[Extension("noddy", ["noddy.c"])])
-
-in a file called :file:`setup.py`; then typing
-
-.. code-block:: shell-session
-
- $ python setup.py build
-
-at a shell should produce a file :file:`noddy.so` in a subdirectory; move to
-that directory and fire up Python --- you should be able to ``import noddy`` and
-play around with Noddy objects.
-
-That wasn't so hard, was it?
-
-Of course, the current Noddy type is pretty uninteresting. It has no data and
-doesn't do anything. It can't even be subclassed.
-
-
-Adding data and methods to the Basic example
---------------------------------------------
-
-Let's extend the basic example to add some data and methods. Let's also make
-the type usable as a base class. We'll create a new module, :mod:`noddy2` that
-adds these capabilities:
-
-.. literalinclude:: ../includes/noddy2.c
-
-
-This version of the module has a number of changes.
-
-We've added an extra include::
-
- #include
-
-This include provides declarations that we use to handle attributes, as
-described a bit later.
-
-The name of the :class:`Noddy` object structure has been shortened to
-:class:`Noddy`. The type object name has been shortened to :class:`NoddyType`.
-
-The :class:`Noddy` type now has three data attributes, *first*, *last*, and
-*number*. The *first* and *last* variables are Python strings containing first
-and last names. The *number* attribute is an integer.
-
-The object structure is updated accordingly::
-
- typedef struct {
- PyObject_HEAD
- PyObject *first;
- PyObject *last;
- int number;
- } Noddy;
-
-Because we now have data to manage, we have to be more careful about object
-allocation and deallocation. At a minimum, we need a deallocation method::
-
- static void
- Noddy_dealloc(Noddy* self)
- {
- Py_XDECREF(self->first);
- Py_XDECREF(self->last);
- Py_TYPE(self)->tp_free((PyObject*)self);
- }
-
-which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member::
-
- (destructor)Noddy_dealloc, /*tp_dealloc*/
-
-This method decrements the reference counts of the two Python attributes. We use
-:c:func:`Py_XDECREF` here because the :attr:`first` and :attr:`last` members
-could be *NULL*. It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type
-to free the object's memory. Note that the object's type might not be
-:class:`NoddyType`, because the object may be an instance of a subclass.
-
-We want to make sure that the first and last names are initialized to empty
-strings, so we provide a new method::
-
- static PyObject *
- Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
- {
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
- }
-
-and install it in the :c:member:`~PyTypeObject.tp_new` member::
-
- Noddy_new, /* tp_new */
-
-The new member is responsible for creating (as opposed to initializing) objects
-of the type. It is exposed in Python as the :meth:`__new__` method. See the
-paper titled "Unifying types and classes in Python" for a detailed discussion of
-the :meth:`__new__` method. One reason to implement a new method is to assure
-the initial values of instance variables. In this case, we use the new method
-to make sure that the initial values of the members :attr:`first` and
-:attr:`last` are not *NULL*. If we didn't care whether the initial values were
-*NULL*, we could have used :c:func:`PyType_GenericNew` as our new method, as we
-did before. :c:func:`PyType_GenericNew` initializes all of the instance variable
-members to *NULL*.
-
-The new method is a static method that is passed the type being instantiated and
-any arguments passed when the type was called, and that returns the new object
-created. New methods always accept positional and keyword arguments, but they
-often ignore the arguments, leaving the argument handling to initializer
-methods. Note that if the type supports subclassing, the type passed may not be
-the type being defined. The new method calls the :c:member:`~PyTypeObject.tp_alloc` slot to
-allocate memory. We don't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather
-:c:func:`PyType_Ready` fills it for us by inheriting it from our base class,
-which is :class:`object` by default. Most types use the default allocation.
-
-.. note::
-
- If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one that calls a base type's
- :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), you must *not* try to determine what method
- to call using method resolution order at runtime. Always statically determine
- what type you are going to call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via
- ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your
- type that also inherit from other Python-defined classes may not work correctly.
- (Specifically, you may not be able to create instances of such subclasses
- without getting a :exc:`TypeError`.)
-
-We provide an initialization function::
-
- static int
- Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
- {
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_XDECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_XDECREF(tmp);
- }
-
- return 0;
- }
-
-by filling the :c:member:`~PyTypeObject.tp_init` slot. ::
-
- (initproc)Noddy_init, /* tp_init */
-
-The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`__init__` method. It
-is used to initialize an object after it's created. Unlike the new method, we
-can't guarantee that the initializer is called. The initializer isn't called
-when unpickling objects and it can be overridden. Our initializer accepts
-arguments to provide initial values for our instance. Initializers always accept
-positional and keyword arguments. Initializers should return either 0 on
-success or -1 on error.
-
-Initializers can be called multiple times. Anyone can call the :meth:`__init__`
-method on our objects. For this reason, we have to be extra careful when
-assigning the new values. We might be tempted, for example to assign the
-:attr:`first` member like this::
-
- if (first) {
- Py_XDECREF(self->first);
- Py_INCREF(first);
- self->first = first;
- }
-
-But this would be risky. Our type doesn't restrict the type of the
-:attr:`first` member, so it could be any kind of object. It could have a
-destructor that causes code to be executed that tries to access the
-:attr:`first` member. To be paranoid and protect ourselves against this
-possibility, we almost always reassign members before decrementing their
-reference counts. When don't we have to do this?
-
-* when we absolutely know that the reference count is greater than 1
-
-* when we know that deallocation of the object [#]_ will not cause any calls
- back into our type's code
-
-* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler when
- garbage-collections is not supported [#]_
-
-We want to expose our instance variables as attributes. There are a
-number of ways to do that. The simplest way is to define member definitions::
-
- static PyMemberDef Noddy_members[] = {
- {"first", T_OBJECT_EX, offsetof(Noddy, first), 0,
- "first name"},
- {"last", T_OBJECT_EX, offsetof(Noddy, last), 0,
- "last name"},
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
- };
-
-and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot::
-
- Noddy_members, /* tp_members */
-
-Each member definition has a member name, type, offset, access flags and
-documentation string. See the :ref:`Generic-Attribute-Management` section below for
-details.
-
-A disadvantage of this approach is that it doesn't provide a way to restrict the
-types of objects that can be assigned to the Python attributes. We expect the
-first and last names to be strings, but any Python objects can be assigned.
-Further, the attributes can be deleted, setting the C pointers to *NULL*. Even
-though we can make sure the members are initialized to non-*NULL* values, the
-members can be set to *NULL* if the attributes are deleted.
-
-We define a single method, :meth:`name`, that outputs the objects name as the
-concatenation of the first and last names. ::
-
- static PyObject *
- Noddy_name(Noddy* self)
- {
- if (self->first == NULL) {
- PyErr_SetString(PyExc_AttributeError, "first");
- return NULL;
- }
-
- if (self->last == NULL) {
- PyErr_SetString(PyExc_AttributeError, "last");
- return NULL;
- }
-
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
- }
-
-The method is implemented as a C function that takes a :class:`Noddy` (or
-:class:`Noddy` subclass) instance as the first argument. Methods always take an
-instance as the first argument. Methods often take positional and keyword
-arguments as well, but in this case we don't take any and don't need to accept
-a positional argument tuple or keyword argument dictionary. This method is
-equivalent to the Python method::
-
- def name(self):
- return "%s %s" % (self.first, self.last)
-
-Note that we have to check for the possibility that our :attr:`first` and
-:attr:`last` members are *NULL*. This is because they can be deleted, in which
-case they are set to *NULL*. It would be better to prevent deletion of these
-attributes and to restrict the attribute values to be strings. We'll see how to
-do that in the next section.
-
-Now that we've defined the method, we need to create an array of method
-definitions::
-
- static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
- };
-
-and assign them to the :c:member:`~PyTypeObject.tp_methods` slot::
-
- Noddy_methods, /* tp_methods */
-
-Note that we used the :const:`METH_NOARGS` flag to indicate that the method is
-passed no arguments.
-
-Finally, we'll make our type usable as a base class. We've written our methods
-carefully so far so that they don't make any assumptions about the type of the
-object being created or used, so all we need to do is to add the
-:const:`Py_TPFLAGS_BASETYPE` to our class flag definition::
-
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
-
-We rename :c:func:`PyInit_noddy` to :c:func:`PyInit_noddy2` and update the module
-name in the :c:type:`PyModuleDef` struct.
-
-Finally, we update our :file:`setup.py` file to build the new module::
-
- from distutils.core import setup, Extension
- setup(name="noddy", version="1.0",
- ext_modules=[
- Extension("noddy", ["noddy.c"]),
- Extension("noddy2", ["noddy2.c"]),
- ])
-
-
-Providing finer control over data attributes
---------------------------------------------
-
-In this section, we'll provide finer control over how the :attr:`first` and
-:attr:`last` attributes are set in the :class:`Noddy` example. In the previous
-version of our module, the instance variables :attr:`first` and :attr:`last`
-could be set to non-string values or even deleted. We want to make sure that
-these attributes always contain strings.
-
-.. literalinclude:: ../includes/noddy3.c
-
-
-To provide greater control, over the :attr:`first` and :attr:`last` attributes,
-we'll use custom getter and setter functions. Here are the functions for
-getting and setting the :attr:`first` attribute::
-
- Noddy_getfirst(Noddy *self, void *closure)
- {
- Py_INCREF(self->first);
- return self->first;
- }
-
- static int
- Noddy_setfirst(Noddy *self, PyObject *value, void *closure)
- {
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The first attribute value must be a str");
- return -1;
- }
-
- Py_DECREF(self->first);
- Py_INCREF(value);
- self->first = value;
-
- return 0;
- }
-
-The getter function is passed a :class:`Noddy` object and a "closure", which is
-void pointer. In this case, the closure is ignored. (The closure supports an
-advanced usage in which definition data is passed to the getter and setter. This
-could, for example, be used to allow a single set of getter and setter functions
-that decide the attribute to get or set based on data in the closure.)
-
-The setter function is passed the :class:`Noddy` object, the new value, and the
-closure. The new value may be *NULL*, in which case the attribute is being
-deleted. In our setter, we raise an error if the attribute is deleted or if the
-attribute value is not a string.
-
-We create an array of :c:type:`PyGetSetDef` structures::
-
- static PyGetSetDef Noddy_getseters[] = {
- {"first",
- (getter)Noddy_getfirst, (setter)Noddy_setfirst,
- "first name",
- NULL},
- {"last",
- (getter)Noddy_getlast, (setter)Noddy_setlast,
- "last name",
- NULL},
- {NULL} /* Sentinel */
- };
-
-and register it in the :c:member:`~PyTypeObject.tp_getset` slot::
-
- Noddy_getseters, /* tp_getset */
-
-to register our attribute getters and setters.
-
-The last item in a :c:type:`PyGetSetDef` structure is the closure mentioned
-above. In this case, we aren't using the closure, so we just pass *NULL*.
-
-We also remove the member definitions for these attributes::
-
- static PyMemberDef Noddy_members[] = {
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
- };
-
-We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to
-be passed::
-
- static int
- Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
- {
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_DECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_DECREF(tmp);
- }
-
- return 0;
- }
-
-With these changes, we can assure that the :attr:`first` and :attr:`last`
-members are never *NULL* so we can remove checks for *NULL* values in almost all
-cases. This means that most of the :c:func:`Py_XDECREF` calls can be converted to
-:c:func:`Py_DECREF` calls. The only place we can't change these calls is in the
-deallocator, where there is the possibility that the initialization of these
-members failed in the constructor.
-
-We also rename the module initialization function and module name in the
-initialization function, as we did before, and we add an extra definition to the
-:file:`setup.py` file.
-
-
-Supporting cyclic garbage collection
-------------------------------------
-
-Python has a cyclic-garbage collector that can identify unneeded objects even
-when their reference counts are not zero. This can happen when objects are
-involved in cycles. For example, consider::
-
- >>> l = []
- >>> l.append(l)
- >>> del l
-
-In this example, we create a list that contains itself. When we delete it, it
-still has a reference from itself. Its reference count doesn't drop to zero.
-Fortunately, Python's cyclic-garbage collector will eventually figure out that
-the list is garbage and free it.
-
-In the second version of the :class:`Noddy` example, we allowed any kind of
-object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. This
-means that :class:`Noddy` objects can participate in cycles::
-
- >>> import noddy2
- >>> n = noddy2.Noddy()
- >>> l = [n]
- >>> n.first = l
-
-This is pretty silly, but it gives us an excuse to add support for the
-cyclic-garbage collector to the :class:`Noddy` example. To support cyclic
-garbage collection, types need to fill two slots and set a class flag that
-enables these slots:
-
-.. literalinclude:: ../includes/noddy4.c
-
-
-The traversal method provides access to subobjects that could participate in
-cycles::
-
- static int
- Noddy_traverse(Noddy *self, visitproc visit, void *arg)
- {
- int vret;
-
- if (self->first) {
- vret = visit(self->first, arg);
- if (vret != 0)
- return vret;
- }
- if (self->last) {
- vret = visit(self->last, arg);
- if (vret != 0)
- return vret;
- }
-
- return 0;
- }
-
-For each subobject that can participate in cycles, we need to call the
-:c:func:`visit` function, which is passed to the traversal method. The
-:c:func:`visit` function takes as arguments the subobject and the extra argument
-*arg* passed to the traversal method. It returns an integer value that must be
-returned if it is non-zero.
-
-Python provides a :c:func:`Py_VISIT` macro that automates calling visit
-functions. With :c:func:`Py_VISIT`, :c:func:`Noddy_traverse` can be simplified::
-
- static int
- Noddy_traverse(Noddy *self, visitproc visit, void *arg)
- {
- Py_VISIT(self->first);
- Py_VISIT(self->last);
- return 0;
- }
-
-.. note::
-
- Note that the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly
- *visit* and *arg* in order to use :c:func:`Py_VISIT`. This is to encourage
- uniformity across these boring implementations.
-
-We also need to provide a method for clearing any subobjects that can
-participate in cycles.
-
-::
-
- static int
- Noddy_clear(Noddy *self)
- {
- PyObject *tmp;
-
- tmp = self->first;
- self->first = NULL;
- Py_XDECREF(tmp);
-
- tmp = self->last;
- self->last = NULL;
- Py_XDECREF(tmp);
-
- return 0;
- }
-
-Notice the use of a temporary variable in :c:func:`Noddy_clear`. We use the
-temporary variable so that we can set each member to *NULL* before decrementing
-its reference count. We do this because, as was discussed earlier, if the
-reference count drops to zero, we might cause code to run that calls back into
-the object. In addition, because we now support garbage collection, we also
-have to worry about code being run that triggers garbage collection. If garbage
-collection is run, our :c:member:`~PyTypeObject.tp_traverse` handler could get called. We can't
-take a chance of having :c:func:`Noddy_traverse` called when a member's reference
-count has dropped to zero and its value hasn't been set to *NULL*.
-
-Python provides a :c:func:`Py_CLEAR` that automates the careful decrementing of
-reference counts. With :c:func:`Py_CLEAR`, the :c:func:`Noddy_clear` function can
-be simplified::
-
- static int
- Noddy_clear(Noddy *self)
- {
- Py_CLEAR(self->first);
- Py_CLEAR(self->last);
- return 0;
- }
-
-Note that :c:func:`Noddy_dealloc` may call arbitrary functions through
-``__del__`` method or weakref callback. It means circular GC can be
-triggered inside the function. Since GC assumes reference count is not zero,
-we need to untrack the object from GC by calling :c:func:`PyObject_GC_UnTrack`
-before clearing members. Here is reimplemented deallocator which uses
-:c:func:`PyObject_GC_UnTrack` and :c:func:`Noddy_clear`.
-
-::
-
- static void
- Noddy_dealloc(Noddy* self)
- {
- PyObject_GC_UnTrack(self);
- Noddy_clear(self);
- Py_TYPE(self)->tp_free((PyObject*)self);
- }
-
-Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags::
-
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */
-
-That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or
-:c:member:`~PyTypeObject.tp_free` slots, we'd need to modify them for cyclic-garbage collection.
-Most extensions will use the versions automatically provided.
-
-
-Subclassing other types
------------------------
-
-It is possible to create new extension types that are derived from existing
-types. It is easiest to inherit from the built in types, since an extension can
-easily use the :class:`PyTypeObject` it needs. It can be difficult to share
-these :class:`PyTypeObject` structures between extension modules.
-
-In this example we will create a :class:`Shoddy` type that inherits from the
-built-in :class:`list` type. The new type will be completely compatible with
-regular lists, but will have an additional :meth:`increment` method that
-increases an internal counter. ::
-
- >>> import shoddy
- >>> s = shoddy.Shoddy(range(3))
- >>> s.extend(s)
- >>> print(len(s))
- 6
- >>> print(s.increment())
- 1
- >>> print(s.increment())
- 2
-
-.. literalinclude:: ../includes/shoddy.c
-
-
-As you can see, the source code closely resembles the :class:`Noddy` examples in
-previous sections. We will break down the main differences between them. ::
-
- typedef struct {
- PyListObject list;
- int state;
- } Shoddy;
-
-The primary difference for derived type objects is that the base type's object
-structure must be the first value. The base type will already include the
-:c:func:`PyObject_HEAD` at the beginning of its structure.
-
-When a Python object is a :class:`Shoddy` instance, its *PyObject\** pointer can
-be safely cast to both *PyListObject\** and *Shoddy\**. ::
-
- static int
- Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds)
- {
- if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0)
- return -1;
- self->state = 0;
- return 0;
- }
-
-In the :attr:`__init__` method for our type, we can see how to call through to
-the :attr:`__init__` method of the base type.
-
-This pattern is important when writing a type with custom :attr:`new` and
-:attr:`dealloc` methods. The :attr:`new` method should not actually create the
-memory for the object with :c:member:`~PyTypeObject.tp_alloc`, that will be handled by the base
-class when calling its :c:member:`~PyTypeObject.tp_new`.
-
-When filling out the :c:func:`PyTypeObject` for the :class:`Shoddy` type, you see
-a slot for :c:func:`tp_base`. Due to cross platform compiler issues, you can't
-fill that field directly with the :c:func:`PyList_Type`; it can be done later in
-the module's :c:func:`init` function. ::
-
- PyMODINIT_FUNC
- PyInit_shoddy(void)
- {
- PyObject *m;
-
- ShoddyType.tp_base = &PyList_Type;
- if (PyType_Ready(&ShoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&shoddymodule);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&ShoddyType);
- PyModule_AddObject(m, "Shoddy", (PyObject *) &ShoddyType);
- return m;
- }
-
-Before calling :c:func:`PyType_Ready`, the type structure must have the
-:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving a new type, it is not
-necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` slot with :c:func:`PyType_GenericNew`
--- the allocate function from the base type will be inherited.
-
-After that, calling :c:func:`PyType_Ready` and adding the type object to the
-module is the same as with the basic :class:`Noddy` examples.
-
+*****************************************
+Defining Extension Types: Assorted Topics
+*****************************************
.. _dnt-type-methods:
-Type Methods
-============
-
This section aims to give a quick fly-by on the various type methods you can
implement and what they do.
@@ -893,21 +15,20 @@ debug builds omitted:
.. literalinclude:: ../includes/typestruct.h
-Now that's a *lot* of methods. Don't worry too much though - if you have a type
-you want to define, the chances are very good that you will only implement a
-handful of these.
+Now that's a *lot* of methods. Don't worry too much though -- if you have
+a type you want to define, the chances are very good that you will only
+implement a handful of these.
As you probably expect by now, we're going to go over this and give more
information about the various handlers. We won't go in the order they are
defined in the structure, because there is a lot of historical baggage that
-impacts the ordering of the fields; be sure your type initialization keeps the
-fields in the right order! It's often easiest to find an example that includes
-all the fields you need (even if they're initialized to ``0``) and then change
-the values to suit your new type. ::
+impacts the ordering of the fields. It's often easiest to find an example
+that includes the fields you need and then change the values to suit your new
+type. ::
const char *tp_name; /* For printing */
-The name of the type - as mentioned in the last section, this will appear in
+The name of the type -- as mentioned in the previous chapter, this will appear in
various places, almost entirely for diagnostic purposes. Try to choose something
that will be helpful in such a situation! ::
@@ -915,7 +36,7 @@ that will be helpful in such a situation! ::
These fields tell the runtime how much memory to allocate when new objects of
this type are created. Python has some built-in support for variable length
-structures (think: strings, lists) which is where the :c:member:`~PyTypeObject.tp_itemsize` field
+structures (think: strings, tuples) which is where the :c:member:`~PyTypeObject.tp_itemsize` field
comes in. This will be dealt with later. ::
const char *tp_doc;
@@ -923,7 +44,7 @@ comes in. This will be dealt with later. ::
Here you can put a string (or its address) that you want returned when the
Python script references ``obj.__doc__`` to retrieve the doc string.
-Now we come to the basic type methods---the ones most extension types will
+Now we come to the basic type methods -- the ones most extension types will
implement.
@@ -947,7 +68,7 @@ object itself needs to be freed here as well. Here is an example of this
function::
static void
- newdatatype_dealloc(newdatatypeobject * obj)
+ newdatatype_dealloc(newdatatypeobject *obj)
{
free(obj->obj_UnderlyingDatatypePtr);
Py_TYPE(obj)->tp_free(obj);
@@ -1035,7 +156,7 @@ example::
static PyObject *
newdatatype_repr(newdatatypeobject * obj)
{
- return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:\%d}}",
+ return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}",
obj->obj_UnderlyingDatatypePtr->size);
}
@@ -1055,7 +176,7 @@ Here is a simple example::
static PyObject *
newdatatype_str(newdatatypeobject * obj)
{
- return PyUnicode_FromFormat("Stringified_newdatatype{{size:\%d}}",
+ return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}",
obj->obj_UnderlyingDatatypePtr->size);
}
@@ -1236,7 +357,7 @@ example that simply raises an exception; if this were really all you wanted, the
static int
newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v)
{
- (void)PyErr_Format(PyExc_RuntimeError, "Read-only attribute: \%s", name);
+ PyErr_Format(PyExc_RuntimeError, "Read-only attribute: %s", name);
return -1;
}
@@ -1321,17 +442,23 @@ these in the :file:`Objects` directory of the Python source distribution. ::
hashfunc tp_hash;
This function, if you choose to provide it, should return a hash number for an
-instance of your data type. Here is a moderately pointless example::
+instance of your data type. Here is a simple example::
- static long
+ static Py_hash_t
newdatatype_hash(newdatatypeobject *obj)
{
- long result;
- result = obj->obj_UnderlyingDatatypePtr->size;
- result = result * 3;
+ Py_hash_t result;
+ result = obj->some_size + 32767 * obj->some_number;
+ if (result == -1)
+ result = -2;
return result;
}
+:c:type:`Py_hash_t` is a signed integer type with a platform-varying width.
+Returning ``-1`` from :c:member:`~PyTypeObject.tp_hash` indicates an error,
+which is why you should be careful to avoid returning it when hash computation
+is successful, as seen above.
+
::
ternaryfunc tp_call;
@@ -1342,27 +469,22 @@ contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is inv
This function takes three arguments:
-#. *arg1* is the instance of the data type which is the subject of the call. If
- the call is ``obj1('hello')``, then *arg1* is ``obj1``.
+#. *self* is the instance of the data type which is the subject of the call.
+ If the call is ``obj1('hello')``, then *self* is ``obj1``.
-#. *arg2* is a tuple containing the arguments to the call. You can use
+#. *args* is a tuple containing the arguments to the call. You can use
:c:func:`PyArg_ParseTuple` to extract the arguments.
-#. *arg3* is a dictionary of keyword arguments that were passed. If this is
+#. *kwds* is a dictionary of keyword arguments that were passed. If this is
non-*NULL* and you support keyword arguments, use
- :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you do not
- want to support keyword arguments and this is non-*NULL*, raise a
+ :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you
+ do not want to support keyword arguments and this is non-*NULL*, raise a
:exc:`TypeError` with a message saying that keyword arguments are not supported.
-Here is a desultory example of the implementation of the call function. ::
+Here is a toy ``tp_call`` implementation::
- /* Implement the call function.
- * obj1 is the instance receiving the call.
- * obj2 is a tuple containing the arguments to the call, in this
- * case 3 strings.
- */
static PyObject *
- newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *other)
+ newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds)
{
PyObject *result;
const char *arg1;
@@ -1373,7 +495,7 @@ Here is a desultory example of the implementation of the call function. ::
return NULL;
}
result = PyUnicode_FromFormat(
- "Returning -- value: [\%d] arg1: [\%s] arg2: [\%s] arg3: [\%s]\n",
+ "Returning -- value: [%d] arg1: [%s] arg2: [%s] arg3: [%s]\n",
obj->obj_UnderlyingDatatypePtr->size,
arg1, arg2, arg3);
return result;
@@ -1385,32 +507,36 @@ Here is a desultory example of the implementation of the call function. ::
getiterfunc tp_iter;
iternextfunc tp_iternext;
-These functions provide support for the iterator protocol. Any object which
-wishes to support iteration over its contents (which may be generated during
-iteration) must implement the ``tp_iter`` handler. Objects which are returned
-by a ``tp_iter`` handler must implement both the ``tp_iter`` and ``tp_iternext``
-handlers. Both handlers take exactly one parameter, the instance for which they
-are being called, and return a new reference. In the case of an error, they
-should set an exception and return *NULL*.
-
-For an object which represents an iterable collection, the ``tp_iter`` handler
-must return an iterator object. The iterator object is responsible for
-maintaining the state of the iteration. For collections which can support
-multiple iterators which do not interfere with each other (as lists and tuples
-do), a new iterator should be created and returned. Objects which can only be
-iterated over once (usually due to side effects of iteration) should implement
-this handler by returning a new reference to themselves, and should also
-implement the ``tp_iternext`` handler. File objects are an example of such an
-iterator.
-
-Iterator objects should implement both handlers. The ``tp_iter`` handler should
-return a new reference to the iterator (this is the same as the ``tp_iter``
-handler for objects which can only be iterated over destructively). The
-``tp_iternext`` handler should return a new reference to the next object in the
-iteration if there is one. If the iteration has reached the end, it may return
-*NULL* without setting an exception or it may set :exc:`StopIteration`; avoiding
-the exception can yield slightly better performance. If an actual error occurs,
-it should set an exception and return *NULL*.
+These functions provide support for the iterator protocol. Both handlers
+take exactly one parameter, the instance for which they are being called,
+and return a new reference. In the case of an error, they should set an
+exception and return *NULL*. :c:member:`~PyTypeObject.tp_iter` corresponds
+to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext`
+corresponds to the Python :meth:`~iterator.__next__` method.
+
+Any :term:`iterable` object must implement the :c:member:`~PyTypeObject.tp_iter`
+handler, which must return an :term:`iterator` object. Here the same guidelines
+apply as for Python classes:
+
+* For collections (such as lists and tuples) which can support multiple
+ independent iterators, a new iterator should be created and returned by
+ each call to :c:member:`~PyTypeObject.tp_iter`.
+* Objects which can only be iterated over once (usually due to side effects of
+ iteration, such as file objects) can implement :c:member:`~PyTypeObject.tp_iter`
+ by returning a new reference to themselves -- and should also therefore
+ implement the :c:member:`~PyTypeObject.tp_iternext` handler.
+
+Any :term:`iterator` object should implement both :c:member:`~PyTypeObject.tp_iter`
+and :c:member:`~PyTypeObject.tp_iternext`. An iterator's
+:c:member:`~PyTypeObject.tp_iter` handler should return a new reference
+to the iterator. Its :c:member:`~PyTypeObject.tp_iternext` handler should
+return a new reference to the next object in the iteration, if there is one.
+If the iteration has reached the end, :c:member:`~PyTypeObject.tp_iternext`
+may return *NULL* without setting an exception, or it may set
+:exc:`StopIteration` *in addition* to returning *NULL*; avoiding
+the exception can yield slightly better performance. If an actual error
+occurs, :c:member:`~PyTypeObject.tp_iternext` should always set an exception
+and return *NULL*.
.. _weakref-support:
@@ -1418,110 +544,76 @@ it should set an exception and return *NULL*.
Weak Reference Support
----------------------
-One of the goals of Python's weak-reference implementation is to allow any type
+One of the goals of Python's weak reference implementation is to allow any type
to participate in the weak reference mechanism without incurring the overhead on
-those objects which do not benefit by weak referencing (such as numbers).
+performance-critical objects (such as numbers).
-For an object to be weakly referencable, the extension must include a
-:c:type:`PyObject\*` field in the instance structure for the use of the weak
-reference mechanism; it must be initialized to *NULL* by the object's
-constructor. It must also set the :c:member:`~PyTypeObject.tp_weaklistoffset` field of the
-corresponding type object to the offset of the field. For example, the instance
-type is defined with the following structure::
+.. seealso::
+ Documentation for the :mod:`weakref` module.
- typedef struct {
- PyObject_HEAD
- PyClassObject *in_class; /* The class object */
- PyObject *in_dict; /* A dictionary */
- PyObject *in_weakreflist; /* List of weak references */
- } PyInstanceObject;
-
-The statically-declared type object for instances is defined this way::
-
- PyTypeObject PyInstance_Type = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- 0,
- "module.instance",
-
- /* Lots of stuff omitted for brevity... */
-
- Py_TPFLAGS_DEFAULT, /* tp_flags */
- 0, /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- offsetof(PyInstanceObject, in_weakreflist), /* tp_weaklistoffset */
- };
+For an object to be weakly referencable, the extension type must do two things:
-The type constructor is responsible for initializing the weak reference list to
-*NULL*::
+#. Include a :c:type:`PyObject\*` field in the C object structure dedicated to
+ the weak reference mechanism. The object's constructor should leave it
+ *NULL* (which is automatic when using the default
+ :c:member:`~PyTypeObject.tp_alloc`).
- static PyObject *
- instance_new() {
- /* Other initialization stuff omitted for brevity */
+#. Set the :c:member:`~PyTypeObject.tp_weaklistoffset` type member
+ to the offset of the aforementioned field in the C object structure,
+ so that the interpreter knows how to access and modify that field.
- self->in_weakreflist = NULL;
+Concretely, here is how a trivial object structure would be augmented
+with the required field::
- return (PyObject *) self;
- }
+ typedef struct {
+ PyObject_HEAD
+ PyObject *weakreflist; /* List of weak references */
+ } TrivialObject;
-The only further addition is that the destructor needs to call the weak
-reference manager to clear any weak references. This is only required if the
-weak reference list is non-*NULL*::
+And the corresponding member in the statically-declared type object::
- static void
- instance_dealloc(PyInstanceObject *inst)
- {
- /* Allocate temporaries if needed, but do not begin
- destruction just yet.
- */
+ static PyTypeObject TrivialType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ /* ... other members omitted for brevity ... */
+ .tp_weaklistoffset = offsetof(TrivialObject, weakreflist),
+ };
- if (inst->in_weakreflist != NULL)
- PyObject_ClearWeakRefs((PyObject *) inst);
+The only further addition is that ``tp_dealloc`` needs to clear any weak
+references (by calling :c:func:`PyObject_ClearWeakRefs`) if the field is
+non-*NULL*::
- /* Proceed with object destruction normally. */
+ static void
+ Trivial_dealloc(TrivialObject *self)
+ {
+ /* Clear weakrefs first before calling any destructors */
+ if (self->weakreflist != NULL)
+ PyObject_ClearWeakRefs((PyObject *) self);
+ /* ... remainder of destruction code omitted for brevity ... */
+ Py_TYPE(self)->tp_free((PyObject *) self);
}
More Suggestions
----------------
-Remember that you can omit most of these functions, in which case you provide
-``0`` as a value. There are type definitions for each of the functions you must
-provide. They are in :file:`object.h` in the Python include directory that
-comes with the source distribution of Python.
-
In order to learn how to implement any specific method for your new data type,
-do the following: Download and unpack the Python source distribution. Go to
-the :file:`Objects` directory, then search the C source files for ``tp_`` plus
-the function you want (for example, ``tp_richcompare``). You will find examples
-of the function you want to implement.
+get the :term:`CPython` source code. Go to the :file:`Objects` directory,
+then search the C source files for ``tp_`` plus the function you want
+(for example, ``tp_richcompare``). You will find examples of the function
+you want to implement.
-When you need to verify that an object is an instance of the type you are
-implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of its use
-might be something like the following::
+When you need to verify that an object is a concrete instance of the type you
+are implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of
+its use might be something like the following::
- if (! PyObject_TypeCheck(some_object, &MyType)) {
+ if (!PyObject_TypeCheck(some_object, &MyType)) {
PyErr_SetString(PyExc_TypeError, "arg #1 not a mything");
return NULL;
}
-.. rubric:: Footnotes
-
-.. [#] This is true when we know that the object is a basic type, like a string or a
- float.
-
-.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler in this example, because our
- type doesn't support garbage collection. Even if a type supports garbage
- collection, there are calls that can be made to "untrack" the object from
- garbage collection, however, these calls are advanced and not covered here.
-
-.. [#] We now know that the first and last members are strings, so perhaps we could be
- less careful about decrementing their reference counts, however, we accept
- instances of string subclasses. Even though deallocating normal strings won't
- call back into our objects, we can't guarantee that deallocating an instance of
- a string subclass won't call back into our objects.
+.. seealso::
+ Download CPython source releases.
+ https://www.python.org/downloads/source/
-.. [#] Even in the third version, we aren't guaranteed to avoid cycles. Instances of
- string subclasses are allowed and string subclasses could allow cycles even if
- normal strings don't.
+ The CPython project on GitHub, where the CPython source code is developed.
+ https://github.com/python/cpython
diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst
new file mode 100644
index 00000000000000..ac48637bbee9be
--- /dev/null
+++ b/Doc/extending/newtypes_tutorial.rst
@@ -0,0 +1,896 @@
+.. highlightlang:: c
+
+.. _defining-new-types:
+
+**********************************
+Defining Extension Types: Tutorial
+**********************************
+
+.. sectionauthor:: Michael Hudson
+.. sectionauthor:: Dave Kuhlman
+.. sectionauthor:: Jim Fulton
+
+
+Python allows the writer of a C extension module to define new types that
+can be manipulated from Python code, much like the built-in :class:`str`
+and :class:`list` types. The code for all extension types follows a
+pattern, but there are some details that you need to understand before you
+can get started. This document is a gentle introduction to the topic.
+
+
+.. _dnt-basics:
+
+The Basics
+==========
+
+The :term:`CPython` runtime sees all Python objects as variables of type
+:c:type:`PyObject\*`, which serves as a "base type" for all Python objects.
+The :c:type:`PyObject` structure itself only contains the object's
+:term:`reference count` and a pointer to the object's "type object".
+This is where the action is; the type object determines which (C) functions
+get called by the interpreter when, for instance, an attribute gets looked up
+on an object, a method called, or it is multiplied by another object. These
+C functions are called "type methods".
+
+So, if you want to define a new extension type, you need to create a new type
+object.
+
+This sort of thing can only be explained by example, so here's a minimal, but
+complete, module that defines a new type named :class:`Custom` inside a C
+extension module :mod:`custom`:
+
+.. note::
+ What we're showing here is the traditional way of defining *static*
+ extension types. It should be adequate for most uses. The C API also
+ allows defining heap-allocated extension types using the
+ :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial.
+
+.. literalinclude:: ../includes/custom.c
+
+Now that's quite a bit to take in at once, but hopefully bits will seem familiar
+from the previous chapter. This file defines three things:
+
+#. What a :class:`Custom` **object** contains: this is the ``CustomObject``
+ struct, which is allocated once for each :class:`Custom` instance.
+#. How the :class:`Custom` **type** behaves: this is the ``CustomType`` struct,
+ which defines a set of flags and function pointers that the interpreter
+ inspects when specific operations are requested.
+#. How to initialize the :mod:`custom` module: this is the ``PyInit_custom``
+ function and the associated ``custommodule`` struct.
+
+The first bit is::
+
+ typedef struct {
+ PyObject_HEAD
+ } CustomObject;
+
+This is what a Custom object will contain. ``PyObject_HEAD`` is mandatory
+at the start of each object struct and defines a field called ``ob_base``
+of type :c:type:`PyObject`, containing a pointer to a type object and a
+reference count (these can be accessed using the macros :c:macro:`Py_REFCNT`
+and :c:macro:`Py_TYPE` respectively). The reason for the macro is to
+abstract away the layout and to enable additional fields in debug builds.
+
+.. note::
+ There is no semicolon above after the :c:macro:`PyObject_HEAD` macro.
+ Be wary of adding one by accident: some compilers will complain.
+
+Of course, objects generally store additional data besides the standard
+``PyObject_HEAD`` boilerplate; for example, here is the definition for
+standard Python floats::
+
+ typedef struct {
+ PyObject_HEAD
+ double ob_fval;
+ } PyFloatObject;
+
+The second bit is the definition of the type object. ::
+
+ static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_new = PyType_GenericNew,
+ };
+
+.. note::
+ We recommend using C99-style designated initializers as above, to
+ avoid listing all the :c:type:`PyTypeObject` fields that you don't care
+ about and also to avoid caring about the fields' declaration order.
+
+The actual definition of :c:type:`PyTypeObject` in :file:`object.h` has
+many more :ref:`fields ` than the definition above. The
+remaining fields will be filled with zeros by the C compiler, and it's
+common practice to not specify them explicitly unless you need them.
+
+We're going to pick it apart, one field at a time::
+
+ PyVarObject_HEAD_INIT(NULL, 0)
+
+This line is mandatory boilerplate to initialize the ``ob_base``
+field mentioned above. ::
+
+ .tp_name = "custom.Custom",
+
+The name of our type. This will appear in the default textual representation of
+our objects and in some error messages, for example:
+
+.. code-block:: pycon
+
+ >>> "" + custom.Custom()
+ Traceback (most recent call last):
+ File "", line 1, in
+ TypeError: can only concatenate str (not "custom.Custom") to str
+
+Note that the name is a dotted name that includes both the module name and the
+name of the type within the module. The module in this case is :mod:`custom` and
+the type is :class:`Custom`, so we set the type name to :class:`custom.Custom`.
+Using the real dotted import path is important to make your type compatible
+with the :mod:`pydoc` and :mod:`pickle` modules. ::
+
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+
+This is so that Python knows how much memory to allocate when creating
+new :class:`Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is
+only used for variable-sized objects and should otherwise be zero.
+
+.. note::
+
+ If you want your type to be subclassable from Python, and your type has the same
+ :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple
+ inheritance. A Python subclass of your type will have to list your type first
+ in its :attr:`~class.__bases__`, or else it will not be able to call your type's
+ :meth:`__new__` method without getting an error. You can avoid this problem by
+ ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its
+ base type does. Most of the time, this will be true anyway, because either your
+ base type will be :class:`object`, or else you will be adding data members to
+ your base type, and therefore increasing its size.
+
+We set the class flags to :const:`Py_TPFLAGS_DEFAULT`. ::
+
+ .tp_flags = Py_TPFLAGS_DEFAULT,
+
+All types should include this constant in their flags. It enables all of the
+members defined until at least Python 3.3. If you need further members,
+you will need to OR the corresponding flags.
+
+We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. ::
+
+ .tp_doc = "Custom objects",
+
+To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new`
+handler. This is the equivalent of the Python method :meth:`__new__`, but
+has to be specified explicitly. In this case, we can just use the default
+implementation provided by the API function :c:func:`PyType_GenericNew`. ::
+
+ .tp_new = PyType_GenericNew,
+
+Everything else in the file should be familiar, except for some code in
+:c:func:`PyInit_custom`::
+
+ if (PyType_Ready(&CustomType) < 0)
+ return;
+
+This initializes the :class:`Custom` type, filling in a number of members
+to the appropriate default values, including :attr:`ob_type` that we initially
+set to *NULL*. ::
+
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+
+This adds the type to the module dictionary. This allows us to create
+:class:`Custom` instances by calling the :class:`Custom` class:
+
+.. code-block:: pycon
+
+ >>> import custom
+ >>> mycustom = custom.Custom()
+
+That's it! All that remains is to build it; put the above code in a file called
+:file:`custom.c` and:
+
+.. code-block:: python
+
+ from distutils.core import setup, Extension
+ setup(name="custom", version="1.0",
+ ext_modules=[Extension("custom", ["custom.c"])])
+
+in a file called :file:`setup.py`; then typing
+
+.. code-block:: shell-session
+
+ $ python setup.py build
+
+at a shell should produce a file :file:`custom.so` in a subdirectory; move to
+that directory and fire up Python --- you should be able to ``import custom`` and
+play around with Custom objects.
+
+That wasn't so hard, was it?
+
+Of course, the current Custom type is pretty uninteresting. It has no data and
+doesn't do anything. It can't even be subclassed.
+
+.. note::
+ While this documentation showcases the standard :mod:`distutils` module
+ for building C extensions, it is recommended in real-world use cases to
+ use the newer and better-maintained ``setuptools`` library. Documentation
+ on how to do this is out of scope for this document and can be found in
+ the `Python Packaging User's Guide `_.
+
+
+Adding data and methods to the Basic example
+============================================
+
+Let's extend the basic example to add some data and methods. Let's also make
+the type usable as a base class. We'll create a new module, :mod:`custom2` that
+adds these capabilities:
+
+.. literalinclude:: ../includes/custom2.c
+
+
+This version of the module has a number of changes.
+
+We've added an extra include::
+
+ #include
+
+This include provides declarations that we use to handle attributes, as
+described a bit later.
+
+The :class:`Custom` type now has three data attributes in its C struct,
+*first*, *last*, and *number*. The *first* and *last* variables are Python
+strings containing first and last names. The *number* attribute is a C integer.
+
+The object structure is updated accordingly::
+
+ typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+ } CustomObject;
+
+Because we now have data to manage, we have to be more careful about object
+allocation and deallocation. At a minimum, we need a deallocation method::
+
+ static void
+ Custom_dealloc(CustomObject *self)
+ {
+ Py_XDECREF(self->first);
+ Py_XDECREF(self->last);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+ }
+
+which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member::
+
+ .tp_dealloc = (destructor) Custom_dealloc,
+
+This method first clears the reference counts of the two Python attributes.
+:c:func:`Py_XDECREF` correctly handles the case where its argument is
+*NULL* (which might happen here if ``tp_new`` failed midway). It then
+calls the :c:member:`~PyTypeObject.tp_free` member of the object's type
+(computed by ``Py_TYPE(self)``) to free the object's memory. Note that
+the object's type might not be :class:`CustomType`, because the object may
+be an instance of a subclass.
+
+.. note::
+ The explicit cast to ``destructor`` above is needed because we defined
+ ``Custom_dealloc`` to take a ``CustomObject *`` argument, but the ``tp_dealloc``
+ function pointer expects to receive a ``PyObject *`` argument. Otherwise,
+ the compiler will emit a warning. This is object-oriented polymorphism,
+ in C!
+
+We want to make sure that the first and last names are initialized to empty
+strings, so we provide a ``tp_new`` implementation::
+
+ static PyObject *
+ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+ {
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+ }
+
+and install it in the :c:member:`~PyTypeObject.tp_new` member::
+
+ .tp_new = Custom_new,
+
+The ``tp_new`` handler is responsible for creating (as opposed to initializing)
+objects of the type. It is exposed in Python as the :meth:`__new__` method.
+It is not required to define a ``tp_new`` member, and indeed many extension
+types will simply reuse :c:func:`PyType_GenericNew` as done in the first
+version of the ``Custom`` type above. In this case, we use the ``tp_new``
+handler to initialize the ``first`` and ``last`` attributes to non-*NULL*
+default values.
+
+``tp_new`` is passed the type being instantiated (not necessarily ``CustomType``,
+if a subclass is instantiated) and any arguments passed when the type was
+called, and is expected to return the instance created. ``tp_new`` handlers
+always accept positional and keyword arguments, but they often ignore the
+arguments, leaving the argument handling to initializer (a.k.a. ``tp_init``
+in C or ``__init__`` in Python) methods.
+
+.. note::
+ ``tp_new`` shouldn't call ``tp_init`` explicitly, as the interpreter
+ will do it itself.
+
+The ``tp_new`` implementation calls the :c:member:`~PyTypeObject.tp_alloc`
+slot to allocate memory::
+
+ self = (CustomObject *) type->tp_alloc(type, 0);
+
+Since memory allocation may fail, we must check the :c:member:`~PyTypeObject.tp_alloc`
+result against *NULL* before proceeding.
+
+.. note::
+ We didn't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather
+ :c:func:`PyType_Ready` fills it for us by inheriting it from our base class,
+ which is :class:`object` by default. Most types use the default allocation
+ strategy.
+
+.. note::
+ If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one
+ that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`),
+ you must *not* try to determine what method to call using method resolution
+ order at runtime. Always statically determine what type you are going to
+ call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via
+ ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your
+ type that also inherit from other Python-defined classes may not work correctly.
+ (Specifically, you may not be able to create instances of such subclasses
+ without getting a :exc:`TypeError`.)
+
+We also define an initialization function which accepts arguments to provide
+initial values for our instance::
+
+ static int
+ Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+ {
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_XDECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_XDECREF(tmp);
+ }
+ return 0;
+ }
+
+by filling the :c:member:`~PyTypeObject.tp_init` slot. ::
+
+ .tp_init = (initproc) Custom_init,
+
+The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the
+:meth:`__init__` method. It is used to initialize an object after it's
+created. Initializers always accept positional and keyword arguments,
+and they should return either ``0`` on success or ``-1`` on error.
+
+Unlike the ``tp_new`` handler, there is no guarantee that ``tp_init``
+is called at all (for example, the :mod:`pickle` module by default
+doesn't call :meth:`__init__` on unpickled instances). It can also be
+called multiple times. Anyone can call the :meth:`__init__` method on
+our objects. For this reason, we have to be extra careful when assigning
+the new attribute values. We might be tempted, for example to assign the
+``first`` member like this::
+
+ if (first) {
+ Py_XDECREF(self->first);
+ Py_INCREF(first);
+ self->first = first;
+ }
+
+But this would be risky. Our type doesn't restrict the type of the
+``first`` member, so it could be any kind of object. It could have a
+destructor that causes code to be executed that tries to access the
+``first`` member; or that destructor could release the
+:term:`Global interpreter Lock` and let arbitrary code run in other
+threads that accesses and modifies our object.
+
+To be paranoid and protect ourselves against this possibility, we almost
+always reassign members before decrementing their reference counts. When
+don't we have to do this?
+
+* when we absolutely know that the reference count is greater than 1;
+
+* when we know that deallocation of the object [#]_ will neither release
+ the :term:`GIL` nor cause any calls back into our type's code;
+
+* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc`
+ handler on a type which doesn't support cyclic garbage collection [#]_.
+
+We want to expose our instance variables as attributes. There are a
+number of ways to do that. The simplest way is to define member definitions::
+
+ static PyMemberDef Custom_members[] = {
+ {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0,
+ "first name"},
+ {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0,
+ "last name"},
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+ };
+
+and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot::
+
+ .tp_members = Custom_members,
+
+Each member definition has a member name, type, offset, access flags and
+documentation string. See the :ref:`Generic-Attribute-Management` section
+below for details.
+
+A disadvantage of this approach is that it doesn't provide a way to restrict the
+types of objects that can be assigned to the Python attributes. We expect the
+first and last names to be strings, but any Python objects can be assigned.
+Further, the attributes can be deleted, setting the C pointers to *NULL*. Even
+though we can make sure the members are initialized to non-*NULL* values, the
+members can be set to *NULL* if the attributes are deleted.
+
+We define a single method, :meth:`Custom.name()`, that outputs the objects name as the
+concatenation of the first and last names. ::
+
+ static PyObject *
+ Custom_name(CustomObject *self)
+ {
+ if (self->first == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "first");
+ return NULL;
+ }
+ if (self->last == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "last");
+ return NULL;
+ }
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+ }
+
+The method is implemented as a C function that takes a :class:`Custom` (or
+:class:`Custom` subclass) instance as the first argument. Methods always take an
+instance as the first argument. Methods often take positional and keyword
+arguments as well, but in this case we don't take any and don't need to accept
+a positional argument tuple or keyword argument dictionary. This method is
+equivalent to the Python method:
+
+.. code-block:: python
+
+ def name(self):
+ return "%s %s" % (self.first, self.last)
+
+Note that we have to check for the possibility that our :attr:`first` and
+:attr:`last` members are *NULL*. This is because they can be deleted, in which
+case they are set to *NULL*. It would be better to prevent deletion of these
+attributes and to restrict the attribute values to be strings. We'll see how to
+do that in the next section.
+
+Now that we've defined the method, we need to create an array of method
+definitions::
+
+ static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+ };
+
+(note that we used the :const:`METH_NOARGS` flag to indicate that the method
+is expecting no arguments other than *self*)
+
+and assign it to the :c:member:`~PyTypeObject.tp_methods` slot::
+
+ .tp_methods = Custom_methods,
+
+Finally, we'll make our type usable as a base class for subclassing. We've
+written our methods carefully so far so that they don't make any assumptions
+about the type of the object being created or used, so all we need to do is
+to add the :const:`Py_TPFLAGS_BASETYPE` to our class flag definition::
+
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+
+We rename :c:func:`PyInit_custom` to :c:func:`PyInit_custom2`, update the
+module name in the :c:type:`PyModuleDef` struct, and update the full class
+name in the :c:type:`PyTypeObject` struct.
+
+Finally, we update our :file:`setup.py` file to build the new module:
+
+.. code-block:: python
+
+ from distutils.core import setup, Extension
+ setup(name="custom", version="1.0",
+ ext_modules=[
+ Extension("custom", ["custom.c"]),
+ Extension("custom2", ["custom2.c"]),
+ ])
+
+
+Providing finer control over data attributes
+============================================
+
+In this section, we'll provide finer control over how the :attr:`first` and
+:attr:`last` attributes are set in the :class:`Custom` example. In the previous
+version of our module, the instance variables :attr:`first` and :attr:`last`
+could be set to non-string values or even deleted. We want to make sure that
+these attributes always contain strings.
+
+.. literalinclude:: ../includes/custom3.c
+
+
+To provide greater control, over the :attr:`first` and :attr:`last` attributes,
+we'll use custom getter and setter functions. Here are the functions for
+getting and setting the :attr:`first` attribute::
+
+ static PyObject *
+ Custom_getfirst(CustomObject *self, void *closure)
+ {
+ Py_INCREF(self->first);
+ return self->first;
+ }
+
+ static int
+ Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
+ {
+ PyObject *tmp;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+ tmp = self->first;
+ Py_INCREF(value);
+ self->first = value;
+ Py_DECREF(tmp);
+ return 0;
+ }
+
+The getter function is passed a :class:`Custom` object and a "closure", which is
+a void pointer. In this case, the closure is ignored. (The closure supports an
+advanced usage in which definition data is passed to the getter and setter. This
+could, for example, be used to allow a single set of getter and setter functions
+that decide the attribute to get or set based on data in the closure.)
+
+The setter function is passed the :class:`Custom` object, the new value, and the
+closure. The new value may be *NULL*, in which case the attribute is being
+deleted. In our setter, we raise an error if the attribute is deleted or if its
+new value is not a string.
+
+We create an array of :c:type:`PyGetSetDef` structures::
+
+ static PyGetSetDef Custom_getsetters[] = {
+ {"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
+ "first name", NULL},
+ {"last", (getter) Custom_getlast, (setter) Custom_setlast,
+ "last name", NULL},
+ {NULL} /* Sentinel */
+ };
+
+and register it in the :c:member:`~PyTypeObject.tp_getset` slot::
+
+ .tp_getset = Custom_getsetters,
+
+The last item in a :c:type:`PyGetSetDef` structure is the "closure" mentioned
+above. In this case, we aren't using a closure, so we just pass *NULL*.
+
+We also remove the member definitions for these attributes::
+
+ static PyMemberDef Custom_members[] = {
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+ };
+
+We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only
+allow strings [#]_ to be passed::
+
+ static int
+ Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+ {
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_DECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_DECREF(tmp);
+ }
+ return 0;
+ }
+
+With these changes, we can assure that the ``first`` and ``last`` members are
+never *NULL* so we can remove checks for *NULL* values in almost all cases.
+This means that most of the :c:func:`Py_XDECREF` calls can be converted to
+:c:func:`Py_DECREF` calls. The only place we can't change these calls is in
+the ``tp_dealloc`` implementation, where there is the possibility that the
+initialization of these members failed in ``tp_new``.
+
+We also rename the module initialization function and module name in the
+initialization function, as we did before, and we add an extra definition to the
+:file:`setup.py` file.
+
+
+Supporting cyclic garbage collection
+====================================
+
+Python has a :term:`cyclic garbage collector (GC) ` that
+can identify unneeded objects even when their reference counts are not zero.
+This can happen when objects are involved in cycles. For example, consider:
+
+.. code-block:: pycon
+
+ >>> l = []
+ >>> l.append(l)
+ >>> del l
+
+In this example, we create a list that contains itself. When we delete it, it
+still has a reference from itself. Its reference count doesn't drop to zero.
+Fortunately, Python's cyclic garbage collector will eventually figure out that
+the list is garbage and free it.
+
+In the second version of the :class:`Custom` example, we allowed any kind of
+object to be stored in the :attr:`first` or :attr:`last` attributes [#]_.
+Besides, in the second and third versions, we allowed subclassing
+:class:`Custom`, and subclasses may add arbitrary attributes. For any of
+those two reasons, :class:`Custom` objects can participate in cycles:
+
+.. code-block:: pycon
+
+ >>> import custom3
+ >>> class Derived(custom3.Custom): pass
+ ...
+ >>> n = Derived()
+ >>> n.some_attribute = n
+
+To allow a :class:`Custom` instance participating in a reference cycle to
+be properly detected and collected by the cyclic GC, our :class:`Custom` type
+needs to fill two additional slots and to enable a flag that enables these slots:
+
+.. literalinclude:: ../includes/custom4.c
+
+
+First, the traversal method lets the cyclic GC know about subobjects that could
+participate in cycles::
+
+ static int
+ Custom_traverse(CustomObject *self, visitproc visit, void *arg)
+ {
+ int vret;
+ if (self->first) {
+ vret = visit(self->first, arg);
+ if (vret != 0)
+ return vret;
+ }
+ if (self->last) {
+ vret = visit(self->last, arg);
+ if (vret != 0)
+ return vret;
+ }
+ return 0;
+ }
+
+For each subobject that can participate in cycles, we need to call the
+:c:func:`visit` function, which is passed to the traversal method. The
+:c:func:`visit` function takes as arguments the subobject and the extra argument
+*arg* passed to the traversal method. It returns an integer value that must be
+returned if it is non-zero.
+
+Python provides a :c:func:`Py_VISIT` macro that automates calling visit
+functions. With :c:func:`Py_VISIT`, we can minimize the amount of boilerplate
+in ``Custom_traverse``::
+
+ static int
+ Custom_traverse(CustomObject *self, visitproc visit, void *arg)
+ {
+ Py_VISIT(self->first);
+ Py_VISIT(self->last);
+ return 0;
+ }
+
+.. note::
+ The :c:member:`~PyTypeObject.tp_traverse` implementation must name its
+ arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`.
+
+Second, we need to provide a method for clearing any subobjects that can
+participate in cycles::
+
+ static int
+ Custom_clear(CustomObject *self)
+ {
+ Py_CLEAR(self->first);
+ Py_CLEAR(self->last);
+ return 0;
+ }
+
+Notice the use of the :c:func:`Py_CLEAR` macro. It is the recommended and safe
+way to clear data attributes of arbitrary types while decrementing
+their reference counts. If you were to call :c:func:`Py_XDECREF` instead
+on the attribute before setting it to *NULL*, there is a possibility
+that the attribute's destructor would call back into code that reads the
+attribute again (*especially* if there is a reference cycle).
+
+.. note::
+ You could emulate :c:func:`Py_CLEAR` by writing::
+
+ PyObject *tmp;
+ tmp = self->first;
+ self->first = NULL;
+ Py_XDECREF(tmp);
+
+ Nevertheless, it is much easier and less error-prone to always
+ use :c:func:`Py_CLEAR` when deleting an attribute. Don't
+ try to micro-optimize at the expense of robustness!
+
+The deallocator ``Custom_dealloc`` may call arbitrary code when clearing
+attributes. It means the circular GC can be triggered inside the function.
+Since the GC assumes reference count is not zero, we need to untrack the object
+from the GC by calling :c:func:`PyObject_GC_UnTrack` before clearing members.
+Here is our reimplemented deallocator using :c:func:`PyObject_GC_UnTrack`
+and ``Custom_clear``::
+
+ static void
+ Custom_dealloc(CustomObject *self)
+ {
+ PyObject_GC_UnTrack(self);
+ Custom_clear(self);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+ }
+
+Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags::
+
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+
+That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or
+:c:member:`~PyTypeObject.tp_free` handlers, we'd need to modify them for cyclic
+garbage collection. Most extensions will use the versions automatically provided.
+
+
+Subclassing other types
+=======================
+
+It is possible to create new extension types that are derived from existing
+types. It is easiest to inherit from the built in types, since an extension can
+easily use the :c:type:`PyTypeObject` it needs. It can be difficult to share
+these :c:type:`PyTypeObject` structures between extension modules.
+
+In this example we will create a :class:`SubList` type that inherits from the
+built-in :class:`list` type. The new type will be completely compatible with
+regular lists, but will have an additional :meth:`increment` method that
+increases an internal counter:
+
+.. code-block:: pycon
+
+ >>> import sublist
+ >>> s = sublist.SubList(range(3))
+ >>> s.extend(s)
+ >>> print(len(s))
+ 6
+ >>> print(s.increment())
+ 1
+ >>> print(s.increment())
+ 2
+
+.. literalinclude:: ../includes/sublist.c
+
+
+As you can see, the source code closely resembles the :class:`Custom` examples in
+previous sections. We will break down the main differences between them. ::
+
+ typedef struct {
+ PyListObject list;
+ int state;
+ } SubListObject;
+
+The primary difference for derived type objects is that the base type's
+object structure must be the first value. The base type will already include
+the :c:func:`PyObject_HEAD` at the beginning of its structure.
+
+When a Python object is a :class:`SubList` instance, its ``PyObject *`` pointer
+can be safely cast to both ``PyListObject *`` and ``SubListObject *``::
+
+ static int
+ SubList_init(SubListObject *self, PyObject *args, PyObject *kwds)
+ {
+ if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0)
+ return -1;
+ self->state = 0;
+ return 0;
+ }
+
+We see above how to call through to the :attr:`__init__` method of the base
+type.
+
+This pattern is important when writing a type with custom
+:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_dealloc`
+members. The :c:member:`~PyTypeObject.tp_new` handler should not actually
+create the memory for the object with its :c:member:`~PyTypeObject.tp_alloc`,
+but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_new`.
+
+The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base`
+specifying the type's concrete base class. Due to cross-platform compiler
+issues, you can't fill that field directly with a reference to
+:c:type:`PyList_Type`; it should be done later in the module initialization
+function::
+
+ PyMODINIT_FUNC
+ PyInit_sublist(void)
+ {
+ PyObject* m;
+ SubListType.tp_base = &PyList_Type;
+ if (PyType_Ready(&SubListType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&sublistmodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&SubListType);
+ PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+ return m;
+ }
+
+Before calling :c:func:`PyType_Ready`, the type structure must have the
+:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving an
+existing type, it is not necessary to fill out the :c:member:`~PyTypeObject.tp_alloc`
+slot with :c:func:`PyType_GenericNew` -- the allocation function from the base
+type will be inherited.
+
+After that, calling :c:func:`PyType_Ready` and adding the type object to the
+module is the same as with the basic :class:`Custom` examples.
+
+
+.. rubric:: Footnotes
+
+.. [#] This is true when we know that the object is a basic type, like a string or a
+ float.
+
+.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler
+ in this example, because our type doesn't support garbage collection.
+
+.. [#] We now know that the first and last members are strings, so perhaps we
+ could be less careful about decrementing their reference counts, however,
+ we accept instances of string subclasses. Even though deallocating normal
+ strings won't call back into our objects, we can't guarantee that deallocating
+ an instance of a string subclass won't call back into our objects.
+
+.. [#] Also, even with our attributes restricted to strings instances, the user
+ could pass arbitrary :class:`str` subclasses and therefore still create
+ reference cycles.
diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index 2e56fbc2f425ec..5e54df61bda9c9 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -2,6 +2,11 @@
Design and History FAQ
======================
+.. only:: html
+
+ .. contents::
+
+
Why does Python use indentation for grouping of statements?
-----------------------------------------------------------
@@ -210,24 +215,25 @@ objects using the ``for`` statement. For example, :term:`file objects
Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))?
----------------------------------------------------------------------------------------------------------------
-The major reason is history. Functions were used for those operations that were
-generic for a group of types and which were intended to work even for objects
-that didn't have methods at all (e.g. tuples). It is also convenient to have a
-function that can readily be applied to an amorphous collection of objects when
-you use the functional features of Python (``map()``, ``zip()`` et al).
+As Guido said:
-In fact, implementing ``len()``, ``max()``, ``min()`` as a built-in function is
-actually less code than implementing them as methods for each type. One can
-quibble about individual cases but it's a part of Python, and it's too late to
-make such fundamental changes now. The functions have to remain to avoid massive
-code breakage.
+ (a) For some operations, prefix notation just reads better than
+ postfix -- prefix (and infix!) operations have a long tradition in
+ mathematics which likes notations where the visuals help the
+ mathematician thinking about a problem. Compare the easy with which we
+ rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of
+ doing the same thing using a raw OO notation.
-.. XXX talk about protocols?
+ (b) When I read code that says len(x) I *know* that it is asking for
+ the length of something. This tells me two things: the result is an
+ integer, and the argument is some kind of container. To the contrary,
+ when I read x.len(), I have to already know that x is some kind of
+ container implementing an interface or inheriting from a class that
+ has a standard len(). Witness the confusion we occasionally have when
+ a class that is not implementing a mapping has a get() or keys()
+ method, or something that isn't a file has a write() method.
-.. note::
-
- For string operations, Python has moved from external functions (the
- ``string`` module) to methods. However, ``len()`` is still a function.
+ -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html
Why is join() a string method instead of a list or tuple method?
@@ -343,7 +349,7 @@ each Python stack frame. Also, extensions can call back into Python at almost
random moments. Therefore, a complete threads implementation requires thread
support for C.
-Answer 2: Fortunately, there is `Stackless Python `_,
+Answer 2: Fortunately, there is `Stackless Python `_,
which has a completely redesigned interpreter loop that avoids the C stack.
@@ -465,10 +471,10 @@ you can always change a list's elements. Only immutable elements can be used as
dictionary keys, and hence only tuples and not lists can be used as keys.
-How are lists implemented?
---------------------------
+How are lists implemented in CPython?
+-------------------------------------
-Python's lists are really variable-length arrays, not Lisp-style linked lists.
+CPython's lists are really variable-length arrays, not Lisp-style linked lists.
The implementation uses a contiguous array of references to other objects, and
keeps a pointer to this array and the array's length in a list head structure.
@@ -481,10 +487,10 @@ when the array must be grown, some extra space is allocated so the next few
times don't require an actual resize.
-How are dictionaries implemented?
----------------------------------
+How are dictionaries implemented in CPython?
+--------------------------------------------
-Python's dictionaries are implemented as resizable hash tables. Compared to
+CPython's dictionaries are implemented as resizable hash tables. Compared to
B-trees, this gives better performance for lookup (the most common operation by
far) under most circumstances, and the implementation is simpler.
@@ -495,11 +501,7 @@ on the key and a per-process seed; for example, "Python" could hash to
to 1142331976. The hash code is then used to calculate a location in an
internal array where the value will be stored. Assuming that you're storing
keys that all have different hash values, this means that dictionaries take
-constant time -- O(1), in computer science notation -- to retrieve a key. It
-also means that no sorted order of the keys is maintained, and traversing the
-array as the ``.keys()`` and ``.items()`` do will output the dictionary's
-content in some arbitrary jumbled order that can change with every invocation of
-a program.
+constant time -- O(1), in Big-O notation -- to retrieve a key.
Why must dictionary keys be immutable?
diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst
index 88996e48035b24..fd04a83df33c3d 100644
--- a/Doc/faq/extending.rst
+++ b/Doc/faq/extending.rst
@@ -62,7 +62,7 @@ How can I execute arbitrary Python statements from C?
The highest-level function to do this is :c:func:`PyRun_SimpleString` which takes
a single string argument to be executed in the context of the module
-``__main__`` and returns 0 for success and -1 when an exception occurred
+``__main__`` and returns ``0`` for success and ``-1`` when an exception occurred
(including ``SyntaxError``). If you want more control, use
:c:func:`PyRun_String`; see the source for :c:func:`PyRun_SimpleString` in
``Python/pythonrun.c``.
diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst
index 0d1cb198da87d8..9d3e199fca6b0e 100644
--- a/Doc/faq/general.rst
+++ b/Doc/faq/general.rst
@@ -117,7 +117,7 @@ programming), software engineering (unit testing, logging, profiling, parsing
Python code), and operating system interfaces (system calls, filesystems, TCP/IP
sockets). Look at the table of contents for :ref:`library-index` to get an idea
of what's available. A wide variety of third-party extensions are also
-available. Consult `the Python Package Index `_ to
+available. Consult `the Python Package Index `_ to
find packages of interest to you.
diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst
index f84feadd780234..ab92a879a88516 100644
--- a/Doc/faq/library.rst
+++ b/Doc/faq/library.rst
@@ -19,7 +19,7 @@ standard library module. (Eventually you'll learn what's in the standard
library and will be able to skip this step.)
For third-party packages, search the `Python Package Index
-`_ or try `Google `_ or
+`_ or try `Google `_ or
another Web search engine. Searching for "Python" plus a keyword or two for
your topic of interest will usually find something helpful.
@@ -74,7 +74,9 @@ interpreter.
Occasionally, a user's environment is so full that the :program:`/usr/bin/env`
program fails; or there's no env program at all. In that case, you can try the
-following hack (due to Alex Rezinsky)::
+following hack (due to Alex Rezinsky):
+
+.. code-block:: sh
#! /bin/sh
""":"
@@ -609,7 +611,7 @@ use ``p.read(n)``.
"expect" library. A Python extension that interfaces to expect is called
"expy" and available from http://expectpy.sourceforge.net. A pure Python
solution that works like expect is `pexpect
- `_.
+ `_.
How do I access the serial (RS232) port?
diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index 1a2f582a31ad17..53f3b7f528c065 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -371,8 +371,8 @@ compute, a common technique is to cache the parameters and the resulting value
of each call to the function, and return the cached value if the same value is
requested again. This is called "memoizing", and can be implemented like this::
- # Callers will never provide a third parameter for this function.
- def expensive(arg1, arg2, _cache={}):
+ # Callers can only provide two parameters and optionally pass _cache by keyword
+ def expensive(arg1, arg2, *, _cache={}):
if (arg1, arg2) in _cache:
return _cache[(arg1, arg2)]
@@ -1315,11 +1315,6 @@ that final assignment still results in an error, because tuples are immutable.
Dictionaries
============
-How can I get a dictionary to store and display its keys in a consistent order?
--------------------------------------------------------------------------------
-
-Use :class:`collections.OrderedDict`.
-
I want to do a complicated sort: can you do a Schwartzian Transform in Python?
------------------------------------------------------------------------------
diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst
index d703f2862221a6..7792cfaa16a19d 100644
--- a/Doc/faq/windows.rst
+++ b/Doc/faq/windows.rst
@@ -1,5 +1,7 @@
:tocdepth: 2
+.. highlightlang:: none
+
.. _windows-faq:
=====================
@@ -39,12 +41,16 @@ or "Command prompt window". Usually you can create such a window from your
Start menu; under Windows 7 the menu selection is :menuselection:`Start -->
Programs --> Accessories --> Command Prompt`. You should be able to recognize
when you have started such a window because you will see a Windows "command
-prompt", which usually looks like this::
+prompt", which usually looks like this:
+
+.. code-block:: doscon
C:\>
The letter may be different, and there might be other things after it, so you
-might just as easily see something like::
+might just as easily see something like:
+
+.. code-block:: doscon
D:\YourName\Projects\Python>
@@ -60,11 +66,15 @@ program. So, how do you arrange for the interpreter to handle your Python?
First, you need to make sure that your command window recognises the word
"python" as an instruction to start the interpreter. If you have opened a
command window, you should try entering the command ``python`` and hitting
-return.::
+return:
+
+.. code-block:: doscon
C:\Users\YourName> python
-You should then see something like::
+You should then see something like:
+
+.. code-block:: pycon
Python 3.3.0 (v3.3.0:bd8afb90ebf2, Sep 29 2012, 10:55:48) [MSC v.1600 32 bit (Intel)] on win32
Type "help", "copyright", "credits" or "license" for more information.
@@ -73,7 +83,9 @@ You should then see something like::
You have started the interpreter in "interactive mode". That means you can enter
Python statements or expressions interactively and have them executed or
evaluated while you wait. This is one of Python's strongest features. Check it
-by entering a few expressions of your choice and seeing the results::
+by entering a few expressions of your choice and seeing the results:
+
+.. code-block:: pycon
>>> print("Hello")
Hello
@@ -317,7 +329,9 @@ present, and ``getch()`` which gets one character without echoing it.
How do I emulate os.kill() in Windows?
--------------------------------------
-Prior to Python 2.7 and 3.2, to terminate a process, you can use :mod:`ctypes`::
+Prior to Python 2.7 and 3.2, to terminate a process, you can use :mod:`ctypes`:
+
+.. code-block:: python
import ctypes
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index dcfe086b38b12f..69960579f2c04f 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -14,8 +14,9 @@ Glossary
``...``
The default Python prompt of the interactive shell when entering code for
- an indented code block or within a pair of matching left and right
- delimiters (parentheses, square brackets or curly braces).
+ an indented code block, when within a pair of matching left and right
+ delimiters (parentheses, square brackets, curly braces or triple quotes),
+ or after specifying a decorator.
2to3
A tool that tries to convert Python 2.x code to Python 3.x code by
@@ -39,6 +40,20 @@ Glossary
and loaders (in the :mod:`importlib.abc` module). You can create your own
ABCs with the :mod:`abc` module.
+ annotation
+ A label associated with a variable, a class
+ attribute or a function parameter or return value,
+ used by convention as a :term:`type hint`.
+
+ Annotations of local variables cannot be accessed at runtime, but
+ annotations of global variables, class attributes, and functions
+ are stored in the :attr:`__annotations__`
+ special attribute of modules, classes, and functions,
+ respectively.
+
+ See :term:`variable annotation`, :term:`function annotation`, :pep:`484`
+ and :pep:`526`, which describe this functionality.
+
argument
A value passed to a :term:`function` (or :term:`method`) when calling the
function. There are two kinds of argument:
@@ -100,7 +115,7 @@ Glossary
location execution state (including local variables and pending
try-statements). When the *asynchronous generator iterator* effectively
resumes with another awaitable returned by :meth:`__anext__`, it
- picks-up where it left-off. See :pep:`492` and :pep:`525`.
+ picks up where it left off. See :pep:`492` and :pep:`525`.
asynchronous iterable
An object, that can be used in an :keyword:`async for` statement.
@@ -108,10 +123,10 @@ Glossary
:meth:`__aiter__` method. Introduced by :pep:`492`.
asynchronous iterator
- An object that implements :meth:`__aiter__` and :meth:`__anext__`
+ An object that implements the :meth:`__aiter__` and :meth:`__anext__`
methods. ``__anext__`` must return an :term:`awaitable` object.
- :keyword:`async for` resolves awaitable returned from asynchronous
- iterator's :meth:`__anext__` method until it raises
+ :keyword:`async for` resolves the awaitables returned by an asynchronous
+ iterator's :meth:`__anext__` method until it raises a
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
attribute
@@ -136,8 +151,8 @@ Glossary
:data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and
:class:`gzip.GzipFile`.
- .. seealso::
- A :term:`text file` reads and writes :class:`str` objects.
+ See also :term:`text file` for a file object able to read and write
+ :class:`str` objects.
bytes-like object
An object that supports the :ref:`bufferobjects` and can
@@ -175,6 +190,10 @@ Glossary
normally contain method definitions which operate on instances of the
class.
+ class variable
+ A variable defined in a class and intended to be modified only at
+ class level (i.e., not in an instance of the class).
+
coercion
The implicit conversion of an instance of one type to another during an
operation which involves two arguments of the same type. For example,
@@ -367,16 +386,20 @@ Glossary
and the :ref:`function` section.
function annotation
- An arbitrary metadata value associated with a function parameter or return
- value. Its syntax is explained in section :ref:`function`. Annotations
- may be accessed via the :attr:`__annotations__` special attribute of a
- function object.
+ An :term:`annotation` of a function parameter or return value.
- See also the :term:`variable annotation` glossary entry.
+ Function annotations are usually used for
+ :term:`type hints `: for example this function is expected to take two
+ :class:`int` arguments and is also expected to have an :class:`int`
+ return value::
- Annotations are meant to provide a standard way for programmers to
- document types of functions they design. See :pep:`484`, which
- describes this functionality.
+ def sum_two_numbers(a: int, b: int) -> int:
+ return a + b
+
+ Function annotation syntax is explained in section :ref:`function`.
+
+ See :term:`variable annotation` and :pep:`484`,
+ which describe this functionality.
__future__
A pseudo-module which programmers can use to enable new language features
@@ -413,8 +436,8 @@ Glossary
Each :keyword:`yield` temporarily suspends processing, remembering the
location execution state (including local variables and pending
- try-statements). When the *generator iterator* resumes, it picks-up where
- it left-off (in contrast to functions which start fresh on every
+ try-statements). When the *generator iterator* resumes, it picks up where
+ it left off (in contrast to functions which start fresh on every
invocation).
.. index:: single: generator expression
@@ -463,7 +486,7 @@ Glossary
hash-based pyc
- A bytecode cache file that uses the the hash rather than the last-modified
+ A bytecode cache file that uses the hash rather than the last-modified
time of the corresponding source file to determine its validity. See
:ref:`pyc-invalidation`.
@@ -603,7 +626,7 @@ Glossary
lambda
An anonymous inline function consisting of a single :term:`expression`
which is evaluated when the function is called. The syntax to create
- a lambda function is ``lambda [arguments]: expression``
+ a lambda function is ``lambda [parameters]: expression``
LBYL
Look before you leap. This coding style explicitly tests for
@@ -620,7 +643,7 @@ Glossary
list
A built-in Python :term:`sequence`. Despite its name it is more akin
to an array in other languages than to a linked list since access to
- elements are O(1).
+ elements is O(1).
list comprehension
A compact way to process all or part of the elements in a sequence and
@@ -837,6 +860,21 @@ Glossary
:class:`str` or :class:`bytes` result instead, respectively. Introduced
by :pep:`519`.
+ PEP
+ Python Enhancement Proposal. A PEP is a design document
+ providing information to the Python community, or describing a new
+ feature for Python or its processes or environment. PEPs should
+ provide a concise technical specification and a rationale for proposed
+ features.
+
+ PEPs are intended to be the primary mechanisms for proposing major new
+ features, for collecting community input on an issue, and for documenting
+ the design decisions that have gone into Python. The PEP author is
+ responsible for building consensus within the community and documenting
+ dissenting opinions.
+
+ See :pep:`1`.
+
portion
A set of files in a single directory (possibly stored in a zip file)
that contribute to a namespace package, as defined in :pep:`420`.
@@ -991,8 +1029,8 @@ Glossary
:data:`sys.stdin`, :data:`sys.stdout`, and instances of
:class:`io.StringIO`.
- .. seealso::
- A :term:`binary file` reads and write :class:`bytes` objects.
+ See also :term:`binary file` for a file object able to read and write
+ :term:`bytes-like objects `.
triple-quoted string
A string which is bound by three instances of either a quotation mark
@@ -1009,6 +1047,43 @@ Glossary
:attr:`~instance.__class__` attribute or can be retrieved with
``type(obj)``.
+ type alias
+ A synonym for a type, created by assigning the type to an identifier.
+
+ Type aliases are useful for simplifying :term:`type hints `.
+ For example::
+
+ from typing import List, Tuple
+
+ def remove_gray_shades(
+ colors: List[Tuple[int, int, int]]) -> List[Tuple[int, int, int]]:
+ pass
+
+ could be made more readable like this::
+
+ from typing import List, Tuple
+
+ Color = Tuple[int, int, int]
+
+ def remove_gray_shades(colors: List[Color]) -> List[Color]:
+ pass
+
+ See :mod:`typing` and :pep:`484`, which describe this functionality.
+
+ type hint
+ An :term:`annotation` that specifies the expected type for a variable, a class
+ attribute, or a function parameter or return value.
+
+ Type hints are optional and are not enforced by Python but
+ they are useful to static type analysis tools, and aid IDEs with code
+ completion and refactoring.
+
+ Type hints of global variables, class attributes, and functions,
+ but not local variables, can be accessed using
+ :func:`typing.get_type_hints`.
+
+ See :mod:`typing` and :pep:`484`, which describe this functionality.
+
universal newlines
A manner of interpreting text streams in which all of the following are
recognized as ending a line: the Unix end-of-line convention ``'\n'``,
@@ -1017,17 +1092,23 @@ Glossary
:func:`bytes.splitlines` for an additional use.
variable annotation
- A type metadata value associated with a module global variable or
- a class attribute. Its syntax is explained in section :ref:`annassign`.
- Annotations are stored in the :attr:`__annotations__` special
- attribute of a class or module object and can be accessed using
- :func:`typing.get_type_hints`.
+ An :term:`annotation` of a variable or a class attribute.
- See also the :term:`function annotation` glossary entry.
+ When annotating a variable or a class attribute, assignment is optional::
- Annotations are meant to provide a standard way for programmers to
- document types of functions they design. See :pep:`484` and :pep:`526`
- which describe this functionality.
+ class C:
+ field: 'annotation'
+
+ Variable annotations are usually used for
+ :term:`type hints `: for example this variable is expected to take
+ :class:`int` values::
+
+ count: int = 0
+
+ Variable annotation syntax is explained in section :ref:`annassign`.
+
+ See :term:`function annotation`, :pep:`484`
+ and :pep:`526`, which describe this functionality.
virtual environment
A cooperatively isolated runtime environment that allows Python users
diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst
index 9d770f5232b440..e78a022b372faa 100644
--- a/Doc/howto/argparse.rst
+++ b/Doc/howto/argparse.rst
@@ -24,7 +24,7 @@ Concepts
Let's show the sort of functionality that we are going to explore in this
introductory tutorial by making use of the :command:`ls` command:
-.. code-block:: sh
+.. code-block:: shell-session
$ ls
cpython devguide prog.py pypy rm-unused-function.patch
@@ -77,7 +77,7 @@ Let us start with a very simple example which does (almost) nothing::
Following is a result of running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
$ python3 prog.py --help
@@ -119,7 +119,7 @@ An example::
And running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
usage: prog.py [-h] echo
@@ -164,7 +164,7 @@ by reading the source code. So, let's make it a bit more useful::
And we get:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py -h
usage: prog.py [-h] echo
@@ -185,7 +185,7 @@ Now, how about doing something even more useful::
Following is a result of running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
Traceback (most recent call last):
@@ -206,7 +206,7 @@ give it as strings, unless we tell it otherwise. So, let's tell
Following is a result of running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -233,7 +233,7 @@ have a look on how to add optional ones::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py --verbosity 1
verbosity turned on
@@ -279,7 +279,7 @@ Let's modify the code accordingly::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py --verbose
verbosity turned on
@@ -325,7 +325,7 @@ versions of the options. It's quite simple::
And here goes:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py -v
verbosity turned on
@@ -359,7 +359,7 @@ Our program keeps growing in complexity::
And now the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
usage: prog.py [-h] [-v] square
@@ -395,7 +395,7 @@ multiple verbosity values, and actually get to use them::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -429,7 +429,7 @@ Let's fix it by restricting the values the ``--verbosity`` option can accept::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 -v 3
usage: prog.py [-h] [-v {0,1,2}] square
@@ -470,7 +470,7 @@ verbosity argument (check the output of ``python --help``)::
We have introduced another action, "count",
to count the number of occurrences of a specific optional arguments:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -537,7 +537,7 @@ Let's fix::
And this is what it gives:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 -vvv
the square of 4 equals 16
@@ -581,7 +581,7 @@ it gets the ``None`` value, and that cannot be compared to an int value
And:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -614,7 +614,7 @@ not just squares::
Output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
usage: prog.py [-h] [-v] x y
@@ -652,7 +652,7 @@ to display *more* text instead::
Output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 2
16
@@ -695,7 +695,7 @@ which will be the opposite of the ``--verbose`` one::
Our program is now simpler, and we've lost some functionality for the sake of
demonstration. Anyways, here's the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 2
4^2 == 16
@@ -739,7 +739,7 @@ Note that slight difference in the usage text. Note the ``[-v | -q]``,
which tells us that we can either use ``-v`` or ``-q``,
but not both at the same time:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py --help
usage: prog.py [-h] [-v | -q] x y
diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
index d3c7d668959e56..695fbb1be192b2 100644
--- a/Doc/howto/clinic.rst
+++ b/Doc/howto/clinic.rst
@@ -22,8 +22,8 @@ Argument Clinic How-To
compatibility for future versions. In other words: if you
maintain an external C extension for CPython, you're welcome
to experiment with Argument Clinic in your own code. But the
- version of Argument Clinic that ships with CPython 3.5 *could*
- be totally incompatible and break all your code.
+ version of Argument Clinic that ships with the next version
+ of CPython *could* be totally incompatible and break all your code.
The Goals Of Argument Clinic
============================
@@ -267,12 +267,16 @@ Let's dive in!
should get its own line. All the parameter lines should be
indented from the function name and the docstring.
- The general form of these parameter lines is as follows::
+ The general form of these parameter lines is as follows:
+
+ .. code-block:: none
name_of_parameter: converter
If the parameter has a default value, add that after the
- converter::
+ converter:
+
+ .. code-block:: none
name_of_parameter: converter = default_value
@@ -925,13 +929,17 @@ Parameter default values
------------------------
Default values for parameters can be any of a number of values.
-At their simplest, they can be string, int, or float literals::
+At their simplest, they can be string, int, or float literals:
+
+.. code-block:: none
foo: str = "abc"
bar: int = 123
bat: float = 45.6
-They can also use any of Python's built-in constants::
+They can also use any of Python's built-in constants:
+
+.. code-block:: none
yep: bool = True
nope: bool = False
@@ -959,7 +967,9 @@ It can be an entire expression, using math operators and looking up attributes
on objects. However, this support isn't exactly simple, because of some
non-obvious semantics.
-Consider the following example::
+Consider the following example:
+
+.. code-block:: none
foo: Py_ssize_t = sys.maxsize - 1
@@ -970,7 +980,9 @@ runtime, when the user asks for the function's signature.
What namespace is available when the expression is evaluated? It's evaluated
in the context of the module the builtin came from. So, if your module has an
-attribute called "``max_widgets``", you may simply use it::
+attribute called "``max_widgets``", you may simply use it:
+
+.. code-block:: none
foo: Py_ssize_t = max_widgets
@@ -982,7 +994,9 @@ it's best to restrict yourself to modules that are preloaded by Python itself.)
Evaluating default values only at runtime means Argument Clinic can't compute
the correct equivalent C default value. So you need to tell it explicitly.
When you use an expression, you must also specify the equivalent expression
-in C, using the ``c_default`` parameter to the converter::
+in C, using the ``c_default`` parameter to the converter:
+
+.. code-block:: none
foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1
@@ -1359,7 +1373,9 @@ Let's start with defining some terminology:
A field, in this context, is a subsection of Clinic's output.
For example, the ``#define`` for the ``PyMethodDef`` structure
is a field, called ``methoddef_define``. Clinic has seven
- different fields it can output per function definition::
+ different fields it can output per function definition:
+
+ .. code-block:: none
docstring_prototype
docstring_definition
@@ -1416,7 +1432,9 @@ Let's start with defining some terminology:
Clinic defines five new directives that let you reconfigure its output.
-The first new directive is ``dump``::
+The first new directive is ``dump``:
+
+.. code-block:: none
dump
@@ -1425,7 +1443,9 @@ the current block, and empties it. This only works with ``buffer`` and
``two-pass`` destinations.
The second new directive is ``output``. The most basic form of ``output``
-is like this::
+is like this:
+
+.. code-block:: none
output
@@ -1433,7 +1453,9 @@ This tells Clinic to output *field* to *destination*. ``output`` also
supports a special meta-destination, called ``everything``, which tells
Clinic to output *all* fields to that *destination*.
-``output`` has a number of other functions::
+``output`` has a number of other functions:
+
+.. code-block:: none
output push
output pop
@@ -1508,7 +1530,9 @@ preset configurations, as follows:
Suppresses the ``impl_prototype``, write the ``docstring_definition``
and ``parser_definition`` to ``buffer``, write everything else to ``block``.
-The third new directive is ``destination``::
+The third new directive is ``destination``:
+
+.. code-block:: none
destination [...]
@@ -1516,7 +1540,9 @@ This performs an operation on the destination named ``name``.
There are two defined subcommands: ``new`` and ``clear``.
-The ``new`` subcommand works like this::
+The ``new`` subcommand works like this:
+
+.. code-block:: none
destination new
@@ -1564,7 +1590,9 @@ There are five destination types:
A two-pass buffer, like the "two-pass" builtin destination above.
-The ``clear`` subcommand works like this::
+The ``clear`` subcommand works like this:
+
+.. code-block:: none
destination clear
@@ -1572,7 +1600,9 @@ It removes all the accumulated text up to this point in the destination.
(I don't know what you'd need this for, but I thought maybe it'd be
useful while someone's experimenting.)
-The fourth new directive is ``set``::
+The fourth new directive is ``set``:
+
+.. code-block:: none
set line_prefix "string"
set line_suffix "string"
@@ -1590,7 +1620,9 @@ Both of these support two format strings:
Turns into the string ``*/``, the end-comment text sequence for C files.
The final new directive is one you shouldn't need to use directly,
-called ``preserve``::
+called ``preserve``:
+
+.. code-block:: none
preserve
@@ -1638,7 +1670,9 @@ like so::
#endif /* HAVE_FUNCTIONNAME */
Then, remove those three lines from the ``PyMethodDef`` structure,
-replacing them with the macro Argument Clinic generated::
+replacing them with the macro Argument Clinic generated:
+
+.. code-block:: none
MODULE_FUNCTIONNAME_METHODDEF
diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst
index 19d65d6996b7f8..cc4b4785b12290 100644
--- a/Doc/howto/curses.rst
+++ b/Doc/howto/curses.rst
@@ -41,7 +41,7 @@ appearance---and the curses library will figure out what control codes
need to be sent to the terminal to produce the right output. curses
doesn't provide many user-interface concepts such as buttons, checkboxes,
or dialogs; if you need such features, consider a user interface library such as
-`Urwid `_.
+`Urwid `_.
The curses library was originally written for BSD Unix; the later System V
versions of Unix from AT&T added many enhancements and new functions. BSD curses
@@ -55,7 +55,7 @@ everything, though.
The Windows version of Python doesn't include the :mod:`curses`
module. A ported version called `UniCurses
-`_ is available. You could
+`_ is available. You could
also try `the Console module `_
written by Fredrik Lundh, which doesn't
use the same API as curses but provides cursor-addressable text output
@@ -432,7 +432,7 @@ User Input
The C curses library offers only very simple input mechanisms. Python's
:mod:`curses` module adds a basic text-input widget. (Other libraries
-such as `Urwid `_ have more extensive
+such as `Urwid `_ have more extensive
collections of widgets.)
There are two methods for getting input from a window:
diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst
index 40601812a77cb5..2efe4537e288d6 100644
--- a/Doc/howto/functional.rst
+++ b/Doc/howto/functional.rst
@@ -198,7 +198,7 @@ for it.
You can experiment with the iteration interface manually:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> it = iter(L)
>>> it #doctest: +ELLIPSIS
<...iterator object at ...>
@@ -229,7 +229,7 @@ iterator. These two statements are equivalent::
Iterators can be materialized as lists or tuples by using the :func:`list` or
:func:`tuple` constructor functions:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> iterator = iter(L)
>>> t = tuple(iterator)
>>> t
@@ -238,10 +238,10 @@ Iterators can be materialized as lists or tuples by using the :func:`list` or
Sequence unpacking also supports iterators: if you know an iterator will return
N elements, you can unpack them into an N-tuple:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> iterator = iter(L)
- >>> a,b,c = iterator
- >>> a,b,c
+ >>> a, b, c = iterator
+ >>> a, b, c
(1, 2, 3)
Built-in functions such as :func:`max` and :func:`min` can take a single
@@ -273,23 +273,24 @@ dictionary's keys::
>>> m = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6,
... 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}
- >>> for key in m: #doctest: +SKIP
+ >>> for key in m:
... print(key, m[key])
- Mar 3
+ Jan 1
Feb 2
- Aug 8
- Sep 9
+ Mar 3
Apr 4
+ May 5
Jun 6
Jul 7
- Jan 1
- May 5
+ Aug 8
+ Sep 9
+ Oct 10
Nov 11
Dec 12
- Oct 10
-Note that the order is essentially random, because it's based on the hash
-ordering of the objects in the dictionary.
+Note that starting with Python 3.7, dictionary iteration order is guaranteed
+to be the same as the insertion order. In earlier versions, the behaviour was
+unspecified and could vary between implementations.
Applying :func:`iter` to a dictionary always loops over the keys, but
dictionaries have methods that return other iterators. If you want to iterate
@@ -301,8 +302,8 @@ The :func:`dict` constructor can accept an iterator that returns a finite stream
of ``(key, value)`` tuples:
>>> L = [('Italy', 'Rome'), ('France', 'Paris'), ('US', 'Washington DC')]
- >>> dict(iter(L)) #doctest: +SKIP
- {'Italy': 'Rome', 'US': 'Washington DC', 'France': 'Paris'}
+ >>> dict(iter(L))
+ {'Italy': 'Rome', 'France': 'Paris', 'US': 'Washington DC'}
Files also support iteration by calling the :meth:`~io.TextIOBase.readline`
method until there are no more lines in the file. This means you can read each
@@ -410,7 +411,7 @@ lengths of all the sequences. If you have two lists of length 3, the output
list is 9 elements long:
>>> seq1 = 'abc'
- >>> seq2 = (1,2,3)
+ >>> seq2 = (1, 2, 3)
>>> [(x, y) for x in seq1 for y in seq2] #doctest: +NORMALIZE_WHITESPACE
[('a', 1), ('a', 2), ('a', 3),
('b', 1), ('b', 2), ('b', 3),
@@ -478,7 +479,7 @@ Here's a sample usage of the ``generate_ints()`` generator:
File "stdin", line 2, in generate_ints
StopIteration
-You could equally write ``for i in generate_ints(5)``, or ``a,b,c =
+You could equally write ``for i in generate_ints(5)``, or ``a, b, c =
generate_ints(3)``.
Inside a generator function, ``return value`` causes ``StopIteration(value)``
@@ -694,17 +695,17 @@ truth values of an iterable's contents. :func:`any` returns ``True`` if any ele
in the iterable is a true value, and :func:`all` returns ``True`` if all of the
elements are true values:
- >>> any([0,1,0])
+ >>> any([0, 1, 0])
True
- >>> any([0,0,0])
+ >>> any([0, 0, 0])
False
- >>> any([1,1,1])
+ >>> any([1, 1, 1])
True
- >>> all([0,1,0])
+ >>> all([0, 1, 0])
False
- >>> all([0,0,0])
+ >>> all([0, 0, 0])
False
- >>> all([1,1,1])
+ >>> all([1, 1, 1])
True
@@ -763,7 +764,7 @@ which defaults to 0, and the interval between numbers, which defaults to 1::
a provided iterable and returns a new iterator that returns its elements from
first to last. The new iterator will repeat these elements infinitely. ::
- itertools.cycle([1,2,3,4,5]) =>
+ itertools.cycle([1, 2, 3, 4, 5]) =>
1, 2, 3, 4, 5, 1, 2, 3, 4, 5, ...
:func:`itertools.repeat(elem, [n]) ` returns the provided
@@ -874,7 +875,7 @@ iterable's results. ::
iterators and returns only those elements of *data* for which the corresponding
element of *selectors* is true, stopping whenever either one is exhausted::
- itertools.compress([1,2,3,4,5], [True, True, False, False, True]) =>
+ itertools.compress([1, 2, 3, 4, 5], [True, True, False, False, True]) =>
1, 2, 5
@@ -1034,7 +1035,7 @@ first calculation. ::
Traceback (most recent call last):
...
TypeError: reduce() of empty sequence with no initial value
- >>> functools.reduce(operator.mul, [1,2,3], 1)
+ >>> functools.reduce(operator.mul, [1, 2, 3], 1)
6
>>> functools.reduce(operator.mul, [], 1)
1
@@ -1044,9 +1045,9 @@ elements of the iterable. This case is so common that there's a special
built-in called :func:`sum` to compute it:
>>> import functools, operator
- >>> functools.reduce(operator.add, [1,2,3,4], 0)
+ >>> functools.reduce(operator.add, [1, 2, 3, 4], 0)
10
- >>> sum([1,2,3,4])
+ >>> sum([1, 2, 3, 4])
10
>>> sum([])
0
@@ -1056,11 +1057,11 @@ write the obvious :keyword:`for` loop::
import functools
# Instead of:
- product = functools.reduce(operator.mul, [1,2,3], 1)
+ product = functools.reduce(operator.mul, [1, 2, 3], 1)
# You can write:
product = 1
- for i in [1,2,3]:
+ for i in [1, 2, 3]:
product *= i
A related function is :func:`itertools.accumulate(iterable, func=operator.add)
@@ -1068,10 +1069,10 @@ A related function is :func:`itertools.accumulate(iterable, func=operator.add)
returning only the final result, :func:`accumulate` returns an iterator that
also yields each partial result::
- itertools.accumulate([1,2,3,4,5]) =>
+ itertools.accumulate([1, 2, 3, 4, 5]) =>
1, 3, 6, 10, 15
- itertools.accumulate([1,2,3,4,5], operator.mul) =>
+ itertools.accumulate([1, 2, 3, 4, 5], operator.mul) =>
1, 2, 6, 24, 120
@@ -1155,7 +1156,7 @@ But it would be best of all if I had simply used a ``for`` loop::
Or the :func:`sum` built-in and a generator expression::
- total = sum(b for a,b in items)
+ total = sum(b for a, b in items)
Many uses of :func:`functools.reduce` are clearer when written as ``for`` loops.
diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst
index b9c51a4a4538ab..b63c43c81f71d2 100644
--- a/Doc/howto/instrumentation.rst
+++ b/Doc/howto/instrumentation.rst
@@ -254,11 +254,15 @@ and the remainder indicates the call/return hierarchy as the script executes.
For a `--enable-shared` build of CPython, the markers are contained within the
libpython shared library, and the probe's dotted path needs to reflect this. For
-example, this line from the above example::
+example, this line from the above example:
+
+.. code-block:: none
probe process("python").mark("function__entry") {
-should instead read::
+should instead read:
+
+.. code-block:: none
probe process("python").library("libpython3.6dm.so.1.0").mark("function__entry") {
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index 4d2d052d291788..b1930a791fca5b 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -72,7 +72,9 @@ Here is the auxiliary module::
def some_function():
module_logger.info('received a call to "some_function"')
-The output looks like this::
+The output looks like this:
+
+.. code-block:: none
2005-03-23 23:47:11,663 - spam_application - INFO -
creating an instance of auxiliary_module.Auxiliary
@@ -127,7 +129,9 @@ shows logging from the main (initial) thread and another thread::
if __name__ == '__main__':
main()
-When run, the script should print something like the following::
+When run, the script should print something like the following:
+
+.. code-block:: none
0 Thread-1 Hi from myfunc
3 MainThread Hello from main
@@ -240,14 +244,18 @@ messages should not. Here's how you can achieve this::
logger2.warning('Jail zesty vixen who grabbed pay from quack.')
logger2.error('The five boxing wizards jump quickly.')
-When you run this, on the console you will see ::
+When you run this, on the console you will see
+
+.. code-block:: none
root : INFO Jackdaws love my big sphinx of quartz.
myapp.area1 : INFO How quickly daft jumping zebras vex.
myapp.area2 : WARNING Jail zesty vixen who grabbed pay from quack.
myapp.area2 : ERROR The five boxing wizards jump quickly.
-and in the file you will see something like ::
+and in the file you will see something like
+
+.. code-block:: none
10-22 22:19 root INFO Jackdaws love my big sphinx of quartz.
10-22 22:19 myapp.area1 DEBUG Quick zephyrs blow, vexing daft Jim.
@@ -515,7 +523,9 @@ module. Here is a basic working example::
main()
First run the server, and then the client. On the client side, nothing is
-printed on the console; on the server side, you should see something like::
+printed on the console; on the server side, you should see something like:
+
+.. code-block:: none
About to start TCP server...
59 root INFO Jackdaws love my big sphinx of quartz.
@@ -675,7 +685,9 @@ script::
lvlname = logging.getLevelName(lvl)
a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters')
-which, when run, produces something like::
+which, when run, produces something like:
+
+.. code-block:: none
2010-09-06 22:38:15,292 a.b.c DEBUG IP: 123.231.231.123 User: fred A debug message
2010-09-06 22:38:15,300 a.b.c INFO IP: 192.168.0.1 User: sheila An info message with some parameters
@@ -976,7 +988,9 @@ logging package provides a :class:`~handlers.RotatingFileHandler`::
print(filename)
The result should be 6 separate files, each with part of the log history for the
-application::
+application:
+
+.. code-block:: none
logging_rotatingfile_example.out
logging_rotatingfile_example.out.1
@@ -1638,11 +1652,11 @@ works::
Inserting a BOM into messages sent to a SysLogHandler
-----------------------------------------------------
-`RFC 5424 `_ requires that a
+:rfc:`5424` requires that a
Unicode message be sent to a syslog daemon as a set of bytes which have the
following structure: an optional pure-ASCII component, followed by a UTF-8 Byte
-Order Mark (BOM), followed by Unicode encoded using UTF-8. (See the `relevant
-section of the specification `_.)
+Order Mark (BOM), followed by Unicode encoded using UTF-8. (See the
+:rfc:`relevant section of the specification <5424#section-6>`.)
In Python 3.1, code was added to
:class:`~logging.handlers.SysLogHandler` to insert a BOM into the message, but
@@ -1652,7 +1666,7 @@ appear before it.
As this behaviour is broken, the incorrect BOM insertion code is being removed
from Python 3.2.4 and later. However, it is not being replaced, and if you
-want to produce RFC 5424-compliant messages which include a BOM, an optional
+want to produce :rfc:`5424`-compliant messages which include a BOM, an optional
pure-ASCII sequence before it and arbitrary Unicode after it, encoded using
UTF-8, then you need to do the following:
@@ -1675,7 +1689,7 @@ UTF-8, then you need to do the following:
The formatted message *will* be encoded using UTF-8 encoding by
``SysLogHandler``. If you follow the above rules, you should be able to produce
-RFC 5424-compliant messages. If you don't, logging may not complain, but your
+:rfc:`5424`-compliant messages. If you don't, logging may not complain, but your
messages will not be RFC 5424-compliant, and your syslog daemon may complain.
@@ -1706,7 +1720,9 @@ which uses JSON to serialise the event in a machine-parseable manner::
logging.basicConfig(level=logging.INFO, format='%(message)s')
logging.info(_('message 1', foo='bar', bar='baz', num=123, fnum=123.456))
-If the above script is run, it prints::
+If the above script is run, it prints:
+
+.. code-block:: none
message 1 >>> {"fnum": 123.456, "num": 123, "bar": "baz", "foo": "bar"}
@@ -1753,7 +1769,9 @@ as in the following complete example::
if __name__ == '__main__':
main()
-When the above script is run, it prints::
+When the above script is run, it prints:
+
+.. code-block:: none
message 1 >>> {"snowman": "\u2603", "set_value": [1, 2, 3]}
@@ -2083,7 +2101,9 @@ most obvious, but you can provide any callable which returns a
This example shows how you can pass configuration data to the callable which
constructs the instance, in the form of keyword parameters. When run, the above
-script will print::
+script will print:
+
+.. code-block:: none
changed: hello
@@ -2150,7 +2170,9 @@ class, as shown in the following example::
if __name__ == '__main__':
main()
-When run, this produces a file with exactly two lines::
+When run, this produces a file with exactly two lines:
+
+.. code-block:: none
28/01/2015 07:21:23|INFO|Sample message|
28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'|
@@ -2312,7 +2334,9 @@ Here's the script::
write_line('Calling decorated foo with True')
assert decorated_foo(True)
-When this script is run, the following output should be observed::
+When this script is run, the following output should be observed:
+
+.. code-block:: none
Calling undecorated foo with False
about to log at DEBUG ...
@@ -2408,7 +2432,9 @@ the following complete example::
logging.config.dictConfig(LOGGING)
logging.warning('The local time is %s', time.asctime())
-When this script is run, it should print something like::
+When this script is run, it should print something like:
+
+.. code-block:: none
2015-10-17 12:53:29,501 The local time is Sat Oct 17 13:53:29 2015
2015-10-17 13:53:29,501 The local time is Sat Oct 17 13:53:29 2015
diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst
index 4ee68b4747eb00..6f47baeee2f5dd 100644
--- a/Doc/howto/logging.rst
+++ b/Doc/howto/logging.rst
@@ -134,7 +134,9 @@ interpreter, and don't just continue from the session described above::
logging.warning('And this, too')
And now if we open the file and look at what we have, we should find the log
-messages::
+messages:
+
+.. code-block:: none
DEBUG:root:This message should go to the log file
INFO:root:So should this
@@ -144,7 +146,9 @@ This example also shows how you can set the logging level which acts as the
threshold for tracking. In this case, because we set the threshold to
``DEBUG``, all of the messages were printed.
-If you want to set the logging level from a command-line option such as::
+If you want to set the logging level from a command-line option such as:
+
+.. code-block:: none
--log=INFO
@@ -208,7 +212,9 @@ could organize logging in it::
def do_something():
logging.info('Doing something')
-If you run *myapp.py*, you should see this in *myapp.log*::
+If you run *myapp.py*, you should see this in *myapp.log*:
+
+.. code-block:: none
INFO:root:Started
INFO:root:Doing something
@@ -258,7 +264,9 @@ specify the format you want to use::
logging.info('So should this')
logging.warning('And this, too')
-which would print::
+which would print:
+
+.. code-block:: none
DEBUG:This message should appear on the console
INFO:So should this
@@ -282,19 +290,23 @@ your format string::
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when this event was logged.')
-which should print something like this::
+which should print something like this:
+
+.. code-block:: none
2010-12-12 11:41:42,612 is when this event was logged.
-The default format for date/time display (shown above) is ISO8601. If you need
-more control over the formatting of the date/time, provide a *datefmt*
-argument to ``basicConfig``, as in this example::
+The default format for date/time display (shown above) is like ISO8601 or
+:rfc:`3339`. If you need more control over the formatting of the date/time, provide
+a *datefmt* argument to ``basicConfig``, as in this example::
import logging
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.warning('is when this event was logged.')
-which would display something like this::
+which would display something like this:
+
+.. code-block:: none
12/12/2010 11:46:36 AM is when this event was logged.
@@ -376,7 +388,9 @@ if no destination is set; and if one is not set, they will set a destination
of the console (``sys.stderr``) and a default format for the displayed
message before delegating to the root logger to do the actual message output.
-The default format set by :func:`basicConfig` for messages is::
+The default format set by :func:`basicConfig` for messages is:
+
+.. code-block:: none
severity:logger name:message
@@ -522,7 +536,9 @@ indicator.
.. method:: logging.Formatter.__init__(fmt=None, datefmt=None, style='%')
If there is no message format string, the default is to use the
-raw message. If there is no date format string, the default date format is::
+raw message. If there is no date format string, the default date format is:
+
+.. code-block:: none
%Y-%m-%d %H:%M:%S
@@ -628,7 +644,9 @@ the names of the objects::
logger.error('error message')
logger.critical('critical message')
-Here is the logging.conf file::
+Here is the logging.conf file:
+
+.. code-block:: ini
[loggers]
keys=root,simpleExample
@@ -713,7 +731,9 @@ construct the dictionary in Python code, receive it in pickled form over a
socket, or use whatever approach makes sense for your application.
Here's an example of the same configuration as above, in YAML format for
-the new dictionary-based approach::
+the new dictionary-based approach:
+
+.. code-block:: yaml
version: 1
formatters:
@@ -730,9 +750,9 @@ the new dictionary-based approach::
level: DEBUG
handlers: [console]
propagate: no
- root:
- level: DEBUG
- handlers: [console]
+ root:
+ level: DEBUG
+ handlers: [console]
For more information about logging using a dictionary, see
:ref:`logging-config-api`.
diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst
index 98c81206741a49..3be6bb380d663b 100644
--- a/Doc/howto/pyporting.rst
+++ b/Doc/howto/pyporting.rst
@@ -427,25 +427,25 @@ to make sure everything functions as expected in both versions of Python.
.. _2to3: https://docs.python.org/3/library/2to3.html
-.. _caniusepython3: https://pypi.python.org/pypi/caniusepython3
+.. _caniusepython3: https://pypi.org/project/caniusepython3
.. _cheat sheet: http://python-future.org/compatible_idioms.html
-.. _coverage.py: https://pypi.python.org/pypi/coverage
+.. _coverage.py: https://pypi.org/project/coverage
.. _Futurize: http://python-future.org/automatic_conversion.html
.. _importlib: https://docs.python.org/3/library/importlib.html#module-importlib
-.. _importlib2: https://pypi.python.org/pypi/importlib2
+.. _importlib2: https://pypi.org/project/importlib2
.. _Modernize: https://python-modernize.readthedocs.io/
.. _mypy: http://mypy-lang.org/
.. _Porting to Python 3: http://python3porting.com/
-.. _Pylint: https://pypi.python.org/pypi/pylint
+.. _Pylint: https://pypi.org/project/pylint
.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.io/en/latest/python3/questions_and_answers.html
.. _pytype: https://github.com/google/pytype
.. _python-future: http://python-future.org/
.. _python-porting: https://mail.python.org/mailman/listinfo/python-porting
-.. _six: https://pypi.python.org/pypi/six
-.. _tox: https://pypi.python.org/pypi/tox
-.. _trove classifier: https://pypi.python.org/pypi?%3Aaction=list_classifiers
+.. _six: https://pypi.org/project/six
+.. _tox: https://pypi.org/project/tox
+.. _trove classifier: https://pypi.org/classifiers
.. _"What's New": https://docs.python.org/3/whatsnew/index.html
diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst
index 87a6b1aba59f9f..b09f748a922733 100644
--- a/Doc/howto/regex.rst
+++ b/Doc/howto/regex.rst
@@ -289,6 +289,8 @@ Putting REs in strings keeps the Python language simpler, but has one
disadvantage which is the topic of the next section.
+.. _the-backslash-plague:
+
The Backslash Plague
--------------------
@@ -327,6 +329,13 @@ backslashes are not handled in any special way in a string literal prefixed with
while ``"\n"`` is a one-character string containing a newline. Regular
expressions will often be written in Python code using this raw string notation.
+In addition, special escape sequences that are valid in regular expressions,
+but not valid as Python string literals, now result in a
+:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`,
+which means the sequences will be invalid if raw string notation or escaping
+the backslashes isn't used.
+
+
+-------------------+------------------+
| Regular String | Raw string |
+===================+==================+
@@ -457,10 +466,16 @@ In actual programs, the most common style is to store the
Two pattern methods return all of the matches for a pattern.
:meth:`~re.Pattern.findall` returns a list of matching strings::
- >>> p = re.compile('\d+')
+ >>> p = re.compile(r'\d+')
>>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping')
['12', '11', '10']
+The ``r`` prefix, making the literal a raw string literal, is needed in this
+example because escape sequences in a normal "cooked" string literal that are
+not recognized by Python, as opposed to regular expressions, now result in a
+:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See
+:ref:`the-backslash-plague`.
+
:meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the
result. The :meth:`~re.Pattern.finditer` method returns a sequence of
:ref:`match object ` instances as an :term:`iterator`::
@@ -771,7 +786,9 @@ Frequently you need to obtain more information than just whether the RE matched
or not. Regular expressions are often used to dissect strings by writing a RE
divided into several subgroups which match different components of interest.
For example, an RFC-822 header line is divided into a header name and a value,
-separated by a ``':'``, like this::
+separated by a ``':'``, like this:
+
+.. code-block:: none
From: author@example.com
User-Agent: Thunderbird 1.5.0.9 (X11/20061227)
@@ -1096,11 +1113,11 @@ following calls::
The module-level function :func:`re.split` adds the RE to be used as the first
argument, but is otherwise the same. ::
- >>> re.split('[\W]+', 'Words, words, words.')
+ >>> re.split(r'[\W]+', 'Words, words, words.')
['Words', 'words', 'words', '']
- >>> re.split('([\W]+)', 'Words, words, words.')
+ >>> re.split(r'([\W]+)', 'Words, words, words.')
['Words', ', ', 'words', ', ', 'words', '.', '']
- >>> re.split('[\W]+', 'Words, words, words.', 1)
+ >>> re.split(r'[\W]+', 'Words, words, words.', 1)
['Words', 'words, words.']
diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index d4b8f8d2204ab7..be1fefb35a71f0 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -30,7 +30,9 @@ spellings such as 'coöperate'.)
For a while people just wrote programs that didn't display accents.
In the mid-1980s an Apple II BASIC program written by a French speaker
-might have lines like these::
+might have lines like these:
+
+.. code-block:: basic
PRINT "MISE A JOUR TERMINEE"
PRINT "PARAMETRES ENREGISTRES"
@@ -463,7 +465,7 @@ The string in this example has the number 57 written in both Thai and
Arabic numerals::
import re
- p = re.compile('\d+')
+ p = re.compile(r'\d+')
s = "Over \u0e55\u0e57 57 flavours"
m = p.search(s)
diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst
index c1fd5cf0680d96..046a88af62f0b3 100644
--- a/Doc/howto/urllib2.rst
+++ b/Doc/howto/urllib2.rst
@@ -56,12 +56,20 @@ The simplest way to use urllib.request is as follows::
with urllib.request.urlopen('http://python.org/') as response:
html = response.read()
-If you wish to retrieve a resource via URL and store it in a temporary location,
-you can do so via the :func:`~urllib.request.urlretrieve` function::
+If you wish to retrieve a resource via URL and store it in a temporary
+location, you can do so via the :func:`shutil.copyfileobj` and
+:func:`tempfile.NamedTemporaryFile` functions::
+ import shutil
+ import tempfile
import urllib.request
- local_filename, headers = urllib.request.urlretrieve('http://python.org/')
- html = open(local_filename)
+
+ with urllib.request.urlopen('http://python.org/') as response:
+ with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+ shutil.copyfileobj(response, tmp_file)
+
+ with open(tmp_file.name) as html:
+ pass
Many uses of urllib will be that simple (note that instead of an 'http:' URL we
could have used a URL starting with 'ftp:', 'file:', etc.). However, it's the
@@ -231,7 +239,7 @@ a different URL, urllib will handle that for you). For those it can't handle,
urlopen will raise an :exc:`HTTPError`. Typical errors include '404' (page not
found), '403' (request forbidden), and '401' (authentication required).
-See section 10 of RFC 2616 for a reference on all the HTTP error codes.
+See section 10 of :rfc:`2616` for a reference on all the HTTP error codes.
The :exc:`HTTPError` instance raised will have an integer 'code' attribute, which
corresponds to the error sent by the server.
@@ -244,7 +252,7 @@ codes in the 100--299 range indicate success, you will usually only see error
codes in the 400--599 range.
:attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of
-response codes in that shows all the response codes used by RFC 2616. The
+response codes in that shows all the response codes used by :rfc:`2616`. The
dictionary is reproduced here for convenience ::
# Table mapping response codes to messages; entries have the
@@ -457,7 +465,9 @@ error code) requesting authentication. This specifies the authentication scheme
and a 'realm'. The header looks like: ``WWW-Authenticate: SCHEME
realm="REALM"``.
-e.g. ::
+e.g.
+
+.. code-block:: none
WWW-Authenticate: Basic realm="cPanel Users"
diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c
new file mode 100644
index 00000000000000..fb2c7b2a430e64
--- /dev/null
+++ b/Doc/includes/custom.c
@@ -0,0 +1,39 @@
+#include
+
+typedef struct {
+ PyObject_HEAD
+ /* Type-specific fields go here. */
+} CustomObject;
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT,
+ .tp_new = PyType_GenericNew,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c
new file mode 100644
index 00000000000000..51ab4b80d68098
--- /dev/null
+++ b/Doc/includes/custom2.c
@@ -0,0 +1,132 @@
+#include
+#include "structmember.h"
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+} CustomObject;
+
+static void
+Custom_dealloc(CustomObject *self)
+{
+ Py_XDECREF(self->first);
+ Py_XDECREF(self->last);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+}
+
+static int
+Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_XDECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_XDECREF(tmp);
+ }
+ return 0;
+}
+
+static PyMemberDef Custom_members[] = {
+ {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0,
+ "first name"},
+ {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0,
+ "last name"},
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
+{
+ if (self->first == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "first");
+ return NULL;
+ }
+ if (self->last == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "last");
+ return NULL;
+ }
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+}
+
+static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+};
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom2.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ .tp_new = Custom_new,
+ .tp_init = (initproc) Custom_init,
+ .tp_dealloc = (destructor) Custom_dealloc,
+ .tp_members = Custom_members,
+ .tp_methods = Custom_methods,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom2",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom2(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c
new file mode 100644
index 00000000000000..09e87355b91afa
--- /dev/null
+++ b/Doc/includes/custom3.c
@@ -0,0 +1,183 @@
+#include
+#include "structmember.h"
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+} CustomObject;
+
+static void
+Custom_dealloc(CustomObject *self)
+{
+ Py_XDECREF(self->first);
+ Py_XDECREF(self->last);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+}
+
+static int
+Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_DECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_DECREF(tmp);
+ }
+ return 0;
+}
+
+static PyMemberDef Custom_members[] = {
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_getfirst(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->first);
+ return self->first;
+}
+
+static int
+Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
+{
+ PyObject *tmp;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+ tmp = self->first;
+ Py_INCREF(value);
+ self->first = value;
+ Py_DECREF(tmp);
+ return 0;
+}
+
+static PyObject *
+Custom_getlast(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->last);
+ return self->last;
+}
+
+static int
+Custom_setlast(CustomObject *self, PyObject *value, void *closure)
+{
+ PyObject *tmp;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The last attribute value must be a string");
+ return -1;
+ }
+ tmp = self->last;
+ Py_INCREF(value);
+ self->last = value;
+ Py_DECREF(tmp);
+ return 0;
+}
+
+static PyGetSetDef Custom_getsetters[] = {
+ {"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
+ "first name", NULL},
+ {"last", (getter) Custom_getlast, (setter) Custom_setlast,
+ "last name", NULL},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
+{
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+}
+
+static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+};
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom3.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ .tp_new = Custom_new,
+ .tp_init = (initproc) Custom_init,
+ .tp_dealloc = (destructor) Custom_dealloc,
+ .tp_members = Custom_members,
+ .tp_methods = Custom_methods,
+ .tp_getset = Custom_getsetters,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom3",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom3(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c
new file mode 100644
index 00000000000000..0994d8fda0e51f
--- /dev/null
+++ b/Doc/includes/custom4.c
@@ -0,0 +1,197 @@
+#include
+#include "structmember.h"
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+} CustomObject;
+
+static int
+Custom_traverse(CustomObject *self, visitproc visit, void *arg)
+{
+ Py_VISIT(self->first);
+ Py_VISIT(self->last);
+ return 0;
+}
+
+static int
+Custom_clear(CustomObject *self)
+{
+ Py_CLEAR(self->first);
+ Py_CLEAR(self->last);
+ return 0;
+}
+
+static void
+Custom_dealloc(CustomObject *self)
+{
+ PyObject_GC_UnTrack(self);
+ Custom_clear(self);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+}
+
+static int
+Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_DECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_DECREF(tmp);
+ }
+ return 0;
+}
+
+static PyMemberDef Custom_members[] = {
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_getfirst(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->first);
+ return self->first;
+}
+
+static int
+Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
+{
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+ Py_INCREF(value);
+ Py_CLEAR(self->first);
+ self->first = value;
+ return 0;
+}
+
+static PyObject *
+Custom_getlast(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->last);
+ return self->last;
+}
+
+static int
+Custom_setlast(CustomObject *self, PyObject *value, void *closure)
+{
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The last attribute value must be a string");
+ return -1;
+ }
+ Py_INCREF(value);
+ Py_CLEAR(self->last);
+ self->last = value;
+ return 0;
+}
+
+static PyGetSetDef Custom_getsetters[] = {
+ {"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
+ "first name", NULL},
+ {"last", (getter) Custom_getlast, (setter) Custom_setlast,
+ "last name", NULL},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
+{
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+}
+
+static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+};
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom4.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+ .tp_new = Custom_new,
+ .tp_init = (initproc) Custom_init,
+ .tp_dealloc = (destructor) Custom_dealloc,
+ .tp_traverse = (traverseproc) Custom_traverse,
+ .tp_clear = (inquiry) Custom_clear,
+ .tp_members = Custom_members,
+ .tp_methods = Custom_methods,
+ .tp_getset = Custom_getsetters,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom4",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom4(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/noddy.c b/Doc/includes/noddy.c
deleted file mode 100644
index 07b5d5a9b83ce0..00000000000000
--- a/Doc/includes/noddy.c
+++ /dev/null
@@ -1,72 +0,0 @@
-#include
-
-typedef struct {
- PyObject_HEAD
- /* Type-specific fields go here. */
-} noddy_NoddyObject;
-
-static PyTypeObject noddy_NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(noddy_NoddyObject), /* tp_basicsize */
- 0, /* tp_itemsize */
- 0, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT, /* tp_flags */
- "Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- 0, /* tp_methods */
- 0, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- 0, /* tp_init */
- 0, /* tp_alloc */
- PyType_GenericNew, /* tp_new */
-};
-
-static PyModuleDef noddymodule = {
- PyModuleDef_HEAD_INIT,
- "noddy",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy(void)
-{
- PyObject* m;
-
- if (PyType_Ready(&noddy_NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddymodule);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&noddy_NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&noddy_NoddyType);
- return m;
-}
diff --git a/Doc/includes/noddy2.c b/Doc/includes/noddy2.c
deleted file mode 100644
index 964155845fee83..00000000000000
--- a/Doc/includes/noddy2.c
+++ /dev/null
@@ -1,172 +0,0 @@
-#include
-#include "structmember.h"
-
-typedef struct {
- PyObject_HEAD
- PyObject *first; /* first name */
- PyObject *last; /* last name */
- int number;
-} Noddy;
-
-static void
-Noddy_dealloc(Noddy* self)
-{
- Py_XDECREF(self->first);
- Py_XDECREF(self->last);
- Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-static PyObject *
-Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
-}
-
-static int
-Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
-{
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_XDECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_XDECREF(tmp);
- }
-
- return 0;
-}
-
-
-static PyMemberDef Noddy_members[] = {
- {"first", T_OBJECT_EX, offsetof(Noddy, first), 0,
- "first name"},
- {"last", T_OBJECT_EX, offsetof(Noddy, last), 0,
- "last name"},
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_name(Noddy* self)
-{
- if (self->first == NULL) {
- PyErr_SetString(PyExc_AttributeError, "first");
- return NULL;
- }
-
- if (self->last == NULL) {
- PyErr_SetString(PyExc_AttributeError, "last");
- return NULL;
- }
-
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
-}
-
-static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
-};
-
-static PyTypeObject NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(Noddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- (destructor)Noddy_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE, /* tp_flags */
- "Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Noddy_methods, /* tp_methods */
- Noddy_members, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Noddy_init, /* tp_init */
- 0, /* tp_alloc */
- Noddy_new, /* tp_new */
-};
-
-static PyModuleDef noddy2module = {
- PyModuleDef_HEAD_INIT,
- "noddy2",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy2(void)
-{
- PyObject* m;
-
- if (PyType_Ready(&NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddy2module);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType);
- return m;
-}
diff --git a/Doc/includes/noddy3.c b/Doc/includes/noddy3.c
deleted file mode 100644
index 8a5a753ca439eb..00000000000000
--- a/Doc/includes/noddy3.c
+++ /dev/null
@@ -1,225 +0,0 @@
-#include
-#include "structmember.h"
-
-typedef struct {
- PyObject_HEAD
- PyObject *first;
- PyObject *last;
- int number;
-} Noddy;
-
-static void
-Noddy_dealloc(Noddy* self)
-{
- Py_XDECREF(self->first);
- Py_XDECREF(self->last);
- Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-static PyObject *
-Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
-}
-
-static int
-Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
-{
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_DECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_DECREF(tmp);
- }
-
- return 0;
-}
-
-static PyMemberDef Noddy_members[] = {
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_getfirst(Noddy *self, void *closure)
-{
- Py_INCREF(self->first);
- return self->first;
-}
-
-static int
-Noddy_setfirst(Noddy *self, PyObject *value, void *closure)
-{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The first attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->first);
- Py_INCREF(value);
- self->first = value;
-
- return 0;
-}
-
-static PyObject *
-Noddy_getlast(Noddy *self, void *closure)
-{
- Py_INCREF(self->last);
- return self->last;
-}
-
-static int
-Noddy_setlast(Noddy *self, PyObject *value, void *closure)
-{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The last attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->last);
- Py_INCREF(value);
- self->last = value;
-
- return 0;
-}
-
-static PyGetSetDef Noddy_getseters[] = {
- {"first",
- (getter)Noddy_getfirst, (setter)Noddy_setfirst,
- "first name",
- NULL},
- {"last",
- (getter)Noddy_getlast, (setter)Noddy_setlast,
- "last name",
- NULL},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_name(Noddy* self)
-{
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
-}
-
-static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
-};
-
-static PyTypeObject NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(Noddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- (destructor)Noddy_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE, /* tp_flags */
- "Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Noddy_methods, /* tp_methods */
- Noddy_members, /* tp_members */
- Noddy_getseters, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Noddy_init, /* tp_init */
- 0, /* tp_alloc */
- Noddy_new, /* tp_new */
-};
-
-static PyModuleDef noddy3module = {
- PyModuleDef_HEAD_INIT,
- "noddy3",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy3(void)
-{
- PyObject* m;
-
- if (PyType_Ready(&NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddy3module);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType);
- return m;
-}
diff --git a/Doc/includes/noddy4.c b/Doc/includes/noddy4.c
deleted file mode 100644
index 08ba4c3d91a030..00000000000000
--- a/Doc/includes/noddy4.c
+++ /dev/null
@@ -1,208 +0,0 @@
-#include
-#include "structmember.h"
-
-typedef struct {
- PyObject_HEAD
- PyObject *first;
- PyObject *last;
- int number;
-} Noddy;
-
-static int
-Noddy_traverse(Noddy *self, visitproc visit, void *arg)
-{
- int vret;
-
- if (self->first) {
- vret = visit(self->first, arg);
- if (vret != 0)
- return vret;
- }
- if (self->last) {
- vret = visit(self->last, arg);
- if (vret != 0)
- return vret;
- }
-
- return 0;
-}
-
-static int
-Noddy_clear(Noddy *self)
-{
- PyObject *tmp;
-
- tmp = self->first;
- self->first = NULL;
- Py_XDECREF(tmp);
-
- tmp = self->last;
- self->last = NULL;
- Py_XDECREF(tmp);
-
- return 0;
-}
-
-static void
-Noddy_dealloc(Noddy* self)
-{
- PyObject_GC_UnTrack(self);
- Noddy_clear(self);
- Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-static PyObject *
-Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
-}
-
-static int
-Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
-{
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_XDECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_XDECREF(tmp);
- }
-
- return 0;
-}
-
-
-static PyMemberDef Noddy_members[] = {
- {"first", T_OBJECT_EX, offsetof(Noddy, first), 0,
- "first name"},
- {"last", T_OBJECT_EX, offsetof(Noddy, last), 0,
- "last name"},
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_name(Noddy* self)
-{
- if (self->first == NULL) {
- PyErr_SetString(PyExc_AttributeError, "first");
- return NULL;
- }
-
- if (self->last == NULL) {
- PyErr_SetString(PyExc_AttributeError, "last");
- return NULL;
- }
-
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
-}
-
-static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
-};
-
-static PyTypeObject NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(Noddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- (destructor)Noddy_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE |
- Py_TPFLAGS_HAVE_GC, /* tp_flags */
- "Noddy objects", /* tp_doc */
- (traverseproc)Noddy_traverse, /* tp_traverse */
- (inquiry)Noddy_clear, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Noddy_methods, /* tp_methods */
- Noddy_members, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Noddy_init, /* tp_init */
- 0, /* tp_alloc */
- Noddy_new, /* tp_new */
-};
-
-static PyModuleDef noddy4module = {
- PyModuleDef_HEAD_INIT,
- "noddy4",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy4(void)
-{
- PyObject* m;
-
- if (PyType_Ready(&NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddy4module);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType);
- return m;
-}
diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c
index ead7bdd23209a3..9caf1fdb20104a 100644
--- a/Doc/includes/run-func.c
+++ b/Doc/includes/run-func.c
@@ -3,7 +3,7 @@
int
main(int argc, char *argv[])
{
- PyObject *pName, *pModule, *pDict, *pFunc;
+ PyObject *pName, *pModule, *pFunc;
PyObject *pArgs, *pValue;
int i;
diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c
deleted file mode 100644
index 0ef4765327776f..00000000000000
--- a/Doc/includes/shoddy.c
+++ /dev/null
@@ -1,99 +0,0 @@
-#include
-
-typedef struct {
- PyListObject list;
- int state;
-} Shoddy;
-
-
-static PyObject *
-Shoddy_increment(Shoddy *self, PyObject *unused)
-{
- self->state++;
- return PyLong_FromLong(self->state);
-}
-
-
-static PyMethodDef Shoddy_methods[] = {
- {"increment", (PyCFunction)Shoddy_increment, METH_NOARGS,
- PyDoc_STR("increment state counter")},
- {NULL},
-};
-
-static int
-Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds)
-{
- if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0)
- return -1;
- self->state = 0;
- return 0;
-}
-
-
-static PyTypeObject ShoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "shoddy.Shoddy", /* tp_name */
- sizeof(Shoddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- 0, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE, /* tp_flags */
- 0, /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Shoddy_methods, /* tp_methods */
- 0, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Shoddy_init, /* tp_init */
- 0, /* tp_alloc */
- 0, /* tp_new */
-};
-
-static PyModuleDef shoddymodule = {
- PyModuleDef_HEAD_INIT,
- "shoddy",
- "Shoddy module",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_shoddy(void)
-{
- PyObject *m;
-
- ShoddyType.tp_base = &PyList_Type;
- if (PyType_Ready(&ShoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&shoddymodule);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&ShoddyType);
- PyModule_AddObject(m, "Shoddy", (PyObject *) &ShoddyType);
- return m;
-}
diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c
new file mode 100644
index 00000000000000..376dddfac09c68
--- /dev/null
+++ b/Doc/includes/sublist.c
@@ -0,0 +1,63 @@
+#include
+
+typedef struct {
+ PyListObject list;
+ int state;
+} SubListObject;
+
+static PyObject *
+SubList_increment(SubListObject *self, PyObject *unused)
+{
+ self->state++;
+ return PyLong_FromLong(self->state);
+}
+
+static PyMethodDef SubList_methods[] = {
+ {"increment", (PyCFunction) SubList_increment, METH_NOARGS,
+ PyDoc_STR("increment state counter")},
+ {NULL},
+};
+
+static int
+SubList_init(SubListObject *self, PyObject *args, PyObject *kwds)
+{
+ if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0)
+ return -1;
+ self->state = 0;
+ return 0;
+}
+
+static PyTypeObject SubListType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "sublist.SubList",
+ .tp_doc = "SubList objects",
+ .tp_basicsize = sizeof(SubListObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ .tp_init = (initproc) SubList_init,
+ .tp_methods = SubList_methods,
+};
+
+static PyModuleDef sublistmodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "sublist",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_sublist(void)
+{
+ PyObject *m;
+ SubListType.tp_base = &PyList_Type;
+ if (PyType_Ready(&SubListType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&sublistmodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&SubListType);
+ PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+ return m;
+}
diff --git a/Doc/includes/test.py b/Doc/includes/test.py
index 9e9d4a67121cae..09ebe3fec0bdbe 100644
--- a/Doc/includes/test.py
+++ b/Doc/includes/test.py
@@ -1,181 +1,168 @@
-"""Test module for the noddy examples
+"""Test module for the custom examples
-Noddy 1:
+Custom 1:
->>> import noddy
->>> n1 = noddy.Noddy()
->>> n2 = noddy.Noddy()
->>> del n1
->>> del n2
+>>> import custom
+>>> c1 = custom.Custom()
+>>> c2 = custom.Custom()
+>>> del c1
+>>> del c2
-Noddy 2
+Custom 2
->>> import noddy2
->>> n1 = noddy2.Noddy('jim', 'fulton', 42)
->>> n1.first
+>>> import custom2
+>>> c1 = custom2.Custom('jim', 'fulton', 42)
+>>> c1.first
'jim'
->>> n1.last
+>>> c1.last
'fulton'
->>> n1.number
+>>> c1.number
42
->>> n1.name()
+>>> c1.name()
'jim fulton'
->>> n1.first = 'will'
->>> n1.name()
+>>> c1.first = 'will'
+>>> c1.name()
'will fulton'
->>> n1.last = 'tell'
->>> n1.name()
+>>> c1.last = 'tell'
+>>> c1.name()
'will tell'
->>> del n1.first
->>> n1.name()
+>>> del c1.first
+>>> c1.name()
Traceback (most recent call last):
...
AttributeError: first
->>> n1.first
+>>> c1.first
Traceback (most recent call last):
...
AttributeError: first
->>> n1.first = 'drew'
->>> n1.first
+>>> c1.first = 'drew'
+>>> c1.first
'drew'
->>> del n1.number
+>>> del c1.number
Traceback (most recent call last):
...
TypeError: can't delete numeric/char attribute
->>> n1.number=2
->>> n1.number
+>>> c1.number=2
+>>> c1.number
2
->>> n1.first = 42
->>> n1.name()
+>>> c1.first = 42
+>>> c1.name()
'42 tell'
->>> n2 = noddy2.Noddy()
->>> n2.name()
+>>> c2 = custom2.Custom()
+>>> c2.name()
' '
->>> n2.first
+>>> c2.first
''
->>> n2.last
+>>> c2.last
''
->>> del n2.first
->>> n2.first
+>>> del c2.first
+>>> c2.first
Traceback (most recent call last):
...
AttributeError: first
->>> n2.first
+>>> c2.first
Traceback (most recent call last):
...
AttributeError: first
->>> n2.name()
+>>> c2.name()
Traceback (most recent call last):
File "", line 1, in ?
AttributeError: first
->>> n2.number
+>>> c2.number
0
->>> n3 = noddy2.Noddy('jim', 'fulton', 'waaa')
+>>> n3 = custom2.Custom('jim', 'fulton', 'waaa')
Traceback (most recent call last):
File "", line 1, in ?
-TypeError: an integer is required
->>> del n1
->>> del n2
+TypeError: an integer is required (got type str)
+>>> del c1
+>>> del c2
-Noddy 3
+Custom 3
->>> import noddy3
->>> n1 = noddy3.Noddy('jim', 'fulton', 42)
->>> n1 = noddy3.Noddy('jim', 'fulton', 42)
->>> n1.name()
+>>> import custom3
+>>> c1 = custom3.Custom('jim', 'fulton', 42)
+>>> c1 = custom3.Custom('jim', 'fulton', 42)
+>>> c1.name()
'jim fulton'
->>> del n1.first
+>>> del c1.first
Traceback (most recent call last):
File "", line 1, in ?
TypeError: Cannot delete the first attribute
->>> n1.first = 42
+>>> c1.first = 42
Traceback (most recent call last):
File "", line 1, in ?
TypeError: The first attribute value must be a string
->>> n1.first = 'will'
->>> n1.name()
+>>> c1.first = 'will'
+>>> c1.name()
'will fulton'
->>> n2 = noddy3.Noddy()
->>> n2 = noddy3.Noddy()
->>> n2 = noddy3.Noddy()
->>> n3 = noddy3.Noddy('jim', 'fulton', 'waaa')
+>>> c2 = custom3.Custom()
+>>> c2 = custom3.Custom()
+>>> c2 = custom3.Custom()
+>>> n3 = custom3.Custom('jim', 'fulton', 'waaa')
Traceback (most recent call last):
File "", line 1, in ?
-TypeError: an integer is required
->>> del n1
->>> del n2
+TypeError: an integer is required (got type str)
+>>> del c1
+>>> del c2
-Noddy 4
+Custom 4
->>> import noddy4
->>> n1 = noddy4.Noddy('jim', 'fulton', 42)
->>> n1.first
+>>> import custom4
+>>> c1 = custom4.Custom('jim', 'fulton', 42)
+>>> c1.first
'jim'
->>> n1.last
+>>> c1.last
'fulton'
->>> n1.number
+>>> c1.number
42
->>> n1.name()
+>>> c1.name()
'jim fulton'
->>> n1.first = 'will'
->>> n1.name()
+>>> c1.first = 'will'
+>>> c1.name()
'will fulton'
->>> n1.last = 'tell'
->>> n1.name()
+>>> c1.last = 'tell'
+>>> c1.name()
'will tell'
->>> del n1.first
->>> n1.name()
+>>> del c1.first
Traceback (most recent call last):
...
-AttributeError: first
->>> n1.first
-Traceback (most recent call last):
-...
-AttributeError: first
->>> n1.first = 'drew'
->>> n1.first
+TypeError: Cannot delete the first attribute
+>>> c1.name()
+'will tell'
+>>> c1.first = 'drew'
+>>> c1.first
'drew'
->>> del n1.number
+>>> del c1.number
Traceback (most recent call last):
...
TypeError: can't delete numeric/char attribute
->>> n1.number=2
->>> n1.number
+>>> c1.number=2
+>>> c1.number
2
->>> n1.first = 42
->>> n1.name()
-'42 tell'
->>> n2 = noddy4.Noddy()
->>> n2 = noddy4.Noddy()
->>> n2 = noddy4.Noddy()
->>> n2 = noddy4.Noddy()
->>> n2.name()
+>>> c1.first = 42
+Traceback (most recent call last):
+...
+TypeError: The first attribute value must be a string
+>>> c1.name()
+'drew tell'
+>>> c2 = custom4.Custom()
+>>> c2 = custom4.Custom()
+>>> c2 = custom4.Custom()
+>>> c2 = custom4.Custom()
+>>> c2.name()
' '
->>> n2.first
+>>> c2.first
''
->>> n2.last
+>>> c2.last
''
->>> del n2.first
->>> n2.first
-Traceback (most recent call last):
-...
-AttributeError: first
->>> n2.first
-Traceback (most recent call last):
-...
-AttributeError: first
->>> n2.name()
-Traceback (most recent call last):
- File "", line 1, in ?
-AttributeError: first
->>> n2.number
+>>> c2.number
0
->>> n3 = noddy4.Noddy('jim', 'fulton', 'waaa')
+>>> n3 = custom4.Custom('jim', 'fulton', 'waaa')
Traceback (most recent call last):
- File "", line 1, in ?
-TypeError: an integer is required
+...
+TypeError: an integer is required (got type str)
Test cyclic gc(?)
@@ -183,15 +170,14 @@
>>> import gc
>>> gc.disable()
->>> x = []
->>> l = [x]
->>> n2.first = l
->>> n2.first
-[[]]
->>> l.append(n2)
->>> del l
->>> del n1
->>> del n2
+>>> class Subclass(custom4.Custom): pass
+...
+>>> s = Subclass()
+>>> s.cycle = [s]
+>>> s.cycle.append(s.cycle)
+>>> x = object()
+>>> s.x = x
+>>> del s
>>> sys.getrefcount(x)
3
>>> ignore = gc.collect()
diff --git a/Doc/includes/tzinfo_examples.py b/Doc/includes/tzinfo_examples.py
index ae5a5092665f09..9b9e32a553e7d8 100644
--- a/Doc/includes/tzinfo_examples.py
+++ b/Doc/includes/tzinfo_examples.py
@@ -1,4 +1,4 @@
-from datetime import tzinfo, timedelta, datetime, timezone
+from datetime import tzinfo, timedelta, datetime
ZERO = timedelta(0)
HOUR = timedelta(hours=1)
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
index 0545b8f33d0378..92cdf2f11443fb 100644
--- a/Doc/install/index.rst
+++ b/Doc/install/index.rst
@@ -283,7 +283,9 @@ Windows, choose :menuselection:`Start --> Programs --> Python X.Y -->
Python (command line)`. Once the interpreter is started, you type Python code
at the prompt. For example, on my Linux system, I type the three Python
statements shown below, and get the output as shown, to find out my
-:file:`{prefix}` and :file:`{exec-prefix}`::
+:file:`{prefix}` and :file:`{exec-prefix}`:
+
+.. code-block:: pycon
Python 2.4 (#26, Aug 7 2004, 17:19:02)
Type "help", "copyright", "credits" or "license" for more information.
@@ -622,7 +624,9 @@ parsing your configuration file(s).
Obviously, specifying the entire installation scheme every time you install a
new module distribution would be very tedious. Thus, you can put these options
-into your Distutils config file (see section :ref:`inst-config-files`)::
+into your Distutils config file (see section :ref:`inst-config-files`):
+
+.. code-block:: ini
[install]
install-base=$HOME
@@ -631,7 +635,9 @@ into your Distutils config file (see section :ref:`inst-config-files`)::
install-scripts=python/scripts
install-data=python/data
-or, equivalently, ::
+or, equivalently,
+
+.. code-block:: ini
[install]
install-base=$HOME/python
@@ -718,7 +724,9 @@ A slightly less convenient way is to edit the :file:`site.py` file in Python's
standard library, and modify ``sys.path``. :file:`site.py` is automatically
imported when the Python interpreter is executed, unless the :option:`-S` switch
is supplied to suppress this behaviour. So you could simply edit
-:file:`site.py` and add two lines to it::
+:file:`site.py` and add two lines to it:
+
+.. code-block:: python
import sys
sys.path.append('/www/python/')
@@ -839,7 +847,9 @@ plus a ``global`` section for global options that affect every command. Each
section consists of one option per line, specified as ``option=value``.
For example, the following is a complete config file that just forces all
-commands to run quietly by default::
+commands to run quietly by default:
+
+.. code-block:: ini
[global]
verbose=0
@@ -853,7 +863,9 @@ distribution.
You could override the default "build base" directory and make the
:command:`build\*` commands always forcibly rebuild all files with the
-following::
+following:
+
+.. code-block:: ini
[build]
build-base=blib
diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst
index f9a224be92b8be..7a69869fb50885 100644
--- a/Doc/installing/index.rst
+++ b/Doc/installing/index.rst
@@ -44,7 +44,7 @@ Key terms
``venv``. It allows virtual environments to be used on versions of
Python prior to 3.4, which either don't provide ``venv`` at all, or
aren't able to automatically install ``pip`` into created environments.
-* The `Python Packaging Index `__ is a public
+* The `Python Packaging Index `__ is a public
repository of open source licensed packages made available for use by
other Python users.
* the `Python Packaging Authority
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
index deb5e10f6ef7e9..fa4b0a9645531c 100644
--- a/Doc/library/2to3.rst
+++ b/Doc/library/2to3.rst
@@ -385,7 +385,7 @@ and off individually. They are described here in more detail.
.. 2to3fixer:: reload
- Converts :func:`reload` to :func:`imp.reload`.
+ Converts :func:`reload` to :func:`importlib.reload`.
.. 2to3fixer:: renames
diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst
index 9522dd62049138..70710761a39555 100644
--- a/Doc/library/abc.rst
+++ b/Doc/library/abc.rst
@@ -160,7 +160,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance:
-The :mod:`abc` module also provides the following decorators:
+The :mod:`abc` module also provides the following decorator:
.. decorator:: abstractmethod
@@ -236,8 +236,15 @@ The :mod:`abc` module also provides the following decorators:
multiple-inheritance.
+The :mod:`abc` module also supports the following legacy decorators:
+
.. decorator:: abstractclassmethod
+ .. versionadded:: 3.2
+ .. deprecated:: 3.3
+ It is now possible to use :class:`classmethod` with
+ :func:`abstractmethod`, making this decorator redundant.
+
A subclass of the built-in :func:`classmethod`, indicating an abstract
classmethod. Otherwise it is similar to :func:`abstractmethod`.
@@ -251,14 +258,14 @@ The :mod:`abc` module also provides the following decorators:
def my_abstract_classmethod(cls, ...):
...
+
+.. decorator:: abstractstaticmethod
+
.. versionadded:: 3.2
.. deprecated:: 3.3
- It is now possible to use :class:`classmethod` with
+ It is now possible to use :class:`staticmethod` with
:func:`abstractmethod`, making this decorator redundant.
-
-.. decorator:: abstractstaticmethod
-
A subclass of the built-in :func:`staticmethod`, indicating an abstract
staticmethod. Otherwise it is similar to :func:`abstractmethod`.
@@ -272,23 +279,17 @@ The :mod:`abc` module also provides the following decorators:
def my_abstract_staticmethod(...):
...
- .. versionadded:: 3.2
- .. deprecated:: 3.3
- It is now possible to use :class:`staticmethod` with
- :func:`abstractmethod`, making this decorator redundant.
-
.. decorator:: abstractproperty
+ .. deprecated:: 3.3
+ It is now possible to use :class:`property`, :meth:`property.getter`,
+ :meth:`property.setter` and :meth:`property.deleter` with
+ :func:`abstractmethod`, making this decorator redundant.
+
A subclass of the built-in :func:`property`, indicating an abstract
property.
- Using this function requires that the class's metaclass is :class:`ABCMeta`
- or is derived from it. A class that has a metaclass derived from
- :class:`ABCMeta` cannot be instantiated unless all of its abstract methods
- and properties are overridden. The abstract properties can be called using
- any of the normal 'super' call mechanisms.
-
This special case is deprecated, as the :func:`property` decorator
is now correctly identified as abstract when applied to an abstract
method::
@@ -322,12 +323,6 @@ The :mod:`abc` module also provides the following decorators:
...
- .. deprecated:: 3.3
- It is now possible to use :class:`property`, :meth:`property.getter`,
- :meth:`property.setter` and :meth:`property.deleter` with
- :func:`abstractmethod`, making this decorator redundant.
-
-
The :mod:`abc` module also provides the following functions:
.. function:: get_cache_token()
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 53e670161dd5b3..260e5cbf700035 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -712,7 +712,7 @@ be positional::
Namespace(bar='BAR', foo='FOO')
>>> parser.parse_args(['--foo', 'FOO'])
usage: PROG [-h] [-f FOO] bar
- PROG: error: too few arguments
+ PROG: error: the following arguments are required: bar
action
@@ -898,7 +898,7 @@ values are:
Namespace(foo=['a', 'b'])
>>> parser.parse_args([])
usage: PROG [-h] foo [foo ...]
- PROG: error: too few arguments
+ PROG: error: the following arguments are required: foo
.. _`argparse.REMAINDER`:
@@ -981,7 +981,7 @@ is used when no command-line argument was present::
Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the
-command-line argument was not present.::
+command-line argument was not present::
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument('--foo', default=argparse.SUPPRESS)
@@ -1577,7 +1577,7 @@ Sub-commands
stored; by default ``None`` and no value is stored
* required_ - Whether or not a subcommand must be provided, by default
- ``True``.
+ ``False``.
* help_ - help for sub-parser group in help output, by default ``None``
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index b7f610ba8b2a5f..9ff422cdf503cb 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees:
Parse the source into an AST node. Equivalent to ``compile(source,
filename, mode, ast.PyCF_ONLY_AST)``.
+ .. warning::
+ It is possible to crash the Python interpreter with a
+ sufficiently large/complex string due to stack depth limitations
+ in Python's AST compiler.
+
.. function:: literal_eval(node_or_string)
@@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees:
capable of evaluating arbitrarily complex expressions, for example involving
operators or indexing.
+ .. warning::
+ It is possible to crash the Python interpreter with a
+ sufficiently large/complex string due to stack depth limitations
+ in Python's AST compiler.
+
.. versionchanged:: 3.2
Now allows bytes and set literals.
@@ -141,10 +151,6 @@ and classes for traversing abstract syntax trees:
.. versionchanged:: 3.5
:class:`AsyncFunctionDef` is now supported.
- .. versionchanged:: 3.7
- The docstring is now exported from the node docstring field, instead of
- the first body statement.
-
.. function:: fix_missing_locations(node)
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index 5f915c5c43921a..317f3fb85c5481 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -124,7 +124,7 @@ keywords to your callback, use :func:`functools.partial`. For example,
parameters in debug mode, whereas ``lambda`` functions have a poor
representation.
-.. method:: AbstractEventLoop.call_soon(callback, \*args)
+.. method:: AbstractEventLoop.call_soon(callback, *args, context=None)
Arrange for a callback to be called as soon as possible. The callback is
called after :meth:`call_soon` returns, when control returns to the event
@@ -137,19 +137,31 @@ keywords to your callback, use :func:`functools.partial`. For example,
Any positional arguments after the callback will be passed to the
callback when it is called.
+ An optional keyword-only *context* argument allows specifying a custom
+ :class:`contextvars.Context` for the *callback* to run in. The current
+ context is used when no *context* is provided.
+
An instance of :class:`asyncio.Handle` is returned, which can be
used to cancel the callback.
:ref:`Use functools.partial to pass keywords to the callback
`.
-.. method:: AbstractEventLoop.call_soon_threadsafe(callback, \*args)
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
+
+.. method:: AbstractEventLoop.call_soon_threadsafe(callback, *args, context=None)
Like :meth:`call_soon`, but thread safe.
See the :ref:`concurrency and multithreading `
section of the documentation.
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
+
.. _asyncio-delayed-calls:
@@ -166,12 +178,12 @@ a different clock than :func:`time.time`.
Timeouts (relative *delay* or absolute *when*) should not exceed one day.
-.. method:: AbstractEventLoop.call_later(delay, callback, *args)
+.. method:: AbstractEventLoop.call_later(delay, callback, *args, context=None)
Arrange for the *callback* to be called after the given *delay*
seconds (either an int or float).
- An instance of :class:`asyncio.Handle` is returned, which can be
+ An instance of :class:`asyncio.TimerHandle` is returned, which can be
used to cancel the callback.
*callback* will be called exactly once per call to :meth:`call_later`.
@@ -182,10 +194,18 @@ a different clock than :func:`time.time`.
is called. If you want the callback to be called with some named
arguments, use a closure or :func:`functools.partial`.
+ An optional keyword-only *context* argument allows specifying a custom
+ :class:`contextvars.Context` for the *callback* to run in. The current
+ context is used when no *context* is provided.
+
:ref:`Use functools.partial to pass keywords to the callback
`.
-.. method:: AbstractEventLoop.call_at(when, callback, *args)
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
+
+.. method:: AbstractEventLoop.call_at(when, callback, *args, context=None)
Arrange for the *callback* to be called at the given absolute timestamp
*when* (an int or float), using the same time reference as
@@ -193,12 +213,16 @@ a different clock than :func:`time.time`.
This method's behavior is the same as :meth:`call_later`.
- An instance of :class:`asyncio.Handle` is returned, which can be
+ An instance of :class:`asyncio.TimerHandle` is returned, which can be
used to cancel the callback.
:ref:`Use functools.partial to pass keywords to the callback
`.
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
+
.. method:: AbstractEventLoop.time()
Return the current time, as a :class:`float` value, according to the
@@ -327,7 +351,7 @@ Creating connections
* *ssl_handshake_timeout* is (for an SSL connection) the time in seconds
to wait for the SSL handshake to complete before aborting the connection.
- ``10.0`` seconds if ``None`` (default).
+ ``60.0`` seconds if ``None`` (default).
.. versionadded:: 3.7
@@ -352,7 +376,7 @@ Creating connections
callable returning a :ref:`protocol ` instance.
This method will try to establish the connection in the background.
- When successful, the it returns a ``(transport, protocol)`` pair.
+ When successful, it returns a ``(transport, protocol)`` pair.
Options changing how the connection is created:
@@ -393,6 +417,9 @@ Creating connections
See :ref:`UDP echo client protocol ` and
:ref:`UDP echo server protocol ` examples.
+ .. versionchanged:: 3.4.4
+ The *family*, *proto*, *flags*, *reuse_address*, *reuse_port,
+ *allow_broadcast*, and *sock* parameters were added.
.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path=None, \*, ssl=None, sock=None, server_hostname=None, ssl_handshake_timeout=None)
@@ -402,7 +429,7 @@ Creating connections
efficiently.
This method will try to establish the connection in the background.
- When successful, the it returns a ``(transport, protocol)`` pair.
+ When successful, it returns a ``(transport, protocol)`` pair.
*path* is the name of a UNIX domain socket, and is required unless a *sock*
parameter is specified. Abstract UNIX sockets, :class:`str`,
@@ -418,7 +445,7 @@ Creating connections
.. versionchanged:: 3.7
- The *path* parameter can now be a :class:`~pathlib.Path` object.
+ The *path* parameter can now be a :term:`path-like object`.
Creating listening connections
@@ -470,7 +497,7 @@ Creating listening connections
* *ssl_handshake_timeout* is (for an SSL server) the time in seconds to wait
for the SSL handshake to complete before aborting the connection.
- ``10.0`` seconds if ``None`` (default).
+ ``60.0`` seconds if ``None`` (default).
* *start_serving* set to ``True`` (the default) causes the created server
to start accepting connections immediately. When set to ``False``,
@@ -509,7 +536,7 @@ Creating listening connections
.. versionadded:: 3.7
- The *ssl_handshake_timeout* parameter.
+ The *ssl_handshake_timeout* and *start_serving* parameters.
.. versionchanged:: 3.7
@@ -532,7 +559,7 @@ Creating listening connections
* *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to
wait for the SSL handshake to complete before aborting the connection.
- ``10.0`` seconds if ``None`` (default).
+ ``60.0`` seconds if ``None`` (default).
When completed it returns a ``(transport, protocol)`` pair.
@@ -601,7 +628,7 @@ TLS Upgrade
* *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to
wait for the SSL handshake to complete before aborting the connection.
- ``10.0`` seconds if ``None`` (default).
+ ``60.0`` seconds if ``None`` (default).
.. versionadded:: 3.7
@@ -1043,7 +1070,7 @@ Server
async def main(host, port):
srv = await asyncio.start_server(
client_connected, host, port)
- await loop.serve_forever()
+ await srv.serve_forever()
asyncio.run(main('127.0.0.1', 0))
@@ -1076,8 +1103,7 @@ Handle
.. class:: Handle
A callback wrapper object returned by :func:`AbstractEventLoop.call_soon`,
- :func:`AbstractEventLoop.call_soon_threadsafe`, :func:`AbstractEventLoop.call_later`,
- and :func:`AbstractEventLoop.call_at`.
+ :func:`AbstractEventLoop.call_soon_threadsafe`.
.. method:: cancel()
@@ -1090,6 +1116,22 @@ Handle
.. versionadded:: 3.7
+.. class:: TimerHandle
+
+ A callback wrapper object returned by :func:`AbstractEventLoop.call_later`,
+ and :func:`AbstractEventLoop.call_at`.
+
+ The class is inherited from :class:`Handle`.
+
+ .. method:: when()
+
+ Return a scheduled callback time as :class:`float` seconds.
+
+ The time is an absolute timestamp, using the same time
+ reference as :meth:`AbstractEventLoop.time`.
+
+ .. versionadded:: 3.7
+
SendfileNotAvailableError
-------------------------
@@ -1099,7 +1141,7 @@ SendfileNotAvailableError
Sendfile syscall is not available, subclass of :exc:`RuntimeError`.
- Raised if the OS does not support senfile syscall for
+ Raised if the OS does not support sendfile syscall for
given socket or file type.
diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst
index 3051fde5b93baf..7f6e9535a8a508 100644
--- a/Doc/library/asyncio-eventloops.rst
+++ b/Doc/library/asyncio-eventloops.rst
@@ -176,12 +176,15 @@ An event loop policy must implement the following interface:
Get the event loop for the current context.
Returns an event loop object implementing the :class:`AbstractEventLoop`
- interface.
+ interface. In case called from coroutine, it returns the currently
+ running event loop.
Raises an exception in case no event loop has been set for the current
context and the current policy does not specify to create one. It must
never return ``None``.
+ .. versionchanged:: 3.6
+
.. method:: set_event_loop(loop)
Set the event loop for the current context to *loop*.
diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index 004cac80d90c2c..9a08a4a49021cc 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -339,7 +339,7 @@ Protocol classes
control of the receive buffer.
.. versionadded:: 3.7
- **Important:** this has been been added to asyncio in Python 3.7
+ **Important:** this has been added to asyncio in Python 3.7
*on a provisional basis*! Treat it as an experimental API that
might be changed or removed in Python 3.8.
@@ -450,7 +450,7 @@ Streaming protocols with manual receive buffer control
------------------------------------------------------
.. versionadded:: 3.7
- **Important:** :class:`BufferedProtocol` has been been added to
+ **Important:** :class:`BufferedProtocol` has been added to
asyncio in Python 3.7 *on a provisional basis*! Consider it as an
experimental API that might be changed or removed in Python 3.8.
@@ -463,16 +463,23 @@ The idea of BufferedProtocol is that it allows to manually allocate
and control the receive buffer. Event loops can then use the buffer
provided by the protocol to avoid unnecessary data copies. This
can result in noticeable performance improvement for protocols that
-receive big amounts of data. Sophisticated protocols can allocate
-the buffer only once at creation time.
+receive big amounts of data. Sophisticated protocols implementations
+can allocate the buffer only once at creation time.
The following callbacks are called on :class:`BufferedProtocol`
instances:
-.. method:: BufferedProtocol.get_buffer()
+.. method:: BufferedProtocol.get_buffer(sizehint)
- Called to allocate a new receive buffer. Must return an object
- that implements the :ref:`buffer protocol `.
+ Called to allocate a new receive buffer.
+
+ *sizehint* is a recommended minimal size for the returned
+ buffer. It is acceptable to return smaller or bigger buffers
+ than what *sizehint* suggests. When set to -1, the buffer size
+ can be arbitrary. It is an error to return a zero-sized buffer.
+
+ Must return an object that implements the
+ :ref:`buffer protocol `.
.. method:: BufferedProtocol.buffer_updated(nbytes)
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
index 099b59ee5823f4..f662e722333725 100644
--- a/Doc/library/asyncio-stream.rst
+++ b/Doc/library/asyncio-stream.rst
@@ -18,7 +18,7 @@ Stream functions
exactly what you want, feel free to copy their code.
-.. coroutinefunction:: open_connection(host=None, port=None, \*, loop=None, limit=None, \*\*kwds)
+.. coroutinefunction:: open_connection(host=None, port=None, \*, loop=None, limit=None, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None, ssl_handshake_timeout=None)
A wrapper for :meth:`~AbstractEventLoop.create_connection()` returning a (reader,
writer) pair.
@@ -26,64 +26,102 @@ Stream functions
The reader returned is a :class:`StreamReader` instance; the writer is
a :class:`StreamWriter` instance.
- The arguments are all the usual arguments to
- :meth:`AbstractEventLoop.create_connection` except *protocol_factory*; most
- common are positional host and port, with various optional keyword arguments
- following.
+ When specified, the *loop* argument determines which event loop to use,
+ and the *limit* argument determines the buffer size limit used by the
+ returned :class:`StreamReader` instance.
- Additional optional keyword arguments are *loop* (to set the event loop
- instance to use) and *limit* (to set the buffer limit passed to the
- :class:`StreamReader`).
+ The rest of the arguments are passed directly to
+ :meth:`AbstractEventLoop.create_connection`.
This function is a :ref:`coroutine `.
-.. coroutinefunction:: start_server(client_connected_cb, host=None, port=None, \*, loop=None, limit=None, \*\*kwds)
+ .. versionadded:: 3.7
+
+ The *ssl_handshake_timeout* parameter.
+
+.. coroutinefunction:: start_server(client_connected_cb, host=None, port=None, \*, loop=None, limit=None, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, ssl_handshake_timeout=None, start_serving=True)
Start a socket server, with a callback for each client connected. The return
value is the same as :meth:`~AbstractEventLoop.create_server()`.
- The *client_connected_cb* parameter is called with two parameters:
- *client_reader*, *client_writer*. *client_reader* is a
- :class:`StreamReader` object, while *client_writer* is a
- :class:`StreamWriter` object. The *client_connected_cb* parameter can
- either be a plain callback function or a :ref:`coroutine function
- `; if it is a coroutine function, it will be automatically
- converted into a :class:`Task`.
+ The *client_connected_cb* callback is called whenever a new client
+ connection is established. It receives a reader/writer pair as two
+ arguments, the first is a :class:`StreamReader` instance,
+ and the second is a :class:`StreamWriter` instance.
+
+ *client_connected_cb* accepts a plain callable or a
+ :ref:`coroutine function `; if it is a coroutine function,
+ it will be automatically converted into a :class:`Task`.
- The rest of the arguments are all the usual arguments to
- :meth:`~AbstractEventLoop.create_server()` except *protocol_factory*; most
- common are positional *host* and *port*, with various optional keyword
- arguments following.
+ When specified, the *loop* argument determines which event loop to use,
+ and the *limit* argument determines the buffer size limit used by the
+ :class:`StreamReader` instance passed to *client_connected_cb*.
- Additional optional keyword arguments are *loop* (to set the event loop
- instance to use) and *limit* (to set the buffer limit passed to the
- :class:`StreamReader`).
+ The rest of the arguments are passed directly to
+ :meth:`~AbstractEventLoop.create_server()`.
This function is a :ref:`coroutine `.
-.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, limit=None, **kwds)
+ .. versionadded:: 3.7
+
+ The *ssl_handshake_timeout* and *start_serving* parameters.
+
+.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, limit=None, ssl=None, sock=None, server_hostname=None, ssl_handshake_timeout=None)
A wrapper for :meth:`~AbstractEventLoop.create_unix_connection()` returning
a (reader, writer) pair.
- See :func:`open_connection` for information about return value and other
- details.
+ When specified, the *loop* argument determines which event loop to use,
+ and the *limit* argument determines the buffer size limit used by the
+ returned :class:`StreamReader` instance.
+
+ The rest of the arguments are passed directly to
+ :meth:`~AbstractEventLoop.create_unix_connection()`.
This function is a :ref:`coroutine `.
Availability: UNIX.
-.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \*, loop=None, limit=None, **kwds)
+ .. versionadded:: 3.7
+
+ The *ssl_handshake_timeout* parameter.
+
+ .. versionchanged:: 3.7
+
+ The *path* parameter can now be a :term:`path-like object`
+
+.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \*, loop=None, limit=None, sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, start_serving=True)
Start a UNIX Domain Socket server, with a callback for each client connected.
- See :func:`start_server` for information about return value and other
- details.
+ The *client_connected_cb* callback is called whenever a new client
+ connection is established. It receives a reader/writer pair as two
+ arguments, the first is a :class:`StreamReader` instance,
+ and the second is a :class:`StreamWriter` instance.
+
+ *client_connected_cb* accepts a plain callable or a
+ :ref:`coroutine function `; if it is a coroutine function,
+ it will be automatically converted into a :class:`Task`.
+
+ When specified, the *loop* argument determines which event loop to use,
+ and the *limit* argument determines the buffer size limit used by the
+ :class:`StreamReader` instance passed to *client_connected_cb*.
+
+ The rest of the arguments are passed directly to
+ :meth:`~AbstractEventLoop.create_unix_server()`.
This function is a :ref:`coroutine `.
Availability: UNIX.
+ .. versionadded:: 3.7
+
+ The *ssl_handshake_timeout* and *start_serving* parameters.
+
+ .. versionchanged:: 3.7
+
+ The *path* parameter can now be a :term:`path-like object`.
+
StreamReader
============
@@ -393,8 +431,7 @@ Simple example querying HTTP headers of the URL passed on the command line::
import urllib.parse
import sys
- @asyncio.coroutine
- def print_http_headers(url):
+ async def print_http_headers(url):
url = urllib.parse.urlsplit(url)
if url.scheme == 'https':
connect = asyncio.open_connection(url.hostname, 443, ssl=True)
@@ -475,4 +512,3 @@ Coroutine waiting until a socket receives data using the
` example uses the low-level
:meth:`AbstractEventLoop.add_reader` method to register the file descriptor of a
socket.
-
diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst
index 280b76400374cb..60e174574b04e9 100644
--- a/Doc/library/asyncio-subprocess.rst
+++ b/Doc/library/asyncio-subprocess.rst
@@ -392,8 +392,7 @@ function::
import asyncio.subprocess
import sys
- @asyncio.coroutine
- def get_date():
+ async def get_date():
code = 'import datetime; print(datetime.datetime.now())'
# Create the subprocess, redirect the standard output into a pipe
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 71dbe06c899f18..2b480d4be3fb21 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -92,7 +92,7 @@ Coroutines (and tasks) can only run when the event loop is running.
used in a callback-style code, wrap its result with :func:`ensure_future`.
-.. function:: asyncio.run(coro, \*, debug=False)
+.. function:: run(coro, \*, debug=False)
This function runs the passed coroutine, taking care of
managing the asyncio event loop and finalizing asynchronous
@@ -108,6 +108,8 @@ Coroutines (and tasks) can only run when the event loop is running.
programs, and should ideally only be called once.
.. versionadded:: 3.7
+ **Important:** this has been been added to asyncio in Python 3.7
+ on a :term:`provisional basis `.
.. _asyncio-hello-world-coroutine:
@@ -273,19 +275,27 @@ Future
:exc:`CancelledError`. If the future isn't done yet, raises
:exc:`InvalidStateError`.
- .. method:: add_done_callback(fn)
+ .. method:: add_done_callback(callback, *, context=None)
Add a callback to be run when the future becomes done.
- The callback is called with a single argument - the future object. If the
+ The *callback* is called with a single argument - the future object. If the
future is already done when this is called, the callback is scheduled
with :meth:`~AbstractEventLoop.call_soon`.
+ An optional keyword-only *context* argument allows specifying a custom
+ :class:`contextvars.Context` for the *callback* to run in. The current
+ context is used when no *context* is provided.
+
:ref:`Use functools.partial to pass parameters to the callback
`. For example,
``fut.add_done_callback(functools.partial(print, "Future:",
flush=True))`` will call ``print("Future:", fut, flush=True)``.
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
+
.. method:: remove_done_callback(fn)
Remove all instances of a callback from the "call when done" list.
@@ -419,13 +429,22 @@ Task
Don't directly create :class:`Task` instances: use the :func:`create_task`
function or the :meth:`AbstractEventLoop.create_task` method.
+ Tasks support the :mod:`contextvars` module. When a Task
+ is created it copies the current context and later runs its coroutine
+ in the copied context. See :pep:`567` for more details.
+
This class is :ref:`not thread safe `.
+ .. versionchanged:: 3.7
+ Added support for the :mod:`contextvars` module.
+
.. classmethod:: all_tasks(loop=None)
Return a set of all tasks for an event loop.
By default all tasks for the current event loop are returned.
+ If *loop* is ``None``, :func:`get_event_loop` function
+ is used to get the current loop.
.. classmethod:: current_task(loop=None)
@@ -535,7 +554,7 @@ Task functions
not provided, the default event loop is used.
-.. function:: current_task(loop=None):
+.. function:: current_task(loop=None)
Return the current running :class:`Task` instance or ``None``, if
no task is running.
@@ -546,12 +565,13 @@ Task functions
.. versionadded:: 3.7
-.. function:: all_tasks(loop=None):
+.. function:: all_tasks(loop=None)
Return a set of :class:`Task` objects created for the loop.
- If *loop* is ``None`` :func:`get_event_loop` is used for getting
- current loop.
+ If *loop* is ``None``, :func:`get_running_loop` is used for getting
+ current loop (contrary to the deprecated :meth:`Task.all_tasks` method
+ that uses :func:`get_event_loop`.)
.. versionadded:: 3.7
@@ -620,6 +640,10 @@ Task functions
outer Future is *not* cancelled in this case. (This is to prevent the
cancellation of one child to cause other children to be cancelled.)
+ .. versionchanged:: 3.7.0
+ If the *gather* itself is cancelled, the cancellation is propagated
+ regardless of *return_exceptions*.
+
.. function:: iscoroutine(obj)
Return ``True`` if *obj* is a :ref:`coroutine object `,
@@ -682,7 +706,7 @@ Task functions
This function is a :ref:`coroutine `.
-.. function:: shield(arg, \*, loop=None)
+.. coroutinefunction:: shield(arg, \*, loop=None)
Wait for a future, shielding it from cancellation.
@@ -744,6 +768,9 @@ Task functions
| | futures finish or are cancelled. |
+-----------------------------+----------------------------------------+
+ Unlike :func:`~asyncio.wait_for`, ``wait()`` will not cancel the futures
+ when a timeout occurs.
+
This function is a :ref:`coroutine `.
Usage::
@@ -766,7 +793,9 @@ Task functions
Returns result of the Future or coroutine. When a timeout occurs, it
cancels the task and raises :exc:`asyncio.TimeoutError`. To avoid the task
- cancellation, wrap it in :func:`shield`.
+ cancellation, wrap it in :func:`shield`. The function will wait until
+ the future is actually cancelled, so the total wait time may exceed
+ the *timeout*.
If the wait is cancelled, the future *fut* is also cancelled.
@@ -776,3 +805,8 @@ Task functions
.. versionchanged:: 3.4.3
If the wait is cancelled, the future *fut* is now also cancelled.
+
+ .. versionchanged:: 3.7
+ When *fut* is cancelled due to a timeout, ``wait_for`` now waits
+ for *fut* to be cancelled. Previously,
+ it raised :exc:`~asyncio.TimeoutError` immediately.
diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst
index ceecf17cba23e3..ad9f5f58bee2aa 100644
--- a/Doc/library/base64.rst
+++ b/Doc/library/base64.rst
@@ -218,14 +218,6 @@ The modern interface provides:
.. versionadded:: 3.4
-.. note::
- Both Base85 and Ascii85 have an expansion factor of 5 to 4 (5 Base85 or
- Ascii85 characters can encode 4 binary bytes), while the better-known
- Base64 has an expansion factor of 6 to 4. They are therefore more
- efficient when space expensive. They differ by details such as the
- character map used for encoding.
-
-
The legacy interface:
.. function:: decode(input, output)
diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst
index 4d3d0e07f1861d..a4efef8465725c 100644
--- a/Doc/library/binascii.rst
+++ b/Doc/library/binascii.rst
@@ -80,7 +80,7 @@ The :mod:`binascii` module defines the following functions:
*quotetabs* is present and true, all tabs and spaces will be encoded. If the
optional argument *istext* is present and true, newlines are not encoded but
trailing whitespace will be encoded. If the optional argument *header* is
- present and true, spaces will be encoded as underscores per RFC1522. If the
+ present and true, spaces will be encoded as underscores per :rfc:`1522`. If the
optional argument *header* is present and false, newline characters will be
encoded as well; otherwise linefeed conversion might corrupt the binary data
stream.
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
index b60e1cc41e4be0..0b1aead9ddf1fd 100644
--- a/Doc/library/cgi.rst
+++ b/Doc/library/cgi.rst
@@ -289,12 +289,14 @@ algorithms implemented in this module in other circumstances.
This function is deprecated in this module. Use :func:`urllib.parse.parse_qs`
instead. It is maintained here only for backward compatibility.
+
.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False)
This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl`
instead. It is maintained here only for backward compatibility.
-.. function:: parse_multipart(fp, pdict, encoding="utf-8")
+
+.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace")
Parse input of type :mimetype:`multipart/form-data` (for file uploads).
Arguments are *fp* for the input file, *pdict* for a dictionary containing
@@ -309,6 +311,10 @@ algorithms implemented in this module in other circumstances.
uploaded --- in that case, use the :class:`FieldStorage` class instead
which is much more flexible.
+ .. versionchanged:: 3.7
+ Added the *encoding* and *errors* parameters. For non-file fields, the
+ value is now a list of strings, not bytes.
+
.. function:: parse_header(string)
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 74b24e10ede647..24008a0b3f00f1 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -1425,7 +1425,7 @@ to the user.
Python supports this conversion in several ways: the ``idna`` codec performs
conversion between Unicode and ACE, separating an input string into labels
-based on the separator characters defined in `section 3.1`_ (1) of :rfc:`3490`
+based on the separator characters defined in :rfc:`section 3.1 of RFC 3490 <3490#section-3.1>`
and converting each label to ACE as required, and conversely separating an input
byte string into labels based on the ``.`` separator and converting any ACE
labels found into unicode. Furthermore, the :mod:`socket` module
@@ -1436,8 +1436,6 @@ parameters, such as :mod:`http.client` and :mod:`ftplib`, accept Unicode host
names (:mod:`http.client` then also transparently sends an IDNA hostname in the
:mailheader:`Host` field if it sends that field at all).
-.. _section 3.1: https://tools.ietf.org/html/rfc3490#section-3.1
-
When receiving host names from the wire (such as in reverse name lookup), no
automatic conversion to Unicode is performed: Applications wishing to present
such host names to the user should decode them to Unicode.
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index 772ff60fe9839a..2a83d30372770b 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -114,7 +114,7 @@ The class can be used to simulate nested scopes and is useful in templating.
for templating is a read-only chain of mappings. It also features
pushing and popping of contexts similar to the
:meth:`~collections.ChainMap.new_child` method and the
- :meth:`~collections.ChainMap.parents` property.
+ :attr:`~collections.ChainMap.parents` property.
* The `Nested Contexts recipe
`_ has options to control
@@ -270,7 +270,7 @@ For example::
Return a list of the *n* most common elements and their counts from the
most common to the least. If *n* is omitted or ``None``,
- :func:`most_common` returns *all* elements in the counter.
+ :meth:`most_common` returns *all* elements in the counter.
Elements with equal counts are ordered arbitrarily:
>>> Counter('abracadabra').most_common(3) # doctest: +SKIP
@@ -357,12 +357,12 @@ or subtracting from an empty counter.
restrictions on its keys and values. The values are intended to be numbers
representing counts, but you *could* store anything in the value field.
- * The :meth:`most_common` method requires only that the values be orderable.
+ * The :meth:`~Counter.most_common` method requires only that the values be orderable.
* For in-place operations such as ``c[key] += 1``, the value type need only
support addition and subtraction. So fractions, floats, and decimals would
work and negative values are supported. The same is also true for
- :meth:`update` and :meth:`subtract` which allow negative and zero values
+ :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values
for both inputs and outputs.
* The multiset methods are designed only for use cases with positive values.
@@ -370,7 +370,7 @@ or subtracting from an empty counter.
are created. There are no type restrictions, but the value type needs to
support addition, subtraction, and comparison.
- * The :meth:`elements` method requires integer counts. It ignores zero and
+ * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and
negative counts.
.. seealso::
@@ -388,9 +388,9 @@ or subtracting from an empty counter.
Section 4.6.3, Exercise 19*.
* To enumerate all distinct multisets of a given size over a given set of
- elements, see :func:`itertools.combinations_with_replacement`:
+ elements, see :func:`itertools.combinations_with_replacement`::
- map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC
+ map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC
:class:`deque` objects
@@ -509,11 +509,14 @@ or subtracting from an empty counter.
.. versionadded:: 3.2
- .. method:: rotate(n)
+ .. method:: rotate(n=1)
- Rotate the deque *n* steps to the right. If *n* is negative, rotate to
- the left. Rotating one step to the right is equivalent to:
- ``d.appendleft(d.pop())``.
+ Rotate the deque *n* steps to the right. If *n* is negative, rotate
+ to the left.
+
+ When the deque is not empty, rotating one step to the right is equivalent
+ to ``d.appendleft(d.pop())``, and rotating one step to the left is
+ equivalent to ``d.append(d.popleft())``.
Deque objects also provide one read-only attribute:
@@ -637,9 +640,9 @@ the :meth:`~deque.rotate` method::
# Remove an exhausted iterator.
iterators.popleft()
-The :meth:`rotate` method provides a way to implement :class:`deque` slicing and
+The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and
deletion. For example, a pure Python implementation of ``del d[n]`` relies on
-the :meth:`rotate` method to position elements to be popped::
+the ``rotate()`` method to position elements to be popped::
def delete_nth(d, n):
d.rotate(-n)
@@ -647,8 +650,8 @@ the :meth:`rotate` method to position elements to be popped::
d.rotate(n)
To implement :class:`deque` slicing, use a similar approach applying
-:meth:`rotate` to bring a target element to the left side of the deque. Remove
-old entries with :meth:`popleft`, add new entries with :meth:`extend`, and then
+:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove
+old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then
reverse the rotation.
With minor variations on that approach, it is easy to implement Forth style
stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``,
@@ -709,7 +712,7 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``,
:class:`defaultdict` Examples
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Using :class:`list` as the :attr:`default_factory`, it is easy to group a
+Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a
sequence of key-value pairs into a dictionary of lists:
>>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)]
@@ -721,7 +724,7 @@ sequence of key-value pairs into a dictionary of lists:
[('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]
When each key is encountered for the first time, it is not already in the
-mapping; so an entry is automatically created using the :attr:`default_factory`
+mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory`
function which returns an empty :class:`list`. The :meth:`list.append`
operation then attaches the value to the new list. When keys are encountered
again, the look-up proceeds normally (returning the list for that key) and the
@@ -735,7 +738,7 @@ simpler and faster than an equivalent technique using :meth:`dict.setdefault`:
>>> sorted(d.items())
[('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]
-Setting the :attr:`default_factory` to :class:`int` makes the
+Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the
:class:`defaultdict` useful for counting (like a bag or multiset in other
languages):
@@ -748,7 +751,7 @@ languages):
[('i', 4), ('m', 1), ('p', 2), ('s', 4)]
When a letter is first encountered, it is missing from the mapping, so the
-:attr:`default_factory` function calls :func:`int` to supply a default count of
+:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of
zero. The increment operation then builds up the count for each letter.
The function :func:`int` which always returns zero is just a special case of
@@ -763,7 +766,7 @@ zero):
>>> '%(name)s %(action)s to %(object)s' % d
'John ran to '
-Setting the :attr:`default_factory` to :class:`set` makes the
+Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the
:class:`defaultdict` useful for building a dictionary of sets:
>>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)]
@@ -970,7 +973,7 @@ The subclass shown above sets ``__slots__`` to an empty tuple. This helps
keep memory requirements low by preventing the creation of instance dictionaries.
Subclassing is not useful for adding new, stored fields. Instead, simply
-create a new named tuple type from the :attr:`_fields` attribute:
+create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute:
>>> Point3D = namedtuple('Point3D', Point._fields + ('z',))
@@ -986,7 +989,7 @@ fields:
.. versionchanged:: 3.5
Property docstrings became writeable.
-Default values can be implemented by using :meth:`_replace` to
+Default values can be implemented by using :meth:`~somenamedtuple._replace` to
customize a prototype instance:
>>> Account = namedtuple('Account', 'owner balance transaction_count')
@@ -1197,15 +1200,22 @@ subclass directly from :class:`str`; however, this class can be easier
to work with because the underlying string is accessible as an
attribute.
-.. class:: UserString([sequence])
+.. class:: UserString(seq)
- Class that simulates a string or a Unicode string object. The instance's
+ Class that simulates a string object. The instance's
content is kept in a regular string object, which is accessible via the
:attr:`data` attribute of :class:`UserString` instances. The instance's
- contents are initially set to a copy of *sequence*. The *sequence* can
- be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a
- subclass) or an arbitrary sequence which can be converted into a string using
- the built-in :func:`str` function.
+ contents are initially set to a copy of *seq*. The *seq* argument can
+ be any object which can be converted into a string using the built-in
+ :func:`str` function.
+
+ In addition to supporting the methods and operations of strings,
+ :class:`UserString` instances provide the following attribute:
+
+ .. attribute:: data
+
+ A real :class:`str` object used to store the contents of the
+ :class:`UserString` class.
.. versionchanged:: 3.5
New methods ``__getnewargs__``, ``__rmod__``, ``casefold``,
diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst
index dd8257720b7c5f..4e55623f5360d4 100644
--- a/Doc/library/configparser.rst
+++ b/Doc/library/configparser.rst
@@ -887,7 +887,7 @@ interpolation if an option used is not defined elsewhere. ::
ConfigParser Objects
--------------------
-.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={})
+.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={})
The main configuration parser. When *defaults* is given, it is initialized
into the dictionary of intrinsic defaults. When *dict_type* is given, it
@@ -949,6 +949,9 @@ ConfigParser Objects
providing consistent behavior across the parser: non-string
keys and values are implicitly converted to strings.
+ .. versionchanged:: 3.7
+ The default *dict_type* is :class:`dict`, since it now preserves
+ insertion order.
.. method:: defaults()
@@ -1120,10 +1123,6 @@ ConfigParser Objects
given *section*. Optional arguments have the same meaning as for the
:meth:`get` method.
- .. versionchanged:: 3.2
- Items present in *vars* no longer appear in the result. The previous
- behaviour mixed actual parser options with variables provided for
- interpolation.
.. method:: set(section, option, value)
@@ -1209,7 +1208,7 @@ ConfigParser Objects
RawConfigParser Objects
-----------------------
-.. class:: RawConfigParser(defaults=None, dict_type=collections.OrderedDict, \
+.. class:: RawConfigParser(defaults=None, dict_type=dict, \
allow_no_value=False, *, delimiters=('=', ':'), \
comment_prefixes=('#', ';'), \
inline_comment_prefixes=None, strict=True, \
@@ -1222,6 +1221,10 @@ RawConfigParser Objects
names, and values via its unsafe ``add_section`` and ``set`` methods,
as well as the legacy ``defaults=`` keyword argument handling.
+ .. versionchanged:: 3.7
+ The default *dict_type* is :class:`dict`, since it now preserves
+ insertion order.
+
.. note::
Consider using :class:`ConfigParser` instead which checks types of
the values to be stored internally. If you don't want interpolation, you
diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst
index 469a3eed606ff0..78f16196369810 100644
--- a/Doc/library/constants.rst
+++ b/Doc/library/constants.rst
@@ -86,10 +86,14 @@ should not be used in programs.
specified exit code.
.. data:: copyright
- license
credits
- Objects that when printed, print a message like "Type license() to see the
- full license text", and when called, display the corresponding text in a
+ Objects that when printed or called, print the text of copyright or
+ credits, respectively.
+
+.. data:: license
+
+ Object that when printed, prints the message "Type license() to see the
+ full license text", and when called, displays the full license text in a
pager-like fashion (one screen at a time).
diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst
index 54d3a8e4296b06..793bd63f673f6a 100644
--- a/Doc/library/contextlib.rst
+++ b/Doc/library/contextlib.rst
@@ -47,22 +47,28 @@ Functions and classes provided:
function for :keyword:`with` statement context managers, without needing to
create a class or separate :meth:`__enter__` and :meth:`__exit__` methods.
- A simple example (this is not recommended as a real way of generating HTML!)::
+ While many objects natively support use in with statements, sometimes a
+ resource needs to be managed that isn't a context manager in its own right,
+ and doesn't implement a ``close()`` method for use with ``contextlib.closing``
+
+ An abstract example would be the following to ensure correct resource
+ management::
from contextlib import contextmanager
@contextmanager
- def tag(name):
- print("<%s>" % name)
- yield
- print("%s>" % name)
+ def managed_resource(*args, **kwds):
+ # Code to acquire resource, e.g.:
+ resource = acquire_resource(*args, **kwds)
+ try:
+ yield resource
+ finally:
+ # Code to release resource, e.g.:
+ release_resource(resource)
- >>> with tag("h1"):
- ... print("foo")
- ...
-
- foo
-
+ >>> with managed_resource(timeout=3600) as resource:
+ ... # Resource is released at the end of this block,
+ ... # even if code in the block raises an exception
The function being decorated must return a :term:`generator`-iterator when
called. This iterator must yield exactly one value, which will be bound to
@@ -152,10 +158,22 @@ Functions and classes provided:
.. function:: nullcontext(enter_result=None)
- Return a context manager that returns enter_result from ``__enter__``, but
+ Return a context manager that returns *enter_result* from ``__enter__``, but
otherwise does nothing. It is intended to be used as a stand-in for an
optional context manager, for example::
+ def myfunction(arg, ignore_exceptions=False):
+ if ignore_exceptions:
+ # Use suppress to ignore all exceptions.
+ cm = contextlib.suppress(Exception)
+ else:
+ # Do not ignore any exceptions, cm has no effect.
+ cm = contextlib.nullcontext()
+ with cm:
+ # Do something
+
+ An example using *enter_result*::
+
def process_file(file_or_path):
if isinstance(file_or_path, str):
# If string, open file
diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst
new file mode 100644
index 00000000000000..8805661c456edb
--- /dev/null
+++ b/Doc/library/contextvars.rst
@@ -0,0 +1,281 @@
+:mod:`contextvars` --- Context Variables
+========================================
+
+.. module:: contextvars
+ :synopsis: Context Variables
+
+.. sectionauthor:: Yury Selivanov
+
+--------------
+
+This module provides APIs to manage, store, and access context-local
+state. The :class:`~contextvars.ContextVar` class is used to declare
+and work with *Context Variables*. The :func:`~contextvars.copy_context`
+function and the :class:`~contextvars.Context` class should be used to
+manage the current context in asynchronous frameworks.
+
+Context managers that have state should use Context Variables
+instead of :func:`threading.local()` to prevent their state from
+bleeding to other code unexpectedly, when used in concurrent code.
+
+See also :pep:`567` for additional details.
+
+.. versionadded:: 3.7
+
+
+Context Variables
+-----------------
+
+.. class:: ContextVar(name, [\*, default])
+
+ This class is used to declare a new Context Variable, e.g.::
+
+ var: ContextVar[int] = ContextVar('var', default=42)
+
+ The required *name* parameter is used for introspection and debug
+ purposes.
+
+ The optional keyword-only *default* parameter is returned by
+ :meth:`ContextVar.get` when no value for the variable is found
+ in the current context.
+
+ **Important:** Context Variables should be created at the top module
+ level and never in closures. :class:`Context` objects hold strong
+ references to context variables which prevents context variables
+ from being properly garbage collected.
+
+ .. attribute:: ContextVar.name
+
+ The name of the variable. This is a read-only property.
+
+ .. versionadded:: 3.7.1
+
+ .. method:: get([default])
+
+ Return a value for the context variable for the current context.
+
+ If there is no value for the variable in the current context,
+ the method will:
+
+ * return the value of the *default* argument of the method,
+ if provided; or
+
+ * return the default value for the context variable,
+ if it was created with one; or
+
+ * raise a :exc:`LookupError`.
+
+ .. method:: set(value)
+
+ Call to set a new value for the context variable in the current
+ context.
+
+ The required *value* argument is the new value for the context
+ variable.
+
+ Returns a :class:`~contextvars.Token` object that can be used
+ to restore the variable to its previous value via the
+ :meth:`ContextVar.reset` method.
+
+ .. method:: reset(token)
+
+ Reset the context variable to the value it had before the
+ :meth:`ContextVar.set` that created the *token* was used.
+
+ For example::
+
+ var = ContextVar('var')
+
+ token = var.set('new value')
+ # code that uses 'var'; var.get() returns 'new value'.
+ var.reset(token)
+
+ # After the reset call the var has no value again, so
+ # var.get() would raise a LookupError.
+
+
+.. class:: contextvars.Token
+
+ *Token* objects are returned by the :meth:`ContextVar.set` method.
+ They can be passed to the :meth:`ContextVar.reset` method to revert
+ the value of the variable to what it was before the corresponding
+ *set*.
+
+ .. attribute:: Token.var
+
+ A read-only property. Points to the :class:`ContextVar` object
+ that created the token.
+
+ .. attribute:: Token.old_value
+
+ A read-only property. Set to the value the variable had before
+ the :meth:`ContextVar.set` method call that created the token.
+ It points to :attr:`Token.MISSING` is the variable was not set
+ before the call.
+
+ .. attribute:: Token.MISSING
+
+ A marker object used by :attr:`Token.old_value`.
+
+
+Manual Context Management
+-------------------------
+
+.. function:: copy_context()
+
+ Returns a copy of the current :class:`~contextvars.Context` object.
+
+ The following snippet gets a copy of the current context and prints
+ all variables and their values that are set in it::
+
+ ctx: Context = copy_context()
+ print(list(ctx.items()))
+
+ The function has an O(1) complexity, i.e. works equally fast for
+ contexts with a few context variables and for contexts that have
+ a lot of them.
+
+
+.. class:: Context()
+
+ A mapping of :class:`ContextVars ` to their values.
+
+ ``Context()`` creates an empty context with no values in it.
+ To get a copy of the current context use the
+ :func:`~contextvars.copy_context` function.
+
+ Context implements the :class:`collections.abc.Mapping` interface.
+
+ .. method:: run(callable, \*args, \*\*kwargs)
+
+ Execute ``callable(*args, **kwargs)`` code in the context object
+ the *run* method is called on. Return the result of the execution
+ or propagate an exception if one occurred.
+
+ Any changes to any context variables that *callable* makes will
+ be contained in the context object::
+
+ var = ContextVar('var')
+ var.set('spam')
+
+ def main():
+ # 'var' was set to 'spam' before
+ # calling 'copy_context()' and 'ctx.run(main)', so:
+ # var.get() == ctx[var] == 'spam'
+
+ var.set('ham')
+
+ # Now, after setting 'var' to 'ham':
+ # var.get() == ctx[var] == 'ham'
+
+ ctx = copy_context()
+
+ # Any changes that the 'main' function makes to 'var'
+ # will be contained in 'ctx'.
+ ctx.run(main)
+
+ # The 'main()' function was run in the 'ctx' context,
+ # so changes to 'var' are contained in it:
+ # ctx[var] == 'ham'
+
+ # However, outside of 'ctx', 'var' is still set to 'spam':
+ # var.get() == 'spam'
+
+ The method raises a :exc:`RuntimeError` when called on the same
+ context object from more than one OS thread, or when called
+ recursively.
+
+ .. method:: copy()
+
+ Return a shallow copy of the context object.
+
+ .. describe:: var in context
+
+ Return ``True`` if the *context* has a value for *var* set;
+ return ``False`` otherwise.
+
+ .. describe:: context[var]
+
+ Return the value of the *var* :class:`ContextVar` variable.
+ If the variable is not set in the context object, a
+ :exc:`KeyError` is raised.
+
+ .. method:: get(var, [default])
+
+ Return the value for *var* if *var* has the value in the context
+ object. Return *default* otherwise. If *default* is not given,
+ return ``None``.
+
+ .. describe:: iter(context)
+
+ Return an iterator over the variables stored in the context
+ object.
+
+ .. describe:: len(proxy)
+
+ Return the number of variables set in the context object.
+
+ .. method:: keys()
+
+ Return a list of all variables in the context object.
+
+ .. method:: values()
+
+ Return a list of all variables' values in the context object.
+
+
+ .. method:: items()
+
+ Return a list of 2-tuples containing all variables and their
+ values in the context object.
+
+
+asyncio support
+---------------
+
+Context variables are natively supported in :mod:`asyncio` and are
+ready to be used without any extra configuration. For example, here
+is a simple echo server, that uses a context variable to make the
+address of a remote client available in the Task that handles that
+client::
+
+ import asyncio
+ import contextvars
+
+ client_addr_var = contextvars.ContextVar('client_addr')
+
+ def render_goodbye():
+ # The address of the currently handled client can be accessed
+ # without passing it explicitly to this function.
+
+ client_addr = client_addr_var.get()
+ return f'Good bye, client @ {client_addr}\n'.encode()
+
+ async def handle_request(reader, writer):
+ addr = writer.transport.get_extra_info('socket').getpeername()
+ client_addr_var.set(addr)
+
+ # In any code that we call is now possible to get
+ # client's address by calling 'client_addr_var.get()'.
+
+ while True:
+ line = await reader.readline()
+ print(line)
+ if not line.strip():
+ break
+ writer.write(line)
+
+ writer.write(render_goodbye())
+ writer.close()
+
+ async def main():
+ srv = await asyncio.start_server(
+ handle_request, '127.0.0.1', 8081)
+
+ async with srv:
+ await srv.serve_forever()
+
+ asyncio.run(main())
+
+ # To test it you can use telnet:
+ # telnet 127.0.0.1 8081
diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst
index e1290d4c9b5c83..049537eff89846 100644
--- a/Doc/library/csv.rst
+++ b/Doc/library/csv.rst
@@ -203,9 +203,7 @@ The :mod:`csv` module defines the following classes:
:class:`writer` instance.
Note that unlike the :class:`DictReader` class, the *fieldnames* parameter
- of the :class:`DictWriter` is not optional. Since Python's :class:`dict`
- objects are not ordered, there is not enough information available to deduce
- the order in which the row should be written to file *f*.
+ of the :class:`DictWriter` class is not optional.
A short usage example::
@@ -452,8 +450,9 @@ read CSV files (assuming they support complex numbers at all).
.. method:: csvwriter.writerows(rows)
- Write all the *rows* parameters (a list of *row* objects as described above) to
- the writer's file object, formatted according to the current dialect.
+ Write all elements in *rows* (an iterable of *row* objects as described
+ above) to the writer's file object, formatted according to the current
+ dialect.
Writer objects have the following public attribute:
diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index edec764d640ed6..500aad8858f2ca 100644
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -1025,6 +1025,22 @@ As we can easily check, our array is sorted now::
1 5 7 33 99
>>>
+The function factories can be used as decorator factories, so we may as well
+write::
+
+ >>> @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
+ ... def py_cmp_func(a, b):
+ ... print("py_cmp_func", a[0], b[0])
+ ... return a[0] - b[0]
+ ...
+ >>> qsort(ia, len(ia), sizeof(c_int), py_cmp_func)
+ py_cmp_func 5 1
+ py_cmp_func 33 99
+ py_cmp_func 7 33
+ py_cmp_func 1 7
+ py_cmp_func 5 7
+ >>>
+
.. note::
Make sure you keep references to :func:`CFUNCTYPE` objects as long as they
@@ -1577,7 +1593,9 @@ Foreign functions can also be created by instantiating function prototypes.
Function prototypes are similar to function prototypes in C; they describe a
function (return type, argument types, calling convention) without defining an
implementation. The factory functions must be called with the desired result
-type and the argument types of the function.
+type and the argument types of the function, and can be used as decorator
+factories, and as such, be applied to functions through the ``@wrapper`` syntax.
+See :ref:`ctypes-callback-functions` for examples.
.. function:: CFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False)
diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index 13717f7b35e6b7..2a2ee2be84af2a 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -685,6 +685,12 @@ the following methods and attributes:
character previously painter at that location. By default, the character
position and attributes are the current settings for the window object.
+ .. note::
+
+ Writing outside the window, subwindow, or pad raises a :exc:`curses.error`.
+ Attempting to write to the lower right corner of a window, subwindow,
+ or pad will cause an exception to be raised after the character is printed.
+
.. method:: window.addnstr(str, n[, attr])
window.addnstr(y, x, str, n[, attr])
@@ -700,6 +706,12 @@ the following methods and attributes:
Paint the character string *str* at ``(y, x)`` with attributes
*attr*, overwriting anything previously on the display.
+ .. note::
+
+ Writing outside the window, subwindow, or pad raises :exc:`curses.error`.
+ Attempting to write to the lower right corner of a window, subwindow,
+ or pad will cause an exception to be raised after the string is printed.
+
.. method:: window.attroff(attr)
diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst
new file mode 100644
index 00000000000000..e9af20a04af167
--- /dev/null
+++ b/Doc/library/dataclasses.rst
@@ -0,0 +1,593 @@
+:mod:`dataclasses` --- Data Classes
+===================================
+
+.. module:: dataclasses
+ :synopsis: Generate special methods on user-defined classes.
+
+.. moduleauthor:: Eric V. Smith
+.. sectionauthor:: Eric V. Smith
+
+**Source code:** :source:`Lib/dataclasses.py`
+
+--------------
+
+This module provides a decorator and functions for automatically
+adding generated :term:`special method`\s such as :meth:`__init__` and
+:meth:`__repr__` to user-defined classes. It was originally described
+in :pep:`557`.
+
+The member variables to use in these generated methods are defined
+using :pep:`526` type annotations. For example this code::
+
+ @dataclass
+ class InventoryItem:
+ '''Class for keeping track of an item in inventory.'''
+ name: str
+ unit_price: float
+ quantity_on_hand: int = 0
+
+ def total_cost(self) -> float:
+ return self.unit_price * self.quantity_on_hand
+
+Will add, among other things, a :meth:`__init__` that looks like::
+
+ def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0):
+ self.name = name
+ self.unit_price = unit_price
+ self.quantity_on_hand = quantity_on_hand
+
+Note that this method is automatically added to the class: it is not
+directly specified in the ``InventoryItem`` definition shown above.
+
+.. versionadded:: 3.7
+
+Module-level decorators, classes, and functions
+-----------------------------------------------
+
+.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False)
+
+ This function is a :term:`decorator` that is used to add generated
+ :term:`special method`\s to classes, as described below.
+
+ The :func:`dataclass` decorator examines the class to find
+ ``field``\s. A ``field`` is defined as class variable that has a
+ type annotation. With two exceptions described below, nothing in
+ :func:`dataclass` examines the type specified in the variable
+ annotation.
+
+ The order of the fields in all of the generated methods is the
+ order in which they appear in the class definition.
+
+ The :func:`dataclass` decorator will add various "dunder" methods to
+ the class, described below. If any of the added methods already
+ exist on the class, the behavior depends on the parameter, as documented
+ below. The decorator returns the same class that is called on; no new
+ class is created.
+
+ If :func:`dataclass` is used just as a simple decorator with no parameters,
+ it acts as if it has the default values documented in this
+ signature. That is, these three uses of :func:`dataclass` are
+ equivalent::
+
+ @dataclass
+ class C:
+ ...
+
+ @dataclass()
+ class C:
+ ...
+
+ @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False)
+ class C:
+ ...
+
+ The parameters to :func:`dataclass` are:
+
+ - ``init``: If true (the default), a :meth:`__init__` method will be
+ generated.
+
+ If the class already defines :meth:`__init__`, this parameter is
+ ignored.
+
+ - ``repr``: If true (the default), a :meth:`__repr__` method will be
+ generated. The generated repr string will have the class name and
+ the name and repr of each field, in the order they are defined in
+ the class. Fields that are marked as being excluded from the repr
+ are not included. For example:
+ ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``.
+
+ If the class already defines :meth:`__repr__`, this parameter is
+ ignored.
+
+ - ``eq``: If true (the default), an :meth:`__eq__` method will be
+ generated. This method compares the class as if it were a tuple
+ of its fields, in order. Both instances in the comparison must
+ be of the identical type.
+
+ If the class already defines :meth:`__eq__`, this parameter is
+ ignored.
+
+ - ``order``: If true (the default is ``False``), :meth:`__lt__`,
+ :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be
+ generated. These compare the class as if it were a tuple of its
+ fields, in order. Both instances in the comparison must be of the
+ identical type. If ``order`` is true and ``eq`` is false, a
+ :exc:`ValueError` is raised.
+
+ If the class already defines any of :meth:`__lt__`,
+ :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then
+ :exc:`TypeError` is raised.
+
+ - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method
+ is generated according to how ``eq`` and ``frozen`` are set.
+
+ :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are
+ added to hashed collections such as dictionaries and sets. Having a
+ :meth:`__hash__` implies that instances of the class are immutable.
+ Mutability is a complicated property that depends on the programmer's
+ intent, the existence and behavior of :meth:`__eq__`, and the values of
+ the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator.
+
+ By default, :func:`dataclass` will not implicitly add a :meth:`__hash__`
+ method unless it is safe to do so. Neither will it add or change an
+ existing explicitly defined :meth:`__hash__` method. Setting the class
+ attribute ``__hash__ = None`` has a specific meaning to Python, as
+ described in the :meth:`__hash__` documentation.
+
+ If :meth:`__hash__` is not explicit defined, or if it is set to ``None``,
+ then :func:`dataclass` *may* add an implicit :meth:`__hash__` method.
+ Although not recommended, you can force :func:`dataclass` to create a
+ :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case
+ if your class is logically immutable but can nonetheless be mutated.
+ This is a specialized use case and should be considered carefully.
+
+ Here are the rules governing implicit creation of a :meth:`__hash__`
+ method. Note that you cannot both have an explicit :meth:`__hash__`
+ method in your dataclass and set ``unsafe_hash=True``; this will result
+ in a :exc:`TypeError`.
+
+ If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will
+ generate a :meth:`__hash__` method for you. If ``eq`` is true and
+ ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it
+ unhashable (which it is, since it is mutable). If ``eq`` is false,
+ :meth:`__hash__` will be left untouched meaning the :meth:`__hash__`
+ method of the superclass will be used (if the superclass is
+ :class:`object`, this means it will fall back to id-based hashing).
+
+ - ``frozen``: If true (the default is False), assigning to fields will
+ generate an exception. This emulates read-only frozen instances. If
+ :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then
+ :exc:`TypeError` is raised. See the discussion below.
+
+ ``field``\s may optionally specify a default value, using normal
+ Python syntax::
+
+ @dataclass
+ class C:
+ a: int # 'a' has no default value
+ b: int = 0 # assign a default value for 'b'
+
+ In this example, both ``a`` and ``b`` will be included in the added
+ :meth:`__init__` method, which will be defined as::
+
+ def __init__(self, a: int, b: int = 0):
+
+ :exc:`TypeError` will be raised if a field without a default value
+ follows a field with a default value. This is true either when this
+ occurs in a single class, or as a result of class inheritance.
+
+.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None)
+
+ For common and simple use cases, no other functionality is
+ required. There are, however, some dataclass features that
+ require additional per-field information. To satisfy this need for
+ additional information, you can replace the default field value
+ with a call to the provided :func:`field` function. For example::
+
+ @dataclass
+ class C:
+ mylist: List[int] = field(default_factory=list)
+
+ c = C()
+ c.mylist += [1, 2, 3]
+
+ As shown above, the ``MISSING`` value is a sentinel object used to
+ detect if the ``default`` and ``default_factory`` parameters are
+ provided. This sentinel is used because ``None`` is a valid value
+ for ``default``. No code should directly use the ``MISSING``
+ value.
+
+ The parameters to :func:`field` are:
+
+ - ``default``: If provided, this will be the default value for this
+ field. This is needed because the :meth:`field` call itself
+ replaces the normal position of the default value.
+
+ - ``default_factory``: If provided, it must be a zero-argument
+ callable that will be called when a default value is needed for
+ this field. Among other purposes, this can be used to specify
+ fields with mutable default values, as discussed below. It is an
+ error to specify both ``default`` and ``default_factory``.
+
+ - ``init``: If true (the default), this field is included as a
+ parameter to the generated :meth:`__init__` method.
+
+ - ``repr``: If true (the default), this field is included in the
+ string returned by the generated :meth:`__repr__` method.
+
+ - ``compare``: If true (the default), this field is included in the
+ generated equality and comparison methods (:meth:`__eq__`,
+ :meth:`__gt__`, et al.).
+
+ - ``hash``: This can be a bool or ``None``. If true, this field is
+ included in the generated :meth:`__hash__` method. If ``None`` (the
+ default), use the value of ``compare``: this would normally be
+ the expected behavior. A field should be considered in the hash
+ if it's used for comparisons. Setting this value to anything
+ other than ``None`` is discouraged.
+
+ One possible reason to set ``hash=False`` but ``compare=True``
+ would be if a field is expensive to compute a hash value for,
+ that field is needed for equality testing, and there are other
+ fields that contribute to the type's hash value. Even if a field
+ is excluded from the hash, it will still be used for comparisons.
+
+ - ``metadata``: This can be a mapping or None. None is treated as
+ an empty dict. This value is wrapped in
+ :func:`~types.MappingProxyType` to make it read-only, and exposed
+ on the :class:`Field` object. It is not used at all by Data
+ Classes, and is provided as a third-party extension mechanism.
+ Multiple third-parties can each have their own key, to use as a
+ namespace in the metadata.
+
+ If the default value of a field is specified by a call to
+ :func:`field()`, then the class attribute for this field will be
+ replaced by the specified ``default`` value. If no ``default`` is
+ provided, then the class attribute will be deleted. The intent is
+ that after the :func:`dataclass` decorator runs, the class
+ attributes will all contain the default values for the fields, just
+ as if the default value itself were specified. For example,
+ after::
+
+ @dataclass
+ class C:
+ x: int
+ y: int = field(repr=False)
+ z: int = field(repr=False, default=10)
+ t: int = 20
+
+ The class attribute ``C.z`` will be ``10``, the class attribute
+ ``C.t`` will be ``20``, and the class attributes ``C.x`` and
+ ``C.y`` will not be set.
+
+.. class:: Field
+
+ :class:`Field` objects describe each defined field. These objects
+ are created internally, and are returned by the :func:`fields`
+ module-level method (see below). Users should never instantiate a
+ :class:`Field` object directly. Its documented attributes are:
+
+ - ``name``: The name of the field.
+
+ - ``type``: The type of the field.
+
+ - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``,
+ ``compare``, and ``metadata`` have the identical meaning and
+ values as they do in the :func:`field` declaration.
+
+ Other attributes may exist, but they are private and must not be
+ inspected or relied on.
+
+.. function:: fields(class_or_instance)
+
+ Returns a tuple of :class:`Field` objects that define the fields for this
+ dataclass. Accepts either a dataclass, or an instance of a dataclass.
+ Raises :exc:`TypeError` if not passed a dataclass or instance of one.
+ Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``.
+
+.. function:: asdict(instance, *, dict_factory=dict)
+
+ Converts the dataclass ``instance`` to a dict (by using the
+ factory function ``dict_factory``). Each dataclass is converted
+ to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts,
+ lists, and tuples are recursed into. For example::
+
+ @dataclass
+ class Point:
+ x: int
+ y: int
+
+ @dataclass
+ class C:
+ mylist: List[Point]
+
+ p = Point(10, 20)
+ assert asdict(p) == {'x': 10, 'y': 20}
+
+ c = C([Point(0, 0), Point(10, 4)])
+ assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]}
+
+ Raises :exc:`TypeError` if ``instance`` is not a dataclass instance.
+
+.. function:: astuple(*, tuple_factory=tuple)
+
+ Converts the dataclass ``instance`` to a tuple (by using the
+ factory function ``tuple_factory``). Each dataclass is converted
+ to a tuple of its field values. dataclasses, dicts, lists, and
+ tuples are recursed into.
+
+ Continuing from the previous example::
+
+ assert astuple(p) == (10, 20)
+ assert astuple(c) == ([(0, 0), (10, 4)],)
+
+ Raises :exc:`TypeError` if ``instance`` is not a dataclass instance.
+
+.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False)
+
+ Creates a new dataclass with name ``cls_name``, fields as defined
+ in ``fields``, base classes as given in ``bases``, and initialized
+ with a namespace as given in ``namespace``. ``fields`` is an
+ iterable whose elements are each either ``name``, ``(name, type)``,
+ or ``(name, type, Field)``. If just ``name`` is supplied,
+ ``typing.Any`` is used for ``type``. The values of ``init``,
+ ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have
+ the same meaning as they do in :func:`dataclass`.
+
+ This function is not strictly required, because any Python
+ mechanism for creating a new class with ``__annotations__`` can
+ then apply the :func:`dataclass` function to convert that class to
+ a dataclass. This function is provided as a convenience. For
+ example::
+
+ C = make_dataclass('C',
+ [('x', int),
+ 'y',
+ ('z', int, field(default=5))],
+ namespace={'add_one': lambda self: self.x + 1})
+
+ Is equivalent to::
+
+ @dataclass
+ class C:
+ x: int
+ y: 'typing.Any'
+ z: int = 5
+
+ def add_one(self):
+ return self.x + 1
+
+.. function:: replace(instance, **changes)
+
+ Creates a new object of the same type of ``instance``, replacing
+ fields with values from ``changes``. If ``instance`` is not a Data
+ Class, raises :exc:`TypeError`. If values in ``changes`` do not
+ specify fields, raises :exc:`TypeError`.
+
+ The newly returned object is created by calling the :meth:`__init__`
+ method of the dataclass. This ensures that
+ :meth:`__post_init__`, if present, is also called.
+
+ Init-only variables without default values, if any exist, must be
+ specified on the call to :func:`replace` so that they can be passed to
+ :meth:`__init__` and :meth:`__post_init__`.
+
+ It is an error for ``changes`` to contain any fields that are
+ defined as having ``init=False``. A :exc:`ValueError` will be raised
+ in this case.
+
+ Be forewarned about how ``init=False`` fields work during a call to
+ :func:`replace`. They are not copied from the source object, but
+ rather are initialized in :meth:`__post_init__`, if they're
+ initialized at all. It is expected that ``init=False`` fields will
+ be rarely and judiciously used. If they are used, it might be wise
+ to have alternate class constructors, or perhaps a custom
+ ``replace()`` (or similarly named) method which handles instance
+ copying.
+
+.. function:: is_dataclass(class_or_instance)
+
+ Returns True if its parameter is a dataclass or an instance of one,
+ otherwise returns False.
+
+ If you need to know if a class is an instance of a dataclass (and
+ not a dataclass itself), then add a further check for ``not
+ isinstance(obj, type)``::
+
+ def is_dataclass_instance(obj):
+ return is_dataclass(obj) and not isinstance(obj, type)
+
+Post-init processing
+--------------------
+
+The generated :meth:`__init__` code will call a method named
+:meth:`__post_init__`, if :meth:`__post_init__` is defined on the
+class. It will normally be called as ``self.__post_init__()``.
+However, if any ``InitVar`` fields are defined, they will also be
+passed to :meth:`__post_init__` in the order they were defined in the
+class. If no :meth:`__init__` method is generated, then
+:meth:`__post_init__` will not automatically be called.
+
+Among other uses, this allows for initializing field values that
+depend on one or more other fields. For example::
+
+ @dataclass
+ class C:
+ a: float
+ b: float
+ c: float = field(init=False)
+
+ def __post_init__(self):
+ self.c = self.a + self.b
+
+See the section below on init-only variables for ways to pass
+parameters to :meth:`__post_init__`. Also see the warning about how
+:func:`replace` handles ``init=False`` fields.
+
+Class variables
+---------------
+
+One of two places where :func:`dataclass` actually inspects the type
+of a field is to determine if a field is a class variable as defined
+in :pep:`526`. It does this by checking if the type of the field is
+``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded
+from consideration as a field and is ignored by the dataclass
+mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the
+module-level :func:`fields` function.
+
+Init-only variables
+-------------------
+
+The other place where :func:`dataclass` inspects a type annotation is to
+determine if a field is an init-only variable. It does this by seeing
+if the type of a field is of type ``dataclasses.InitVar``. If a field
+is an ``InitVar``, it is considered a pseudo-field called an init-only
+field. As it is not a true field, it is not returned by the
+module-level :func:`fields` function. Init-only fields are added as
+parameters to the generated :meth:`__init__` method, and are passed to
+the optional :meth:`__post_init__` method. They are not otherwise used
+by dataclasses.
+
+For example, suppose a field will be initialzed from a database, if a
+value is not provided when creating the class::
+
+ @dataclass
+ class C:
+ i: int
+ j: int = None
+ database: InitVar[DatabaseType] = None
+
+ def __post_init__(self, database):
+ if self.j is None and database is not None:
+ self.j = database.lookup('j')
+
+ c = C(10, database=my_database)
+
+In this case, :func:`fields` will return :class:`Field` objects for ``i`` and
+``j``, but not for ``database``.
+
+Frozen instances
+----------------
+
+It is not possible to create truly immutable Python objects. However,
+by passing ``frozen=True`` to the :meth:`dataclass` decorator you can
+emulate immutability. In that case, dataclasses will add
+:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These
+methods will raise a :exc:`FrozenInstanceError` when invoked.
+
+There is a tiny performance penalty when using ``frozen=True``:
+:meth:`__init__` cannot use simple assignment to initialize fields, and
+must use :meth:`object.__setattr__`.
+
+Inheritance
+-----------
+
+When the dataclass is being created by the :meth:`dataclass` decorator,
+it looks through all of the class's base classes in reverse MRO (that
+is, starting at :class:`object`) and, for each dataclass that it finds,
+adds the fields from that base class to an ordered mapping of fields.
+After all of the base class fields are added, it adds its own fields
+to the ordered mapping. All of the generated methods will use this
+combined, calculated ordered mapping of fields. Because the fields
+are in insertion order, derived classes override base classes. An
+example::
+
+ @dataclass
+ class Base:
+ x: Any = 15.0
+ y: int = 0
+
+ @dataclass
+ class C(Base):
+ z: int = 10
+ x: int = 15
+
+The final list of fields is, in order, ``x``, ``y``, ``z``. The final
+type of ``x`` is ``int``, as specified in class ``C``.
+
+The generated :meth:`__init__` method for ``C`` will look like::
+
+ def __init__(self, x: int = 15, y: int = 0, z: int = 10):
+
+Default factory functions
+-------------------------
+
+ If a :func:`field` specifies a ``default_factory``, it is called with
+ zero arguments when a default value for the field is needed. For
+ example, to create a new instance of a list, use::
+
+ mylist: list = field(default_factory=list)
+
+ If a field is excluded from :meth:`__init__` (using ``init=False``)
+ and the field also specifies ``default_factory``, then the default
+ factory function will always be called from the generated
+ :meth:`__init__` function. This happens because there is no other
+ way to give the field an initial value.
+
+Mutable default values
+----------------------
+
+ Python stores default member variable values in class attributes.
+ Consider this example, not using dataclasses::
+
+ class C:
+ x = []
+ def add(self, element):
+ self.x.append(element)
+
+ o1 = C()
+ o2 = C()
+ o1.add(1)
+ o2.add(2)
+ assert o1.x == [1, 2]
+ assert o1.x is o2.x
+
+ Note that the two instances of class ``C`` share the same class
+ variable ``x``, as expected.
+
+ Using dataclasses, *if* this code was valid::
+
+ @dataclass
+ class D:
+ x: List = []
+ def add(self, element):
+ self.x += element
+
+ it would generate code similar to::
+
+ class D:
+ x = []
+ def __init__(self, x=x):
+ self.x = x
+ def add(self, element):
+ self.x += element
+
+ assert D().x is D().x
+
+ This has the same issue as the original example using class ``C``.
+ That is, two instances of class ``D`` that do not specify a value for
+ ``x`` when creating a class instance will share the same copy of
+ ``x``. Because dataclasses just use normal Python class creation
+ they also share this behavior. There is no general way for Data
+ Classes to detect this condition. Instead, dataclasses will raise a
+ :exc:`TypeError` if it detects a default parameter of type ``list``,
+ ``dict``, or ``set``. This is a partial solution, but it does protect
+ against many common errors.
+
+ Using default factory functions is a way to create new instances of
+ mutable types as default values for fields::
+
+ @dataclass
+ class D:
+ x: list = field(default_factory=list)
+
+ assert D().x is not D().x
+
+Exceptions
+----------
+
+.. exception:: FrozenInstanceError
+
+ Raised when an implicitly defined :meth:`__setattr__` or
+ :meth:`__delattr__` is called on a dataclass which was defined with
+ ``frozen=True``.
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index c1b164ebc1f23a..53c181c4014a9c 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -241,7 +241,7 @@ Supported operations:
+--------------------------------+-----------------------------------------------+
| ``t1 = t2 - t3`` | Difference of *t2* and *t3*. Afterwards *t1* |
| | == *t2* - *t3* and *t2* == *t1* + *t3* are |
-| | true. (1) |
+| | true. (1)(6) |
+--------------------------------+-----------------------------------------------+
| ``t1 = t2 * i or t1 = i * t2`` | Delta multiplied by an integer. |
| | Afterwards *t1* // i == *t2* is true, |
@@ -317,6 +317,11 @@ Notes:
>>> print(_)
-1 day, 19:00:00
+(6)
+ The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except
+ when t3 is equal to ``timedelta.max``; in that case the former will produce a result
+ while the latter will overflow.
+
In addition to the operations listed above :class:`timedelta` objects support
certain additions and subtractions with :class:`date` and :class:`.datetime`
objects (see below).
@@ -513,8 +518,6 @@ Notes:
:const:`MINYEAR` or larger than :const:`MAXYEAR`.
(2)
- This isn't quite equivalent to date1 + (-timedelta), because -timedelta in
- isolation can overflow in cases where date1 - timedelta does not.
``timedelta.seconds`` and ``timedelta.microseconds`` are ignored.
(3)
@@ -961,8 +964,6 @@ Supported operations:
Computes the datetime2 such that datetime2 + timedelta == datetime1. As for
addition, the result has the same :attr:`~.datetime.tzinfo` attribute as the input
datetime, and no time zone adjustments are done even if the input is aware.
- This isn't quite equivalent to datetime1 + (-timedelta), because -timedelta
- in isolation can overflow in cases where datetime1 - timedelta does not.
(3)
Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if
@@ -1058,8 +1059,7 @@ Instance methods:
If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its
:meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self*
- is naive (``self.tzinfo is None``), it is presumed to represent time in the
- system timezone.
+ is naive, it is presumed to represent time in the system timezone.
If called without arguments (or with ``tz=None``) the system local
timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted
@@ -2209,8 +2209,8 @@ Notes:
:meth:`utcoffset` is transformed into a string of the form
±HHMM[SS[.uuuuuu]], where HH is a 2-digit string giving the number of UTC
offset hours, and MM is a 2-digit string giving the number of UTC offset
- minutes, SS is a 2-digit string string giving the number of UTC offset
- seconds and uuuuuu is a 2-digit string string giving the number of UTC
+ minutes, SS is a 2-digit string giving the number of UTC offset
+ seconds and uuuuuu is a 2-digit string giving the number of UTC
offset microseconds. The uuuuuu part is omitted when the offset is a
whole number of minutes and both the uuuuuu and the SS parts are omitted
when the offset is a whole number of minutes. For example, if
diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst
index 32e80b2cf6ed22..0150f5d5c6e8c3 100644
--- a/Doc/library/dbm.rst
+++ b/Doc/library/dbm.rst
@@ -347,6 +347,11 @@ The module defines the following:
database has to be created. It defaults to octal ``0o666`` (and will be modified
by the prevailing umask).
+ .. warning::
+ It is possible to crash the Python interpreter when loading a database
+ with a sufficiently large/complex entry due to stack depth limitations in
+ Python's AST compiler.
+
.. versionchanged:: 3.5
:func:`.open` always creates a new database when the flag has the value
``'n'``.
diff --git a/Doc/library/depgraph-output.png b/Doc/library/depgraph-output.png
deleted file mode 100644
index 960bb1b5639e7f..00000000000000
Binary files a/Doc/library/depgraph-output.png and /dev/null differ
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 48c42d119d5dc9..535b36efbf2ec8 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -339,12 +339,16 @@ The Python compiler currently generates the following bytecode instructions.
Duplicates the reference on top of the stack.
+ .. versionadded:: 3.2
+
.. opcode:: DUP_TOP_TWO
Duplicates the two references on top of the stack, leaving them in the
same order.
+ .. versionadded:: 3.2
+
**Unary operations**
@@ -555,11 +559,14 @@ the original TOS1.
the CO_ITERABLE_COROUTINE flag, or resolves
``o.__await__``.
+ .. versionadded:: 3.5
+
.. opcode:: GET_AITER
Implements ``TOS = TOS.__aiter__()``.
+ .. versionadded:: 3.5
.. versionchanged:: 3.7
Returning awaitable objects from ``__aiter__`` is no longer
supported.
@@ -570,17 +577,23 @@ the original TOS1.
Implements ``PUSH(get_awaitable(TOS.__anext__()))``. See ``GET_AWAITABLE``
for details about ``get_awaitable``
+ .. versionadded:: 3.5
+
.. opcode:: BEFORE_ASYNC_WITH
Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the
stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack.
+ .. versionadded:: 3.5
+
.. opcode:: SETUP_ASYNC_WITH
Creates a new frame object.
+ .. versionadded:: 3.5
+
**Miscellaneous opcodes**
@@ -618,6 +631,8 @@ the original TOS1.
Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict
comprehensions.
+ .. versionadded:: 3.1
+
For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD`
instructions, while the added value or key/value pair is popped off, the
container object remains on the stack so that it is available for further
@@ -640,6 +655,7 @@ iterations of the loop.
.. versionadded:: 3.3
+
.. opcode:: SETUP_ANNOTATIONS
Checks whether ``__annotations__`` is defined in ``locals()``, if not it is
@@ -649,6 +665,7 @@ iterations of the loop.
.. versionadded:: 3.6
+
.. opcode:: IMPORT_STAR
Loads all symbols not starting with ``'_'`` directly from the module TOS to
@@ -694,6 +711,8 @@ iterations of the loop.
store it in (a) variable(s) (:opcode:`STORE_FAST`, :opcode:`STORE_NAME`, or
:opcode:`UNPACK_SEQUENCE`).
+ .. versionadded:: 3.2
+
.. opcode:: WITH_CLEANUP_START
@@ -924,23 +943,31 @@ All of the following opcodes use their arguments.
If TOS is true, sets the bytecode counter to *target*. TOS is popped.
+ .. versionadded:: 3.1
+
.. opcode:: POP_JUMP_IF_FALSE (target)
If TOS is false, sets the bytecode counter to *target*. TOS is popped.
+ .. versionadded:: 3.1
+
.. opcode:: JUMP_IF_TRUE_OR_POP (target)
If TOS is true, sets the bytecode counter to *target* and leaves TOS on the
stack. Otherwise (TOS is false), TOS is popped.
+ .. versionadded:: 3.1
+
.. opcode:: JUMP_IF_FALSE_OR_POP (target)
If TOS is false, sets the bytecode counter to *target* and leaves TOS on the
stack. Otherwise (TOS is true), TOS is popped.
+ .. versionadded:: 3.1
+
.. opcode:: JUMP_ABSOLUTE (target)
@@ -1013,6 +1040,8 @@ All of the following opcodes use their arguments.
consulting the cell. This is used for loading free variables in class
bodies.
+ .. versionadded:: 3.4
+
.. opcode:: STORE_DEREF (i)
@@ -1025,6 +1054,8 @@ All of the following opcodes use their arguments.
Empties the cell contained in slot *i* of the cell and free variable storage.
Used by the :keyword:`del` statement.
+ .. versionadded:: 3.2
+
.. opcode:: RAISE_VARARGS (argc)
diff --git a/Doc/library/distribution.rst b/Doc/library/distribution.rst
index 3e6e84b42a2e51..8d4befe41b329c 100644
--- a/Doc/library/distribution.rst
+++ b/Doc/library/distribution.rst
@@ -4,7 +4,7 @@ Software Packaging and Distribution
These libraries help you with publishing and installing Python software.
While these modules are designed to work in conjunction with the
-`Python Package Index `__, they can also be used
+`Python Package Index `__, they can also be used
with a local index server, or without any index server at all.
.. toctree::
diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst
index f56836ae2d2781..e09c7c0e402bbc 100644
--- a/Doc/library/email.contentmanager.rst
+++ b/Doc/library/email.contentmanager.rst
@@ -126,9 +126,6 @@ Currently the email package provides only one concrete content manager,
set_content(msg, <'EmailMessage'>, cte=None, \
disposition=None, filename=None, cid=None, \
params=None, headers=None)
- set_content(msg, <'list'>, subtype='mixed', \
- disposition=None, filename=None, cid=None, \
- params=None, headers=None)
Add headers and payload to *msg*:
@@ -144,12 +141,6 @@ Currently the email package provides only one concrete content manager,
specified or ``rfc822`` if it is not. If *subtype* is
``partial``, raise an error (``bytes`` objects must be used to
construct ``message/partial`` parts).
- * For *<'list'>*, which should be a list of
- :class:`~email.message.EmailMessage` objects, set the ``maintype``
- to ``multipart``, and the ``subtype`` to *subtype* if it is
- specified, and ``mixed`` if it is not. If the message parts in
- the *<'list'>* have :mailheader:`MIME-Version` headers, remove
- them.
If *charset* is provided (which is valid only for ``str``), encode the
string to bytes using the specified character set. The default is
diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst
index 5838767b18f742..511ad163583197 100644
--- a/Doc/library/email.errors.rst
+++ b/Doc/library/email.errors.rst
@@ -108,3 +108,7 @@ All defect classes are subclassed from :class:`email.errors.MessageDefect`.
* :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64
encoded bytes, characters outside the base64 alphabet were encountered.
The characters are ignored, but the resulting decoded bytes may be invalid.
+
+* :class:`InvalidBase64LengthDefect` -- When decoding a block of base64 encoded
+ bytes, the number of non-padding base64 characters was invalid (1 more than
+ a multiple of 4). The encoded block was kept as-is.
diff --git a/Doc/library/email.examples.rst b/Doc/library/email.examples.rst
index 84e9aee0bc4841..fc964622809d0e 100644
--- a/Doc/library/email.examples.rst
+++ b/Doc/library/email.examples.rst
@@ -12,7 +12,7 @@ text content and the addresses may contain unicode characters):
.. literalinclude:: ../includes/email-simple.py
-Parsing RFC822 headers can easily be done by the using the classes
+Parsing :rfc:`822` headers can easily be done by the using the classes
from the :mod:`~email.parser` module:
.. literalinclude:: ../includes/email-headers.py
diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst
index cc8e8225a203ab..2575a51300705f 100644
--- a/Doc/library/email.generator.rst
+++ b/Doc/library/email.generator.rst
@@ -185,7 +185,7 @@ to be using :class:`BytesGenerator`, and not :class:`Generator`.
Convert any bytes with the high bit set as needed using an
ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is,
transform parts with non-ASCII :mailheader:`Cotnent-Transfer-Encoding`
- (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatibile
+ (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible
:mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII
bytes in headers using the MIME ``unknown-8bit`` character set, thus
rendering them RFC-compliant.
diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst
index dea409d223da4c..e0cab6a6949743 100644
--- a/Doc/library/email.parser.rst
+++ b/Doc/library/email.parser.rst
@@ -246,7 +246,7 @@ in the top-level :mod:`email` package namespace.
Removed the *strict* argument. Added the *policy* keyword.
-.. function:: message_from_binary_file(fp, _class=None, *,
+.. function:: message_from_binary_file(fp, _class=None, *, \
policy=policy.compat32)
Return a message object structure tree from an open binary :term:`file
diff --git a/Doc/library/email.rst b/Doc/library/email.rst
index c4187dd0098d1a..1033d8c130eb75 100644
--- a/Doc/library/email.rst
+++ b/Doc/library/email.rst
@@ -133,7 +133,7 @@ Legacy API:
.. seealso::
Module :mod:`smtplib`
- SMTP (Simple Mail Transport Protcol) client
+ SMTP (Simple Mail Transport Protocol) client
Module :mod:`poplib`
POP (Post Office Protocol) client
diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst
index ed22180dc3f89f..c797f63326d1a2 100644
--- a/Doc/library/ensurepip.rst
+++ b/Doc/library/ensurepip.rst
@@ -78,9 +78,6 @@ options:
Providing both of the script selection options will trigger an exception.
-.. versionchanged:: 3.7.0
- The exit status is non-zero if the command fails.
-
Module API
----------
diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst
index fc65a3d078f19c..a6aa863a1cc91b 100644
--- a/Doc/library/enum.rst
+++ b/Doc/library/enum.rst
@@ -4,10 +4,10 @@
.. module:: enum
:synopsis: Implementation of an enumeration class.
-.. :moduleauthor:: Ethan Furman
-.. :sectionauthor:: Barry Warsaw ,
-.. :sectionauthor:: Eli Bendersky ,
-.. :sectionauthor:: Ethan Furman
+.. moduleauthor:: Ethan Furman
+.. sectionauthor:: Barry Warsaw
+.. sectionauthor:: Eli Bendersky
+.. sectionauthor:: Ethan Furman
.. versionadded:: 3.4
@@ -976,7 +976,7 @@ Enum Classes
The :class:`EnumMeta` metaclass is responsible for providing the
:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that
allow one to do things with an :class:`Enum` class that fail on a typical
-class, such as `list(Color)` or `some_var in Color`. :class:`EnumMeta` is
+class, such as `list(Color)` or `some_enum_var in Color`. :class:`EnumMeta` is
responsible for ensuring that various other methods on the final :class:`Enum`
class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`,
:meth:`__str__` and :meth:`__repr__`).
diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
index 42b99cc47e05f1..e33b88673f7533 100644
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -525,7 +525,7 @@ The following exceptions are the exceptions that are usually raised.
.. exception:: ValueError
- Raised when a built-in operation or function receives an argument that has the
+ Raised when an operation or function receives an argument that has the
right type but an inappropriate value, and the situation is not described by a
more precise exception such as :exc:`IndexError`.
diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
index d0c4cd07c57187..94ebd87639c501 100644
--- a/Doc/library/faulthandler.rst
+++ b/Doc/library/faulthandler.rst
@@ -152,10 +152,10 @@ these functions again each time that the file is replaced.
Example
-------
-.. highlight:: sh
-
Example of a segmentation fault on Linux with and without enabling the fault
-handler::
+handler:
+
+.. code-block:: shell-session
$ python3 -c "import ctypes; ctypes.string_at(0)"
Segmentation fault
diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index 7291dfe84811c6..6c39f9a59fc1fd 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -295,7 +295,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
If optional *rest* is given, a ``REST`` command is sent to the server, passing
*rest* as an argument. *rest* is usually a byte offset into the requested file,
telling the server to restart sending the file's bytes at the requested offset,
- skipping over the initial bytes. Note however that RFC 959 requires only that
+ skipping over the initial bytes. Note however that :rfc:`959` requires only that
*rest* be a string containing characters in the printable range from ASCII code
33 to ASCII code 126. The :meth:`transfercmd` method, therefore, converts
*rest* to a string, but no check is performed on the string's contents. If the
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index bfb813cf39060a..949a54d6d2d548 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -98,7 +98,7 @@ are always available. They are listed here in alphabetical order.
>>> f'{14:#b}', f'{14:b}'
('0b1110', '1110')
- See also :func:`format` for more information.
+ See also :func:`format` for more information.
.. class:: bool([x])
@@ -240,8 +240,8 @@ are always available. They are listed here in alphabetical order.
interactive statement (in the latter case, expression statements that
evaluate to something other than ``None`` will be printed).
- The optional arguments *flags* and *dont_inherit* control which future
- statements (see :pep:`236`) affect the compilation of *source*. If neither
+ The optional arguments *flags* and *dont_inherit* control which :ref:`future
+ statements ` affect the compilation of *source*. If neither
is present (or both are zero) the code is compiled with those future
statements that are in effect in the code that is calling :func:`compile`. If the
*flags* argument is given and *dont_inherit* is not (or is zero) then the
@@ -274,6 +274,12 @@ are always available. They are listed here in alphabetical order.
character. This is to facilitate detection of incomplete and complete
statements in the :mod:`code` module.
+ .. warning::
+
+ It is possible to crash the Python interpreter with a
+ sufficiently large/complex string when compiling to an AST
+ object due to stack depth limitations in Python's AST compiler.
+
.. versionchanged:: 3.2
Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode
does not have to end in a newline anymore. Added the *optimize* parameter.
@@ -428,8 +434,10 @@ are always available. They are listed here in alphabetical order.
The *expression* argument is parsed and evaluated as a Python expression
(technically speaking, a condition list) using the *globals* and *locals*
dictionaries as global and local namespace. If the *globals* dictionary is
- present and lacks '__builtins__', the current globals are copied into *globals*
- before *expression* is parsed. This means that *expression* normally has full
+ present and does not contain a value for the key ``__builtins__``, a
+ reference to the dictionary of the built-in module :mod:`builtins` is
+ inserted under that key before *expression* is parsed.
+ This means that *expression* normally has full
access to the standard :mod:`builtins` module and restricted environments are
propagated. If the *locals* dictionary is omitted it defaults to the *globals*
dictionary. If both dictionaries are omitted, the expression is executed in the
@@ -640,11 +648,11 @@ are always available. They are listed here in alphabetical order.
dictionary lookup. Numeric values that compare equal have the same hash
value (even if they are of different types, as is the case for 1 and 1.0).
- .. note::
+ .. note::
- For objects with custom :meth:`__hash__` methods, note that :func:`hash`
- truncates the return value based on the bit width of the host machine.
- See :meth:`__hash__` for details.
+ For objects with custom :meth:`__hash__` methods, note that :func:`hash`
+ truncates the return value based on the bit width of the host machine.
+ See :meth:`__hash__` for details.
.. function:: help([object])
@@ -665,8 +673,8 @@ are always available. They are listed here in alphabetical order.
.. function:: hex(x)
Convert an integer number to a lowercase hexadecimal string prefixed with
- "0x". If x is not a Python :class:`int` object, it has to define an
- __index__() method that returns an integer. Some examples:
+ "0x". If *x* is not a Python :class:`int` object, it has to define an
+ :meth:`__index__` method that returns an integer. Some examples:
>>> hex(255)
'0xff'
@@ -724,9 +732,10 @@ are always available. They are listed here in alphabetical order.
int(x, base=10)
Return an integer object constructed from a number or string *x*, or return
- ``0`` if no arguments are given. If *x* is a number, return
- :meth:`x.__int__() `. For floating point numbers, this
- truncates towards zero.
+ ``0`` if no arguments are given. If *x* defines :meth:`__int__`,
+ ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__trunc__`,
+ it returns ``x.__trunc__()``.
+ For floating point numbers, this truncates towards zero.
If *x* is not a number or if *base* is given, then *x* must be a string,
:class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer
@@ -964,6 +973,11 @@ are always available. They are listed here in alphabetical order.
encoding. (For reading and writing raw bytes use binary mode and leave
*encoding* unspecified.) The available modes are:
+ .. _filemodes:
+
+ .. index::
+ pair: file; modes
+
========= ===============================================================
Character Meaning
========= ===============================================================
@@ -1322,11 +1336,12 @@ are always available. They are listed here in alphabetical order.
equally close, rounding is done toward the even choice (so, for example,
both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is
``2``). Any integer value is valid for *ndigits* (positive, zero, or
- negative). The return value is an integer if called with one argument,
- otherwise of the same type as *number*.
+ negative). The return value is an integer if *ndigits* is omitted or
+ ``None``.
+ Otherwise the return value has the same type as *number*.
- For a general Python object ``number``, ``round(number, ndigits)`` delegates to
- ``number.__round__(ndigits)``.
+ For a general Python object ``number``, ``round`` delegates to
+ ``number.__round__``.
.. note::
diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst
index 92240c7606774a..153d8fb7045623 100644
--- a/Doc/library/gc.rst
+++ b/Doc/library/gc.rst
@@ -33,34 +33,6 @@ The :mod:`gc` module provides the following functions:
Disable automatic garbage collection.
-.. class:: ensure_disabled()
-
- Return a context manager object that disables the garbage collector and reenables the previous
- state upon completion of the block. This is basically equivalent to::
-
- from gc import enable, disable, isenabled
-
- @contextmanager
- def ensure_disabled():
- was_enabled_previously = isenabled()
- gc.disable()
- yield
- if was_enabled_previously:
- gc.enable()
-
- And lets you write code like this::
-
- with ensure_disabled():
- run_some_timing()
-
- with ensure_disabled():
- # do_something_that_has_real_time_guarantees
- # such as a pair trade, robotic braking, etc
-
- without needing to explicitly enable and disable the garbage collector yourself.
- This context manager is implemented in C to assure atomicity, thread safety and speed.
-
-
.. function:: isenabled()
Returns true if automatic collection is enabled.
diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst
index a8a5a500cbcfbf..25bd4b7f9a874c 100644
--- a/Doc/library/glob.rst
+++ b/Doc/library/glob.rst
@@ -48,7 +48,7 @@ For example, ``'[?]'`` matches the character ``'?'``.
Support for recursive globs using "``**``".
-.. function:: iglob(pathname, recursive=False)
+.. function:: iglob(pathname, *, recursive=False)
Return an :term:`iterator` which yields the same values as :func:`glob`
without actually storing them all simultaneously.
diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst
index eda18adc9e5ec8..baf6b0af242cf5 100644
--- a/Doc/library/hashlib.rst
+++ b/Doc/library/hashlib.rst
@@ -101,7 +101,7 @@ More condensed:
.. function:: new(name[, data])
- Is a generic constructor that takes the string name of the desired
+ Is a generic constructor that takes the string *name* of the desired
algorithm as its first parameter. It also exists to allow access to the
above listed hashes as well as any other algorithms that your OpenSSL
library may offer. The named constructors are much faster than :func:`new`
@@ -162,10 +162,10 @@ A hash object has the following attributes:
A hash object has the following methods:
-.. method:: hash.update(arg)
+.. method:: hash.update(data)
- Update the hash object with the object *arg*, which must be interpretable as
- a buffer of bytes. Repeated calls are equivalent to a single call with the
+ Update the hash object with the :term:`bytes-like object`.
+ Repeated calls are equivalent to a single call with the
concatenation of all the arguments: ``m.update(a); m.update(b)`` is
equivalent to ``m.update(a+b)``.
@@ -206,7 +206,7 @@ by the SHAKE algorithm.
.. method:: shake.digest(length)
Return the digest of the data passed to the :meth:`update` method so far.
- This is a bytes object of size ``length`` which may contain bytes in
+ This is a bytes object of size *length* which may contain bytes in
the whole range from 0 to 255.
@@ -262,9 +262,10 @@ include a `salt `_.
The function provides scrypt password-based key derivation function as
defined in :rfc:`7914`.
- *password* and *salt* must be bytes-like objects. Applications and
- libraries should limit *password* to a sensible length (e.g. 1024). *salt*
- should be about 16 or more bytes from a proper source, e.g. :func:`os.urandom`.
+ *password* and *salt* must be :term:`bytes-like objects
+ `. Applications and libraries should limit *password*
+ to a sensible length (e.g. 1024). *salt* should be about 16 or more
+ bytes from a proper source, e.g. :func:`os.urandom`.
*n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization
factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB).
@@ -283,7 +284,7 @@ BLAKE2
.. index::
single: blake2b, blake2s
-BLAKE2_ is a cryptographic hash function defined in RFC-7693_ that comes in two
+BLAKE2_ is a cryptographic hash function defined in :rfc:`7693` that comes in two
flavors:
* **BLAKE2b**, optimized for 64-bit platforms and produces digests of any size
@@ -305,11 +306,11 @@ Creating hash objects
New hash objects are created by calling constructor functions:
-.. function:: blake2b(data=b'', digest_size=64, key=b'', salt=b'', \
+.. function:: blake2b(data=b'', *, digest_size=64, key=b'', salt=b'', \
person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \
node_depth=0, inner_size=0, last_node=False)
-.. function:: blake2s(data=b'', digest_size=32, key=b'', salt=b'', \
+.. function:: blake2s(data=b'', *, digest_size=32, key=b'', salt=b'', \
person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \
node_depth=0, inner_size=0, last_node=False)
@@ -317,8 +318,8 @@ New hash objects are created by calling constructor functions:
These functions return the corresponding hash objects for calculating
BLAKE2b or BLAKE2s. They optionally take these general parameters:
-* *data*: initial chunk of data to hash, which must be interpretable as buffer
- of bytes.
+* *data*: initial chunk of data to hash, which must be
+ :term:`bytes-like object`. It can be passed only as positional argument.
* *digest_size*: size of output digest in bytes.
@@ -427,7 +428,7 @@ object, and, finally, get the digest out of the object by calling
As a shortcut, you can pass the first chunk of data to update directly to the
-constructor as the first argument (or as *data* keyword argument):
+constructor as the positional argument:
>>> from hashlib import blake2b
>>> blake2b(b'Hello world').hexdigest()
@@ -546,7 +547,7 @@ on the hash function used in digital signatures.
preparer, generates all or part of a message to be signed by a second
party, the message signer. If the message preparer is able to find
cryptographic hash function collisions (i.e., two messages producing the
- same hash value), then she might prepare meaningful versions of the message
+ same hash value), then they might prepare meaningful versions of the message
that would produce the same hash value and digital signature, but with
different results (e.g., transferring $1,000,000 to an account, rather than
$10). Cryptographic hash functions have been designed with collision
@@ -707,7 +708,6 @@ Domain Dedication 1.0 Universal:
* *Alexandr Sokolovskiy*
-.. _RFC-7693: https://tools.ietf.org/html/rfc7693
.. _BLAKE2: https://blake2.net
.. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code
.. _BLAKE: https://131002.net/blake/
diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst
index fcda86cf797800..731624ba94e335 100644
--- a/Doc/library/hmac.rst
+++ b/Doc/library/hmac.rst
@@ -27,7 +27,7 @@ This module implements the HMAC algorithm as described by :rfc:`2104`.
Parameter *msg* can be of any type supported by :mod:`hashlib`.
Parameter *digestmod* can be the name of a hash algorithm.
- .. deprecated:: 3.4
+ .. deprecated-removed:: 3.4 3.8
MD5 as implicit default digest for *digestmod* is deprecated.
diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst
index 5370601544c766..d8da6683a3a52f 100644
--- a/Doc/library/http.cookiejar.rst
+++ b/Doc/library/http.cookiejar.rst
@@ -88,7 +88,7 @@ The following classes are provided:
:class:`DefaultCookiePolicy` objects.
:class:`DefaultCookiePolicy` implements the standard accept / reject rules for
- Netscape and RFC 2965 cookies. By default, RFC 2109 cookies (ie. cookies
+ Netscape and :rfc:`2965` cookies. By default, :rfc:`2109` cookies (ie. cookies
received in a :mailheader:`Set-Cookie` header with a version cookie-attribute of
1) are treated according to the RFC 2965 rules. However, if RFC 2965 handling
is turned off or :attr:`rfc2109_as_netscape` is ``True``, RFC 2109 cookies are
@@ -100,7 +100,7 @@ The following classes are provided:
.. class:: Cookie()
- This class represents Netscape, RFC 2109 and RFC 2965 cookies. It is not
+ This class represents Netscape, :rfc:`2109` and :rfc:`2965` cookies. It is not
expected that users of :mod:`http.cookiejar` construct their own :class:`Cookie`
instances. Instead, if necessary, call :meth:`make_cookies` on a
:class:`CookieJar` instance.
@@ -123,14 +123,14 @@ The following classes are provided:
the one sketched out in ``cookie_spec.html``.
:rfc:`2109` - HTTP State Management Mechanism
- Obsoleted by RFC 2965. Uses :mailheader:`Set-Cookie` with version=1.
+ Obsoleted by :rfc:`2965`. Uses :mailheader:`Set-Cookie` with version=1.
:rfc:`2965` - HTTP State Management Mechanism
The Netscape protocol with the bugs fixed. Uses :mailheader:`Set-Cookie2` in
place of :mailheader:`Set-Cookie`. Not widely used.
http://kristol.org/cookie/errata.html
- Unfinished errata to RFC 2965.
+ Unfinished errata to :rfc:`2965`.
:rfc:`2964` - Use of HTTP State Management
@@ -320,7 +320,7 @@ writing.
.. note::
- This loses information about RFC 2965 cookies, and also about newer or
+ This loses information about :rfc:`2965` cookies, and also about newer or
non-standard cookie-attributes such as ``port``.
.. warning::
@@ -410,13 +410,13 @@ be assigned to.
.. attribute:: CookiePolicy.rfc2965
- Implement RFC 2965 protocol.
+ Implement :rfc:`2965` protocol.
.. attribute:: CookiePolicy.hide_cookie2
Don't add :mailheader:`Cookie2` header to requests (the presence of this header
- indicates to the server that we understand RFC 2965 cookies).
+ indicates to the server that we understand :rfc:`2965` cookies).
The most useful way to define a :class:`CookiePolicy` class is by subclassing
from :class:`DefaultCookiePolicy` and overriding some or all of the methods
@@ -431,7 +431,7 @@ DefaultCookiePolicy Objects
Implements the standard rules for accepting and returning cookies.
-Both RFC 2965 and Netscape cookies are covered. RFC 2965 handling is switched
+Both :rfc:`2965` and Netscape cookies are covered. RFC 2965 handling is switched
off by default.
The easiest way to provide your own policy is to override this class and call
@@ -510,11 +510,11 @@ all be assigned to.
.. attribute:: DefaultCookiePolicy.rfc2109_as_netscape
- If true, request that the :class:`CookieJar` instance downgrade RFC 2109 cookies
+ If true, request that the :class:`CookieJar` instance downgrade :rfc:`2109` cookies
(ie. cookies received in a :mailheader:`Set-Cookie` header with a version
cookie-attribute of 1) to Netscape cookies by setting the version attribute of
the :class:`Cookie` instance to 0. The default value is :const:`None`, in which
- case RFC 2109 cookies are downgraded if and only if RFC 2965 handling is turned
+ case RFC 2109 cookies are downgraded if and only if :rfc:`2965` handling is turned
off. Therefore, RFC 2109 cookies are downgraded by default.
@@ -527,11 +527,11 @@ General strictness switches:
and isn't guaranteed to work!
-RFC 2965 protocol strictness switches:
+:rfc:`2965` protocol strictness switches:
.. attribute:: DefaultCookiePolicy.strict_rfc2965_unverifiable
- Follow RFC 2965 rules on unverifiable transactions (usually, an unverifiable
+ Follow :rfc:`2965` rules on unverifiable transactions (usually, an unverifiable
transaction is one resulting from a redirect or a request for an image hosted on
another site). If this is false, cookies are *never* blocked on the basis of
verifiability
@@ -541,7 +541,7 @@ Netscape protocol strictness switches:
.. attribute:: DefaultCookiePolicy.strict_ns_unverifiable
- Apply RFC 2965 rules on unverifiable transactions even to Netscape cookies.
+ Apply :rfc:`2965` rules on unverifiable transactions even to Netscape cookies.
.. attribute:: DefaultCookiePolicy.strict_ns_domain
@@ -581,7 +581,7 @@ both flags are set).
.. attribute:: DefaultCookiePolicy.DomainRFC2965Match
- When setting cookies, require a full RFC 2965 domain-match.
+ When setting cookies, require a full :rfc:`2965` domain-match.
The following attributes are provided for convenience, and are the most useful
combinations of the above flags:
@@ -605,7 +605,7 @@ Cookie Objects
standard cookie-attributes specified in the various cookie standards. The
correspondence is not one-to-one, because there are complicated rules for
assigning default values, because the ``max-age`` and ``expires``
-cookie-attributes contain equivalent information, and because RFC 2109 cookies
+cookie-attributes contain equivalent information, and because :rfc:`2109` cookies
may be 'downgraded' by :mod:`http.cookiejar` from version 1 to version 0 (Netscape)
cookies.
@@ -616,8 +616,8 @@ internal consistency, so you should know what you're doing if you do that.
.. attribute:: Cookie.version
- Integer or :const:`None`. Netscape cookies have :attr:`version` 0. RFC 2965 and
- RFC 2109 cookies have a ``version`` cookie-attribute of 1. However, note that
+ Integer or :const:`None`. Netscape cookies have :attr:`version` 0. :rfc:`2965` and
+ :rfc:`2109` cookies have a ``version`` cookie-attribute of 1. However, note that
:mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which
case :attr:`version` is 0.
@@ -673,7 +673,7 @@ internal consistency, so you should know what you're doing if you do that.
.. attribute:: Cookie.rfc2109
- ``True`` if this cookie was received as an RFC 2109 cookie (ie. the cookie
+ ``True`` if this cookie was received as an :rfc:`2109` cookie (ie. the cookie
arrived in a :mailheader:`Set-Cookie` header, and the value of the Version
cookie-attribute in that header was 1). This attribute is provided because
:mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in
@@ -745,7 +745,7 @@ cookies (assumes Unix/Netscape convention for location of the cookies file)::
r = opener.open("http://example.com/")
The next example illustrates the use of :class:`DefaultCookiePolicy`. Turn on
-RFC 2965 cookies, be more strict about domains when setting and returning
+:rfc:`2965` cookies, be more strict about domains when setting and returning
Netscape cookies, and block some domains from setting cookies or having them
returned::
diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst
index c98843de02cba3..0bd7f778cec0cf 100644
--- a/Doc/library/http.server.rst
+++ b/Doc/library/http.server.rst
@@ -33,9 +33,19 @@ handler. Code to create and run the server looks like this::
:attr:`server_port`. The server is accessible by the handler, typically
through the handler's :attr:`server` instance variable.
+.. class:: ThreadingHTTPServer(server_address, RequestHandlerClass)
-The :class:`HTTPServer` must be given a *RequestHandlerClass* on instantiation,
-of which this module provides three different variants:
+ This class is identical to HTTPServer but uses threads to handle
+ requests by using the :class:`~socketserver.ThreadingMixIn`. This
+ is useful to handle web browsers pre-opening sockets, on which
+ :class:`HTTPServer` would wait indefinitely.
+
+ .. versionadded:: 3.7
+
+
+The :class:`HTTPServer` and :class:`ThreadingHTTPServer` must be given
+a *RequestHandlerClass* on instantiation, of which this module
+provides three different variants:
.. class:: BaseHTTPRequestHandler(request, client_address, server)
diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
index af153593e3c46d..58e61931aaf499 100644
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -274,7 +274,8 @@ Configure IDLE
Code Context (toggle)(Editor Window only)
Open a pane at the top of the edit window which shows the block context
- of the code which has scrolled above the top of the window.
+ of the code which has scrolled above the top of the window. Clicking a
+ line in this pane exposes that line at the top of the editor.
Window menu (Shell and Editor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
index 1632eb7aed3620..2e2c59c9ce0056 100644
--- a/Doc/library/imaplib.rst
+++ b/Doc/library/imaplib.rst
@@ -342,7 +342,7 @@ An :class:`IMAP4` instance has the following methods:
.. method:: IMAP4.namespace()
- Returns IMAP namespaces as defined in RFC2342.
+ Returns IMAP namespaces as defined in :rfc:`2342`.
.. method:: IMAP4.noop()
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index 5fa1d7d869dd9d..6f4da11989551e 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -249,7 +249,7 @@ ABC hierarchy::
.. abstractmethod:: find_module(fullname, path=None)
- An abstact method for finding a :term:`loader` for the specified
+ An abstract method for finding a :term:`loader` for the specified
module. Originally specified in :pep:`302`, this method was meant
for use in :data:`sys.meta_path` and in the path-based import subsystem.
@@ -369,9 +369,9 @@ ABC hierarchy::
An abstract base class for a :term:`loader`.
See :pep:`302` for the exact definition for a loader.
- For loaders that wish to support resource reading, they should
- implement a ``get_resource_reader(fullname)`` method as specified
- by :class:`importlib.abc.ResourceReader`.
+ Loaders that wish to support resource reading should implement a
+ ``get_resource_reader(fullname)`` method as specified by
+ :class:`importlib.abc.ResourceReader`.
.. versionchanged:: 3.7
Introduced the optional ``get_resource_reader()`` method.
@@ -530,7 +530,7 @@ ABC hierarchy::
.. abstractmethod:: contents()
- Returns an :term:`iterator` of strings over the contents of
+ Returns an :term:`iterable` of strings over the contents of
the package. Do note that it is not required that all names
returned by the iterator be actual resources, e.g. it is
acceptable to return names for which :meth:`is_resource` would
@@ -544,7 +544,7 @@ ABC hierarchy::
the file system then those subdirectory names can be used
directly.
- The abstract method returns an iterator of no items.
+ The abstract method returns an iterable of no items.
.. class:: ResourceLoader
@@ -770,7 +770,7 @@ ABC hierarchy::
Concrete implementation of :meth:`Loader.exec_module`.
- .. versionadded:: 3.4
+ .. versionadded:: 3.4
.. method:: load_module(fullname)
@@ -813,8 +813,25 @@ Resources are roughly akin to files inside directories, though it's important
to keep in mind that this is just a metaphor. Resources and packages **do
not** have to exist as physical files and directories on the file system.
-Loaders can support resources by implementing the :class:`ResourceReader`
-abstract base class.
+.. note::
+
+ This module provides functionality similar to `pkg_resources
+ `_ `Basic
+ Resource Access
+ `_
+ without the performance overhead of that package. This makes reading
+ resources included in packages easier, with more stable and consistent
+ semantics.
+
+ The standalone backport of this module provides more information
+ on `using importlib.resources
+ `_ and
+ `migrating from pkg_resources to importlib.resources
+ `_.
+
+Loaders that wish to support resource reading should implement a
+``get_resource_reader(fullname)`` method as specified by
+:class:`importlib.abc.ResourceReader`.
The following types are defined.
@@ -909,9 +926,9 @@ The following functions are available.
.. function:: contents(package)
- Return an iterator over the named items within the package. The iterator
+ Return an iterable over the named items within the package. The iterable
returns :class:`str` resources (e.g. files) and non-resources
- (e.g. directories). The iterator does not recurse into subdirectories.
+ (e.g. directories). The iterable does not recurse into subdirectories.
*package* is either a name or a module object which conforms to the
``Package`` requirements.
@@ -1013,7 +1030,7 @@ find and load modules.
.. class:: WindowsRegistryFinder
:term:`Finder` for modules declared in the Windows registry. This class
- implements the :class:`importlib.abc.Finder` ABC.
+ implements the :class:`importlib.abc.MetaPathFinder` ABC.
Only class methods are defined by this class to alleviate the need for
instantiation.
@@ -1064,7 +1081,12 @@ find and load modules.
.. classmethod:: invalidate_caches()
Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all
- finders stored in :attr:`sys.path_importer_cache`.
+ finders stored in :data:`sys.path_importer_cache` that define the method.
+ Otherwise entries in :data:`sys.path_importer_cache` set to ``None`` are
+ deleted.
+
+ .. versionchanged:: 3.7
+ Entries of ``None`` in :data:`sys.path_importer_cache` are deleted.
.. versionchanged:: 3.4
Calls objects in :data:`sys.path_hooks` with the current working
@@ -1291,7 +1313,7 @@ find and load modules.
Name of the place from which the module is loaded, e.g. "builtin" for
built-in modules and the filename for modules loaded from source.
Normally "origin" should be set, but it may be ``None`` (the default)
- which indicates it is unspecified.
+ which indicates it is unspecified (e.g. for namespace packages).
.. attribute:: submodule_search_locations
diff --git a/Doc/library/index.rst b/Doc/library/index.rst
index a925e10ee4988f..b8fbf44ae49f13 100644
--- a/Doc/library/index.rst
+++ b/Doc/library/index.rst
@@ -30,7 +30,7 @@ optional components.
In addition to the standard library, there is a growing collection of
several thousand components (from individual programs and modules to
packages and entire application development frameworks), available from
-the `Python Package Index `_.
+the `Python Package Index `_.
.. toctree::
@@ -55,6 +55,7 @@ the `Python Package Index `_.
crypto.rst
allos.rst
concurrency.rst
+ contextvars.rst
ipc.rst
netdata.rst
markup.rst
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index 5c71d900fd258d..98649de7310f0b 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -799,7 +799,7 @@ Text I/O
.. versionadded:: 3.1
- .. method:: read(size)
+ .. method:: read(size=-1)
Read and return at most *size* characters from the stream as a single
:class:`str`. If *size* is negative or ``None``, reads until EOF.
@@ -889,7 +889,7 @@ Text I/O
characters written are translated to the given string.
If *line_buffering* is ``True``, :meth:`flush` is implied when a call to
- write contains a newline character.
+ write contains a newline character or a carriage return.
If *write_through* is ``True``, calls to :meth:`write` are guaranteed
not to be buffered: any data written on the :class:`TextIOWrapper`
diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst
index 75c9107bd5eb31..b7b502aff15e0b 100644
--- a/Doc/library/ipaddress.rst
+++ b/Doc/library/ipaddress.rst
@@ -91,7 +91,8 @@ Address objects
The :class:`IPv4Address` and :class:`IPv6Address` objects share a lot of common
attributes. Some attributes that are only meaningful for IPv6 addresses are
also implemented by :class:`IPv4Address` objects, in order to make it easier to
-write code that handles both IP versions correctly.
+write code that handles both IP versions correctly. Address objects are
+:term:`hashable`, so they can be used as keys in dictionaries.
.. class:: IPv4Address(address)
@@ -236,7 +237,7 @@ write code that handles both IP versions correctly.
groups consisting entirely of zeroes included.
- For the following attributes, see the corresponding documention of the
+ For the following attributes, see the corresponding documentation of the
:class:`IPv4Address` class:
.. attribute:: packed
@@ -368,6 +369,8 @@ All attributes implemented by address objects are implemented by network
objects as well. In addition, network objects implement additional attributes.
All of these are common between :class:`IPv4Network` and :class:`IPv6Network`,
so to avoid duplication they are only documented for :class:`IPv4Network`.
+Network objects are :term:`hashable`, so they can be used as keys in
+dictionaries.
.. class:: IPv4Network(address, strict=True)
@@ -377,8 +380,9 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
a slash (``/``). The IP address is the network address, and the mask
can be either a single number, which means it's a *prefix*, or a string
representation of an IPv4 address. If it's the latter, the mask is
- interpreted as a *net mask* if it starts with a non-zero field, or as
- a *host mask* if it starts with a zero field. If no mask is provided,
+ interpreted as a *net mask* if it starts with a non-zero field, or as a
+ *host mask* if it starts with a zero field, with the single exception of
+ an all-zero mask which is treated as a *net mask*. If no mask is provided,
it's considered to be ``/32``.
For example, the following *address* specifications are equivalent:
@@ -408,7 +412,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
Unless stated otherwise, all network methods accepting other network/address
objects will raise :exc:`TypeError` if the argument's IP version is
- incompatible to ``self``
+ incompatible to ``self``.
.. versionchanged:: 3.5
@@ -418,7 +422,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
.. attribute:: max_prefixlen
Refer to the corresponding attribute documentation in
- :class:`IPv4Address`
+ :class:`IPv4Address`.
.. attribute:: is_multicast
.. attribute:: is_private
@@ -428,7 +432,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
.. attribute:: is_link_local
These attributes are true for the network as a whole if they are true
- for both the network address and the broadcast address
+ for both the network address and the broadcast address.
.. attribute:: network_address
@@ -442,7 +446,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
.. attribute:: hostmask
- The host mask, as a string.
+ The host mask, as an :class:`IPv4Address` object.
+
+ .. attribute:: netmask
+
+ The net mask, as an :class:`IPv4Address` object.
.. attribute:: with_prefixlen
.. attribute:: compressed
@@ -477,12 +485,16 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
Returns an iterator over the usable hosts in the network. The usable
hosts are all the IP addresses that belong to the network, except the
- network address itself and the network broadcast address.
+ network address itself and the network broadcast address. For networks
+ with a mask length of 31, the network address and network broadcast
+ address are also included in the result.
>>> list(ip_network('192.0.2.0/29').hosts()) #doctest: +NORMALIZE_WHITESPACE
[IPv4Address('192.0.2.1'), IPv4Address('192.0.2.2'),
IPv4Address('192.0.2.3'), IPv4Address('192.0.2.4'),
IPv4Address('192.0.2.5'), IPv4Address('192.0.2.6')]
+ >>> list(ip_network('192.0.2.0/31').hosts())
+ [IPv4Address('192.0.2.0'), IPv4Address('192.0.2.1')]
.. method:: overlaps(other)
@@ -586,15 +598,14 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
Construct an IPv6 network definition. *address* can be one of the following:
- 1. A string consisting of an IP address and an optional mask, separated by
- a slash (``/``). The IP address is the network address, and the mask
- can be either a single number, which means it's a *prefix*, or a string
- representation of an IPv6 address. If it's the latter, the mask is
- interpreted as a *net mask*. If no mask is provided, it's considered to
- be ``/128``.
+ 1. A string consisting of an IP address and an optional prefix length,
+ separated by a slash (``/``). The IP address is the network address,
+ and the prefix length must be a single number, the *prefix*. If no
+ prefix length is provided, it's considered to be ``/128``.
- For example, the following *address* specifications are equivalent:
- ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``.
+ Note that currently expanded netmasks are not supported. That means
+ ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::``
+ not.
2. An integer that fits into 128 bits. This is equivalent to a
single-address network, with the network address being *address* and
@@ -631,6 +642,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
.. attribute:: network_address
.. attribute:: broadcast_address
.. attribute:: hostmask
+ .. attribute:: netmask
.. attribute:: with_prefixlen
.. attribute:: compressed
.. attribute:: exploded
@@ -639,6 +651,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
.. attribute:: num_addresses
.. attribute:: prefixlen
.. method:: hosts()
+
+ Returns an iterator over the usable hosts in the network. The usable
+ hosts are all the IP addresses that belong to the network, except the
+ Subnet-Router anycast address. For networks with a mask length of 127,
+ the Subnet-Router anycast address is also included in the result.
+
.. method:: overlaps(other)
.. method:: address_exclude(network)
.. method:: subnets(prefixlen_diff=1, new_prefix=None)
@@ -648,12 +666,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
.. method:: compare_networks(other)
Refer to the corresponding attribute documentation in
- :class:`IPv4Network`
+ :class:`IPv4Network`.
.. attribute:: is_site_local
These attribute is true for the network as a whole if it is true
- for both the network address and the broadcast address
+ for both the network address and the broadcast address.
Operators
@@ -667,8 +685,8 @@ IPv6).
Logical operators
"""""""""""""""""
-Network objects can be compared with the usual set of logical operators,
-similarly to address objects.
+Network objects can be compared with the usual set of logical operators.
+Network objects are ordered first by network address, then by net mask.
Iteration
@@ -718,6 +736,9 @@ Network objects can act as containers of addresses. Some examples::
Interface objects
-----------------
+Interface objects are :term:`hashable`, so they can be used as keys in
+dictionaries.
+
.. class:: IPv4Interface(address)
Construct an IPv4 interface. The meaning of *address* is as in the
@@ -789,6 +810,30 @@ Interface objects
:class:`IPv4Interface`.
+Operators
+^^^^^^^^^
+
+Interface objects support some operators. Unless stated otherwise, operators
+can only be applied between compatible objects (i.e. IPv4 with IPv4, IPv6 with
+IPv6).
+
+
+Logical operators
+"""""""""""""""""
+
+Interface objects can be compared with the usual set of logical operators.
+
+For equality comparison (``==`` and ``!=``), both the IP address and network
+must be the same for the objects to be equal. An interface will not compare
+equal to any address or network object.
+
+For ordering (``<``, ``>``, etc) the rules are different. Interface and
+address objects with the same IP version can be compared, and the address
+objects will always sort before the interface objects. Two interface objects
+are first compared by their networks and, if those are the same, then by their
+IP addresses.
+
+
Other Module Level Functions
----------------------------
@@ -854,7 +899,7 @@ The module also provides the following module level functions:
doesn't make sense. There are some times however, where you may wish to
have :mod:`ipaddress` sort these anyway. If you need to do this, you can use
- this function as the ``key`` argument to :func:`sorted()`.
+ this function as the *key* argument to :func:`sorted()`.
*obj* is either a network or address object.
@@ -872,4 +917,4 @@ module defines the following exceptions:
.. exception:: NetmaskValueError(ValueError)
- Any value error related to the netmask.
+ Any value error related to the net mask.
diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index 0b3829f19faf9c..959424ff914390 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -436,15 +436,24 @@ loops that truncate the stream.
# islice('ABCDEFG', 2, None) --> C D E F G
# islice('ABCDEFG', 0, None, 2) --> A C E G
s = slice(*args)
- it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1))
+ start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1
+ it = iter(range(start, stop, step))
try:
nexti = next(it)
except StopIteration:
+ # Consume *iterable* up to the *start* position.
+ for i, element in zip(range(start), iterable):
+ pass
return
- for i, element in enumerate(iterable):
- if i == nexti:
- yield element
- nexti = next(it)
+ try:
+ for i, element in enumerate(iterable):
+ if i == nexti:
+ yield element
+ nexti = next(it)
+ except StopIteration:
+ # Consume to *stop*.
+ for i, element in zip(range(i + 1, stop), iterable):
+ pass
If *start* is ``None``, then iteration starts at zero. If *step* is ``None``,
then the step defaults to one.
@@ -679,6 +688,11 @@ which incur interpreter overhead.
"Return first n items of the iterable as a list"
return list(islice(iterable, n))
+ def prepend(value, iterator):
+ "Prepend a single value in front of an iterator"
+ # prepend(1, [2, 3, 4]) -> 1 2 3 4
+ return chain([value], iterator)
+
def tabulate(function, start=0):
"Return function(0), function(1), ..."
return map(function, count(start))
@@ -688,8 +702,8 @@ which incur interpreter overhead.
# tail(3, 'ABCDEFG') --> E F G
return iter(collections.deque(iterable, maxlen=n))
- def consume(iterator, n):
- "Advance the iterator n-steps ahead. If n is none, consume entirely."
+ def consume(iterator, n=None):
+ "Advance the iterator n-steps ahead. If n is None, consume entirely."
# Use functions that consume iterators at C speed.
if n is None:
# feed the entire iterator into a zero-length deque
diff --git a/Doc/library/json.rst b/Doc/library/json.rst
index 829218d558439b..8ce493d63995d4 100644
--- a/Doc/library/json.rst
+++ b/Doc/library/json.rst
@@ -100,9 +100,9 @@ Extending :class:`JSONEncoder`::
['[2.0', ', 1.0', ']']
-.. highlight:: bash
+Using :mod:`json.tool` from the shell to validate and pretty-print:
-Using :mod:`json.tool` from the shell to validate and pretty-print::
+.. code-block:: shell-session
$ echo '{"json":"obj"}' | python -m json.tool
{
@@ -113,8 +113,6 @@ Using :mod:`json.tool` from the shell to validate and pretty-print::
See :ref:`json-commandline` for detailed documentation.
-.. highlight:: python3
-
.. note::
JSON is a subset of `YAML `_ 1.2. The JSON produced by
@@ -190,6 +188,11 @@ Basic Usage
.. versionchanged:: 3.6
All optional parameters are now :ref:`keyword-only `.
+ .. note::
+
+ Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol,
+ so trying to serialize multiple objects with repeated calls to
+ :func:`dump` using the same *fp* will result in an invalid JSON file.
.. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \
check_circular=True, allow_nan=True, cls=None, \
@@ -200,12 +203,6 @@ Basic Usage
table `. The arguments have the same meaning as in
:func:`dump`.
- .. note::
-
- Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol,
- so trying to serialize multiple objects with repeated calls to
- :func:`dump` using the same *fp* will result in an invalid JSON file.
-
.. note::
Keys in key/value pairs of JSON are always of the type :class:`str`. When
@@ -217,9 +214,9 @@ Basic Usage
.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw)
- Deserialize *fp* (a ``.read()``-supporting :term:`file-like object`
- containing a JSON document) to a Python object using this :ref:`conversion
- table `.
+ Deserialize *fp* (a ``.read()``-supporting :term:`text file` or
+ :term:`binary file` containing a JSON document) to a Python object using
+ this :ref:`conversion table `.
*object_hook* is an optional function that will be called with the result of
any object literal decoded (a :class:`dict`). The return value of
@@ -230,10 +227,8 @@ Basic Usage
*object_pairs_hook* is an optional function that will be called with the
result of any object literal decoded with an ordered list of pairs. The
return value of *object_pairs_hook* will be used instead of the
- :class:`dict`. This feature can be used to implement custom decoders that
- rely on the order that the key and value pairs are decoded (for example,
- :func:`collections.OrderedDict` will remember the order of insertion). If
- *object_hook* is also defined, the *object_pairs_hook* takes priority.
+ :class:`dict`. This feature can be used to implement custom decoders.
+ If *object_hook* is also defined, the *object_pairs_hook* takes priority.
.. versionchanged:: 3.1
Added support for *object_pairs_hook*.
@@ -266,6 +261,10 @@ Basic Usage
.. versionchanged:: 3.6
All optional parameters are now :ref:`keyword-only `.
+ .. versionchanged:: 3.6
+ *fp* can now be a :term:`binary file`. The input encoding should be
+ UTF-8, UTF-16 or UTF-32.
+
.. function:: loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw)
Deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray`
@@ -325,10 +324,8 @@ Encoders and Decoders
*object_pairs_hook*, if specified will be called with the result of every
JSON object decoded with an ordered list of pairs. The return value of
*object_pairs_hook* will be used instead of the :class:`dict`. This
- feature can be used to implement custom decoders that rely on the order
- that the key and value pairs are decoded (for example,
- :func:`collections.OrderedDict` will remember the order of insertion). If
- *object_hook* is also defined, the *object_pairs_hook* takes priority.
+ feature can be used to implement custom decoders. If *object_hook* is also
+ defined, the *object_pairs_hook* takes priority.
.. versionchanged:: 3.1
Added support for *object_pairs_hook*.
@@ -651,8 +648,6 @@ when serializing Python :class:`int` values of extremely large magnitude, or
when serializing instances of "exotic" numerical types such as
:class:`decimal.Decimal`.
-.. highlight:: bash
-
.. _json-commandline:
Command Line Interface
@@ -669,7 +664,9 @@ The :mod:`json.tool` module provides a simple command line interface to validate
and pretty-print JSON objects.
If the optional ``infile`` and ``outfile`` arguments are not
-specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively::
+specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively:
+
+.. code-block:: shell-session
$ echo '{"json": "obj"}' | python -m json.tool
{
@@ -688,7 +685,9 @@ Command line options
.. cmdoption:: infile
- The JSON file to be validated or pretty-printed::
+ The JSON file to be validated or pretty-printed:
+
+ .. code-block:: shell-session
$ python -m json.tool mp_films.json
[
diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst
index 06378379c331cf..de805eb955dfbe 100644
--- a/Doc/library/logging.config.rst
+++ b/Doc/library/logging.config.rst
@@ -538,7 +538,9 @@ target handler, and the system will resolve to the handler from the
id. If, however, a user defines a ``my.package.MyHandler`` which has
an ``alternate`` handler, the configuration system would not know that
the ``alternate`` referred to a handler. To cater for this, a generic
-resolution system allows the user to specify::
+resolution system allows the user to specify:
+
+.. code-block:: yaml
handlers:
file:
@@ -552,7 +554,9 @@ The literal string ``'cfg://handlers.file'`` will be resolved in an
analogous way to strings with the ``ext://`` prefix, but looking
in the configuration itself rather than the import namespace. The
mechanism allows access by dot or by index, in a similar way to
-that provided by ``str.format``. Thus, given the following snippet::
+that provided by ``str.format``. Thus, given the following snippet:
+
+.. code-block:: yaml
handlers:
email:
@@ -786,11 +790,10 @@ Sections which specify formatter configuration are typified by the following.
The ``format`` entry is the overall format string, and the ``datefmt`` entry is
the :func:`strftime`\ -compatible date/time format string. If empty, the
-package substitutes ISO8601 format date/times, which is almost equivalent to
-specifying the date format string ``'%Y-%m-%d %H:%M:%S'``. The ISO8601 format
-also specifies milliseconds, which are appended to the result of using the above
-format string, with a comma separator. An example time in ISO8601 format is
-``2003-01-23 00:29:50,411``.
+package substitutes something which is almost equivalent to specifying the date
+format string ``'%Y-%m-%d %H:%M:%S'``. This format also specifies milliseconds,
+which are appended to the result of using the above format string, with a comma
+separator. An example time in this format is ``2003-01-23 00:29:50,411``.
The ``class`` entry is optional. It indicates the name of the formatter's class
(as a dotted module and class name.) This option is useful for instantiating a
diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
index 0974286e55dc10..bdf16a8177e994 100644
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -524,7 +524,7 @@ over UDP sockets.
.. versionchanged:: 3.4
If ``port`` is specified as ``None``, a Unix domain socket is created
- using the value in ``host`` - otherwise, a TCP socket is created.
+ using the value in ``host`` - otherwise, a UDP socket is created.
.. method:: emit()
@@ -596,7 +596,7 @@ supports sending logging messages to a remote or local Unix syslog.
(See: :issue:`12168`.) In earlier versions, the message sent to the
syslog daemons was always terminated with a NUL byte, because early
versions of these daemons expected a NUL terminated message - even
- though it's not in the relevant specification (RFC 5424). More recent
+ though it's not in the relevant specification (:rfc:`5424`). More recent
versions of these daemons don't expect the NUL byte but strip it off
if it's there, and even more recent daemons (which adhere more closely
to RFC 5424) pass the NUL byte on as part of the message.
diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index 1ed129c00d4907..a2188c860eb458 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -179,7 +179,9 @@ is the module's name in the Python package namespace.
You can specify *stack_info* independently of *exc_info*, e.g. to just show
how you got to a certain point in your code, even when no exceptions were
- raised. The stack frames are printed following a header line which says::
+ raised. The stack frames are printed following a header line which says:
+
+ .. code-block:: none
Stack (most recent call last):
@@ -198,7 +200,9 @@ is the module's name in the Python package namespace.
logger = logging.getLogger('tcpserver')
logger.warning('Protocol problem: %s', 'connection reset', extra=d)
- would print something like ::
+ would print something like
+
+ .. code-block:: none
2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset
@@ -328,7 +332,7 @@ is the module's name in the Python package namespace.
.. versionadded:: 3.2
.. versionchanged:: 3.7
- Loggers can now be picked and unpickled.
+ Loggers can now be pickled and unpickled.
.. _levels:
@@ -511,8 +515,8 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on
Returns a new instance of the :class:`Formatter` class. The instance is
initialized with a format string for the message as a whole, as well as a
format string for the date/time portion of a message. If no *fmt* is
- specified, ``'%(message)s'`` is used. If no *datefmt* is specified, the
- ISO8601 date format is used.
+ specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format
+ is used which is described in the :meth:`formatTime` documentation.
The *style* parameter can be one of '%', '{' or '$' and determines how
the format string will be merged with its data: using one of %-formatting,
@@ -552,8 +556,10 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on
formatters to provide for any specific requirement, but the basic behavior
is as follows: if *datefmt* (a string) is specified, it is used with
:func:`time.strftime` to format the creation time of the
- record. Otherwise, the ISO8601 format is used. The resulting string is
- returned.
+ record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the
+ uuu part is a millisecond value and the other letters are as per the
+ :func:`time.strftime` documentation. An example time in this format is
+ ``2003-01-23 00:29:50,411``. The resulting string is returned.
This function uses a user-configurable function to convert the creation
time to a tuple. By default, :func:`time.localtime` is used; to change
@@ -564,8 +570,8 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on
attribute in the ``Formatter`` class.
.. versionchanged:: 3.3
- Previously, the default ISO 8601 format was hard-coded as in this
- example: ``2010-09-06 22:38:15,292`` where the part before the comma is
+ Previously, the default format was hard-coded as in this example:
+ ``2010-09-06 22:38:15,292`` where the part before the comma is
handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the
part after the comma is a millisecond value. Because strptime does not
have a format placeholder for milliseconds, the millisecond value is
@@ -939,7 +945,9 @@ functions.
You can specify *stack_info* independently of *exc_info*, e.g. to just show
how you got to a certain point in your code, even when no exceptions were
- raised. The stack frames are printed following a header line which says::
+ raised. The stack frames are printed following a header line which says:
+
+ .. code-block:: none
Stack (most recent call last):
@@ -957,7 +965,9 @@ functions.
d = {'clientip': '192.168.0.1', 'user': 'fbloggs'}
logging.warning('Protocol problem: %s', 'connection reset', extra=d)
- would print something like::
+ would print something like:
+
+ .. code-block:: none
2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset
@@ -1119,52 +1129,54 @@ functions.
+--------------+---------------------------------------------+
| Format | Description |
+==============+=============================================+
- | ``filename`` | Specifies that a FileHandler be created, |
+ | *filename* | Specifies that a FileHandler be created, |
| | using the specified filename, rather than a |
| | StreamHandler. |
+--------------+---------------------------------------------+
- | ``filemode`` | Specifies the mode to open the file, if |
- | | filename is specified (if filemode is |
- | | unspecified, it defaults to 'a'). |
+ | *filemode* | If *filename* is specified, open the file |
+ | | in this :ref:`mode `. Defaults |
+ | | to ``'a'``. |
+--------------+---------------------------------------------+
- | ``format`` | Use the specified format string for the |
+ | *format* | Use the specified format string for the |
| | handler. |
+--------------+---------------------------------------------+
- | ``datefmt`` | Use the specified date/time format. |
+ | *datefmt* | Use the specified date/time format, as |
+ | | accepted by :func:`time.strftime`. |
+--------------+---------------------------------------------+
- | ``style`` | If ``format`` is specified, use this style |
- | | for the format string. One of '%', '{' or |
- | | '$' for %-formatting, :meth:`str.format` or |
- | | :class:`string.Template` respectively, and |
- | | defaulting to '%' if not specified. |
+ | *style* | If *format* is specified, use this style |
+ | | for the format string. One of ``'%'``, |
+ | | ``'{'`` or ``'$'`` for :ref:`printf-style |
+ | | `, |
+ | | :meth:`str.format` or |
+ | | :class:`string.Template` respectively. |
+ | | Defaults to ``'%'``. |
+--------------+---------------------------------------------+
- | ``level`` | Set the root logger level to the specified |
- | | level. |
+ | *level* | Set the root logger level to the specified |
+ | | :ref:`level `. |
+--------------+---------------------------------------------+
- | ``stream`` | Use the specified stream to initialize the |
+ | *stream* | Use the specified stream to initialize the |
| | StreamHandler. Note that this argument is |
- | | incompatible with 'filename' - if both are |
- | | present, a ``ValueError`` is raised. |
+ | | incompatible with *filename* - if both |
+ | | are present, a ``ValueError`` is raised. |
+--------------+---------------------------------------------+
- | ``handlers`` | If specified, this should be an iterable of |
+ | *handlers* | If specified, this should be an iterable of |
| | already created handlers to add to the root |
| | logger. Any handlers which don't already |
| | have a formatter set will be assigned the |
| | default formatter created in this function. |
| | Note that this argument is incompatible |
- | | with 'filename' or 'stream' - if both are |
- | | present, a ``ValueError`` is raised. |
+ | | with *filename* or *stream* - if both |
+ | | are present, a ``ValueError`` is raised. |
+--------------+---------------------------------------------+
.. versionchanged:: 3.2
- The ``style`` argument was added.
+ The *style* argument was added.
.. versionchanged:: 3.3
- The ``handlers`` argument was added. Additional checks were added to
+ The *handlers* argument was added. Additional checks were added to
catch situations where incompatible arguments are specified (e.g.
- ``handlers`` together with ``stream`` or ``filename``, or ``stream``
- together with ``filename``).
-
+ *handlers* together with *stream* or *filename*, or *stream*
+ together with *filename*).
.. function:: shutdown()
diff --git a/Doc/library/math.rst b/Doc/library/math.rst
index 55eb41b86f5d16..33aec573a1f281 100644
--- a/Doc/library/math.rst
+++ b/Doc/library/math.rst
@@ -203,7 +203,7 @@ Number-theoretic and representation functions
Return the :class:`~numbers.Real` value *x* truncated to an
:class:`~numbers.Integral` (usually an integer). Delegates to
- ``x.__trunc__()``.
+ :meth:`x.__trunc__() `.
Note that :func:`frexp` and :func:`modf` have a different call/return pattern
diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index 184119df591883..ca09a6a3ca9955 100644
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -127,7 +127,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
:class:`~mmap.mmap` can also be used as a context manager in a :keyword:`with`
- statement.::
+ statement::
import mmap
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index 8b3081cb80c972..504f3a1c3c33bf 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -735,8 +735,9 @@ For an example of the usage of queues for interprocess communication see
.. function:: Pipe([duplex])
- Returns a pair ``(conn1, conn2)`` of :class:`Connection` objects representing
- the ends of a pipe.
+ Returns a pair ``(conn1, conn2)`` of
+ :class:`~multiprocessing.connection.Connection` objects representing the
+ ends of a pipe.
If *duplex* is ``True`` (the default) then the pipe is bidirectional. If
*duplex* is ``False`` then the pipe is unidirectional: ``conn1`` can only be
@@ -1021,10 +1022,13 @@ Miscellaneous
Connection Objects
~~~~~~~~~~~~~~~~~~
+.. currentmodule:: multiprocessing.connection
+
Connection objects allow the sending and receiving of picklable objects or
strings. They can be thought of as message oriented connected sockets.
-Connection objects are usually created using :func:`Pipe` -- see also
+Connection objects are usually created using
+:func:`Pipe ` -- see also
:ref:`multiprocessing-listeners-clients`.
.. class:: Connection
@@ -1159,6 +1163,8 @@ For example:
Synchronization primitives
~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. currentmodule:: multiprocessing
+
Generally synchronization primitives are not as necessary in a multiprocess
program as they are in a multithreaded program. See the documentation for
:mod:`threading` module.
@@ -2269,7 +2275,7 @@ Listeners and Clients
:synopsis: API for dealing with sockets.
Usually message passing between processes is done using queues or by using
-:class:`~multiprocessing.Connection` objects returned by
+:class:`~Connection` objects returned by
:func:`~multiprocessing.Pipe`.
However, the :mod:`multiprocessing.connection` module allows some extra
@@ -2299,7 +2305,7 @@ multiple connections at the same time.
.. function:: Client(address[, family[, authkey]])
Attempt to set up a connection to the listener which is using address
- *address*, returning a :class:`~multiprocessing.Connection`.
+ *address*, returning a :class:`~Connection`.
The type of the connection is determined by *family* argument, but this can
generally be omitted since it can usually be inferred from the format of
@@ -2349,8 +2355,8 @@ multiple connections at the same time.
.. method:: accept()
Accept a connection on the bound socket or named pipe of the listener
- object and return a :class:`~multiprocessing.Connection` object. If
- authentication is attempted and fails, then
+ object and return a :class:`~Connection` object.
+ If authentication is attempted and fails, then
:exc:`~multiprocessing.AuthenticationError` is raised.
.. method:: close()
@@ -2386,7 +2392,7 @@ multiple connections at the same time.
For both Unix and Windows, an object can appear in *object_list* if
it is
- * a readable :class:`~multiprocessing.Connection` object;
+ * a readable :class:`~multiprocessing.connection.Connection` object;
* a connected and readable :class:`socket.socket` object; or
* the :attr:`~multiprocessing.Process.sentinel` attribute of a
:class:`~multiprocessing.Process` object.
@@ -2509,10 +2515,10 @@ an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address.
Authentication keys
~~~~~~~~~~~~~~~~~~~
-When one uses :meth:`Connection.recv `, the
+When one uses :meth:`Connection.recv `, the
data received is automatically
-unpickled. Unfortunately unpickling data from an untrusted source is a security
-risk. Therefore :class:`Listener` and :func:`Client` use the :mod:`hmac` module
+unpickled. Unfortunately unpickling data from an untrusted source is a security
+risk. Therefore :class:`Listener` and :func:`Client` use the :mod:`hmac` module
to provide digest authentication.
An authentication key is a byte string which can be thought of as a
diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst
index 2c3cd8dafe6e9c..d8ef8a692a9533 100644
--- a/Doc/library/nntplib.rst
+++ b/Doc/library/nntplib.rst
@@ -542,7 +542,7 @@ them have been superseded by newer commands in :rfc:`3977`.
is supplied, then the returned *list* is an empty list. This is an optional NNTP
extension, and may not be supported by all servers.
- RFC2980 says "It is suggested that this extension be deprecated". Use
+ :rfc:`2980` says "It is suggested that this extension be deprecated". Use
:meth:`descriptions` or :meth:`description` instead.
diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst
index 2ef187db2dcbfa..e9b82ee2ac13c7 100644
--- a/Doc/library/optparse.rst
+++ b/Doc/library/optparse.rst
@@ -567,7 +567,7 @@ An option group is obtained using the class :class:`OptionGroup`:
where
- * parser is the :class:`OptionParser` instance the group will be insterted in
+ * parser is the :class:`OptionParser` instance the group will be inserted in
to
* title is the group title
* description, optional, is a long description of the group
@@ -1677,7 +1677,7 @@ The callback function should raise :exc:`OptionValueError` if there are any
problems with the option or its argument(s). :mod:`optparse` catches this and
terminates the program, printing the error message you supply to stderr. Your
message should be clear, concise, accurate, and mention the option at fault.
-Otherwise, the user will have a hard time figuring out what he did wrong.
+Otherwise, the user will have a hard time figuring out what they did wrong.
.. _optparse-callback-example-1:
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index bae432d33b0bd4..d937451be51e6e 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -1082,20 +1082,81 @@ or `the MSDN `_ on Windo
.. versionadded:: 3.3
-.. function:: pread(fd, buffersize, offset)
+.. function:: pread(fd, n, offset)
- Read from a file descriptor, *fd*, at a position of *offset*. It will read up
- to *buffersize* number of bytes. The file offset remains unchanged.
+ Read at most *n* bytes from file descriptor *fd* at a position of *offset*,
+ leaving the file offset unchanged.
+
+ Return a bytestring containing the bytes read. If the end of the file
+ referred to by *fd* has been reached, an empty bytes object is returned.
Availability: Unix.
.. versionadded:: 3.3
+.. function:: preadv(fd, buffers, offset, flags=0)
+
+ Read from a file descriptor *fd* at a position of *offset* into mutable
+ :term:`bytes-like objects ` *buffers*, leaving the file
+ offset unchanged. Transfer data into each buffer until it is full and then
+ move on to the next buffer in the sequence to hold the rest of the data.
+
+ The flags argument contains a bitwise OR of zero or more of the following
+ flags:
+
+ - :data:`RWF_HIPRI`
+ - :data:`RWF_NOWAIT`
+
+ Return the total number of bytes actually read which can be less than the
+ total capacity of all the objects.
+
+ The operating system may set a limit (:func:`sysconf` value
+ ``'SC_IOV_MAX'``) on the number of buffers that can be used.
+
+ Combine the functionality of :func:`os.readv` and :func:`os.pread`.
+
+ Availability: Linux 2.6.30 and newer, FreeBSD 6.0 and newer,
+ OpenBSD 2.7 and newer. Using flags requires Linux 4.6 or newer.
+
+ .. versionadded:: 3.7
+
+
+.. data:: RWF_NOWAIT
+
+ Do not wait for data which is not immediately available. If this flag is
+ specified, the system call will return instantly if it would have to read
+ data from the backing storage or wait for a lock.
+
+ If some data was successfully read, it will return the number of bytes read.
+ If no bytes were read, it will return ``-1`` and set errno to
+ :data:`errno.EAGAIN`.
+
+ Availability: Linux 4.14 and newer.
+
+ .. versionadded:: 3.7
+
+
+.. data:: RWF_HIPRI
+
+ High priority read/write. Allows block-based filesystems to use polling
+ of the device, which provides lower latency, but may use additional
+ resources.
+
+ Currently, on Linux, this feature is usable only on a file descriptor opened
+ using the :data:`O_DIRECT` flag.
+
+ Availability: Linux 4.6 and newer.
+
+ .. versionadded:: 3.7
+
+
.. function:: pwrite(fd, str, offset)
- Write *bytestring* to a file descriptor, *fd*, from *offset*,
- leaving the file offset unchanged.
+ Write the bytestring in *str* to file descriptor *fd* at position of
+ *offset*, leaving the file offset unchanged.
+
+ Return the number of bytes actually written.
Availability: Unix.
@@ -1104,48 +1165,57 @@ or `the MSDN `_ on Windo
.. function:: pwritev(fd, buffers, offset, flags=0)
- Combines the functionality of :func:`os.writev` and :func:`os.pwrite`. It
- writes the contents of *buffers* to file descriptor *fd* at offset *offset*.
- *buffers* must be a sequence of :term:`bytes-like objects `.
- Buffers are processed in array order. Entire contents of first buffer is written
- before proceeding to second, and so on. The operating system may set a limit
- (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.
- :func:`~os.pwritev` writes the contents of each object to the file descriptor
- and returns the total number of bytes written.
+ Write the *buffers* contents to file descriptor *fd* at a offset *offset*,
+ leaving the file offset unchanged. *buffers* must be a sequence of
+ :term:`bytes-like objects `. Buffers are processed in
+ array order. Entire contents of the first buffer is written before
+ proceeding to the second, and so on.
- The *flags* argument contains a bitwise OR of zero or more of the following
+ The flags argument contains a bitwise OR of zero or more of the following
flags:
- - RWF_DSYNC
- - RWF_SYNC
+ - :data:`RWF_DSYNC`
+ - :data:`RWF_SYNC`
+
+ Return the total number of bytes actually written.
+
+ The operating system may set a limit (:func:`sysconf` value
+ ``'SC_IOV_MAX'``) on the number of buffers that can be used.
- Using non-zero flags requires Linux 4.7 or newer.
+ Combine the functionality of :func:`os.writev` and :func:`os.pwrite`.
- Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer,
- OpenBSD (version 2.7 and newer).
+ Availability: Linux 2.6.30 and newer, FreeBSD 6.0 and newer,
+ OpenBSD 2.7 and newer. Using flags requires Linux 4.7 or newer.
.. versionadded:: 3.7
-.. data:: RWF_DSYNC (since Linux 4.7)
- Provide a per-write equivalent of the O_DSYNC open(2) flag. This flag
- is meaningful only for pwritev2(), and its effect applies only to the
- data range written by the system call.
+
+.. data:: RWF_DSYNC
+
+ Provide a per-write equivalent of the :data:`O_DSYNC` ``open(2)`` flag. This
+ flag effect applies only to the data range written by the system call.
+
+ Availability: Linux 4.7 and newer.
.. versionadded:: 3.7
-.. data:: RWF_SYNC (since Linux 4.7)
- Provide a per-write equivalent of the O_SYNC open(2) flag. This flag is
- meaningful only for pwritev2(), and its effect applies only to the data
- range written by the system call.
+
+.. data:: RWF_SYNC
+
+ Provide a per-write equivalent of the :data:`O_SYNC` ``open(2)`` flag. This
+ flag effect applies only to the data range written by the system call.
+
+ Availability: Linux 4.7 and newer.
.. versionadded:: 3.7
.. function:: read(fd, n)
- Read at most *n* bytes from file descriptor *fd*. Return a bytestring containing the
- bytes read. If the end of the file referred to by *fd* has been reached, an
- empty bytes object is returned.
+ Read at most *n* bytes from file descriptor *fd*.
+
+ Return a bytestring containing the bytes read. If the end of the file
+ referred to by *fd* has been reached, an empty bytes object is returned.
.. note::
@@ -1224,60 +1294,19 @@ or `the MSDN `_ on Windo
.. function:: readv(fd, buffers)
Read from a file descriptor *fd* into a number of mutable :term:`bytes-like
- objects ` *buffers*. :func:`~os.readv` will transfer data
- into each buffer until it is full and then move on to the next buffer in the
- sequence to hold the rest of the data. :func:`~os.readv` returns the total
- number of bytes read (which may be less than the total capacity of all the
- objects).
-
- Availability: Unix.
-
- .. versionadded:: 3.3
-
-
-.. function:: preadv(fd, buffers, offset, flags=0)
-
- Combines the functionality of :func:`os.readv` and :func:`os.pread`. It
- reads from a file descriptor *fd* into a number of mutable :term:`bytes-like
- objects ` *buffers*. As :func:`os.readv`, it will transfer
- data into each buffer until it is full and then move on to the next buffer in
- the sequence to hold the rest of the data. Its fourth argument, *offset*,
- specifies the file offset at which the input operation is to be performed.
- :func:`~os.preadv` return the total number of bytes read (which can be less than
- the total capacity of all the objects).
-
- The flags argument contains a bitwise OR of zero or more of the following
- flags:
+ objects ` *buffers*. Transfer data into each buffer until
+ it is full and then move on to the next buffer in the sequence to hold the
+ rest of the data.
- - RWF_HIPRI
- - RWF_NOWAIT
+ Return the total number of bytes actually read which can be less than the
+ total capacity of all the objects.
- Using non-zero flags requires Linux 4.6 or newer.
+ The operating system may set a limit (:func:`sysconf` value
+ ``'SC_IOV_MAX'``) on the number of buffers that can be used.
- Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer,
- OpenBSD (version 2.7 and newer).
-
- .. versionadded:: 3.7
-
-
-.. data:: RWF_HIPRI (since Linux 4.6)
- High priority read/write. Allows block-based filesystems to use polling
- of the device, which provides lower latency, but may use additional
- resources. (Currently, this feature is usable only on a file descriptor
- opened using the O_DIRECT flag.)
-
- .. versionadded:: 3.7
-
-
-.. data:: RWF_NOWAIT (since Linux 4.14)
- Do not wait for data which is not immediately available. If this flag
- is specified, the preadv2() system call will return instantly
- if it would have to read data from the backing storage or wait for a lock.
- If some data was successfully read, it will return the number of bytes
- read. If no bytes were read, it will return -1 and set errno to EAGAIN.
- Currently, this flag is meaningful only for preadv2().
+ Availability: Unix.
- .. versionadded:: 3.7
+ .. versionadded:: 3.3
.. function:: tcgetpgrp(fd)
@@ -1307,8 +1336,9 @@ or `the MSDN `_ on Windo
.. function:: write(fd, str)
- Write the bytestring in *str* to file descriptor *fd*. Return the number of
- bytes actually written.
+ Write the bytestring in *str* to file descriptor *fd*.
+
+ Return the number of bytes actually written.
.. note::
@@ -1326,14 +1356,15 @@ or `the MSDN `_ on Windo
.. function:: writev(fd, buffers)
- Write the contents of *buffers* to file descriptor *fd*. *buffers* must be a
- sequence of :term:`bytes-like objects `. Buffers are
- processed in array order. Entire contents of first buffer is written before
- proceeding to second, and so on. The operating system may set a limit
- (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.
+ Write the contents of *buffers* to file descriptor *fd*. *buffers* must be
+ a sequence of :term:`bytes-like objects `. Buffers are
+ processed in array order. Entire contents of the first buffer is written
+ before proceeding to the second, and so on.
- :func:`~os.writev` writes the contents of each object to the file descriptor
- and returns the total number of bytes written.
+ Returns the total number of bytes actually written.
+
+ The operating system may set a limit (:func:`sysconf` value
+ ``'SC_IOV_MAX'``) on the number of buffers that can be used.
Availability: Unix.
@@ -2366,7 +2397,13 @@ features:
.. attribute:: st_ino
- Inode number.
+ Platform dependent, but if non-zero, uniquely identifies the
+ file for a given value of ``st_dev``. Typically:
+
+ * the inode number on Unix,
+ * the `file index
+ `_ on
+ Windows
.. attribute:: st_dev
@@ -2525,6 +2562,10 @@ features:
.. versionadded:: 3.5
Added the :attr:`st_file_attributes` member on Windows.
+ .. versionchanged:: 3.5
+ Windows now returns the file index as :attr:`st_ino` when
+ available.
+
.. versionadded:: 3.7
Added the :attr:`st_fstype` member to Solaris/derivatives.
@@ -2812,7 +2853,7 @@ features:
no effect on the behavior of the walk, because in bottom-up mode the directories
in *dirnames* are generated before *dirpath* itself is generated.
- By default, errors from the :func:`listdir` call are ignored. If optional
+ By default, errors from the :func:`scandir` call are ignored. If optional
argument *onerror* is specified, it should be a function; it will be called with
one argument, an :exc:`OSError` instance. It can report the error to continue
with the walk, or raise the exception to abort the walk. Note that the filename
diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst
index ec40c0b93abf7b..522bb7e092427b 100644
--- a/Doc/library/ossaudiodev.rst
+++ b/Doc/library/ossaudiodev.rst
@@ -14,7 +14,7 @@ the standard audio interface for Linux and recent versions of FreeBSD.
.. Things will get more complicated for future Linux versions, since
ALSA is in the standard kernel as of 2.5.x. Presumably if you
use ALSA, you'll have to make sure its OSS compatibility layer
- is active to use ossaudiodev, but you're gonna need it for the vast
+ is active to use ossaudiodev, but you're going to need it for the vast
majority of Linux audio apps anyway.
Sounds like things are also complicated for other BSDs. In response
@@ -447,4 +447,3 @@ The remaining methods are specific to audio mixing:
microphone input::
mixer.setrecsrc (1 << ossaudiodev.SOUND_MIXER_MIC)
-
diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index 3d28a1bea30315..ec604f6815937d 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -559,7 +559,8 @@ Pure paths provide the following methods and properties:
.. method:: PurePath.with_suffix(suffix)
Return a new path with the :attr:`suffix` changed. If the original path
- doesn't have a suffix, the new *suffix* is appended instead::
+ doesn't have a suffix, the new *suffix* is appended instead. If the
+ *suffix* is an empty string, the original suffix is removed::
>>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz')
>>> p.with_suffix('.bz2')
@@ -567,6 +568,9 @@ Pure paths provide the following methods and properties:
>>> p = PureWindowsPath('README')
>>> p.with_suffix('.txt')
PureWindowsPath('README.txt')
+ >>> p = PureWindowsPath('README.txt')
+ >>> p.with_suffix('')
+ PureWindowsPath('README')
.. _concrete-paths:
@@ -911,7 +915,8 @@ call fails (for example because the path doesn't exist):
>>> p.read_text()
'Text file contents'
- The optional parameters have the same meaning as in :func:`open`.
+ The file is opened and then closed. The optional parameters have the same
+ meaning as in :func:`open`.
.. versionadded:: 3.5
diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst
index c6720cb5997436..a72876f3f5a8c0 100644
--- a/Doc/library/pdb.rst
+++ b/Doc/library/pdb.rst
@@ -344,7 +344,7 @@ by the local file.
Specifying any command resuming execution
(currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`,
:pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations)
- terminates the command :pdbcmd:`list` (as if
+ terminates the command list (as if
that command was immediately followed by end). This is because any time you
resume execution (even with a simple next or step), you may encounter another
breakpoint—which could have its own command list, leading to ambiguities about
diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst
index d0c4cf937c8ac5..2b10ee2eb8ee30 100644
--- a/Doc/library/pickle.rst
+++ b/Doc/library/pickle.rst
@@ -510,7 +510,7 @@ methods:
.. method:: object.__getnewargs__()
- This method serve a similar purpose as :meth:`__getnewargs_ex__`, but
+ This method serves a similar purpose as :meth:`__getnewargs_ex__`, but
supports only positional arguments. It must return a tuple of arguments
``args`` which will be passed to the :meth:`__new__` method upon unpickling.
diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst
index eea0abbae4d41e..7d29dc186b67c7 100644
--- a/Doc/library/platform.rst
+++ b/Doc/library/platform.rst
@@ -248,7 +248,8 @@ Unix Platforms
This is another name for :func:`linux_distribution`.
- .. deprecated-removed:: 3.5 3.7
+ .. deprecated-removed:: 3.5 3.8
+ See alternative like the `distro `_ package.
.. function:: linux_distribution(distname='', version='', id='', supported_dists=('SuSE','debian','redhat','mandrake',...), full_distribution_name=1)
@@ -266,9 +267,10 @@ Unix Platforms
parameters. ``id`` is the item in parentheses after the version number. It
is usually the version codename.
- .. deprecated-removed:: 3.5 3.7
+ .. deprecated-removed:: 3.5 3.8
+ See alternative like the `distro `_ package.
-.. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=2048)
+.. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=16384)
Tries to determine the libc version against which the file executable (defaults
to the Python interpreter) is linked. Returns a tuple of strings ``(lib,
diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst
index 20c086c8b524a5..8bd6b63a8ee50b 100644
--- a/Doc/library/plistlib.rst
+++ b/Doc/library/plistlib.rst
@@ -63,9 +63,7 @@ This module defines the following functions:
:class:`Data`.
The *dict_type* is the type used for dictionaries that are read from the
- plist file. The exact structure of the plist can be recovered by using
- :class:`collections.OrderedDict` (although the order of keys shouldn't be
- important in plist files).
+ plist file.
XML data for the :data:`FMT_XML` format is parsed using the Expat parser
from :mod:`xml.parsers.expat` -- see its documentation for possible
diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst
index 65d94fe386c817..aa97d3eabafabd 100644
--- a/Doc/library/pprint.rst
+++ b/Doc/library/pprint.rst
@@ -212,12 +212,12 @@ Example
-------
To demonstrate several uses of the :func:`pprint` function and its parameters,
-let's fetch information about a project from `PyPI `_::
+let's fetch information about a project from `PyPI `_::
>>> import json
>>> import pprint
>>> from urllib.request import urlopen
- >>> with urlopen('http://pypi.python.org/pypi/Twisted/json') as url:
+ >>> with urlopen('http://pypi.org/project/Twisted/json') as url:
... http_info = url.info()
... raw_data = url.read().decode(http_info.get_content_charset())
>>> project_info = json.loads(raw_data)
@@ -248,9 +248,9 @@ In its basic form, :func:`pprint` shows the whole object::
'maintainer': '',
'maintainer_email': '',
'name': 'Twisted',
- 'package_url': 'http://pypi.python.org/pypi/Twisted',
+ 'package_url': 'http://pypi.org/project/Twisted',
'platform': 'UNKNOWN',
- 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0',
+ 'release_url': 'http://pypi.org/project/Twisted/12.3.0',
'requires_python': None,
'stable_version': None,
'summary': 'An asynchronous networking framework written in Python',
@@ -264,7 +264,7 @@ In its basic form, :func:`pprint` shows the whole object::
'python_version': 'source',
'size': 2615733,
'upload_time': '2012-12-26T12:47:03',
- 'url': 'https://pypi.python.org/packages/source/T/Twisted/Twisted-12.3.0.tar.bz2'},
+ 'url': 'https://pypi.org/packages/source/T/Twisted/Twisted-12.3.0.tar.bz2'},
{'comment_text': '',
'downloads': 5224,
'filename': 'Twisted-12.3.0.win32-py2.7.msi',
@@ -274,7 +274,7 @@ In its basic form, :func:`pprint` shows the whole object::
'python_version': '2.7',
'size': 2916352,
'upload_time': '2012-12-26T12:48:15',
- 'url': 'https://pypi.python.org/packages/2.7/T/Twisted/Twisted-12.3.0.win32-py2.7.msi'}]}
+ 'url': 'https://pypi.org/packages/2.7/T/Twisted/Twisted-12.3.0.win32-py2.7.msi'}]}
The result can be limited to a certain *depth* (ellipsis is used for deeper
contents)::
@@ -301,9 +301,9 @@ contents)::
'maintainer': '',
'maintainer_email': '',
'name': 'Twisted',
- 'package_url': 'http://pypi.python.org/pypi/Twisted',
+ 'package_url': 'http://pypi.org/project/Twisted',
'platform': 'UNKNOWN',
- 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0',
+ 'release_url': 'http://pypi.org/project/Twisted/12.3.0',
'requires_python': None,
'stable_version': None,
'summary': 'An asynchronous networking framework written in Python',
@@ -339,9 +339,9 @@ cannot be split, the specified width will be exceeded::
'maintainer': '',
'maintainer_email': '',
'name': 'Twisted',
- 'package_url': 'http://pypi.python.org/pypi/Twisted',
+ 'package_url': 'http://pypi.org/project/Twisted',
'platform': 'UNKNOWN',
- 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0',
+ 'release_url': 'http://pypi.org/project/Twisted/12.3.0',
'requires_python': None,
'stable_version': None,
'summary': 'An asynchronous networking '
diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst
index a6dc56f43cbcf8..926d7757e8eb34 100644
--- a/Doc/library/profile.rst
+++ b/Doc/library/profile.rst
@@ -297,6 +297,11 @@ functions:
Profile ``func(*args, **kwargs)``
+Note that profiling will only work if the called command/function actually
+returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call
+during the called command/function execution) no profiling results will be
+printed.
+
.. _profile-stats:
The :class:`Stats` Class
@@ -317,11 +322,12 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class.
corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific,
there is *no* file compatibility guaranteed with future versions of this
profiler, and there is no compatibility with files produced by other
- profilers. If several files are provided, all the statistics for identical
- functions will be coalesced, so that an overall view of several processes can
- be considered in a single report. If additional files need to be combined
- with data in an existing :class:`~pstats.Stats` object, the
- :meth:`~pstats.Stats.add` method can be used.
+ profilers, or the same profiler run on a different operating system. If
+ several files are provided, all the statistics for identical functions will
+ be coalesced, so that an overall view of several processes can be considered
+ in a single report. If additional files need to be combined with data in an
+ existing :class:`~pstats.Stats` object, the :meth:`~pstats.Stats.add` method
+ can be used.
Instead of reading the profile data from a file, a :class:`cProfile.Profile`
or :class:`profile.Profile` object can be used as the profile data source.
diff --git a/Doc/library/python.rst b/Doc/library/python.rst
index 440dc6632b83a9..f39613f572884f 100644
--- a/Doc/library/python.rst
+++ b/Doc/library/python.rst
@@ -16,6 +16,7 @@ overview:
builtins.rst
__main__.rst
warnings.rst
+ dataclasses.rst
contextlib.rst
abc.rst
atexit.rst
diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index 9b175f4e96756b..8228d912996063 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -45,7 +45,7 @@ fine-tuning parameters.
.. seealso::
- The third-party `regex `_ module,
+ The third-party `regex `_ module,
which has an API compatible with the standard library :mod:`re` module,
but offers additional functionality and a more thorough Unicode support.
@@ -67,8 +67,8 @@ string *pq* will match AB. This holds unless *A* or *B* contain low precedence
operations; boundary conditions between *A* and *B*; or have numbered group
references. Thus, complex expressions can easily be constructed from simpler
primitive expressions like the ones described here. For details of the theory
-and implementation of regular expressions, consult the Friedl book referenced
-above, or almost any textbook about compiler construction.
+and implementation of regular expressions, consult the Friedl book [Frie09]_,
+or almost any textbook about compiler construction.
A brief explanation of the format of regular expressions follows. For further
information and a gentler presentation, consult the :ref:`regex-howto`.
@@ -345,7 +345,7 @@ The special characters are:
This example looks for a word following a hyphen:
- >>> m = re.search('(?<=-)\w+', 'spam-egg')
+ >>> m = re.search(r'(?<=-)\w+', 'spam-egg')
>>> m.group(0)
'egg'
@@ -489,14 +489,6 @@ three digits in length.
Unknown escapes consisting of ``'\'`` and an ASCII letter now are errors.
-.. seealso::
-
- Mastering Regular Expressions
- Book on regular expressions by Jeffrey Friedl, published by O'Reilly. The
- second edition of the book no longer covers Python at all, but the first
- edition covered writing good regular expression patterns in great detail.
-
-
.. _contents-of-module-re:
@@ -1455,8 +1447,8 @@ Finding all Adverbs
^^^^^^^^^^^^^^^^^^^
:func:`findall` matches *all* occurrences of a pattern, not just the first
-one as :func:`search` does. For example, if one was a writer and wanted to
-find all of the adverbs in some text, he or she might use :func:`findall` in
+one as :func:`search` does. For example, if a writer wanted to
+find all of the adverbs in some text, they might use :func:`findall` in
the following manner::
>>> text = "He was carefully disguised but captured quickly by police."
@@ -1470,8 +1462,8 @@ Finding all Adverbs and their Positions
If one wants more information about all matches of a pattern than the matched
text, :func:`finditer` is useful as it provides :ref:`match objects
` instead of strings. Continuing with the previous example, if
-one was a writer who wanted to find all of the adverbs *and their positions* in
-some text, he or she would use :func:`finditer` in the following manner::
+a writer wanted to find all of the adverbs *and their positions* in
+some text, they would use :func:`finditer` in the following manner::
>>> text = "He was carefully disguised but captured quickly by police."
>>> for m in re.finditer(r"\w+ly", text):
@@ -1582,3 +1574,9 @@ The tokenizer produces the following output::
Token(typ='END', value=';', line=4, column=27)
Token(typ='ENDIF', value='ENDIF', line=5, column=4)
Token(typ='END', value=';', line=5, column=9)
+
+
+.. [Frie09] Friedl, Jeffrey. Mastering Regular Expressions. 3rd ed., O'Reilly
+ Media, 2009. The third edition of the book no longer covers Python at all,
+ but the first edition covered writing good regular expression patterns in
+ great detail.
diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst
index 837816e952ad56..16c28cf7d02fb9 100644
--- a/Doc/library/readline.rst
+++ b/Doc/library/readline.rst
@@ -17,11 +17,18 @@ made using this module affect the behaviour of both the interpreter's
interactive prompt and the prompts offered by the built-in :func:`input`
function.
+Readline keybindings may be configured via an initialization file, typically
+``.inputrc`` in your home directory. See `Readline Init File
+`_
+in the GNU Readline manual for information about the format and
+allowable constructs of that file, and the capabilities of the
+Readline library in general.
+
.. note::
The underlying Readline library API may be implemented by
the ``libedit`` library instead of GNU readline.
- On MacOS X the :mod:`readline` module detects which library is being used
+ On macOS the :mod:`readline` module detects which library is being used
at run time.
The configuration file for ``libedit`` is different from that
@@ -29,12 +36,13 @@ function.
you can check for the text "libedit" in :const:`readline.__doc__`
to differentiate between GNU readline and libedit.
-Readline keybindings may be configured via an initialization file, typically
-``.inputrc`` in your home directory. See `Readline Init File
-`_
-in the GNU Readline manual for information about the format and
-allowable constructs of that file, and the capabilities of the
-Readline library in general.
+ If you use *editline*/``libedit`` readline emulation on macOS, the
+ initialization file located in your home directory is named
+ ``.editrc``. For example, the following content in ``~/.editrc`` will
+ turn ON *vi* keybindings and TAB completion::
+
+ python:bind -v
+ python:bind ^I rl_complete
Init file
diff --git a/Doc/library/select.rst b/Doc/library/select.rst
index e252e7adb9246a..5ffb21583fa599 100644
--- a/Doc/library/select.rst
+++ b/Doc/library/select.rst
@@ -57,7 +57,16 @@ The module defines the following:
(Only supported on Linux 2.5.44 and newer.) Return an edge polling object,
which can be used as Edge or Level Triggered interface for I/O
- events. *sizehint* and *flags* are deprecated and completely ignored.
+ events.
+
+ *sizehint* informs epoll about the expected number of events to be
+ registered. It must be positive, or `-1` to use the default. It is only
+ used on older systems where :c:func:`epoll_create1` is not available;
+ otherwise it has no effect (though its value is still checked).
+
+ *flags* is deprecated and completely ignored. However, when supplied, its
+ value must be ``0`` or ``select.EPOLL_CLOEXEC``, otherwise ``OSError`` is
+ raised.
See the :ref:`epoll-objects` section below for the methods supported by
epolling objects.
diff --git a/Doc/library/site.rst b/Doc/library/site.rst
index 43daf790b77cb0..5b5ed93a1b7b63 100644
--- a/Doc/library/site.rst
+++ b/Doc/library/site.rst
@@ -97,12 +97,12 @@ not mentioned in either path configuration file.
After these path manipulations, an attempt is made to import a module named
:mod:`sitecustomize`, which can perform arbitrary site-specific customizations.
It is typically created by a system administrator in the site-packages
-directory. If this import fails with an :exc:`ImportError` exception, it is
-silently ignored. If Python is started without output streams available, as
+directory. If this import fails with an :exc:`ImportError` or its subclass
+exception, and the exception's :attr:`name` attribute equals to ``'sitecustomize'``,
+it is silently ignored. If Python is started without output streams available, as
with :file:`pythonw.exe` on Windows (which is used by default to start IDLE),
-attempted output from :mod:`sitecustomize` is ignored. Any exception other
-than :exc:`ImportError` causes a silent and perhaps mysterious failure of the
-process.
+attempted output from :mod:`sitecustomize` is ignored. Any other exception
+causes a silent and perhaps mysterious failure of the process.
.. index:: module: usercustomize
@@ -110,7 +110,9 @@ After this, an attempt is made to import a module named :mod:`usercustomize`,
which can perform arbitrary user-specific customizations, if
:data:`ENABLE_USER_SITE` is true. This file is intended to be created in the
user site-packages directory (see below), which is part of ``sys.path`` unless
-disabled by :option:`-s`. An :exc:`ImportError` will be silently ignored.
+disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or
+its subclass exception, and the exception's :attr:`name` attribute equals to
+``'usercustomize'``, it is silently ignored.
Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are
empty, and the path manipulations are skipped; however the import of
@@ -220,7 +222,7 @@ Module contents
The :mod:`site` module also provides a way to get the user directories from the
command line:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 -m site --user-site
/home/user/.local/lib/python3.3/site-packages
@@ -243,7 +245,7 @@ If it is called without arguments, it will print the contents of
If both options are given, user base and user site will be printed (always in
this order), separated by :data:`os.pathsep`.
-If any option is given, the script will exit with one of these values: ``O`` if
+If any option is given, the script will exit with one of these values: ``0`` if
the user site-packages directory is enabled, ``1`` if it was disabled by the
user, ``2`` if it is disabled for security reasons or by an administrator, and a
value greater than 2 if there is an error.
diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index 86e769e6a1f8dc..e5effd0306a401 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -271,7 +271,7 @@ An :class:`SMTP` instance has the following methods:
.. method:: SMTP.ehlo_or_helo_if_needed()
- This method call :meth:`ehlo` and or :meth:`helo` if there has been no
+ This method calls :meth:`ehlo` and/or :meth:`helo` if there has been no
previous ``EHLO`` or ``HELO`` command this session. It tries ESMTP ``EHLO``
first.
@@ -379,16 +379,23 @@ An :class:`SMTP` instance has the following methods:
commands that follow will be encrypted. You should then call :meth:`ehlo`
again.
- If *keyfile* and *certfile* are provided, these are passed to the :mod:`socket`
- module's :func:`ssl` function.
+ If *keyfile* and *certfile* are provided, they are used to create an
+ :class:`ssl.SSLContext`.
- Optional *context* parameter is a :class:`ssl.SSLContext` object; This is
+ Optional *context* parameter is an :class:`ssl.SSLContext` object; This is
an alternative to using a keyfile and a certfile and if specified both
*keyfile* and *certfile* should be ``None``.
If there has been no previous ``EHLO`` or ``HELO`` command this session,
this method tries ESMTP ``EHLO`` first.
+ .. deprecated:: 3.6
+
+ *keyfile* and *certfile* are deprecated in favor of *context*.
+ Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let
+ :func:`ssl.create_default_context` select the system's trusted CA
+ certificates for you.
+
:exc:`SMTPHeloError`
The server didn't reply properly to the ``HELO`` greeting.
diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
index 7f0d4ede7ccf4c..cea01755777165 100644
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -70,6 +70,13 @@ created. Socket addresses are represented as follows:
notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``,
and *port* is an integer.
+ - For IPv4 addresses, two special forms are accepted instead of a host
+ address: ``''`` represents :const:`INADDR_ANY`, which is used to bind to all
+ interfaces, and the string ``''`` represents
+ :const:`INADDR_BROADCAST`. This behavior is not compatible with IPv6,
+ therefore, you may want to avoid these if you intend to support IPv6 with your
+ Python programs.
+
- For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo,
scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo``
and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For
@@ -77,6 +84,11 @@ created. Socket addresses are represented as follows:
backward compatibility. Note, however, omission of *scopeid* can cause problems
in manipulating scoped IPv6 addresses.
+ .. versionchanged:: 3.7
+ For multicast addresses (with *scopeid* meaningful) *address* may not contain
+ ``%scope`` (or ``zone id``) part. This information is superfluous and may
+ be safely omitted (recommended).
+
- :const:`AF_NETLINK` sockets are represented as pairs ``(pid, groups)``.
- Linux-only support for TIPC is available using the :const:`AF_TIPC`
@@ -166,12 +178,6 @@ created. Socket addresses are represented as follows:
.. XXX document them!
-For IPv4 addresses, two special forms are accepted instead of a host address:
-the empty string represents :const:`INADDR_ANY`, and the string
-``''`` represents :const:`INADDR_BROADCAST`. This behavior is not
-compatible with IPv6, therefore, you may want to avoid these if you intend
-to support IPv6 with your Python programs.
-
If you use a hostname in the *host* portion of IPv4/v6 socket address, the
program may show a nondeterministic behavior, as Python uses the first address
returned from the DNS resolution. The socket address will be resolved
@@ -315,9 +321,16 @@ Constants
``SO_DOMAIN``, ``SO_PROTOCOL``, ``SO_PEERSEC``, ``SO_PASSSEC``,
``TCP_USER_TIMEOUT``, ``TCP_CONGESTION`` were added.
+ .. versionchanged:: 3.6.5
+ On Windows, ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` appear if run-time Windows
+ supports.
+
.. versionchanged:: 3.7
``TCP_NOTSENT_LOWAT`` was added.
+ On Windows, ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` appear if run-time Windows
+ supports.
+
.. data:: AF_CAN
PF_CAN
SOL_CAN_*
@@ -635,6 +648,10 @@ The :mod:`socket` module also offers various network-related services:
.. versionchanged:: 3.2
parameters can now be passed using keyword arguments.
+ .. versionchanged:: 3.7
+ for IPv6 multicast addresses, string representing an address will not
+ contain ``%scope`` part.
+
.. function:: getfqdn([name])
Return a fully qualified domain name for *name*. If *name* is omitted or empty,
@@ -693,6 +710,8 @@ The :mod:`socket` module also offers various network-related services:
or numeric address representation in *host*. Similarly, *port* can contain a
string port name or a numeric port number.
+ For IPv6 addresses, ``%scope`` is appended to the host part if *sockaddr*
+ contains meaningful *scopeid*. Usually this happens for multicast addresses.
.. function:: getprotobyname(protocolname)
@@ -1193,6 +1212,10 @@ to sockets.
an exception, the method now retries the system call instead of raising
an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+ .. versionchanged:: 3.7
+ For multicast IPv6 address, first item of *address* does not contain
+ ``%scope`` part anymore. In order to get full IPv6 address use
+ :func:`getnameinfo`.
.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]])
diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst
index 5600027383754d..1b3062da6df9d9 100644
--- a/Doc/library/socketserver.rst
+++ b/Doc/library/socketserver.rst
@@ -116,10 +116,13 @@ server classes.
only available on POSIX platforms that support :func:`~os.fork`.
:meth:`socketserver.ForkingMixIn.server_close` waits until all child
- processes complete.
+ processes complete, except if
+ :attr:`socketserver.ForkingMixIn.block_on_close` attribute is false.
:meth:`socketserver.ThreadingMixIn.server_close` waits until all non-daemon
- threads complete. Use daemonic threads by setting
+ threads complete, except if
+ :attr:`socketserver.ThreadingMixIn.block_on_close` attribute is false. Use
+ daemonic threads by setting
:data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads
complete.
@@ -128,6 +131,8 @@ server classes.
:meth:`socketserver.ForkingMixIn.server_close` and
:meth:`socketserver.ThreadingMixIn.server_close` now waits until all
child processes and non-daemonic threads complete.
+ Add a new :attr:`socketserver.ForkingMixIn.block_on_close` class
+ attribute to opt-in for the pre-3.7 behaviour.
.. class:: ForkingTCPServer
diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index e7676a9f3a5082..e6aeee76a0d271 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -236,8 +236,8 @@ Module functions and constants
Registers a callable to convert a bytestring from the database into a custom
Python type. The callable will be invoked for all database values that are of
the type *typename*. Confer the parameter *detect_types* of the :func:`connect`
- function for how the type detection works. Note that the case of *typename* and
- the name of the type in your query must match!
+ function for how the type detection works. Note that *typename* and the name of
+ the type in your query are matched in case-insensitive manner.
.. function:: register_adapter(type, callable)
@@ -281,7 +281,7 @@ Connection Objects
.. attribute:: isolation_level
- Get or set the current isolation level. :const:`None` for autocommit mode or
+ Get or set the current default isolation level. :const:`None` for autocommit mode or
one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section
:ref:`sqlite3-controlling-transactions` for a more detailed explanation.
@@ -532,6 +532,56 @@ Connection Objects
f.write('%s\n' % line)
+ .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250)
+
+ This method makes a backup of a SQLite database even while it's being accessed
+ by other clients, or concurrently by the same connection. The copy will be
+ written into the mandatory argument *target*, that must be another
+ :class:`Connection` instance.
+
+ By default, or when *pages* is either ``0`` or a negative integer, the entire
+ database is copied in a single step; otherwise the method performs a loop
+ copying up to *pages* pages at a time.
+
+ If *progress* is specified, it must either be ``None`` or a callable object that
+ will be executed at each iteration with three integer arguments, respectively
+ the *status* of the last iteration, the *remaining* number of pages still to be
+ copied and the *total* number of pages.
+
+ The *name* argument specifies the database name that will be copied: it must be
+ a string containing either ``"main"``, the default, to indicate the main
+ database, ``"temp"`` to indicate the temporary database or the name specified
+ after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached
+ database.
+
+ The *sleep* argument specifies the number of seconds to sleep by between
+ successive attempts to backup remaining pages, can be specified either as an
+ integer or a floating point value.
+
+ Example 1, copy an existing database into another::
+
+ import sqlite3
+
+ def progress(status, remaining, total):
+ print(f'Copied {total-remaining} of {total} pages...')
+
+ con = sqlite3.connect('existing_db.db')
+ with sqlite3.connect('backup.db') as bck:
+ con.backup(bck, pages=1, progress=progress)
+
+ Example 2, copy an existing database into a transient copy::
+
+ import sqlite3
+
+ source = sqlite3.connect('existing_db.db')
+ dest = sqlite3.connect(':memory:')
+ source.backup(dest)
+
+ Availability: SQLite 3.6.11 or higher
+
+ .. versionadded:: 3.7
+
+
.. _sqlite3-cursor-objects:
Cursor Objects
@@ -771,6 +821,20 @@ Exceptions
exists, syntax error in the SQL statement, wrong number of parameters
specified, etc. It is a subclass of :exc:`DatabaseError`.
+.. exception:: OperationalError
+
+ Exception raised for errors that are related to the database's operation
+ and not necessarily under the control of the programmer, e.g. an unexpected
+ disconnect occurs, the data source name is not found, a transaction could
+ not be processed, etc. It is a subclass of :exc:`DatabaseError`.
+
+.. exception:: NotSupportedError
+
+ Exception raised in case a method or database API was used which is not
+ supported by the database, e.g. calling the :meth:`~Connection.rollback`
+ method on a connection that does not support transaction or has
+ transactions turned off. It is a subclass of :exc:`DatabaseError`.
+
.. _sqlite3-types:
@@ -939,22 +1003,30 @@ timestamp converter.
Controlling Transactions
------------------------
-By default, the :mod:`sqlite3` module opens transactions implicitly before a
-Data Modification Language (DML) statement (i.e.
-``INSERT``/``UPDATE``/``DELETE``/``REPLACE``).
+The underlying ``sqlite3`` library operates in ``autocommit`` mode by default,
+but the Python :mod:`sqlite3` module by default does not.
-You can control which kind of ``BEGIN`` statements sqlite3 implicitly executes
-(or none at all) via the *isolation_level* parameter to the :func:`connect`
-call, or via the :attr:`isolation_level` property of connections.
+``autocommit`` mode means that statements that modify the database take effect
+immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit``
+mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the
+outermost transaction, turns ``autocommit`` mode back on.
-If you want **autocommit mode**, then set :attr:`isolation_level` to ``None``.
-
-Otherwise leave it at its default, which will result in a plain "BEGIN"
-statement, or set it to one of SQLite's supported isolation levels: "DEFERRED",
-"IMMEDIATE" or "EXCLUSIVE".
+The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement
+implicitly before a Data Modification Language (DML) statement (i.e.
+``INSERT``/``UPDATE``/``DELETE``/``REPLACE``).
-The current transaction state is exposed through the
-:attr:`Connection.in_transaction` attribute of the connection object.
+You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly
+executes via the *isolation_level* parameter to the :func:`connect`
+call, or via the :attr:`isolation_level` property of connections.
+If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is
+equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE``
+and ``EXCLUSIVE``.
+
+You can disable the :mod:`sqlite3` module's implicit transaction management by
+setting :attr:`isolation_level` to ``None``. This will leave the underlying
+``sqlite3`` library operating in ``autocommit`` mode. You can then completely
+control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``,
+``SAVEPOINT``, and ``RELEASE`` statements in your code.
.. versionchanged:: 3.6
:mod:`sqlite3` used to implicitly commit an open transaction before DDL
diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index aa1075d4b02b09..99d1d774b4aed4 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -49,6 +49,9 @@ For more sophisticated applications, the :class:`ssl.SSLContext` class
helps manage settings and certificates, which can then be inherited
by SSL sockets created through the :meth:`SSLContext.wrap_socket` method.
+.. versionchanged:: 3.5.3
+ Updated to support linking with OpenSSL 1.1.0
+
.. versionchanged:: 3.6
OpenSSL 0.9.8, 1.0.0 and 1.0.1 are deprecated and no longer supported.
@@ -59,6 +62,120 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method.
Functions, Constants, and Exceptions
------------------------------------
+
+Socket creation
+^^^^^^^^^^^^^^^
+
+Since Python 3.2 and 2.7.9, it is recommended to use the
+:meth:`SSLContext.wrap_socket` of an :class:`SSLContext` instance to wrap
+sockets as :class:`SSLSocket` objects. The helper functions
+:func:`create_default_context` returns a new context with secure default
+settings. The old :func:`wrap_socket` function is deprecated since it is
+both inefficient and has no support for server name indication (SNI) and
+hostname matching.
+
+Client socket example with default context and IPv4/IPv6 dual stack::
+
+ import socket
+ import ssl
+
+ hostname = 'www.python.org'
+ context = ssl.create_default_context()
+
+ with socket.create_connection((hostname, 443)) as sock:
+ with context.wrap_socket(sock, server_hostname=hostname) as ssock:
+ print(ssock.version())
+
+
+Client socket example with custom context and IPv4::
+
+ hostname = 'www.python.org'
+ # PROTOCOL_TLS_CLIENT requires valid cert chain and hostname
+ context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+ context.load_verify_locations('path/to/cabundle.pem')
+
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock:
+ with context.wrap_socket(sock, server_hostname=hostname) as ssock:
+ print(ssock.version())
+
+
+Server socket example listening on localhost IPv4::
+
+ context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ context.load_cert_chain('/path/to/certchain.pem', '/path/to/private.key')
+
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock:
+ sock.bind(('127.0.0.1', 8443))
+ sock.listen(5)
+ with context.wrap_socket(sock, server_side=True) as ssock:
+ conn, addr = ssock.accept()
+ ...
+
+
+Context creation
+^^^^^^^^^^^^^^^^
+
+A convenience function helps create :class:`SSLContext` objects for common
+purposes.
+
+.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None)
+
+ Return a new :class:`SSLContext` object with default settings for
+ the given *purpose*. The settings are chosen by the :mod:`ssl` module,
+ and usually represent a higher security level than when calling the
+ :class:`SSLContext` constructor directly.
+
+ *cafile*, *capath*, *cadata* represent optional CA certificates to
+ trust for certificate verification, as in
+ :meth:`SSLContext.load_verify_locations`. If all three are
+ :const:`None`, this function can choose to trust the system's default
+ CA certificates instead.
+
+ The settings are: :data:`PROTOCOL_TLS`, :data:`OP_NO_SSLv2`, and
+ :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and
+ without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH`
+ as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED`
+ and either loads CA certificates (when at least one of *cafile*, *capath* or
+ *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load
+ default CA certificates.
+
+ .. note::
+ The protocol, options, cipher and other settings may change to more
+ restrictive values anytime without prior deprecation. The values
+ represent a fair balance between compatibility and security.
+
+ If your application needs specific settings, you should create a
+ :class:`SSLContext` and apply the settings yourself.
+
+ .. note::
+ If you find that when certain older clients or servers attempt to connect
+ with a :class:`SSLContext` created by this function that they get an error
+ stating "Protocol or cipher suite mismatch", it may be that they only
+ support SSL3.0 which this function excludes using the
+ :data:`OP_NO_SSLv3`. SSL3.0 is widely considered to be `completely broken
+ `_. If you still wish to continue to
+ use this function but still allow SSL 3.0 connections you can re-enable
+ them using::
+
+ ctx = ssl.create_default_context(Purpose.CLIENT_AUTH)
+ ctx.options &= ~ssl.OP_NO_SSLv3
+
+ .. versionadded:: 3.4
+
+ .. versionchanged:: 3.4.4
+
+ RC4 was dropped from the default cipher string.
+
+ .. versionchanged:: 3.6
+
+ ChaCha20/Poly1305 was added to the default cipher string.
+
+ 3DES was dropped from the default cipher string.
+
+
+Exceptions
+^^^^^^^^^^
+
.. exception:: SSLError
Raised to signal an error from the underlying SSL implementation
@@ -152,173 +269,6 @@ Functions, Constants, and Exceptions
The exception is now an alias for :exc:`SSLCertVerificationError`.
-Socket creation
-^^^^^^^^^^^^^^^
-
-The following function allows for standalone socket creation. Starting from
-Python 3.2, it can be more flexible to use :meth:`SSLContext.wrap_socket`
-instead.
-
-.. function:: wrap_socket(sock, keyfile=None, certfile=None, server_side=False, cert_reqs=CERT_NONE, ssl_version={see docs}, ca_certs=None, do_handshake_on_connect=True, suppress_ragged_eofs=True, ciphers=None)
-
- Takes an instance ``sock`` of :class:`socket.socket`, and returns an instance
- of :class:`ssl.SSLSocket`, a subtype of :class:`socket.socket`, which wraps
- the underlying socket in an SSL context. ``sock`` must be a
- :data:`~socket.SOCK_STREAM` socket; other socket types are unsupported.
-
- For client-side sockets, the context construction is lazy; if the
- underlying socket isn't connected yet, the context construction will be
- performed after :meth:`connect` is called on the socket. For
- server-side sockets, if the socket has no remote peer, it is assumed
- to be a listening socket, and the server-side SSL wrapping is
- automatically performed on client connections accepted via the
- :meth:`accept` method. :func:`wrap_socket` may raise :exc:`SSLError`.
-
- The ``keyfile`` and ``certfile`` parameters specify optional files which
- contain a certificate to be used to identify the local side of the
- connection. See the discussion of :ref:`ssl-certificates` for more
- information on how the certificate is stored in the ``certfile``.
-
- The parameter ``server_side`` is a boolean which identifies whether
- server-side or client-side behavior is desired from this socket.
-
- The parameter ``cert_reqs`` specifies whether a certificate is required from
- the other side of the connection, and whether it will be validated if
- provided. It must be one of the three values :const:`CERT_NONE`
- (certificates ignored), :const:`CERT_OPTIONAL` (not required, but validated
- if provided), or :const:`CERT_REQUIRED` (required and validated). If the
- value of this parameter is not :const:`CERT_NONE`, then the ``ca_certs``
- parameter must point to a file of CA certificates.
-
- The ``ca_certs`` file contains a set of concatenated "certification
- authority" certificates, which are used to validate certificates passed from
- the other end of the connection. See the discussion of
- :ref:`ssl-certificates` for more information about how to arrange the
- certificates in this file.
-
- The parameter ``ssl_version`` specifies which version of the SSL protocol to
- use. Typically, the server chooses a particular protocol version, and the
- client must adapt to the server's choice. Most of the versions are not
- interoperable with the other versions. If not specified, the default is
- :data:`PROTOCOL_TLS`; it provides the most compatibility with other
- versions.
-
- Here's a table showing which versions in a client (down the side) can connect
- to which versions in a server (along the top):
-
- .. table::
-
- ======================== ============ ============ ============= ========= =========== ===========
- *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2**
- ------------------------ ------------ ------------ ------------- --------- ----------- -----------
- *SSLv2* yes no no [1]_ no no no
- *SSLv3* no yes no [2]_ no no no
- *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes
- *TLSv1* no no yes yes no no
- *TLSv1.1* no no yes no yes no
- *TLSv1.2* no no yes no no yes
- ======================== ============ ============ ============= ========= =========== ===========
-
- .. rubric:: Footnotes
- .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default.
- .. [2] :class:`SSLContext` disables SSLv3 with :data:`OP_NO_SSLv3` by default.
- .. [3] TLS 1.3 protocol will be available with :data:`PROTOCOL_TLS` in
- OpenSSL >= 1.1.1. There is no dedicated PROTOCOL constant for just
- TLS 1.3.
-
- .. note::
-
- Which connections succeed will vary depending on the version of
- OpenSSL. For example, before OpenSSL 1.0.0, an SSLv23 client
- would always attempt SSLv2 connections.
-
- The *ciphers* parameter sets the available ciphers for this SSL object.
- It should be a string in the `OpenSSL cipher list format
- `_.
-
- The parameter ``do_handshake_on_connect`` specifies whether to do the SSL
- handshake automatically after doing a :meth:`socket.connect`, or whether the
- application program will call it explicitly, by invoking the
- :meth:`SSLSocket.do_handshake` method. Calling
- :meth:`SSLSocket.do_handshake` explicitly gives the program control over the
- blocking behavior of the socket I/O involved in the handshake.
-
- The parameter ``suppress_ragged_eofs`` specifies how the
- :meth:`SSLSocket.recv` method should signal unexpected EOF from the other end
- of the connection. If specified as :const:`True` (the default), it returns a
- normal EOF (an empty bytes object) in response to unexpected EOF errors
- raised from the underlying socket; if :const:`False`, it will raise the
- exceptions back to the caller.
-
- .. versionchanged:: 3.2
- New optional argument *ciphers*.
-
-Context creation
-^^^^^^^^^^^^^^^^
-
-A convenience function helps create :class:`SSLContext` objects for common
-purposes.
-
-.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None)
-
- Return a new :class:`SSLContext` object with default settings for
- the given *purpose*. The settings are chosen by the :mod:`ssl` module,
- and usually represent a higher security level than when calling the
- :class:`SSLContext` constructor directly.
-
- *cafile*, *capath*, *cadata* represent optional CA certificates to
- trust for certificate verification, as in
- :meth:`SSLContext.load_verify_locations`. If all three are
- :const:`None`, this function can choose to trust the system's default
- CA certificates instead.
-
- The settings are: :data:`PROTOCOL_TLS`, :data:`OP_NO_SSLv2`, and
- :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and
- without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH`
- as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED`
- and either loads CA certificates (when at least one of *cafile*, *capath* or
- *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load
- default CA certificates.
-
- .. note::
- The protocol, options, cipher and other settings may change to more
- restrictive values anytime without prior deprecation. The values
- represent a fair balance between compatibility and security.
-
- If your application needs specific settings, you should create a
- :class:`SSLContext` and apply the settings yourself.
-
- .. note::
- If you find that when certain older clients or servers attempt to connect
- with a :class:`SSLContext` created by this function that they get an error
- stating "Protocol or cipher suite mismatch", it may be that they only
- support SSL3.0 which this function excludes using the
- :data:`OP_NO_SSLv3`. SSL3.0 is widely considered to be `completely broken
- `_. If you still wish to continue to
- use this function but still allow SSL 3.0 connections you can re-enable
- them using::
-
- ctx = ssl.create_default_context(Purpose.CLIENT_AUTH)
- ctx.options &= ~ssl.OP_NO_SSLv3
-
- .. versionadded:: 3.4
-
- .. versionchanged:: 3.4.4
-
- RC4 was dropped from the default cipher string.
-
- .. versionchanged:: 3.6
-
- ChaCha20/Poly1305 was added to the default cipher string.
-
- 3DES was dropped from the default cipher string.
-
- .. versionchanged:: 3.7
-
- TLS 1.3 cipher suites TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384,
- and TLS_CHACHA20_POLY1305_SHA256 were added to the default cipher string.
-
-
Random generation
^^^^^^^^^^^^^^^^^
@@ -474,9 +424,10 @@ Certificate handling
PEM-encoded string. If ``ssl_version`` is specified, uses that version of
the SSL protocol to attempt to connect to the server. If ``ca_certs`` is
specified, it should be a file containing a list of root certificates, the
- same format as used for the same parameter in :func:`wrap_socket`. The call
- will attempt to validate the server certificate against that set of root
- certificates, and will fail if the validation attempt fails.
+ same format as used for the same parameter in
+ :meth:`SSLContext.wrap_socket`. The call will attempt to validate the
+ server certificate against that set of root certificates, and will fail
+ if the validation attempt fails.
.. versionchanged:: 3.3
This function is now IPv6-compatible.
@@ -552,6 +503,33 @@ Certificate handling
.. versionadded:: 3.4
+.. function:: wrap_socket(sock, keyfile=None, certfile=None, \
+ server_side=False, cert_reqs=CERT_NONE, ssl_version=PROTOCOL_TLS, \
+ ca_certs=None, do_handshake_on_connect=True, \
+ suppress_ragged_eofs=True, ciphers=None)
+
+ Takes an instance ``sock`` of :class:`socket.socket`, and returns an instance
+ of :class:`ssl.SSLSocket`, a subtype of :class:`socket.socket`, which wraps
+ the underlying socket in an SSL context. ``sock`` must be a
+ :data:`~socket.SOCK_STREAM` socket; other socket types are unsupported.
+
+ Internally, function creates a :class:`SSLContext` with protocol
+ *ssl_version* and :attr:`SSLContext.options` set to *cert_reqs*. If
+ parameters *keyfile*, *certfile*, *ca_certs* or *ciphers* are set, then
+ the values are passed to :meth:`SSLContext.load_cert_chain`,
+ :meth:`SSLContext.load_verify_locations`, and
+ :meth:`SSLContext.set_ciphers`.
+
+ The arguments *server_side*, *do_handshake_on_connect*, and
+ *suppress_ragged_eofs* have the same meaning as
+ :meth:`SSLContext.wrap_socket`.
+
+ .. deprecated:: 3.7
+
+ Since Python 3.2 and 2.7.9, it is recommended to use the
+ :meth:`SSLContext.wrap_socket` instead of :func:`wrap_socket`. The
+ top-level function is limited and creates an insecure client socket
+ without server name indication or hostname matching.
Constants
^^^^^^^^^
@@ -563,20 +541,28 @@ Constants
.. data:: CERT_NONE
Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs``
- parameter to :func:`wrap_socket`. In this mode (the default), no
- certificates will be required from the other side of the socket connection.
- If a certificate is received from the other end, no attempt to validate it
- is made.
+ parameter to :func:`wrap_socket`. Except for :const:`PROTOCOL_TLS_CLIENT`,
+ it is the default mode. With client-side sockets, just about any
+ cert is accepted. Validation errors, such as untrusted or expired cert,
+ are ignored and do not abort the TLS/SSL handshake.
+
+ In server mode, no certificate is requested from the client, so the client
+ does not send any for client cert authentication.
See the discussion of :ref:`ssl-security` below.
.. data:: CERT_OPTIONAL
Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs``
- parameter to :func:`wrap_socket`. In this mode no certificates will be
- required from the other side of the socket connection; but if they
- are provided, validation will be attempted and an :class:`SSLError`
- will be raised on failure.
+ parameter to :func:`wrap_socket`. In client mode, :const:`CERT_OPTIONAL`
+ has the same meaning as :const:`CERT_REQUIRED`. It is recommended to
+ use :const:`CERT_REQUIRED` for client-side sockets instead.
+
+ In server mode, a client certificate request is sent to the client. The
+ client may either ignore the request or send a certificate in order
+ perform TLS client cert authentication. If the client chooses to send
+ a certificate, it is verified. Any verification error immediately aborts
+ the TLS handshake.
Use of this setting requires a valid set of CA certificates to
be passed, either to :meth:`SSLContext.load_verify_locations` or as a
@@ -588,6 +574,15 @@ Constants
parameter to :func:`wrap_socket`. In this mode, certificates are
required from the other side of the socket connection; an :class:`SSLError`
will be raised if no certificate is provided, or if its validation fails.
+ This mode is **not** sufficient to verify a certificate in client mode as
+ it does not match hostnames. :attr:`~SSLContext.check_hostname` must be
+ enabled as well to verify the authenticity of a cert.
+ :const:`PROTOCOL_TLS_CLIENT` uses :const:`CERT_REQUIRED` and
+ enables :attr:`~SSLContext.check_hostname` by default.
+
+ With server socket, this mode provides mandatory TLS client cert
+ authentication. A client certificate request is sent to the client and
+ the client must provide a valid and trusted certificate.
Use of this setting requires a valid set of CA certificates to
be passed, either to :meth:`SSLContext.load_verify_locations` or as a
@@ -782,6 +777,11 @@ Constants
.. versionadded:: 3.2
+ .. deprecated:: 3.7
+ The option is deprecated since OpenSSL 1.1.0, use the new
+ :attr:`SSLContext.minimum_version` and
+ :attr:`SSLContext.maximum_version` instead.
+
.. data:: OP_NO_TLSv1_1
Prevents a TLSv1.1 connection. This option is only applicable in conjunction
@@ -790,6 +790,9 @@ Constants
.. versionadded:: 3.4
+ .. deprecated:: 3.7
+ The option is deprecated since OpenSSL 1.1.0.
+
.. data:: OP_NO_TLSv1_2
Prevents a TLSv1.2 connection. This option is only applicable in conjunction
@@ -798,6 +801,9 @@ Constants
.. versionadded:: 3.4
+ .. deprecated:: 3.7
+ The option is deprecated since OpenSSL 1.1.0.
+
.. data:: OP_NO_TLSv1_3
Prevents a TLSv1.3 connection. This option is only applicable in conjunction
@@ -808,6 +814,19 @@ Constants
.. versionadded:: 3.7
+ .. deprecated:: 3.7
+ The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15,
+ 3.6.3 and 3.7.0 for backwards compatibility with OpenSSL 1.0.2.
+
+.. data:: OP_NO_RENEGOTIATION
+
+ Disable all renegotiation in TLSv1.2 and earlier. Do not send
+ HelloRequest messages, and ignore renegotiation requests via ClientHello.
+
+ This option is only available with OpenSSL 1.1.0h and later.
+
+ .. versionadded:: 3.7
+
.. data:: OP_CIPHER_SERVER_PREFERENCE
Use the server's cipher ordering preference, rather than the client's.
@@ -831,6 +850,15 @@ Constants
.. versionadded:: 3.3
+.. data:: OP_ENABLE_MIDDLEBOX_COMPAT
+
+ Send dummy Change Cipher Spec (CCS) messages in TLS 1.3 handshake to make
+ a TLS 1.3 connection look more like a TLS 1.2 connection.
+
+ This option is only available with OpenSSL 1.1.1 and later.
+
+ .. versionadded:: 3.8
+
.. data:: OP_NO_COMPRESSION
Disable compression on the SSL channel. This is useful if the application
@@ -867,7 +895,7 @@ Constants
.. data:: HAS_ECDH
- Whether the OpenSSL library has built-in support for Elliptic Curve-based
+ Whether the OpenSSL library has built-in support for the Elliptic Curve-based
Diffie-Hellman key exchange. This should be true unless the feature was
explicitly disabled by the distributor.
@@ -882,7 +910,7 @@ Constants
.. data:: HAS_NPN
- Whether the OpenSSL library has built-in support for *Next Protocol
+ Whether the OpenSSL library has built-in support for the *Next Protocol
Negotiation* as described in the `Application Layer Protocol
Negotiation `_.
When true, you can use the :meth:`SSLContext.set_npn_protocols` method to advertise
@@ -890,6 +918,36 @@ Constants
.. versionadded:: 3.3
+.. data:: HAS_SSLv2
+
+ Whether the OpenSSL library has built-in support for the SSL 2.0 protocol.
+
+ .. versionadded:: 3.7
+
+.. data:: HAS_SSLv3
+
+ Whether the OpenSSL library has built-in support for the SSL 3.0 protocol.
+
+ .. versionadded:: 3.7
+
+.. data:: HAS_TLSv1
+
+ Whether the OpenSSL library has built-in support for the TLS 1.0 protocol.
+
+ .. versionadded:: 3.7
+
+.. data:: HAS_TLSv1_1
+
+ Whether the OpenSSL library has built-in support for the TLS 1.1 protocol.
+
+ .. versionadded:: 3.7
+
+.. data:: HAS_TLSv1_2
+
+ Whether the OpenSSL library has built-in support for the TLS 1.2 protocol.
+
+ .. versionadded:: 3.7
+
.. data:: HAS_TLSv1_3
Whether the OpenSSL library has built-in support for the TLS 1.3 protocol.
@@ -976,6 +1034,27 @@ Constants
.. versionadded:: 3.6
+.. class:: TLSVersion
+
+ :class:`enum.IntEnum` collection of SSL and TLS versions for
+ :attr:`SSLContext.maximum_version` and :attr:`SSLContext.minimum_version`.
+
+ .. versionadded:: 3.7
+
+.. attribute:: TLSVersion.MINIMUM_SUPPORTED
+.. attribute:: TLSVersion.MAXIMUM_SUPPORTED
+
+ The minimum or maximum supported SSL or TLS version. These are magic
+ constants. Their values don't reflect the lowest and highest available
+ TLS/SSL versions.
+
+.. attribute:: TLSVersion.SSLv3
+.. attribute:: TLSVersion.TLSv1
+.. attribute:: TLSVersion.TLSv1_1
+.. attribute:: TLSVersion.TLSv1_2
+.. attribute:: TLSVersion.TLSv1_3
+
+ SSL 3.0 to TLS 1.3.
SSL Sockets
-----------
@@ -1009,7 +1088,7 @@ SSL Sockets
the specification of normal, OS-level sockets. See especially the
:ref:`notes on non-blocking sockets `.
- Usually, :class:`SSLSocket` are not created directly, but using the
+ Instances of :class:`SSLSocket` must be created using the
:meth:`SSLContext.wrap_socket` method.
.. versionchanged:: 3.5
@@ -1024,6 +1103,11 @@ SSL Sockets
It is deprecated to create a :class:`SSLSocket` instance directly, use
:meth:`SSLContext.wrap_socket` to wrap a socket.
+ .. versionchanged:: 3.7
+ :class:`SSLSocket` instances must to created with
+ :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible
+ to create instances directly. This was never documented or officially
+ supported.
SSL sockets also have the following additional methods and attributes:
@@ -1248,7 +1332,7 @@ SSL sockets also have the following additional methods and attributes:
.. attribute:: SSLSocket.context
The :class:`SSLContext` object this SSL socket is tied to. If the SSL
- socket was created using the top-level :func:`wrap_socket` function
+ socket was created using the deprecated :func:`wrap_socket` function
(rather than :meth:`SSLContext.wrap_socket`), this is a custom context
object created for this SSL socket.
@@ -1268,6 +1352,12 @@ SSL sockets also have the following additional methods and attributes:
.. versionadded:: 3.2
+ .. versionchanged:: 3.7
+ The attribute is now always ASCII text. When ``server_hostname`` is
+ an internationalized domain name (IDN), this attribute now stores the
+ A-label form (``"xn--pythn-mua.org"``), rather than the U-label form
+ (``"pythön.org"``).
+
.. attribute:: SSLSocket.session
The :class:`SSLSession` for this SSL connection. The session is available
@@ -1295,9 +1385,36 @@ to speed up repeated connections from the same clients.
.. class:: SSLContext(protocol=PROTOCOL_TLS)
Create a new SSL context. You may pass *protocol* which must be one
- of the ``PROTOCOL_*`` constants defined in this module.
- :data:`PROTOCOL_TLS` is currently recommended for maximum
- interoperability and default value.
+ of the ``PROTOCOL_*`` constants defined in this module. The parameter
+ specifies which version of the SSL protocol to use. Typically, the
+ server chooses a particular protocol version, and the client must adapt
+ to the server's choice. Most of the versions are not interoperable
+ with the other versions. If not specified, the default is
+ :data:`PROTOCOL_TLS`; it provides the most compatibility with other
+ versions.
+
+ Here's a table showing which versions in a client (down the side) can connect
+ to which versions in a server (along the top):
+
+ .. table::
+
+ ======================== ============ ============ ============= ========= =========== ===========
+ *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2**
+ ------------------------ ------------ ------------ ------------- --------- ----------- -----------
+ *SSLv2* yes no no [1]_ no no no
+ *SSLv3* no yes no [2]_ no no no
+ *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes
+ *TLSv1* no no yes yes no no
+ *TLSv1.1* no no yes no yes no
+ *TLSv1.2* no no yes no no yes
+ ======================== ============ ============ ============= ========= =========== ===========
+
+ .. rubric:: Footnotes
+ .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default.
+ .. [2] :class:`SSLContext` disables SSLv3 with :data:`OP_NO_SSLv3` by default.
+ .. [3] TLS 1.3 protocol will be available with :data:`PROTOCOL_TLS` in
+ OpenSSL >= 1.1.1. There is no dedicated PROTOCOL constant for just
+ TLS 1.3.
.. seealso::
:func:`create_default_context` lets the :mod:`ssl` module choose
@@ -1490,7 +1607,7 @@ to speed up repeated connections from the same clients.
Set the available ciphers for sockets created with this context.
It should be a string in the `OpenSSL cipher list format
- `_.
+ `_.
If no cipher can be selected (because compile-time options or other
configuration forbids use of all the specified ciphers), an
:class:`SSLError` will be raised.
@@ -1499,6 +1616,9 @@ to speed up repeated connections from the same clients.
when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will
give the currently selected cipher.
+ OpenSSL 1.1.1 has TLS 1.3 cipher suites enabled by default. The suites
+ cannot be disabled with :meth:`~SSLContext.set_ciphers`.
+
.. method:: SSLContext.set_alpn_protocols(protocols)
Specify which protocols the socket should advertise during the SSL/TLS
@@ -1532,23 +1652,24 @@ to speed up repeated connections from the same clients.
.. versionadded:: 3.3
-.. method:: SSLContext.set_servername_callback(server_name_callback)
+.. attribute:: SSLContext.sni_callback
Register a callback function that will be called after the TLS Client Hello
handshake message has been received by the SSL/TLS server when the TLS client
specifies a server name indication. The server name indication mechanism
is specified in :rfc:`6066` section 3 - Server Name Indication.
- Only one callback can be set per ``SSLContext``. If *server_name_callback*
- is ``None`` then the callback is disabled. Calling this function a
+ Only one callback can be set per ``SSLContext``. If *sni_callback*
+ is set to ``None`` then the callback is disabled. Calling this function a
subsequent time will disable the previously registered callback.
- The callback function, *server_name_callback*, will be called with three
+ The callback function will be called with three
arguments; the first being the :class:`ssl.SSLSocket`, the second is a string
that represents the server name that the client is intending to communicate
(or :const:`None` if the TLS Client Hello does not contain a server name)
and the third argument is the original :class:`SSLContext`. The server name
- argument is the IDNA decoded server name.
+ argument is text. For internationalized domain name, the server
+ name is an IDN A-label (``"xn--pythn-mua.org"``).
A typical use of this callback is to change the :class:`ssl.SSLSocket`'s
:attr:`SSLSocket.context` attribute to a new object of type
@@ -1563,28 +1684,38 @@ to speed up repeated connections from the same clients.
the TLS connection has progressed beyond the TLS Client Hello and therefore
will not contain return meaningful values nor can they be called safely.
- The *server_name_callback* function must return ``None`` to allow the
+ The *sni_callback* function must return ``None`` to allow the
TLS negotiation to continue. If a TLS failure is required, a constant
:const:`ALERT_DESCRIPTION_* ` can be
returned. Other return values will result in a TLS fatal error with
:const:`ALERT_DESCRIPTION_INTERNAL_ERROR`.
- If there is an IDNA decoding error on the server name, the TLS connection
- will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS
- alert message to the client.
-
- If an exception is raised from the *server_name_callback* function the TLS
+ If an exception is raised from the *sni_callback* function the TLS
connection will terminate with a fatal TLS alert message
:const:`ALERT_DESCRIPTION_HANDSHAKE_FAILURE`.
This method will raise :exc:`NotImplementedError` if the OpenSSL library
had OPENSSL_NO_TLSEXT defined when it was built.
+ .. versionadded:: 3.7
+
+.. attribute:: SSLContext.set_servername_callback(server_name_callback)
+
+ This is a legacy API retained for backwards compatibility. When possible,
+ you should use :attr:`sni_callback` instead. The given *server_name_callback*
+ is similar to *sni_callback*, except that when the server hostname is an
+ IDN-encoded internationalized domain name, the *server_name_callback*
+ receives a decoded U-label (``"pythön.org"``).
+
+ If there is an decoding error on the server name, the TLS connection will
+ terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS
+ alert message to the client.
+
.. versionadded:: 3.4
.. method:: SSLContext.load_dh_params(dhfile)
- Load the key generation parameters for Diffie-Helman (DH) key exchange.
+ Load the key generation parameters for Diffie-Hellman (DH) key exchange.
Using DH key exchange improves forward secrecy at the expense of
computational resources (both on the server and on the client).
The *dhfile* parameter should be the path to a file containing DH
@@ -1619,14 +1750,21 @@ to speed up repeated connections from the same clients.
server_hostname=None, session=None)
Wrap an existing Python socket *sock* and return an instance of
- :attr:`SSLContext.sslsocket_class` (default :class:`SSLSocket`).
- *sock* must be a :data:`~socket.SOCK_STREAM` socket; other socket
- types are unsupported.
+ :attr:`SSLContext.sslsocket_class` (default :class:`SSLSocket`). The
+ returned SSL socket is tied to the context, its settings and certificates.
+ *sock* must be a :data:`~socket.SOCK_STREAM` socket; other
+ socket types are unsupported.
- The returned SSL socket is tied to the context, its settings and
- certificates. The parameters *server_side*, *do_handshake_on_connect*
- and *suppress_ragged_eofs* have the same meaning as in the top-level
- :func:`wrap_socket` function.
+ The parameter ``server_side`` is a boolean which identifies whether
+ server-side or client-side behavior is desired from this socket.
+
+ For client-side sockets, the context construction is lazy; if the
+ underlying socket isn't connected yet, the context construction will be
+ performed after :meth:`connect` is called on the socket. For
+ server-side sockets, if the socket has no remote peer, it is assumed
+ to be a listening socket, and the server-side SSL wrapping is
+ automatically performed on client connections accepted via the
+ :meth:`accept` method. The method may raise :exc:`SSLError`.
On client connections, the optional parameter *server_hostname* specifies
the hostname of the service which we are connecting to. This allows a
@@ -1634,6 +1772,20 @@ to speed up repeated connections from the same clients.
quite similarly to HTTP virtual hosts. Specifying *server_hostname* will
raise a :exc:`ValueError` if *server_side* is true.
+ The parameter ``do_handshake_on_connect`` specifies whether to do the SSL
+ handshake automatically after doing a :meth:`socket.connect`, or whether the
+ application program will call it explicitly, by invoking the
+ :meth:`SSLSocket.do_handshake` method. Calling
+ :meth:`SSLSocket.do_handshake` explicitly gives the program control over the
+ blocking behavior of the socket I/O involved in the handshake.
+
+ The parameter ``suppress_ragged_eofs`` specifies how the
+ :meth:`SSLSocket.recv` method should signal unexpected EOF from the other end
+ of the connection. If specified as :const:`True` (the default), it returns a
+ normal EOF (an empty bytes object) in response to unexpected EOF errors
+ raised from the underlying socket; if :const:`False`, it will raise the
+ exceptions back to the caller.
+
*session*, see :attr:`~SSLSocket.session`.
.. versionchanged:: 3.5
@@ -1707,7 +1859,7 @@ to speed up repeated connections from the same clients.
import socket, ssl
- context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+ context = ssl.SSLContext()
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = True
context.load_default_certs()
@@ -1729,6 +1881,37 @@ to speed up repeated connections from the same clients.
This features requires OpenSSL 0.9.8f or newer.
+.. attribute:: SSLContext.maximum_version
+
+ A :class:`TLSVersion` enum member representing the highest supported
+ TLS version. The value defaults to :attr:`TLSVersion.MAXIMUM_SUPPORTED`.
+ The attribute is read-only for protocols other than :attr:`PROTOCOL_TLS`,
+ :attr:`PROTOCOL_TLS_CLIENT`, and :attr:`PROTOCOL_TLS_SERVER`.
+
+ The attributes :attr:`~SSLContext.maximum_version`,
+ :attr:`~SSLContext.minimum_version` and
+ :attr:`SSLContext.options` all affect the supported SSL
+ and TLS versions of the context. The implementation does not prevent
+ invalid combination. For example a context with
+ :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and
+ :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2`
+ will not be able to establish a TLS 1.2 connection.
+
+ .. note::
+
+ This attribute is not available unless the ssl module is compiled
+ with OpenSSL 1.1.0g or newer.
+
+.. attribute:: SSLContext.minimum_version
+
+ Like :attr:`SSLContext.maximum_version` except it is the lowest
+ supported version or :attr:`TLSVersion.MINIMUM_SUPPORTED`.
+
+ .. note::
+
+ This attribute is not available unless the ssl module is compiled
+ with OpenSSL 1.1.0g or newer.
+
.. attribute:: SSLContext.options
An integer representing the set of SSL options enabled on this context.
@@ -1808,7 +1991,7 @@ message with one of the parts, you can decrypt it with the other part, and
A certificate contains information about two principals. It contains the name
of a *subject*, and the subject's public key. It also contains a statement by a
-second principal, the *issuer*, that the subject is who he claims to be, and
+second principal, the *issuer*, that the subject is who they claim to be, and
that this is indeed the subject's public key. The issuer's statement is signed
with the issuer's private key, which only the issuer knows. However, anyone can
verify the issuer's statement by finding the issuer's public key, decrypting the
@@ -1952,7 +2135,7 @@ If you prefer to tune security settings yourself, you might create
a context from scratch (but beware that you might not get the settings
right)::
- >>> context = ssl.SSLContext(ssl.PROTOCOL_TLS)
+ >>> context = ssl.SSLContext()
>>> context.verify_mode = ssl.CERT_REQUIRED
>>> context.check_hostname = True
>>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt")
@@ -2002,9 +2185,9 @@ Visual inspection shows that the certificate does identify the desired service
(('commonName', 'www.python.org'),)),
'subjectAltName': (('DNS', 'www.python.org'),
('DNS', 'python.org'),
- ('DNS', 'pypi.python.org'),
+ ('DNS', 'pypi.org'),
('DNS', 'docs.python.org'),
- ('DNS', 'testpypi.python.org'),
+ ('DNS', 'testpypi.org'),
('DNS', 'bugs.python.org'),
('DNS', 'wiki.python.org'),
('DNS', 'hg.python.org'),
@@ -2195,11 +2378,12 @@ provided.
but does not provide any network IO itself. IO needs to be performed through
separate "BIO" objects which are OpenSSL's IO abstraction layer.
- An :class:`SSLObject` instance can be created using the
- :meth:`~SSLContext.wrap_bio` method. This method will create the
- :class:`SSLObject` instance and bind it to a pair of BIOs. The *incoming*
- BIO is used to pass data from Python to the SSL protocol instance, while the
- *outgoing* BIO is used to pass data the other way around.
+ This class has no public constructor. An :class:`SSLObject` instance
+ must be created using the :meth:`~SSLContext.wrap_bio` method. This
+ method will create the :class:`SSLObject` instance and bind it to a
+ pair of BIOs. The *incoming* BIO is used to pass data from Python to the
+ SSL protocol instance, while the *outgoing* BIO is used to pass data the
+ other way around.
The following methods are available:
@@ -2251,6 +2435,12 @@ provided.
:meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created
via an :class:`SSLContext`.
+ .. versionchanged:: 3.7
+ :class:`SSLObject` instances must to created with
+ :meth:`~SSLContext.wrap_bio`. In earlier versions, it was possible to
+ create instances directly. This was never documented or officially
+ supported.
+
An SSLObject communicates with the outside world using memory buffers. The
class :class:`MemoryBIO` provides a memory buffer that can be used for this
purpose. It wraps an OpenSSL memory BIO (Basic IO) object:
@@ -2364,11 +2554,6 @@ In server mode, if you want to authenticate your clients using the SSL layer
(rather than using a higher-level authentication mechanism), you'll also have
to specify :const:`CERT_REQUIRED` and similarly check the client certificate.
- .. note::
-
- In client mode, :const:`CERT_OPTIONAL` and :const:`CERT_REQUIRED` are
- equivalent unless anonymous ciphers are enabled (they are disabled
- by default).
Protocol versions
'''''''''''''''''
@@ -2417,33 +2602,76 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or
:func:`~ssl.RAND_pseudo_bytes` is sufficient.
+.. _ssl-tlsv1_3:
+
+TLS 1.3
+-------
+
+.. versionadded:: 3.7
+
+Python has provisional and experimental support for TLS 1.3 with OpenSSL
+1.1.1. The new protocol behaves slightly differently than previous version
+of TLS/SSL. Some new TLS 1.3 features are not yet available.
+
+- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and
+ ChaCha20 cipher suites are enabled by default. The method
+ :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3
+ ciphers yet, but :meth:`SSLContext.get_cipers` returns them.
+- Session tickets are no longer sent as part of the initial handshake and
+ are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession`
+ are not compatible with TLS 1.3.
+- Client-side certificates are also no longer verified during the initial
+ handshake. A server can request a certificate at any time. Clients
+ process certificate requests while they send or receive application data
+ from the server.
+- TLS 1.3 features like early data, deferred TLS client cert request,
+ signature algorithm configuration, and rekeying are not supported yet.
+
+
+.. _ssl-libressl:
+
+LibreSSL support
+----------------
+
+LibreSSL is a fork of OpenSSL 1.0.1. The ssl module has limited support for
+LibreSSL. Some features are not available when the ssl module is compiled
+with LibreSSL.
+
+* LibreSSL >= 2.6.1 no longer supports NPN. The methods
+ :meth:`SSLContext.set_npn_protocols` and
+ :meth:`SSLSocket.selected_npn_protocol` are not available.
+* :meth:`SSLContext.set_default_verify_paths` ignores the env vars
+ :envvar:`SSL_CERT_FILE` and :envvar:`SSL_CERT_PATH` although
+ :func:`get_default_verify_paths` still reports them.
+
+
.. seealso::
Class :class:`socket.socket`
Documentation of underlying :mod:`socket` class
`SSL/TLS Strong Encryption: An Introduction `_
- Intro from the Apache webserver documentation
+ Intro from the Apache HTTP Server documentation
- `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_
+ :rfc:`RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <1422>`
Steve Kent
- `RFC 4086: Randomness Requirements for Security `_
+ :rfc:`RFC 4086: Randomness Requirements for Security <4086>`
Donald E., Jeffrey I. Schiller
- `RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile `_
+ :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>`
D. Cooper
- `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 `_
+ :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>`
T. Dierks et. al.
- `RFC 6066: Transport Layer Security (TLS) Extensions `_
+ :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>`
D. Eastlake
`IANA TLS: Transport Layer Security (TLS) Parameters `_
IANA
- `RFC 7525: Recommendations for Secure Use of Transport Layer Security (TLS) and Datagram Transport Layer Security (DTLS) `_
+ :rfc:`RFC 7525: Recommendations for Secure Use of Transport Layer Security (TLS) and Datagram Transport Layer Security (DTLS) <7525>`
IETF
`Mozilla's Server Side TLS recommendations `_
diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index bc3817836b9398..26bb592b23812b 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -169,6 +169,10 @@ However, for reading convenience, most of the examples show sorted sequences.
This is suited for when your data is discrete, and you don't mind that the
median may not be an actual data point.
+ If your data is ordinal (supports order operations) but not numeric (doesn't
+ support addition), you should use :func:`median_low` or :func:`median_high`
+ instead.
+
.. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped`
diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index ad7f578e086069..6f46972750870c 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -382,7 +382,7 @@ modules.
.. _bitstring-ops:
Bitwise Operations on Integer Types
---------------------------------------
+-----------------------------------
.. index::
triple: operations on; integer; types
@@ -396,9 +396,9 @@ Bitwise Operations on Integer Types
operator: >>
operator: ~
-Bitwise operations only make sense for integers. Negative numbers are treated
-as their 2's complement value (this assumes that there are enough bits so that
-no overflow occurs during the operation).
+Bitwise operations only make sense for integers. The result of bitwise
+operations is calculated as though carried out in two's complement with an
+infinite number of sign bits.
The priorities of the binary bitwise operations are all lower than the numeric
operations and higher than the comparisons; the unary operation ``~`` has the
@@ -409,13 +409,13 @@ This table lists the bitwise operations sorted in ascending priority:
+------------+--------------------------------+----------+
| Operation | Result | Notes |
+============+================================+==========+
-| ``x | y`` | bitwise :dfn:`or` of *x* and | |
+| ``x | y`` | bitwise :dfn:`or` of *x* and | (4) |
| | *y* | |
+------------+--------------------------------+----------+
-| ``x ^ y`` | bitwise :dfn:`exclusive or` of | |
+| ``x ^ y`` | bitwise :dfn:`exclusive or` of | (4) |
| | *x* and *y* | |
+------------+--------------------------------+----------+
-| ``x & y`` | bitwise :dfn:`and` of *x* and | |
+| ``x & y`` | bitwise :dfn:`and` of *x* and | (4) |
| | *y* | |
+------------+--------------------------------+----------+
| ``x << n`` | *x* shifted left by *n* bits | (1)(2) |
@@ -438,6 +438,12 @@ Notes:
A right shift by *n* bits is equivalent to division by ``pow(2, n)`` without
overflow check.
+(4)
+ Performing these calculations with at least one extra sign extension bit in
+ a finite two's complement representation (a working bit-width of
+ ``1 + max(x.bit_length(), y.bit_length()`` or more) is sufficient to get the
+ same result as if there were an infinite number of sign bits.
+
Additional Methods on Integer Types
-----------------------------------
@@ -1365,7 +1371,7 @@ objects that compare equal might have different :attr:`~range.start`,
.. seealso::
* The `linspace recipe `_
- shows how to implement a lazy version of range that suitable for floating
+ shows how to implement a lazy version of range suitable for floating
point applications.
.. index::
@@ -1600,13 +1606,14 @@ expression support in the :mod:`re` module).
that can be specified in format strings.
.. note::
- When formatting a number (:class:`int`, :class:`float`, :class:`float`
- and subclasses) with the ``n`` type (ex: ``'{:n}'.format(1234)``), the
- function sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC``
- locale to decode ``decimal_point`` and ``thousands_sep`` fields of
- :c:func:`localeconv` if they are non-ASCII or longer than 1 byte, and the
- ``LC_NUMERIC`` locale is different than the ``LC_CTYPE`` locale. This
- temporary change affects other threads.
+ When formatting a number (:class:`int`, :class:`float`, :class:`complex`,
+ :class:`decimal.Decimal` and subclasses) with the ``n`` type
+ (ex: ``'{:n}'.format(1234)``), the function temporarily sets the
+ ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale to decode
+ ``decimal_point`` and ``thousands_sep`` fields of :c:func:`localeconv` if
+ they are non-ASCII or longer than 1 byte, and the ``LC_NUMERIC`` locale is
+ different than the ``LC_CTYPE`` locale. This temporary change affects
+ other threads.
.. versionchanged:: 3.7
When formatting a number with the ``n`` type, the function sets
@@ -2051,7 +2058,7 @@ expression support in the :mod:`re` module).
.. method:: str.upper()
Return a copy of the string with all the cased characters [4]_ converted to
- uppercase. Note that ``str.upper().isupper()`` might be ``False`` if ``s``
+ uppercase. Note that ``s.upper().isupper()`` might be ``False`` if ``s``
contains uncased characters or if the Unicode category of the resulting
character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter,
titlecase).
@@ -2315,7 +2322,7 @@ data and are closely related to string objects in a variety of other ways.
While bytes literals and representations are based on ASCII text, bytes
objects actually behave like immutable sequences of integers, with each
value in the sequence restricted such that ``0 <= x < 256`` (attempts to
- violate this restriction will trigger :exc:`ValueError`. This is done
+ violate this restriction will trigger :exc:`ValueError`). This is done
deliberately to emphasise that while many binary formats include ASCII based
elements and can be usefully manipulated with some text-oriented algorithms,
this is not generally the case for arbitrary binary data (blindly applying
@@ -3388,7 +3395,10 @@ Notes:
The bytearray version of this method does *not* operate in place - it
always produces a new object, even if no changes were made.
-.. seealso:: :pep:`461`.
+.. seealso::
+
+ :pep:`461` - Adding % formatting to bytes and bytearray
+
.. versionadded:: 3.5
.. _typememoryview:
@@ -4198,12 +4208,17 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098:
.. method:: popitem()
- Remove and return an arbitrary ``(key, value)`` pair from the dictionary.
+ Remove and return a ``(key, value)`` pair from the dictionary.
+ Pairs are returned in :abbr:`LIFO (last-in, first-out)` order.
:meth:`popitem` is useful to destructively iterate over a dictionary, as
often used in set algorithms. If the dictionary is empty, calling
:meth:`popitem` raises a :exc:`KeyError`.
+ .. versionchanged:: 3.7
+ LIFO order is now guaranteed. In prior versions, :meth:`popitem` would
+ return an arbitrary key/value pair.
+
.. method:: setdefault(key[, default])
If *key* is in the dictionary, return its value. If not, insert *key*
@@ -4229,6 +4244,28 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098:
value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise
:exc:`TypeError`.
+ Dictionaries preserve insertion order. Note that updating a key does not
+ affect the order. Keys added after deletion are inserted at the end. ::
+
+ >>> d = {"one": 1, "two": 2, "three": 3, "four": 4}
+ >>> d
+ {'one': 1, 'two': 2, 'three': 3, 'four': 4}
+ >>> list(d)
+ ['one', 'two', 'three', 'four']
+ >>> list(d.values())
+ [1, 2, 3, 4]
+ >>> d["one"] = 42
+ >>> d
+ {'one': 42, 'two': 2, 'three': 3, 'four': 4}
+ >>> del d["two"]
+ >>> d["two"] = None
+ >>> d
+ {'one': 42, 'three': 3, 'four': 4, 'two': None}
+
+ .. versionchanged:: 3.7
+ Dictionary order is guaranteed to be insertion order. This behavior was
+ implementation detail of CPython from 3.6.
+
.. seealso::
:class:`types.MappingProxyType` can be used to create a read-only view
of a :class:`dict`.
@@ -4256,17 +4293,17 @@ support membership tests:
Return an iterator over the keys, values or items (represented as tuples of
``(key, value)``) in the dictionary.
- Keys and values are iterated over in an arbitrary order which is non-random,
- varies across Python implementations, and depends on the dictionary's history
- of insertions and deletions. If keys, values and items views are iterated
- over with no intervening modifications to the dictionary, the order of items
- will directly correspond. This allows the creation of ``(value, key)`` pairs
+ Keys and values are iterated over in insertion order.
+ This allows the creation of ``(value, key)`` pairs
using :func:`zip`: ``pairs = zip(d.values(), d.keys())``. Another way to
create the same list is ``pairs = [(v, k) for (k, v) in d.items()]``.
Iterating views while adding or deleting entries in the dictionary may raise
a :exc:`RuntimeError` or fail to iterate over all entries.
+ .. versionchanged:: 3.7
+ Dictionary order is guaranteed to be insertion order.
+
.. describe:: x in dictview
Return ``True`` if *x* is in the underlying dictionary's keys, values or
@@ -4293,9 +4330,9 @@ An example of dictionary view usage::
>>> print(n)
504
- >>> # keys and values are iterated over in the same order
+ >>> # keys and values are iterated over in the same order (insertion order)
>>> list(keys)
- ['eggs', 'bacon', 'sausage', 'spam']
+ ['eggs', 'sausage', 'bacon', 'spam']
>>> list(values)
[2, 1, 1, 500]
@@ -4303,7 +4340,7 @@ An example of dictionary view usage::
>>> del dishes['eggs']
>>> del dishes['sausage']
>>> list(keys)
- ['spam', 'bacon']
+ ['bacon', 'spam']
>>> # set operations
>>> keys & {'eggs', 'bacon', 'salad'}
@@ -4679,3 +4716,4 @@ types, where they are relevant. Some of these are not reported by the
.. [5] To format only a tuple you should therefore provide a singleton tuple whose only
element is the tuple to be formatted.
+
diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index fc3d94bfbfca00..0fec3df3e3c14e 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -202,9 +202,9 @@ The grammar for a replacement field is as follows:
.. productionlist:: sf
replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}"
field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")*
- arg_name: [`identifier` | `integer`]
+ arg_name: [`identifier` | `digit`+]
attribute_name: `identifier`
- element_index: `integer` | `index_string`
+ element_index: `digit`+ | `index_string`
index_string: +
conversion: "r" | "s" | "a"
format_spec:
@@ -231,8 +231,11 @@ attribute using :func:`getattr`, while an expression of the form ``'[index]'``
does an index lookup using :func:`__getitem__`.
.. versionchanged:: 3.1
- The positional argument specifiers can be omitted, so ``'{} {}'`` is
- equivalent to ``'{0} {1}'``.
+ The positional argument specifiers can be omitted for :meth:`str.format`,
+ so ``'{} {}'.format(a, b)`` is equivalent to ``'{0} {1}'.format(a, b)``.
+
+.. versionchanged:: 3.4
+ The positional argument specifiers can be omitted for :class:`Formatter`.
Some simple format string examples::
@@ -304,9 +307,9 @@ The general form of a *standard format specifier* is:
fill:
align: "<" | ">" | "=" | "^"
sign: "+" | "-" | " "
- width: `integer`
+ width: `digit`+
grouping_option: "_" | ","
- precision: `integer`
+ precision: `digit`+
type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
If a valid *align* value is specified, it can be preceded by a *fill*
@@ -461,11 +464,11 @@ The available presentation types for floating point and decimal values are:
| ``'E'`` | Exponent notation. Same as ``'e'`` except it uses an |
| | upper case 'E' as the separator character. |
+---------+----------------------------------------------------------+
- | ``'f'`` | Fixed point. Displays the number as a fixed-point |
- | | number. The default precision is ``6``. |
+ | ``'f'`` | Fixed-point notation. Displays the number as a |
+ | | fixed-point number. The default precision is ``6``. |
+---------+----------------------------------------------------------+
- | ``'F'`` | Fixed point. Same as ``'f'``, but converts ``nan`` to |
- | | ``NAN`` and ``inf`` to ``INF``. |
+ | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts |
+ | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. |
+---------+----------------------------------------------------------+
| ``'g'`` | General format. For a given precision ``p >= 1``, |
| | this rounds the number to ``p`` significant digits and |
diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst
index e7fae5631d87f6..330032ba1c0ba6 100644
--- a/Doc/library/stringprep.rst
+++ b/Doc/library/stringprep.rst
@@ -26,7 +26,7 @@ define which tables it uses, and what other optional parts of the ``stringprep``
procedure are part of the profile. One example of a ``stringprep`` profile is
``nameprep``, which is used for internationalized domain names.
-The module :mod:`stringprep` only exposes the tables from RFC 3454. As these
+The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these
tables would be very large to represent them as dictionaries or lists, the
module uses the Unicode character database internally. The module source code
itself was generated using the ``mkstringprep.py`` utility.
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 27d4288f67b33d..19f6ff38bb2ba8 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -38,8 +38,8 @@ compatibility with older versions, see the :ref:`call-function-trio` section.
.. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\
- shell=False, cwd=None, timeout=None, check=False, \
- encoding=None, errors=None)
+ capture_output=False, shell=False, cwd=None, timeout=None, \
+ check=False, encoding=None, errors=None, text=None, env=None)
Run the command described by *args*. Wait for command to complete, then
return a :class:`CompletedProcess` instance.
@@ -78,6 +78,11 @@ compatibility with older versions, see the :ref:`call-function-trio` section.
The *universal_newlines* argument is equivalent to *text* and is provided
for backwards compatibility. By default, file objects are opened in binary mode.
+ If *env* is not ``None``, it must be a mapping that defines the environment
+ variables for the new process; these are used instead of the default
+ behavior of inheriting the current process' environment. It is passed directly
+ to :class:`Popen`.
+
Examples::
>>> subprocess.run(["ls", "-l"]) # doesn't capture output
@@ -267,7 +272,8 @@ default values. The arguments that are most commonly needed are:
.. index::
single: universal newlines; subprocess module
- If *encoding* or *errors* are specified, or *universal_newlines* is true,
+ If *encoding* or *errors* are specified, or *text* (also known as
+ *universal_newlines*) is true,
the file objects *stdin*, *stdout* and *stderr* will be opened in text
mode using the *encoding* and *errors* specified in the call or the
defaults for :class:`io.TextIOWrapper`.
@@ -284,6 +290,9 @@ default values. The arguments that are most commonly needed are:
.. versionadded:: 3.6
Added *encoding* and *errors* parameters.
+ .. versionadded:: 3.7
+ Added the *text* parameter as an alias for *universal_newlines*.
+
.. note::
The newlines attribute of the file objects :attr:`Popen.stdin`,
@@ -328,19 +337,19 @@ functions.
cwd=None, env=None, universal_newlines=False, \
startupinfo=None, creationflags=0, restore_signals=True, \
start_new_session=False, pass_fds=(), *, \
- encoding=None, errors=None)
+ encoding=None, errors=None, text=None)
Execute a child program in a new process. On POSIX, the class uses
:meth:`os.execvp`-like behavior to execute the child program. On Windows,
the class uses the Windows ``CreateProcess()`` function. The arguments to
:class:`Popen` are as follows.
- *args* should be a sequence of program arguments or else a single string or
- :term:`path-like object`. By default, the program to execute is the first
- item in *args* if *args* is a sequence. If *args* is a string, the
- interpretation is platform-dependent and described below. See the *shell*
- and *executable* arguments for additional differences from the default
- behavior. Unless otherwise stated, it is recommended to pass *args* as a sequence.
+ *args* should be a sequence of program arguments or else a single string.
+ By default, the program to execute is the first item in *args* if *args* is
+ a sequence. If *args* is a string, the interpretation is
+ platform-dependent and described below. See the *shell* and *executable*
+ arguments for additional differences from the default behavior. Unless
+ otherwise stated, it is recommended to pass *args* as a sequence.
On POSIX, if *args* is a string, the string is interpreted as the name or
path of the program to execute. However, this can only be done if not
@@ -455,7 +464,10 @@ functions.
common use of *preexec_fn* to call os.setsid() in the child.
If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and
- :const:`2` will be closed before the child process is executed.
+ :const:`2` will be closed before the child process is executed. Otherwise
+ when *close_fds* is false, file descriptors obey their inheritable flag
+ as described in :ref:`fd_inheritance`.
+
On Windows, if *close_fds* is true then no handles will be inherited by the
child process unless explicitly passed in the ``handle_list`` element of
:attr:`STARTUPINFO.lpAttributeList`, or by standard handle redirection.
@@ -511,15 +523,18 @@ functions.
.. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly
- If *encoding* or *errors* are specified, the file objects *stdin*, *stdout*
- and *stderr* are opened in text mode with the specified encoding and
- *errors*, as described above in :ref:`frequently-used-arguments`. If
- *universal_newlines* is ``True``, they are opened in text mode with default
- encoding. Otherwise, they are opened as binary streams.
+ If *encoding* or *errors* are specified, or *text* is true, the file objects
+ *stdin*, *stdout* and *stderr* are opened in text mode with the specified
+ encoding and *errors*, as described above in :ref:`frequently-used-arguments`.
+ The *universal_newlines* argument is equivalent to *text* and is provided
+ for backwards compatibility. By default, file objects are opened in binary mode.
.. versionadded:: 3.6
*encoding* and *errors* were added.
+ .. versionadded:: 3.7
+ *text* was added as a more readable alias for *universal_newlines*.
+
If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is
passed to the underlying ``CreateProcess`` function.
*creationflags*, if given, can be one or more of the following flags:
@@ -551,10 +566,6 @@ functions.
Popen destructor now emits a :exc:`ResourceWarning` warning if the child
process is still running.
- .. versionchanged:: 3.7
- *args*, or the first element of *args* if *args* is a sequence, can now
- be a :term:`path-like object`.
-
Exceptions
^^^^^^^^^^
@@ -1087,6 +1098,9 @@ calls these functions.
.. versionchanged:: 3.4
Support for the *input* keyword argument was added.
+ .. versionchanged:: 3.6
+ *encoding* and *errors* were added. See :func:`run` for details.
+
.. _subprocess-replacements:
Replacing Older Functions with the :mod:`subprocess` Module
diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst
index 2450716a1d9120..9cd07158e7f628 100644
--- a/Doc/library/tarfile.rst
+++ b/Doc/library/tarfile.rst
@@ -451,7 +451,8 @@ be finalized; only the internally used file object will be closed. See the
(directory, fifo, symbolic link, etc.). If given, *arcname* specifies an
alternative name for the file in the archive. Directories are added
recursively by default. This can be avoided by setting *recursive* to
- :const:`False`. If *filter* is given, it
+ :const:`False`. Recursion adds entries in sorted order.
+ If *filter* is given, it
should be a function that takes a :class:`TarInfo` object argument and
returns the changed :class:`TarInfo` object. If it instead returns
:const:`None` the :class:`TarInfo` object will be excluded from the
@@ -460,6 +461,9 @@ be finalized; only the internally used file object will be closed. See the
.. versionchanged:: 3.2
Added the *filter* parameter.
+ .. versionchanged:: 3.7
+ Recursion adds entries in sorted order.
+
.. method:: TarFile.addfile(tarinfo, fileobj=None)
diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst
index c59aca1e189086..0d0da4d62e4725 100644
--- a/Doc/library/tempfile.rst
+++ b/Doc/library/tempfile.rst
@@ -253,7 +253,7 @@ to specify the directory and this is the recommended approach.
default value for the *dir* argument to the functions defined in this
module.
- If ``tempdir`` is unset or ``None`` at any call to any of the above
+ If ``tempdir`` is ``None`` (the default) at any call to any of the above
functions except :func:`gettempprefix` it is initialized following the
algorithm described in :func:`gettempdir`.
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
index 01ba1ec7062c98..aeeed0042fce90 100644
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -237,12 +237,127 @@ The :mod:`test.support` module defines the following constants:
``True`` if the running interpreter is Jython.
+.. data:: is_android
+
+ ``True`` if the system is Android.
+
+
+.. data:: unix_shell
+
+ Path for shell if not on Windows; otherwise ``None``.
+
+
+.. data:: FS_NONASCII
+
+ A non-ASCII character encodable by :func:`os.fsencode`.
+
+
.. data:: TESTFN
Set to a name that is safe to use as the name of a temporary file. Any
temporary file that is created should be closed and unlinked (removed).
+.. data:: TESTFN_UNICODE
+
+ Set to a non-ASCII name for a temporary file.
+
+
+.. data:: TESTFN_ENCODING
+
+ Set to :func:`sys.getfilesystemencoding`.
+
+
+.. data:: TESTFN_UNENCODABLE
+
+ Set to a filename (str type) that should not be able to be encoded by file
+ system encoding in strict mode. It may be ``None`` if it's not possible to
+ generate such a filename.
+
+
+.. data:: TESTFN_UNDECODABLE
+
+ Set to a filename (bytes type) that should not be able to be decoded by
+ file system encoding in strict mode. It may be ``None`` if it's not
+ possible to generate such a filename.
+
+
+.. data:: TESTFN_NONASCII
+
+ Set to a filename containing the :data:`FS_NONASCII` character.
+
+
+.. data:: IPV6_ENABLED
+
+ Set to ``True`` if IPV6 is enabled on this host, ``False`` otherwise.
+
+
+.. data:: SAVEDCWD
+
+ Set to :func:`os.getcwd`.
+
+
+.. data:: PGO
+
+ Set when tests can be skipped when they are not useful for PGO.
+
+
+.. data:: PIPE_MAX_SIZE
+
+ A constant that is likely larger than the underlying OS pipe buffer size,
+ to make writes blocking.
+
+
+.. data:: SOCK_MAX_SIZE
+
+ A constant that is likely larger than the underlying OS socket buffer size,
+ to make writes blocking.
+
+
+.. data:: TEST_SUPPORT_DIR
+
+ Set to the top level directory that contains :mod:`test.support`.
+
+
+.. data:: TEST_HOME_DIR
+
+ Set to the top level directory for the test package.
+
+
+.. data:: TEST_DATA_DIR
+
+ Set to the ``data`` directory within the test package.
+
+
+.. data:: MAX_Py_ssize_t
+
+ Set to :data:`sys.maxsize` for big memory tests.
+
+
+.. data:: max_memuse
+
+ Set by :func:`set_memlimit` as the memory limit for big memory tests.
+ Limited by :data:`MAX_Py_ssize_t`.
+
+
+.. data:: real_max_memuse
+
+ Set by :func:`set_memlimit` as the memory limit for big memory tests. Not
+ limited by :data:`MAX_Py_ssize_t`.
+
+
+.. data:: MISSING_C_DOCSTRINGS
+
+ Return ``True`` if running on CPython, not on Windows, and configuration
+ not set with ``WITH_DOC_STRINGS``.
+
+
+.. data:: HAVE_DOCSTRINGS
+
+ Check for presence of docstrings.
+
+
+
The :mod:`test.support` module defines the following functions:
.. function:: forget(module_name)
@@ -251,6 +366,38 @@ The :mod:`test.support` module defines the following functions:
byte-compiled files of the module.
+.. function:: unload(name)
+
+ Delete *name* from ``sys.modules``.
+
+
+.. function:: unlink(filename)
+
+ Call :func:`os.unlink` on *filename*. On Windows platforms, this is
+ wrapped with a wait loop that checks for the existence fo the file.
+
+
+.. function:: rmdir(filename)
+
+ Call :func:`os.rmdir` on *filename*. On Windows platforms, this is
+ wrapped with a wait loop that checks for the existence of the file.
+
+
+.. function:: rmtree(path)
+
+ Call :func:`shutil.rmtree` on *path* or call :func:`os.lstat` and
+ :func:`os.rmdir` to remove a path and its contents. On Windows platforms,
+ this is wrapped with a wait loop that checks for the existence of the files.
+
+
+.. function:: make_legacy_pyc(source)
+
+ Move a PEP 3147/488 pyc file to its legacy pyc location and return the file
+ system path to the legacy pyc file. The *source* value is the file system
+ path to the source file. It does not need to exist, however the PEP
+ 3147/488 pyc file must exist.
+
+
.. function:: is_resource_enabled(resource)
Return ``True`` if *resource* is enabled and available. The list of
@@ -258,6 +405,16 @@ The :mod:`test.support` module defines the following functions:
tests.
+.. function:: python_is_optimized()
+
+ Return ``True`` if Python was not built with ``-O0`` or ``-Og``.
+
+
+.. function:: with_pymalloc()
+
+ Return :data:`_testcapi.WITH_PYMALLOC`.
+
+
.. function:: requires(resource, msg=None)
Raise :exc:`ResourceDenied` if *resource* is not available. *msg* is the
@@ -266,14 +423,44 @@ The :mod:`test.support` module defines the following functions:
Used when tests are executed by :mod:`test.regrtest`.
+.. function:: system_must_validate_cert(f)
+
+ Raise :exc:`unittest.SkipTest` on TLS certification validation failures.
+
+
+.. function:: sortdict(dict)
+
+ Return a repr of *dict* with keys sorted.
+
+
.. function:: findfile(filename, subdir=None)
Return the path to the file named *filename*. If no match is found
*filename* is returned. This does not equal a failure since it could be the
path to the file.
- Setting *subdir* indicates a relative path to use to find the file
- rather than looking directly in the path directories.
+ Setting *subdir* indicates a relative path to use to find the file
+ rather than looking directly in the path directories.
+
+
+.. function:: create_empty_file(filename)
+
+ Create an empty file with *filename*. If it already exists, truncate it.
+
+
+.. function:: fd_count()
+
+ Count the number of open file descriptors.
+
+
+.. function:: match_test(test)
+
+ Match *test* to patterns set in :func:`set_match_tests`.
+
+
+.. function:: set_match_tests(patterns)
+
+ Define match test with regular expression *patterns*.
.. function:: run_unittest(\*classes)
@@ -293,14 +480,32 @@ The :mod:`test.support` module defines the following functions:
This will run all tests defined in the named module.
-.. function:: run_doctest(module, verbosity=None)
+.. function:: run_doctest(module, verbosity=None, optionflags=0)
Run :func:`doctest.testmod` on the given *module*. Return
``(failure_count, test_count)``.
If *verbosity* is ``None``, :func:`doctest.testmod` is run with verbosity
set to :data:`verbose`. Otherwise, it is run with verbosity set to
- ``None``.
+ ``None``. *optionflags* is passed as ``optionflags`` to
+ :func:`doctest.testmod`.
+
+
+.. function:: setswitchinterval(interval)
+
+ Set the :func:`sys.setswitchinterval` to the given *interval*. Defines
+ a minimum interval for Android systems to prevent the system from hanging.
+
+
+.. function:: check_impl_detail(**guards)
+
+ Use this check to guard CPython's implementation-specific tests or to
+ run them only on the implementations guarded by the arguments::
+
+ check_impl_detail() # Only on CPython (default).
+ check_impl_detail(jython=True) # Only on Jython.
+ check_impl_detail(cpython=False) # Everywhere except CPython.
+
.. function:: check_warnings(\*filters, quiet=True)
@@ -368,6 +573,50 @@ The :mod:`test.support` module defines the following functions:
New optional arguments *filters* and *quiet*.
+.. function:: check_no_resource_warning(testcase)
+
+ Context manager to check that no :exc:`ResourceWarning` was raised. You
+ must remove the object which may emit :exc:`ResourceWarning` before the
+ end of the context manager.
+
+
+.. function:: set_memlimit(limit)
+
+ Set the values for :data:`max_memuse` and :data:`real_max_memuse` for big
+ memory tests.
+
+
+.. function:: record_original_stdout(stdout)
+
+ Store the value from *stdout*. It is meant to hold the stdout at the
+ time the regrtest began.
+
+
+.. function:: get_original_stdout
+
+ Return the original stdout set by :func:`record_original_stdout` or
+ ``sys.stdout`` if it's not set.
+
+
+.. function:: strip_python_strerr(stderr)
+
+ Strip the *stderr* of a Python process from potential debug output
+ emitted by the interpreter. This will typically be run on the result of
+ :meth:`subprocess.Popen.communicate`.
+
+
+.. function:: args_from_interpreter_flags()
+
+ Return a list of command line arguments reproducing the current settings
+ in ``sys.flags`` and ``sys.warnoptions``.
+
+
+.. function:: optim_args_from_interpreter_flags()
+
+ Return a list of command line arguments reproducing the current
+ optimization settings in ``sys.flags``.
+
+
.. function:: captured_stdin()
captured_stdout()
captured_stderr()
@@ -434,17 +683,125 @@ The :mod:`test.support` module defines the following functions:
A context manager that temporarily sets the process umask.
+.. function:: transient_internet(resource_name, *, timeout=30.0, errnos=())
+
+ A context manager that raises :exc:`ResourceDenied` when various issues
+ with the internet connection manifest themselves as exceptions.
+
+
+.. function:: disable_faulthandler()
+
+ A context manager that replaces ``sys.stderr`` with ``sys.__stderr__``.
+
+
+.. function:: gc_collect()
+
+ Force as many objects as possible to be collected. This is needed because
+ timely deallocation is not guaranteed by the garbage collector. This means
+ that ``__del__`` methods may be called later than expected and weakrefs
+ may remain alive for longer than expected.
+
+
+.. function:: disable_gc()
+
+ A context manager that disables the garbage collector upon entry and
+ reenables it upon exit.
+
+
+.. function:: swap_attr(obj, attr, new_val)
+
+ Context manager to swap out an attribute with a new object.
+
+ Usage::
+
+ with swap_attr(obj, "attr", 5):
+ ...
+
+ This will set ``obj.attr`` to 5 for the duration of the ``with`` block,
+ restoring the old value at the end of the block. If ``attr`` doesn't
+ exist on ``obj``, it will be created and then deleted at the end of the
+ block.
+
+ The old value (or ``None`` if it doesn't exist) will be assigned to the
+ target of the "as" clause, if there is one.
+
+
+.. function:: swap_item(obj, attr, new_val)
+
+ Context manager to swap out an item with a new object.
+
+ Usage::
+
+ with swap_item(obj, "item", 5):
+ ...
+
+ This will set ``obj["item"]`` to 5 for the duration of the ``with`` block,
+ restoring the old value at the end of the block. If ``item`` doesn't
+ exist on ``obj``, it will be created and then deleted at the end of the
+ block.
+
+ The old value (or ``None`` if it doesn't exist) will be assigned to the
+ target of the "as" clause, if there is one.
+
+
+.. function:: wait_threads_exit(timeout=60.0)
+
+ Context manager to wait until all threads created in the ``with`` statement
+ exit.
+
+
+.. function:: start_threads(threads, unlock=None)
+
+ Context manager to start *threads*. It attempts to join the threads upon
+ exit.
+
+
+.. function:: calcobjsize(fmt)
+
+ Return :func:`struct.calcsize` for ``nP{fmt}0n`` or, if ``gettotalrefcount``
+ exists, ``2PnP{fmt}0P``.
+
+
+.. function:: calcvobjsize(fmt)
+
+ Return :func:`struct.calcsize` for ``nPn{fmt}0n`` or, if ``gettotalrefcount``
+ exists, ``2PnPn{fmt}0P``.
+
+
+.. function:: checksizeof(test, o, size)
+
+ For testcase *test*, assert that the ``sys.getsizeof`` for *o* plus the GC
+ header size equals *size*.
+
+
.. function:: can_symlink()
Return ``True`` if the OS supports symbolic links, ``False``
otherwise.
+.. function:: can_xattr()
+
+ Return ``True`` if the OS supports xattr, ``False``
+ otherwise.
+
+
.. decorator:: skip_unless_symlink
A decorator for running tests that require support for symbolic links.
+.. decorator:: skip_unless_xattr
+
+ A decorator for running tests that require support for xattr.
+
+
+.. decorator:: skip_unless_bind_unix_socket
+
+ A decorator for running tests that require a functional bind() for Unix
+ sockets.
+
+
.. decorator:: anticipate_failure(condition)
A decorator to conditionally mark tests with
@@ -460,20 +817,145 @@ The :mod:`test.support` module defines the following functions:
sequentially, and the first valid locale will be used.
+.. decorator:: run_with_tz(tz)
+
+ A decorator for running a function in a specific timezone, correctly
+ resetting it after it has finished.
+
+
+.. decorator:: requires_freebsd_version(*min_version)
+
+ Decorator for the minimum version when running test on FreeBSD. If the
+ FreeBSD version is less than the minimum, raise :exc:`unittest.SkipTest`.
+
+
+.. decorator:: requires_linux_version(*min_version)
+
+ Decorator for the minimum version when running test on Linux. If the
+ Linux version is less than the minimum, raise :exc:`unittest.SkipTest`.
+
+
+.. decorator:: requires_mac_version(*min_version)
+
+ Decorator for the minimum version when running test on Mac OS X. If the
+ MAC OS X version is less than the minimum, raise :exc:`unittest.SkipTest`.
+
+
+.. decorator:: requires_IEEE_754
+
+ Decorator for skipping tests on non-IEEE 754 platforms.
+
+
+.. decorator:: requires_zlib
+
+ Decorator for skipping tests if :mod:`zlib` doesn't exist.
+
+
+.. decorator:: requires_gzip
+
+ Decorator for skipping tests if :mod:`gzip` doesn't exist.
+
+
+.. decorator:: requires_bz2
+
+ Decorator for skipping tests if :mod:`bz2` doesn't exist.
+
+
+.. decorator:: requires_lzma
+
+ Decorator for skipping tests if :mod:`lzma` doesn't exist.
+
+
+.. decorator:: requires_resource(resource)
+
+ Decorator for skipping tests if *resource* is not available.
+
+
+.. decorator:: requires_docstrings
+
+ Decorator for only running the test if :data:`HAVE_DOCSTRINGS`.
+
+
+.. decorator:: cpython_only(test)
+
+ Decorator for tests only applicable to CPython.
+
+
+.. decorator:: impl_detail(msg=None, **guards)
+
+ Decorator for invoking :func:`check_impl_detail` on *guards*. If that
+ returns ``False``, then uses *msg* as the reason for skipping the test.
+
+
+.. decorator:: no_tracing(func)
+
+ Decorator to temporarily turn off tracing for the duration of the test.
+
+
+.. decorator:: refcount_test(test)
+
+ Decorator for tests which involve reference counting. The decorator does
+ not run the test if it is not run by CPython. Any trace function is unset
+ for the duration of the test to prevent unexpected refcounts caused by
+ the trace function.
+
+
+.. decorator:: reap_threads(func)
+
+ Decorator to ensure the threads are cleaned up even if the test fails.
+
+
+.. decorator:: bigmemtest(size, memuse, dry_run=True)
+
+ Decorator for bigmem tests.
+
+ *size* is a requested size for the test (in arbitrary, test-interpreted
+ units.) *memuse* is the number of bytes per unit for the test, or a good
+ estimate of it. For example, a test that needs two byte buffers, of 4 GiB
+ each, could be decorated with ``@bigmemtest(size=_4G, memuse=2)``.
+
+ The *size* argument is normally passed to the decorated test method as an
+ extra argument. If *dry_run* is ``True``, the value passed to the test
+ method may be less than the requested value. If *dry_run* is ``False``, it
+ means the test doesn't support dummy runs when ``-M`` is not specified.
+
+
+.. decorator:: bigaddrspacetest(f)
+
+ Decorator for tests that fill the address space. *f* is the function to
+ wrap.
+
+
.. function:: make_bad_fd()
Create an invalid file descriptor by opening and closing a temporary file,
and returning its descriptor.
-.. function:: import_module(name, deprecated=False)
+.. function:: check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None)
+
+ Test for syntax errors in *statement* by attempting to compile *statement*.
+ *testcase* is the :mod:`unittest` instance for the test. *errtext* is the
+ text of the error raised by :exc:`SyntaxError`. If *lineno* is not None,
+ compares to the line of the :exc:`SyntaxError`. If *offset* is not None,
+ compares to the offset of the :exc:`SyntaxError`.
+
+
+.. function:: open_urlresource(url, *args, **kw)
+
+ Open *url*. If open fails, raises :exc:`TestFailed`.
+
+
+.. function:: import_module(name, deprecated=False, *, required_on())
This function imports and returns the named module. Unlike a normal
import, this function raises :exc:`unittest.SkipTest` if the module
cannot be imported.
Module and package deprecation messages are suppressed during this import
- if *deprecated* is ``True``.
+ if *deprecated* is ``True``. If a module is required on a platform but
+ optional for others, set *required_on* to an iterable of platform prefixes
+ which will be compared against :data:`sys.platform`.
.. versionadded:: 3.1
@@ -514,6 +996,47 @@ The :mod:`test.support` module defines the following functions:
.. versionadded:: 3.1
+.. function:: modules_setup()
+
+ Return a copy of :data:`sys.modules`.
+
+
+.. function:: modules_cleanup(oldmodules)
+
+ Remove modules except for *oldmodules* and ``encodings`` in order to
+ preserve internal cache.
+
+
+.. function:: threading_setup()
+
+ Return current thread count and copy of dangling threads.
+
+
+.. function:: threading_cleanup(*original_values)
+
+ Cleanup up threads not specified in *original_values*. Designed to emit
+ a warning if a test leaves running threads in the background.
+
+
+.. function:: join_thread(thread, timeout=30.0)
+
+ Join a *thread* within *timeout*. Raise an :exc:`AssertionError` if thread
+ is still alive after *timeout* seconds.
+
+
+.. function:: reap_children()
+
+ Use this at the end of ``test_main`` whenever sub-processes are started.
+ This will help ensure that no extra children (zombies) stick around to
+ hog resources and create problems when looking for refleaks.
+
+
+.. function:: get_attribute(obj, name)
+
+ Get an attribute, raising :exc:`unittest.SkipTest` if :exc:`AttributeError`
+ is raised.
+
+
.. function:: bind_port(sock, host=HOST)
Bind the socket to a free port and return the port number. Relies on
@@ -533,6 +1056,12 @@ The :mod:`test.support` module defines the following functions:
test.
+.. function:: bind_unix_socket(sock, addr)
+
+ Bind a unix socket, raising :exc:`unittest.SkipTest` if
+ :exc:`PermissionError` is raised.
+
+
.. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM)
Returns an unused port that should be suitable for binding. This is
@@ -571,6 +1100,11 @@ The :mod:`test.support` module defines the following functions:
return load_package_tests(os.path.dirname(__file__), *args)
+.. function:: fs_is_case_insensitive(directory)
+
+ Return ``True`` if the file system for *directory* is case-insensitive.
+
+
.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=())
Returns the set of attributes, functions or methods of *ref_api* not
@@ -583,6 +1117,33 @@ The :mod:`test.support` module defines the following functions:
.. versionadded:: 3.5
+.. function:: patch(test_instance, object_to_patch, attr_name, new_value)
+
+ Override *object_to_patch.attr_name* with *new_value*. Also add
+ cleanup procedure to *test_instance* to restore *object_to_patch* for
+ *attr_name*. The *attr_name* should be a valid attribute for
+ *object_to_patch*.
+
+
+.. function:: run_in_subinterp(code)
+
+ Run *code* in subinterpreter. Raise :exc:`unittest.SkipTest` if
+ :mod:`tracemalloc` is enabled.
+
+
+.. function:: check_free_after_iterating(test, iter, cls, args=())
+
+ Assert that *iter* is deallocated after iterating.
+
+
+.. function:: missing_compiler_executable(cmd_names=[])
+
+ Check for the existence of the compiler executables whose names are listed
+ in *cmd_names* or all the compiler executables when *cmd_names* is empty
+ and return the first missing executable or ``None`` when none is found
+ missing.
+
+
.. function:: check__all__(test_case, module, name_of_module=None, extra=(), blacklist=())
Assert that the ``__all__`` variable of *module* contains all public names.
@@ -592,7 +1153,7 @@ The :mod:`test.support` module defines the following functions:
*module*.
The *name_of_module* argument can specify (as a string or tuple thereof) what
- module(s) an API could be defined in in order to be detected as a public
+ module(s) an API could be defined in order to be detected as a public
API. One case for this is when *module* imports part of its public API from
other modules, possibly a C backend (like ``csv`` and its ``_csv``).
@@ -673,7 +1234,169 @@ The :mod:`test.support` module defines the following classes:
On both platforms, the old value is restored by :meth:`__exit__`.
+.. class:: CleanImport(*module_names)
+
+ A context manager to force import to return a new module reference. This
+ is useful for testing module-level behaviors, such as the emission of a
+ DeprecationWarning on import. Example usage::
+
+ with CleanImport('foo'):
+ importlib.import_module('foo') # New reference.
+
+
+.. class:: DirsOnSysPath(*paths)
+
+ A context manager to temporarily add directories to sys.path.
+
+ This makes a copy of :data:`sys.path`, appends any directories given
+ as positional arguments, then reverts :data:`sys.path` to the copied
+ settings when the context ends.
+
+ Note that *all* :data:`sys.path` modifications in the body of the
+ context manager, including replacement of the object,
+ will be reverted at the end of the block.
+
+
+.. class:: SaveSignals()
+
+ Class to save and restore signal handlers registered by the Python signal
+ handler.
+
+
+.. class:: Matcher()
+
+ .. method:: matches(self, d, **kwargs)
+
+ Try to match a single dict with the supplied arguments.
+
+
+ .. method:: match_value(self, k, dv, v)
+
+ Try to match a single stored value (*dv*) with a supplied value (*v*).
+
+
.. class:: WarningsRecorder()
Class used to record warnings for unit tests. See documentation of
:func:`check_warnings` above for more details.
+
+
+.. class:: BasicTestRunner()
+
+ .. method:: run(test)
+
+ Run *test* and return the result.
+
+
+.. class:: TestHandler(logging.handlers.BufferingHandler)
+
+ Class for logging support.
+
+
+.. class:: FakePath(path)
+
+ Simple :term:`path-like object`. It implements the :meth:`__fspath__`
+ method which just returns the *path* argument. If *path* is an exception,
+ it will be raised in :meth:`!__fspath__`.
+
+
+:mod:`test.support.script_helper` --- Utilities for the Python execution tests
+==============================================================================
+
+.. module:: test.support.script_helper
+ :synopsis: Support for Python's script execution tests.
+
+
+The :mod:`test.support.script_helper` module provides support for Python's
+script execution tests.
+
+.. function:: interpreter_requires_environment()
+
+ Return ``True`` if ``sys.executable interpreter`` requires environment
+ variables in order to be able to run at all.
+
+ This is designed to be used with ``@unittest.skipIf()`` to annotate tests
+ that need to use an ``assert_python*()`` function to launch an isolated
+ mode (``-I``) or no environment mode (``-E``) sub-interpreter process.
+
+ A normal build & test does not run into this situation but it can happen
+ when trying to run the standard library test suite from an interpreter that
+ doesn't have an obvious home with Python's current home finding logic.
+
+ Setting :envvar:`PYTHONHOME` is one way to get most of the testsuite to run
+ in that situation. :envvar:`PYTHONPATH` or :envvar:`PYTHONUSERSITE` are
+ other common environment variables that might impact whether or not the
+ interpreter can start.
+
+
+.. function:: run_python_until_end(*args, **env_vars)
+
+ Set up the environment based on *env_vars* for running the interpreter
+ in a subprocess. The values can include ``__isolated``, ``__cleanenv``,
+ ``__cwd``, and ``TERM``.
+
+
+.. function:: assert_python_ok(*args, **env_vars)
+
+ Assert that running the interpreter with *args* and optional environment
+ variables *env_vars* succeeds (``rc == 0``) and return a ``(return code,
+ stdout, stderr)`` tuple.
+
+ If the ``__cleanenv`` keyword is set, *env_vars* is used as a fresh
+ environment.
+
+ Python is started in isolated mode (command line option ``-I``),
+ except if the ``__isolated`` keyword is set to ``False``.
+
+
+.. function:: assert_python_failure(*args, **env_vars)
+
+ Assert that running the interpreter with *args* and optional environment
+ variables *env_vars* fails (``rc != 0``) and return a ``(return code,
+ stdout, stderr)`` tuple.
+
+ See :func:`assert_python_ok` for more options.
+
+
+.. function:: spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw)
+
+ Run a Python subprocess with the given arguments.
+
+ *kw* is extra keyword args to pass to :func:`subprocess.Popen`. Returns a
+ :class:`subprocess.Popen` object.
+
+
+.. function:: kill_python(p)
+
+ Run the given :class:`subprocess.Popen` process until completion and return
+ stdout.
+
+
+.. function:: make_script(script_dir, script_basename, source, omit_suffix=False)
+
+ Create script containing *source* in path *script_dir* and *script_basename*.
+ If *omit_suffix* is ``False``, append ``.py`` to the name. Return the full
+ script path.
+
+
+.. function:: make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None)
+
+ Create zip file at *zip_dir* and *zip_basename* with extension ``zip`` which
+ contains the files in *script_name*. *name_in_zip* is the archive name.
+ Return a tuple containing ``(full path, full path of archive name)``.
+
+
+.. function:: make_pkg(pkg_dir, init_source='')
+
+ Create a directory named *pkg_dir* containing an ``__init__`` file with
+ *init_source* as its contents.
+
+
+.. function:: make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, \
+ source, depth=1, compiled=False)
+
+ Create a zip package directory with a path of *zip_dir* and *zip_basename*
+ containing an empty ``__init__`` file and a file *script_basename*
+ containing the *source*. If *compiled* is ``True``, both source files will
+ be compiled and added to the zip package. Return a tuple of the full zip
+ path and the archive name for the zip file.
diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst
index 8bbadf11608b34..93ca940ef5bdcb 100644
--- a/Doc/library/timeit.rst
+++ b/Doc/library/timeit.rst
@@ -25,7 +25,7 @@ Basic Examples
The following example shows how the :ref:`timeit-command-line-interface`
can be used to compare three different expressions:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 -m timeit '"-".join(str(n) for n in range(100))'
10000 loops, best of 5: 30.2 usec per loop
@@ -69,7 +69,7 @@ The module defines three convenience functions and a public class:
The optional *globals* parameter was added.
-.. function:: repeat(stmt='pass', setup='pass', timer=, repeat=3, number=1000000, globals=None)
+.. function:: repeat(stmt='pass', setup='pass', timer=, repeat=5, number=1000000, globals=None)
Create a :class:`Timer` instance with the given statement, *setup* code and
*timer* function and run its :meth:`.repeat` method with the given *repeat*
@@ -79,6 +79,9 @@ The module defines three convenience functions and a public class:
.. versionchanged:: 3.5
The optional *globals* parameter was added.
+ .. versionchanged:: 3.7
+ Default value of *repeat* changed from 3 to 5.
+
.. function:: default_timer()
The default timer, which is always :func:`time.perf_counter`.
@@ -150,7 +153,7 @@ The module defines three convenience functions and a public class:
.. versionadded:: 3.6
- .. method:: Timer.repeat(repeat=3, number=1000000)
+ .. method:: Timer.repeat(repeat=5, number=1000000)
Call :meth:`.timeit` a few times.
@@ -171,6 +174,9 @@ The module defines three convenience functions and a public class:
should be interested in. After that, you should look at the entire
vector and apply common sense rather than statistics.
+ .. versionchanged:: 3.7
+ Default value of *repeat* changed from 3 to 5.
+
.. method:: Timer.print_exc(file=None)
@@ -208,7 +214,7 @@ Where the following options are understood:
.. cmdoption:: -r N, --repeat=N
- how many times to repeat the timer (default 3)
+ how many times to repeat the timer (default 5)
.. cmdoption:: -s S, --setup=S
@@ -246,7 +252,7 @@ successive powers of 10 until the total time is at least 0.2 seconds.
:func:`default_timer` measurements can be affected by other programs running on
the same machine, so the best thing to do when accurate timing is necessary is
to repeat the timing a few times and use the best time. The :option:`-r`
-option is good for this; the default of 3 repetitions is probably enough in
+option is good for this; the default of 5 repetitions is probably enough in
most cases. You can use :func:`time.process_time` to measure CPU time.
.. note::
@@ -264,7 +270,7 @@ Examples
It is possible to provide a setup statement that is executed only once at the beginning:
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m timeit -s 'text = "sample string"; char = "g"' 'char in text'
5000000 loops, best of 5: 0.0877 usec per loop
@@ -293,7 +299,7 @@ The following examples show how to time expressions that contain multiple lines.
Here we compare the cost of using :func:`hasattr` vs. :keyword:`try`/:keyword:`except`
to test for missing and present object attributes:
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m timeit 'try:' ' str.__bool__' 'except AttributeError:' ' pass'
20000 loops, best of 5: 15.7 usec per loop
diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst
index 6d90e43e33b266..d9c1c35aa294bd 100644
--- a/Doc/library/tkinter.rst
+++ b/Doc/library/tkinter.rst
@@ -13,12 +13,17 @@
The :mod:`tkinter` package ("Tk interface") is the standard Python interface to
the Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix
platforms, as well as on Windows systems. (Tk itself is not part of Python; it
-is maintained at ActiveState.) You can check that :mod:`tkinter` is properly
-installed on your system by running ``python -m tkinter`` from the command line;
-this should open a window demonstrating a simple Tk interface.
+is maintained at ActiveState.)
+
+Running ``python -m tkinter`` from the command line should open a window
+demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is
+properly installed on your system, and also showing what version of Tcl/Tk is
+installed, so you can read the Tcl/Tk documentation specific to that version.
.. seealso::
+ Tkinter documentation:
+
`Python Tkinter Resources `_
The Python Tkinter Topic Guide provides a great deal of information on using Tk
from Python and links to other sources of information on Tk.
@@ -32,9 +37,6 @@ this should open a window demonstrating a simple Tk interface.
`Tkinter docs from effbot `_
Online reference for tkinter supported by effbot.org.
- `Tcl/Tk manual `_
- Official manual for the latest tcl/tk version.
-
`Programming Python `_
Book by Mark Lutz, has excellent coverage of Tkinter.
@@ -42,7 +44,25 @@ this should open a window demonstrating a simple Tk interface.
Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter.
`Python and Tkinter Programming `_
- The book by John Grayson (ISBN 1-884777-81-3).
+ Book by John Grayson (ISBN 1-884777-81-3).
+
+ Tcl/Tk documentation:
+
+ `Tk commands `_
+ Most commands are available as :mod:`tkinter` or :mod:`tkinter.ttk` classes.
+ Change '8.6' to match the version of your Tcl/Tk installation.
+
+ `Tcl/Tk recent man pages `_
+ Recent Tcl/Tk manuals on www.tcl.tk.
+
+ `ActiveState Tcl Home Page `_
+ The Tk/Tcl development is largely taking place at ActiveState.
+
+ `Tcl and the Tk Toolkit `_
+ Book by John Ousterhout, the inventor of Tcl.
+
+ `Practical Programming in Tcl and Tk `_
+ Brent Welch's encyclopedic book.
Tkinter Modules
@@ -175,21 +195,6 @@ documentation that exists. Here are some hints:
place to go when nothing else makes sense.
-.. seealso::
-
- `Tcl/Tk 8.6 man pages `_
- The Tcl/Tk manual on www.tcl.tk.
-
- `ActiveState Tcl Home Page `_
- The Tk/Tcl development is largely taking place at ActiveState.
-
- `Tcl and the Tk Toolkit `_
- The book by John Ousterhout, the inventor of Tcl.
-
- `Practical Programming in Tcl and Tk `_
- Brent Welch's encyclopedic book.
-
-
A Simple Hello World Program
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -783,12 +788,13 @@ Menu indexes (menu.invoke(), menu.entryconfig(), etc.)
Images
^^^^^^
-Bitmap/Pixelmap images can be created through the subclasses of
-:class:`tkinter.Image`:
+Images of different formats can be created through the corresponding subclass
+of :class:`tkinter.Image`:
-* :class:`BitmapImage` can be used for X11 bitmap data.
+* :class:`BitmapImage` for images in XBM format.
-* :class:`PhotoImage` can be used for GIF and PPM/PGM color bitmaps.
+* :class:`PhotoImage` for images in PGM, PPM, GIF and PNG formats. The latter
+ is supported starting with Tk 8.6.
Either type of image is created through either the ``file`` or the ``data``
option (other options are available as well).
@@ -799,6 +805,10 @@ reference to the image. When the last Python reference to the image object is
deleted, the image data is deleted as well, and Tk will display an empty box
wherever the image was used.
+.. seealso::
+
+ The `Pillow `_ package adds support for
+ formats such as BMP, JPEG, TIFF, and WebP, among others.
.. _tkinter-file-handlers:
diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst
index debbc81ce1a390..76ecfcce497775 100644
--- a/Doc/library/tkinter.ttk.rst
+++ b/Doc/library/tkinter.ttk.rst
@@ -66,13 +66,13 @@ for improved styling effects.
Ttk Widgets
-----------
-Ttk comes with 17 widgets, eleven of which already existed in tkinter:
+Ttk comes with 18 widgets, twelve of which already existed in tkinter:
:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`,
:class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`,
-:class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`. The other six are
-new: :class:`Combobox`, :class:`Notebook`, :class:`Progressbar`,
-:class:`Separator`, :class:`Sizegrip` and :class:`Treeview`. And all them are
-subclasses of :class:`Widget`.
+:class:`Radiobutton`, :class:`Scale`, :class:`Scrollbar`, and :class:`Spinbox`.
+The other six are new: :class:`Combobox`, :class:`Notebook`,
+:class:`Progressbar`, :class:`Separator`, :class:`Sizegrip` and
+:class:`Treeview`. And all them are subclasses of :class:`Widget`.
Using the Ttk widgets gives the application an improved look and feel.
As discussed above, there are differences in how the styling is coded.
@@ -381,6 +381,87 @@ ttk.Combobox
Sets the value of the combobox to *value*.
+Spinbox
+-------
+The :class:`ttk.Spinbox` widget is a :class:`ttk.Entry` enhanced with increment
+and decrement arrows. It can be used for numbers or lists of string values.
+This widget is a subclass of :class:`Entry`.
+
+Besides the methods inherited from :class:`Widget`: :meth:`Widget.cget`,
+:meth:`Widget.configure`, :meth:`Widget.identify`, :meth:`Widget.instate`
+and :meth:`Widget.state`, and the following inherited from :class:`Entry`:
+:meth:`Entry.bbox`, :meth:`Entry.delete`, :meth:`Entry.icursor`,
+:meth:`Entry.index`, :meth:`Entry.insert`, :meth:`Entry.xview`,
+it has some other methods, described at :class:`ttk.Spinbox`.
+
+Options
+^^^^^^^
+
+This widget accepts the following specific options:
+
+ .. tabularcolumns:: |l|L|
+
++----------------------+------------------------------------------------------+
+| Option | Description |
++======================+======================================================+
+| from | Float value. If set, this is the minimum value to |
+| | which the decrement button will decrement. Must be |
+| | spelled as ``from_`` when used as an argument, since |
+| | ``from`` is a Python keyword. |
++----------------------+------------------------------------------------------+
+| to | Float value. If set, this is the maximum value to |
+| | which the increment button will increment. |
++----------------------+------------------------------------------------------+
+| increment | Float value. Specifies the amount which the |
+| | increment/decrement buttons change the |
+| | value. Defaults to 1.0. |
++----------------------+------------------------------------------------------+
+| values | Sequence of string or float values. If specified, |
+| | the increment/decrement buttons will cycle through |
+| | the items in this sequence rather than incrementing |
+| | or decrementing numbers. |
+| | |
++----------------------+------------------------------------------------------+
+| wrap | Boolean value. If ``True``, increment and decrement |
+| | buttons will cycle from the ``to`` value to the |
+| | ``from`` value or the ``from`` value to the ``to`` |
+| | value, respectively. |
++----------------------+------------------------------------------------------+
+| format | String value. This specifies the format of numbers |
+| | set by the increment/decrement buttons. It must be |
+| | in the form "%W.Pf", where W is the padded width of |
+| | the value, P is the precision, and '%' and 'f' are |
+| | literal. |
++----------------------+------------------------------------------------------+
+| command | Python callable. Will be called with no arguments |
+| | whenever either of the increment or decrement buttons|
+| | are pressed. |
+| | |
++----------------------+------------------------------------------------------+
+
+
+Virtual events
+^^^^^^^^^^^^^^
+
+The spinbox widget generates an **<>** virtual event when the
+user presses , and a **<>** virtual event when the user
+presses .
+
+ttk.Spinbox
+^^^^^^^^^^^^
+
+.. class:: Spinbox
+
+ .. method:: get()
+
+ Returns the current value of the spinbox.
+
+
+ .. method:: set(value)
+
+ Sets the value of the spinbox to *value*.
+
+
Notebook
--------
diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst
index 02a0428f21bc76..4c0a0ceef7dc4e 100644
--- a/Doc/library/tokenize.rst
+++ b/Doc/library/tokenize.rst
@@ -218,7 +218,7 @@ will be tokenized to the following output where the first column is the range
of the line/column coordinates where the token is found, the second column is
the name of the token, and the final column is the value of the token (if any)
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m tokenize hello.py
0,0-0,0: ENCODING 'utf-8'
@@ -244,7 +244,7 @@ the name of the token, and the final column is the value of the token (if any)
The exact token type names can be displayed using the :option:`-e` option:
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m tokenize -e hello.py
0,0-0,0: ENCODING 'utf-8'
diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst
index 55d331c996a187..7ac3cacd3d1c93 100644
--- a/Doc/library/traceback.rst
+++ b/Doc/library/traceback.rst
@@ -88,14 +88,16 @@ The module defines the following functions:
.. function:: extract_tb(tb, limit=None)
- Return a list of "pre-processed" stack trace entries extracted from the
- traceback object *tb*. It is useful for alternate formatting of
- stack traces. The optional *limit* argument has the same meaning as for
- :func:`print_tb`. A "pre-processed" stack trace entry is a 4-tuple
- (*filename*, *line number*, *function name*, *text*) representing the
- information that is usually printed for a stack trace. The *text* is a
- string with leading and trailing whitespace stripped; if the source is
- not available it is ``None``.
+ Return a :class:`StackSummary` object representing a list of "pre-processed"
+ stack trace entries extracted from the traceback object *tb*. It is useful
+ for alternate formatting of stack traces. The optional *limit* argument has
+ the same meaning as for :func:`print_tb`. A "pre-processed" stack trace
+ entry is a :class:`FrameSummary` object containing attributes
+ :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`,
+ :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the
+ information that is usually printed for a stack trace. The
+ :attr:`~FrameSummary.line` is a string with leading and trailing
+ whitespace stripped; if the source is not available it is ``None``.
.. function:: extract_stack(f=None, limit=None)
@@ -107,12 +109,12 @@ The module defines the following functions:
.. function:: format_list(extracted_list)
- Given a list of tuples as returned by :func:`extract_tb` or
- :func:`extract_stack`, return a list of strings ready for printing. Each
- string in the resulting list corresponds to the item with the same index in
- the argument list. Each string ends in a newline; the strings may contain
- internal newlines as well, for those items whose source text line is not
- ``None``.
+ Given a list of tuples or :class:`FrameSummary` objects as returned by
+ :func:`extract_tb` or :func:`extract_stack`, return a list of strings ready
+ for printing. Each string in the resulting list corresponds to the item with
+ the same index in the argument list. Each string ends in a newline; the
+ strings may contain internal newlines as well, for those items whose source
+ text line is not ``None``.
.. function:: format_exception_only(etype, value)
@@ -293,9 +295,9 @@ capture data for later printing in a lightweight fashion.
.. classmethod:: from_list(a_list)
- Construct a :class:`StackSummary` object from a supplied old-style list
- of tuples. Each tuple should be a 4-tuple with filename, lineno, name,
- line as the elements.
+ Construct a :class:`StackSummary` object from a supplied list of
+ :class:`FrameSummary` objects or old-style list of tuples. Each tuple
+ should be a 4-tuple with filename, lineno, name, line as the elements.
.. method:: format()
diff --git a/Doc/library/types.rst b/Doc/library/types.rst
index bbc1d1301d2c2a..e17070022cd2ed 100644
--- a/Doc/library/types.rst
+++ b/Doc/library/types.rst
@@ -59,7 +59,7 @@ Dynamic Type Creation
The default value for the ``namespace`` element of the returned
tuple has changed. Now an insertion-order-preserving mapping is
- used when the metaclass does not have a ``__prepare__`` method,
+ used when the metaclass does not have a ``__prepare__`` method.
.. seealso::
@@ -69,6 +69,23 @@ Dynamic Type Creation
:pep:`3115` - Metaclasses in Python 3000
Introduced the ``__prepare__`` namespace hook
+.. function:: resolve_bases(bases)
+
+ Resolve MRO entries dynamically as specified by :pep:`560`.
+
+ This function looks for items in *bases* that are not instances of
+ :class:`type`, and returns a tuple where each such object that has
+ an ``__mro_entries__`` method is replaced with an unpacked result of
+ calling this method. If a *bases* item is an instance of :class:`type`,
+ or it doesn't have an ``__mro_entries__`` method, then it is included in
+ the return tuple unchanged.
+
+ .. versionadded:: 3.7
+
+.. seealso::
+
+ :pep:`560` - Core support for typing module and generic types
+
Standard Interpreter Types
--------------------------
@@ -198,16 +215,23 @@ Standard names are defined for the following types:
Defaults to ``None``. Previously the attribute was optional.
-.. data:: TracebackType
+.. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno)
The type of traceback objects such as found in ``sys.exc_info()[2]``.
+ See :ref:`the language reference ` for details of the
+ available attributes and operations, and guidance on creating tracebacks
+ dynamically.
+
.. data:: FrameType
The type of frame objects such as found in ``tb.tb_frame`` if ``tb`` is a
traceback object.
+ See :ref:`the language reference ` for details of the
+ available attributes and operations.
+
.. data:: GetSetDescriptorType
diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst
index 9c4777ac2fe580..23a64156c59297 100644
--- a/Doc/library/typing.rst
+++ b/Doc/library/typing.rst
@@ -666,6 +666,12 @@ The module defines the following classes, functions and decorators:
.. versionadded:: 3.6
+.. class:: AsyncContextManager(Generic[T_co])
+
+ A generic version of :class:`contextlib.AbstractAsyncContextManager`.
+
+ .. versionadded:: 3.6
+
.. class:: Dict(dict, MutableMapping[KT, VT])
A generic version of :class:`dict`.
@@ -931,6 +937,18 @@ The module defines the following classes, functions and decorators:
* Every type is compatible with :data:`Any`.
* :data:`Any` is compatible with every type.
+.. data:: NoReturn
+
+ Special type indicating that a function never returns.
+ For example::
+
+ from typing import NoReturn
+
+ def stop() -> NoReturn:
+ raise RuntimeError('no way')
+
+ .. versionadded:: 3.6.5
+
.. data:: Union
Union type; ``Union[X, Y]`` means either X or Y.
@@ -955,16 +973,15 @@ The module defines the following classes, functions and decorators:
Union[int, str] == Union[str, int]
- * When a class and its subclass are present, the latter is skipped, e.g.::
-
- Union[int, object] == object
-
* You cannot subclass or instantiate a union.
* You cannot write ``Union[X][Y]``.
* You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``.
+ .. versionchanged:: 3.7
+ Don't remove explicit subclasses from unions at runtime.
+
.. data:: Optional
Optional type.
@@ -973,10 +990,18 @@ The module defines the following classes, functions and decorators:
Note that this is not the same concept as an optional argument,
which is one that has a default. An optional argument with a
- default needn't use the ``Optional`` qualifier on its type
- annotation (although it is inferred if the default is ``None``).
- A mandatory argument may still have an ``Optional`` type if an
- explicit value of ``None`` is allowed.
+ default does not require the ``Optional`` qualifier on its type
+ annotation just because it is optional. For example::
+
+ def foo(arg: int = 0) -> None:
+ ...
+
+ On the other hand, if an explicit value of ``None`` is allowed, the
+ use of ``Optional`` is appropriate, whether the argument is optional
+ or not. For example::
+
+ def foo(arg: Optional[int] = None) -> None:
+ ...
.. data:: Tuple
diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index 7a5a1f8ef18ee1..59548f3e8b4ac5 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -17,8 +17,8 @@
This module provides access to the Unicode Character Database (UCD) which
defines character properties for all Unicode characters. The data contained in
-this database is compiled from the `UCD version 10.0.0
-`_.
+this database is compiled from the `UCD version 11.0.0
+`_.
The module uses the same names and symbols as defined by Unicode
Standard Annex #44, `"Unicode Character Database"
@@ -168,6 +168,6 @@ Examples:
.. rubric:: Footnotes
-.. [#] http://www.unicode.org/Public/10.0.0/ucd/NameAliases.txt
+.. [#] http://www.unicode.org/Public/11.0.0/ucd/NameAliases.txt
-.. [#] http://www.unicode.org/Public/10.0.0/ucd/NamedSequences.txt
+.. [#] http://www.unicode.org/Public/11.0.0/ucd/NamedSequences.txt
diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index ac9dd3b6f793a2..396853cc6125b2 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -35,7 +35,7 @@ is based on the 'action -> assertion' pattern instead of 'record -> replay'
used by many mocking frameworks.
There is a backport of :mod:`unittest.mock` for earlier versions of Python,
-available as `mock on PyPI `_.
+available as `mock on PyPI `_.
Quick Guide
@@ -1825,12 +1825,12 @@ sentinel
.. data:: sentinel
- The ``sentinel`` object provides a convenient way of providing unique
- objects for your tests.
+ The ``sentinel`` object provides a convenient way of providing unique
+ objects for your tests.
- Attributes are created on demand when you access them by name. Accessing
- the same attribute will always return the same object. The objects
- returned have a sensible repr so that test failure messages are readable.
+ Attributes are created on demand when you access them by name. Accessing
+ the same attribute will always return the same object. The objects
+ returned have a sensible repr so that test failure messages are readable.
.. versionchanged:: 3.7
The ``sentinel`` attributes now preserve their identity when they are
@@ -2070,22 +2070,22 @@ mock_open
.. function:: mock_open(mock=None, read_data=None)
- A helper function to create a mock to replace the use of :func:`open`. It works
- for :func:`open` called directly or used as a context manager.
-
- The *mock* argument is the mock object to configure. If ``None`` (the
- default) then a :class:`MagicMock` will be created for you, with the API limited
- to methods or attributes available on standard file handles.
-
- *read_data* is a string for the :meth:`~io.IOBase.read`,
- :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods
- of the file handle to return. Calls to those methods will take data from
- *read_data* until it is depleted. The mock of these methods is pretty
- simplistic: every time the *mock* is called, the *read_data* is rewound to
- the start. If you need more control over the data that you are feeding to
- the tested code you will need to customize this mock for yourself. When that
- is insufficient, one of the in-memory filesystem packages on `PyPI
- `_ can offer a realistic filesystem for testing.
+ A helper function to create a mock to replace the use of :func:`open`. It works
+ for :func:`open` called directly or used as a context manager.
+
+ The *mock* argument is the mock object to configure. If ``None`` (the
+ default) then a :class:`MagicMock` will be created for you, with the API limited
+ to methods or attributes available on standard file handles.
+
+ *read_data* is a string for the :meth:`~io.IOBase.read`,
+ :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods
+ of the file handle to return. Calls to those methods will take data from
+ *read_data* until it is depleted. The mock of these methods is pretty
+ simplistic: every time the *mock* is called, the *read_data* is rewound to
+ the start. If you need more control over the data that you are feeding to
+ the tested code you will need to customize this mock for yourself. When that
+ is insufficient, one of the in-memory filesystem packages on `PyPI
+ `_ can offer a realistic filesystem for testing.
.. versionchanged:: 3.4
Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support.
diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
index 93ccd0fd61139c..224adf08567ec8 100644
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -361,8 +361,9 @@ testing code::
Note that in order to test something, we use one of the :meth:`assert\*`
methods provided by the :class:`TestCase` base class. If the test fails, an
-exception will be raised, and :mod:`unittest` will identify the test case as a
-:dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`.
+exception will be raised with an explanatory message, and :mod:`unittest`
+will identify the test case as a :dfn:`failure`. Any other exceptions will be
+treated as :dfn:`errors`.
Tests can be numerous, and their set-up can be repetitive. Luckily, we
can factor out set-up code by implementing a method called
@@ -408,13 +409,18 @@ after the test method has been run::
If :meth:`~TestCase.setUp` succeeded, :meth:`~TestCase.tearDown` will be
run whether the test method succeeded or not.
-Such a working environment for the testing code is called a :dfn:`fixture`.
+Such a working environment for the testing code is called a
+:dfn:`test fixture`. A new TestCase instance is created as a unique
+test fixture used to execute each individual test method. Thus
+:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`~TestCase.__init__`
+will be called once per test.
-Test case instances are grouped together according to the features they test.
-:mod:`unittest` provides a mechanism for this: the :dfn:`test suite`,
-represented by :mod:`unittest`'s :class:`TestSuite` class. In most cases,
-calling :func:`unittest.main` will do the right thing and collect all the
-module's test cases for you, and then execute them.
+It is recommended that you use TestCase implementations to group tests together
+according to the features they test. :mod:`unittest` provides a mechanism for
+this: the :dfn:`test suite`, represented by :mod:`unittest`'s
+:class:`TestSuite` class. In most cases, calling :func:`unittest.main` will do
+the right thing and collect all the module's test cases for you and execute
+them.
However, should you want to customize the building of your test suite,
you can do it yourself::
@@ -2316,7 +2322,7 @@ handling functionality within test frameworks.
When called without arguments this function removes the control-c handler
if it has been installed. This function can also be used as a test decorator
- to temporarily remove the handler whilst the test is being executed::
+ to temporarily remove the handler while the test is being executed::
@unittest.removeHandler
def test_signal_handling(self):
diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst
index 5517b04f5ddf71..f7d47ed76aca18 100644
--- a/Doc/library/urllib.error.rst
+++ b/Doc/library/urllib.error.rst
@@ -41,8 +41,7 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate:
.. attribute:: code
- An HTTP status code as defined in `RFC 2616
- `_. This numeric value corresponds
+ An HTTP status code as defined in :rfc:`2616`. This numeric value corresponds
to a value found in the dictionary of codes as found in
:attr:`http.server.BaseHTTPRequestHandler.responses`.
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
index c0a388c5153eae..2d3488bb5eef54 100644
--- a/Doc/library/urllib.parse.rst
+++ b/Doc/library/urllib.parse.rst
@@ -468,7 +468,7 @@ task isn't already covered by the URL parsing functions above.
*string* may be either a :class:`str` or a :class:`bytes`.
.. versionchanged:: 3.7
- Moved from RFC 2396 to RFC 3986 for quoting URL strings. "~" is now
+ Moved from :rfc:`2396` to :rfc:`3986` for quoting URL strings. "~" is now
included in the set of reserved characters.
The optional *encoding* and *errors* parameters specify how to deal with
diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst
index 413d8b6ffdfe0a..cbbec0cd831183 100644
--- a/Doc/library/urllib.request.rst
+++ b/Doc/library/urllib.request.rst
@@ -232,7 +232,7 @@ The following classes are provided:
containing the image.
*unverifiable* should indicate whether the request is unverifiable,
- as defined by RFC 2965. It defaults to ``False``. An unverifiable
+ as defined by :rfc:`2965`. It defaults to ``False``. An unverifiable
request is one whose URL the user did not have the option to
approve. For example, if the request is for an image in an HTML
document, and the user had no option to approve the automatic
@@ -504,7 +504,7 @@ request.
.. attribute:: Request.unverifiable
boolean, indicates whether the request is unverifiable as defined
- by RFC 2965.
+ by :rfc:`2965`.
.. attribute:: Request.method
@@ -1125,7 +1125,7 @@ UnknownHandler Objects
HTTPErrorProcessor Objects
--------------------------
-.. method:: HTTPErrorProcessor.http_response()
+.. method:: HTTPErrorProcessor.http_response(request, response)
Process HTTP error responses.
@@ -1137,7 +1137,7 @@ HTTPErrorProcessor Objects
:exc:`~urllib.error.HTTPError` if no other handler handles the error.
-.. method:: HTTPErrorProcessor.https_response()
+.. method:: HTTPErrorProcessor.https_response(request, response)
Process HTTPS error responses.
@@ -1339,9 +1339,9 @@ some point in the future.
The second argument, if present, specifies the file location to copy to (if
absent, the location will be a tempfile with a generated name). The third
- argument, if present, is a hook function that will be called once on
+ argument, if present, is a callable that will be called once on
establishment of the network connection and once after each block read
- thereafter. The hook will be passed three arguments; a count of blocks
+ thereafter. The callable will be passed three arguments; a count of blocks
transferred so far, a block size in bytes, and the total size of the file. The
third argument may be ``-1`` on older FTP servers which do not return a file
size in response to a retrieval request.
diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst
index 8ec75a79acfa71..415e25bef6de7d 100644
--- a/Doc/library/uuid.rst
+++ b/Doc/library/uuid.rst
@@ -1,5 +1,5 @@
-:mod:`uuid` --- UUID objects according to RFC 4122
-==================================================
+:mod:`uuid` --- UUID objects according to :rfc:`4122`
+=====================================================
.. module:: uuid
:synopsis: UUID objects (universally unique identifiers) according to RFC 4122
@@ -45,12 +45,13 @@ which relays any information about the UUID's safety, using this enumeration:
.. class:: UUID(hex=None, bytes=None, bytes_le=None, fields=None, int=None, version=None, *, is_safe=SafeUUID.unknown)
Create a UUID from either a string of 32 hexadecimal digits, a string of 16
- bytes as the *bytes* argument, a string of 16 bytes in little-endian order as
- the *bytes_le* argument, a tuple of six integers (32-bit *time_low*, 16-bit
- *time_mid*, 16-bit *time_hi_version*, 8-bit *clock_seq_hi_variant*, 8-bit
- *clock_seq_low*, 48-bit *node*) as the *fields* argument, or a single 128-bit
- integer as the *int* argument. When a string of hex digits is given, curly
- braces, hyphens, and a URN prefix are all optional. For example, these
+ bytes in big-endian order as the *bytes* argument, a string of 16 bytes in
+ little-endian order as the *bytes_le* argument, a tuple of six integers
+ (32-bit *time_low*, 16-bit *time_mid*, 16-bit *time_hi_version*,
+ 8-bit *clock_seq_hi_variant*, 8-bit *clock_seq_low*, 48-bit *node*) as the
+ *fields* argument, or a single 128-bit integer as the *int* argument.
+ When a string of hex digits is given, curly braces, hyphens,
+ and a URN prefix are all optional. For example, these
expressions all yield the same UUID::
UUID('{12345678-1234-5678-1234-567812345678}')
@@ -64,7 +65,7 @@ which relays any information about the UUID's safety, using this enumeration:
Exactly one of *hex*, *bytes*, *bytes_le*, *fields*, or *int* must be given.
The *version* argument is optional; if given, the resulting UUID will have its
- variant and version number set according to RFC 4122, overriding bits in the
+ variant and version number set according to :rfc:`4122`, overriding bits in the
given *hex*, *bytes*, *bytes_le*, *fields*, or *int*.
Comparison of UUID objects are made by way of comparing their
@@ -127,7 +128,7 @@ which relays any information about the UUID's safety, using this enumeration:
.. attribute:: UUID.urn
- The UUID as a URN as specified in RFC 4122.
+ The UUID as a URN as specified in :rfc:`4122`.
.. attribute:: UUID.variant
@@ -158,7 +159,7 @@ The :mod:`uuid` module defines the following functions:
runs, it may launch a separate program, which could be quite slow. If all
attempts to obtain the hardware address fail, we choose a random 48-bit
number with the multicast bit (least significant bit of the first octet)
- set to 1 as recommended in RFC 4122. "Hardware address" means the MAC
+ set to 1 as recommended in :rfc:`4122`. "Hardware address" means the MAC
address of a network interface. On a machine with multiple network
interfaces, universally administered MAC addresses (i.e. where the second
least significant bit of the first octet is *unset*) will be preferred over
diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst
index 6707be7fc820dd..def926b0852690 100644
--- a/Doc/library/venv.rst
+++ b/Doc/library/venv.rst
@@ -17,12 +17,18 @@
The :mod:`venv` module provides support for creating lightweight "virtual
environments" with their own site directories, optionally isolated from system
-site directories. Each virtual environment has its own Python binary (allowing
-creation of environments with various Python versions) and can have its own
-independent set of installed Python packages in its site directories.
+site directories. Each virtual environment has its own Python binary (which
+matches the version of the binary that was used to create this environment) and
+can have its own independent set of installed Python packages in its site
+directories.
See :pep:`405` for more information about Python virtual environments.
+.. seealso::
+
+ `Python Packaging User Guide: Creating and using virtual environments
+ `__
+
.. note::
The ``pyvenv`` script has been deprecated as of Python 3.6 in favor of using
``python3 -m venv`` to help prevent any potential confusion as to which
@@ -109,8 +115,7 @@ creation according to their needs, the :class:`EnvBuilder` class.
* ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the
Python binary (and any necessary DLLs or other binaries,
- e.g. ``pythonw.exe``), rather than copying. Defaults to ``True`` on Linux and
- Unix systems, but ``False`` on Windows.
+ e.g. ``pythonw.exe``), rather than copying.
* ``upgrade`` -- a Boolean value which, if true, will upgrade an existing
environment with the running Python - for use when that Python has been
diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst
index b0ef3465b04fab..2d9b7b359e8060 100644
--- a/Doc/library/wsgiref.rst
+++ b/Doc/library/wsgiref.rst
@@ -397,7 +397,7 @@ Paste" library.
Wrap *application* and return a new WSGI application object. The returned
application will forward all requests to the original *application*, and will
check that both the *application* and the server invoking it are conforming to
- the WSGI specification and to RFC 2616.
+ the WSGI specification and to :rfc:`2616`.
Any detected nonconformance results in an :exc:`AssertionError` being raised;
note, however, that how these errors are handled is server-dependent. For
diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index 61808596a55532..6298f1396ae294 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -297,7 +297,7 @@ If the XML input has `namespaces
with prefixes in the form ``prefix:sometag`` get expanded to
``{uri}sometag`` where the *prefix* is replaced by the full *URI*.
Also, if there is a `default namespace
-`__,
+`__,
that full URI gets prepended to all of the non-prefixed tags.
Here is an XML example that incorporates two namespaces, one with the
diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst
index d833b7fe99774e..63c24f80ac87c4 100644
--- a/Doc/library/xml.rst
+++ b/Doc/library/xml.rst
@@ -130,8 +130,8 @@ but will not be included in any bugfix releases of
Python because they break backward compatibility.
-.. _defusedxml: https://pypi.python.org/pypi/defusedxml/
-.. _defusedexpat: https://pypi.python.org/pypi/defusedexpat/
+.. _defusedxml: https://pypi.org/project/defusedxml/
+.. _defusedexpat: https://pypi.org/project/defusedexpat/
.. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs
.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb
.. _DTD: https://en.wikipedia.org/wiki/Document_type_definition
diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst
index 390828ef3d9859..27d92e324722ea 100644
--- a/Doc/library/xmlrpc.client.rst
+++ b/Doc/library/xmlrpc.client.rst
@@ -145,7 +145,7 @@ between conformable Python objects and XML on the wire.
.. versionchanged:: 3.6
Added support of type tags with prefixes (e.g. ``ex:nil``).
- Added support of unmarsalling additional types used by Apache XML-RPC
+ Added support of unmarshalling additional types used by Apache XML-RPC
implementation for numerics: ``i1``, ``i2``, ``i8``, ``biginteger``,
``float`` and ``bigdecimal``.
See http://ws.apache.org/xmlrpc/types.html for a description.
@@ -328,7 +328,7 @@ Binary Objects
Write the XML-RPC base 64 encoding of this binary item to the *out* stream object.
The encoded data will have newlines every 76 characters as per
- `RFC 2045 section 6.8 `_,
+ :rfc:`RFC 2045 section 6.8 <2045#section-6.8>`,
which was the de facto standard base64 specification when the
XML-RPC spec was written.
diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst
index 9076593c4b1c22..26b0f19fdeb648 100644
--- a/Doc/library/zipapp.rst
+++ b/Doc/library/zipapp.rst
@@ -27,7 +27,7 @@ can be used to create an executable archive from a directory containing
Python code. When run, the archive will execute the ``main`` function from
the module ``myapp`` in the archive.
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m zipapp myapp -m "myapp:main"
$ python myapp.pyz
@@ -41,7 +41,7 @@ Command-Line Interface
When called as a program from the command line, the following form is used:
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m zipapp source [options]
@@ -189,7 +189,7 @@ Examples
Pack up a directory into an archive, and run it.
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m zipapp myapp
$ python myapp.pyz
@@ -203,7 +203,7 @@ The same can be done using the :func:`create_archive` functon::
To make the application directly executable on POSIX, specify an interpreter
to use.
-.. code-block:: sh
+.. code-block:: shell-session
$ python -m zipapp myapp -p "/usr/bin/env python"
$ ./myapp.pyz
@@ -229,6 +229,12 @@ fits in memory::
>>> with open('myapp.pyz', 'wb') as f:
>>> f.write(temp.getvalue())
+
+.. _zipapp-specifying-the-interpreter:
+
+Specifying the Interpreter
+--------------------------
+
Note that if you specify an interpreter and then distribute your application
archive, you need to ensure that the interpreter used is portable. The Python
launcher for Windows supports most common forms of POSIX ``#!`` line, but there
@@ -245,6 +251,169 @@ are other issues to consider:
exact version like "/usr/bin/env python3.4" as you will need to change your
shebang line for users of Python 3.5, for example.
+Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3",
+depending on whether your code is written for Python 2 or 3.
+
+
+Creating Standalone Applications with zipapp
+--------------------------------------------
+
+Using the :mod:`zipapp` module, it is possible to create self-contained Python
+programs, which can be distributed to end users who only need to have a
+suitable version of Python installed on their system. The key to doing this
+is to bundle all of the application's dependencies into the archive, along
+with the application code.
+
+The steps to create a standalone archive are as follows:
+
+1. Create your application in a directory as normal, so you have a ``myapp``
+ directory containing a ``__main__.py`` file, and any supporting application
+ code.
+
+2. Install all of your application's dependencies into the ``myapp`` directory,
+ using pip:
+
+ .. code-block:: shell-session
+
+ $ python -m pip install -r requirements.txt --target myapp
+
+ (this assumes you have your project requirements in a ``requirements.txt``
+ file - if not, you can just list the dependencies manually on the pip command
+ line).
+
+3. Optionally, delete the ``.dist-info`` directories created by pip in the
+ ``myapp`` directory. These hold metadata for pip to manage the packages, and
+ as you won't be making any further use of pip they aren't required -
+ although it won't do any harm if you leave them.
+
+4. Package the application using:
+
+ .. code-block:: shell-session
+
+ $ python -m zipapp -p "interpreter" myapp
+
+This will produce a standalone executable, which can be run on any machine with
+the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter`
+for details. It can be shipped to users as a single file.
+
+On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the
+file to remove the ``.pyz`` extension if you prefer a "plain" command name. On
+Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that
+the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions
+when installed.
+
+
+Making a Windows executable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On Windows, registration of the ``.pyz`` extension is optional, and
+furthermore, there are certain places that don't recognise registered
+extensions "transparently" (the simplest example is that
+``subprocess.run(['myapp'])`` won't find your application - you need to
+explicitly specify the extension).
+
+On Windows, therefore, it is often preferable to create an executable from the
+zipapp. This is relatively easy, although it does require a C compiler. The
+basic approach relies on the fact that zipfiles can have arbitrary data
+prepended, and Windows exe files can have arbitrary data appended. So by
+creating a suitable launcher and tacking the ``.pyz`` file onto the end of it,
+you end up with a single-file executable that runs your application.
+
+A suitable launcher can be as simple as the following::
+
+ #define Py_LIMITED_API 1
+ #include "Python.h"
+
+ #define WIN32_LEAN_AND_MEAN
+ #include
+
+ #ifdef WINDOWS
+ int WINAPI wWinMain(
+ HINSTANCE hInstance, /* handle to current instance */
+ HINSTANCE hPrevInstance, /* handle to previous instance */
+ LPWSTR lpCmdLine, /* pointer to command line */
+ int nCmdShow /* show state of window */
+ )
+ #else
+ int wmain()
+ #endif
+ {
+ wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*));
+ myargv[0] = __wargv[0];
+ memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *));
+ return Py_Main(__argc+1, myargv);
+ }
+
+If you define the ``WINDOWS`` preprocessor symbol, this will generate a
+GUI executable, and without it, a console executable.
+
+To compile the executable, you can either just use the standard MSVC
+command line tools, or you can take advantage of the fact that distutils
+knows how to compile Python source::
+
+ >>> from distutils.ccompiler import new_compiler
+ >>> import distutils.sysconfig
+ >>> import sys
+ >>> import os
+ >>> from pathlib import Path
+
+ >>> def compile(src):
+ >>> src = Path(src)
+ >>> cc = new_compiler()
+ >>> exe = src.stem
+ >>> cc.add_include_dir(distutils.sysconfig.get_python_inc())
+ >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs'))
+ >>> # First the CLI executable
+ >>> objs = cc.compile([str(src)])
+ >>> cc.link_executable(objs, exe)
+ >>> # Now the GUI executable
+ >>> cc.define_macro('WINDOWS')
+ >>> objs = cc.compile([str(src)])
+ >>> cc.link_executable(objs, exe + 'w')
+
+ >>> if __name__ == "__main__":
+ >>> compile("zastub.c")
+
+The resulting launcher uses the "Limited ABI", so it will run unchanged with
+any version of Python 3.x. All it needs is for Python (``python3.dll``) to be
+on the user's ``PATH``.
+
+For a fully standalone distribution, you can distribute the launcher with your
+application appended, bundled with the Python "embedded" distribution. This
+will run on any PC with the appropriate architecture (32 bit or 64 bit).
+
+
+Caveats
+~~~~~~~
+
+There are some limitations to the process of bundling your application into
+a single file. In most, if not all, cases they can be addressed without
+needing major changes to your application.
+
+1. If your application depends on a package that includes a C extension, that
+ package cannot be run from a zip file (this is an OS limitation, as executable
+ code must be present in the filesystem for the OS loader to load it). In this
+ case, you can exclude that dependency from the zipfile, and either require
+ your users to have it installed, or ship it alongside your zipfile and add code
+ to your ``__main__.py`` to include the directory containing the unzipped
+ module in ``sys.path``. In this case, you will need to make sure to ship
+ appropriate binaries for your target architecture(s) (and potentially pick the
+ correct version to add to ``sys.path`` at runtime, based on the user's machine).
+
+2. If you are shipping a Windows executable as described above, you either need to
+ ensure that your users have ``python3.dll`` on their PATH (which is not the
+ default behaviour of the installer) or you should bundle your application with
+ the embedded distribution.
+
+3. The suggested launcher above uses the Python embedding API. This means that in
+ your application, ``sys.executable`` will be your application, and *not* a
+ conventional Python interpreter. Your code and its dependencies need to be
+ prepared for this possibility. For example, if your application uses the
+ :mod:`multiprocessing` module, it will need to call
+ :func:`multiprocessing.set_executable` to let the module know where to find the
+ standard Python interpreter.
+
+
The Python Zip Application Archive Format
-----------------------------------------
diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst
index 7c9a8c80225491..c0f2a89a3a17fa 100644
--- a/Doc/library/zipfile.rst
+++ b/Doc/library/zipfile.rst
@@ -491,7 +491,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the
:file:`\*.pyc` are added at the top level. If the directory is a
package directory, then all :file:`\*.pyc` are added under the package
name as a file path, and if any subdirectories are package directories,
- all of these are added recursively.
+ all of these are added recursively in sorted order.
*basename* is intended for internal use only.
@@ -524,6 +524,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the
.. versionchanged:: 3.6.2
The *pathname* parameter accepts a :term:`path-like object`.
+ .. versionchanged:: 3.7
+ Recursion sorts directory entries.
+
.. _zipinfo-objects:
diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst
index 3d742ab35b9cc9..8a531c92b8ff15 100644
--- a/Doc/library/zlib.rst
+++ b/Doc/library/zlib.rst
@@ -51,9 +51,9 @@ The available exception and functions in this module are:
Compresses the bytes in *data*, returning a bytes object containing compressed data.
*level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression;
- ``1`` is fastest and produces the least compression, ``9`` is slowest and
- produces the most. ``0`` is no compression. The default value is ``-1``
- (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default
+ ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION)
+ is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression.
+ The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default
compromise between speed and compression (currently equivalent to level 6).
Raises the :exc:`error` exception if any error occurs.
@@ -61,23 +61,25 @@ The available exception and functions in this module are:
*level* can now be used as a keyword parameter.
-.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict])
+.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict])
Returns a compression object, to be used for compressing data streams that won't
fit into memory at once.
*level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``.
- A value of ``1`` is fastest and produces the least compression, while a value of
- ``9`` is slowest and produces the most. ``0`` is no compression. The default
- value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default
- compromise between speed and compression (currently equivalent to level 6).
+ A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression,
+ while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most.
+ ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION).
+ Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression
+ (currently equivalent to level 6).
*method* is the compression algorithm. Currently, the only supported value is
- ``DEFLATED``.
+ :const:`DEFLATED`.
The *wbits* argument controls the size of the history buffer (or the
"window size") used when compressing data, and whether a header and
- trailer is included in the output. It can take several ranges of values:
+ trailer is included in the output. It can take several ranges of values,
+ defaulting to ``15`` (MAX_WBITS):
* +9 to +15: The base-two logarithm of the window size, which
therefore ranges between 512 and 32768. Larger values produce
@@ -97,7 +99,8 @@ The available exception and functions in this module are:
Higher values use more memory, but are faster and produce smaller output.
*strategy* is used to tune the compression algorithm. Possible values are
- ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``.
+ :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`,
+ :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2).
*zdict* is a predefined compression dictionary. This is a sequence of bytes
(such as a :class:`bytes` object) containing subsequences that are expected
@@ -175,7 +178,7 @@ The available exception and functions in this module are:
.. versionchanged:: 3.6
*wbits* and *bufsize* can be used as keyword arguments.
-.. function:: decompressobj(wbits=15[, zdict])
+.. function:: decompressobj(wbits=MAX_WBITS[, zdict])
Returns a decompression object, to be used for decompressing data streams that
won't fit into memory at once.
@@ -213,13 +216,13 @@ Compression objects support the following methods:
All pending input is processed, and a bytes object containing the remaining compressed
output is returned. *mode* can be selected from the constants
- :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`,
- defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and
- :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while
- :const:`Z_FINISH` finishes the compressed stream and prevents compressing any
- more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`,
- the :meth:`compress` method cannot be called again; the only realistic action is
- to delete the object.
+ :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`,
+ :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`,
+ defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants
+ allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the
+ compressed stream and prevents compressing any more data. After calling :meth:`flush`
+ with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again;
+ the only realistic action is to delete the object.
.. method:: Compress.copy()
diff --git a/Doc/make.bat b/Doc/make.bat
index 3f201de1db796c..d28dae78e86d4f 100644
--- a/Doc/make.bat
+++ b/Doc/make.bat
@@ -6,17 +6,27 @@ pushd %~dp0
set this=%~n0
call ..\PCbuild\find_python.bat %PYTHON%
-if not defined SPHINXBUILD if defined PYTHON (
+
+if not defined PYTHON set PYTHON=py
+
+if not defined SPHINXBUILD (
%PYTHON% -c "import sphinx" > nul 2> nul
if errorlevel 1 (
echo Installing sphinx with %PYTHON%
%PYTHON% -m pip install sphinx
if errorlevel 1 exit /B
)
- set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
+ set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sys.exit(sphinx.main())"
+)
+
+%PYTHON% -c "import python_docs_theme" > nul 2> nul
+if errorlevel 1 (
+ echo Installing python-docs-theme with %PYTHON%
+ %PYTHON% -m pip install python-docs-theme
+ if errorlevel 1 exit /B
)
-if not defined BLURB if defined PYTHON (
+if not defined BLURB (
%PYTHON% -c "import blurb" > nul 2> nul
if errorlevel 1 (
echo Installing blurb with %PYTHON%
@@ -26,10 +36,6 @@ if not defined BLURB if defined PYTHON (
set BLURB=%PYTHON% -m blurb
)
-if not defined PYTHON set PYTHON=py
-if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
-if not defined BLURB set BLURB=blurb
-
if "%1" NEQ "htmlhelp" goto :skiphhcsearch
if exist "%HTMLHELP%" goto :skiphhcsearch
@@ -125,6 +131,9 @@ if exist ..\Misc\NEWS (
if NOT "%PAPER%" == "" (
set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
)
+if "%1" EQU "htmlhelp" (
+ set SPHINXOPTS=-D html_theme_options.body_max_width=none %SPHINXOPTS%
+)
cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . "%BUILDDIR%\%1" %2 %3 %4 %5 %6 %7 %8 %9"
if "%1" EQU "htmlhelp" (
@@ -150,7 +159,7 @@ cmd /C %this% html
if EXIST "%BUILDDIR%\html\index.html" (
echo.Opening "%BUILDDIR%\html\index.html" in the default web browser...
- start "%BUILDDIR%\html\index.html"
+ start "" "%BUILDDIR%\html\index.html"
)
goto end
diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst
index d7792f1eaf825f..aeb4ada4fc4346 100644
--- a/Doc/reference/compound_stmts.rst
+++ b/Doc/reference/compound_stmts.rst
@@ -91,7 +91,7 @@ The :keyword:`if` statement is used for conditional execution:
.. productionlist::
if_stmt: "if" `expression` ":" `suite`
- : ( "elif" `expression` ":" `suite` )*
+ : ("elif" `expression` ":" `suite`)*
: ["else" ":" `suite`]
It selects exactly one of the suites by evaluating the expressions one by one
@@ -203,7 +203,7 @@ returns the list ``[0, 1, 2]``.
single: mutable sequence; loop over
There is a subtlety when the sequence is being modified by the loop (this can
- only occur for mutable sequences, i.e. lists). An internal counter is used
+ only occur for mutable sequences, e.g. lists). An internal counter is used
to keep track of which item is used next, and this is incremented on each
iteration. When this counter has reached the length of the sequence the loop
terminates. This means that if the suite deletes the current (or a previous)
@@ -235,7 +235,7 @@ The :keyword:`try` statement specifies exception handlers and/or cleanup code
for a group of statements:
.. productionlist::
- try_stmt: try1_stmt | try2_stmt
+ try_stmt: `try1_stmt` | `try2_stmt`
try1_stmt: "try" ":" `suite`
: ("except" [`expression` ["as" `identifier`]] ":" `suite`)+
: ["else" ":" `suite`]
@@ -383,7 +383,7 @@ This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally`
usage patterns to be encapsulated for convenient reuse.
.. productionlist::
- with_stmt: "with" with_item ("," with_item)* ":" `suite`
+ with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite`
with_item: `expression` ["as" `target`]
The execution of the :keyword:`with` statement with one "item" proceeds as follows:
@@ -467,14 +467,15 @@ A function definition defines a user-defined function object (see section
:ref:`types`):
.. productionlist::
- funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" ["->" `expression`] ":" `suite`
+ funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")"
+ : ["->" `expression`] ":" `suite`
decorators: `decorator`+
decorator: "@" `dotted_name` ["(" [`argument_list` [","]] ")"] NEWLINE
dotted_name: `identifier` ("." `identifier`)*
parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]]
: | `parameter_list_starargs`
parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]]
- : | "**" `parameter` [","]
+ : | "**" `parameter` [","]
parameter: `identifier` [":" `expression`]
defparameter: `parameter` ["=" `expression`]
funcname: `identifier`
@@ -683,6 +684,8 @@ can be used to create instance variables with different implementation details.
:pep:`3129` - Class Decorators
+.. _async:
+
Coroutines
==========
@@ -695,7 +698,8 @@ Coroutine function definition
-----------------------------
.. productionlist::
- async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" ["->" `expression`] ":" `suite`
+ async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")"
+ : ["->" `expression`] ":" `suite`
.. index::
keyword: async
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 25b95c115438b3..735ecbf66939f1 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -768,7 +768,7 @@ Custom classes
When a class attribute reference (for class :class:`C`, say) would yield a
class method object, it is transformed into an instance method object whose
- :attr:`__self__` attributes is :class:`C`. When it would yield a static
+ :attr:`__self__` attribute is :class:`C`. When it would yield a static
method object, it is transformed into the object wrapped by the static method
object. See section :ref:`descriptors` for another way in which attributes
retrieved from a class may differ from those actually contained in its
@@ -950,7 +950,7 @@ Internal types
.. index:: object: frame
Frame objects represent execution frames. They may occur in traceback objects
- (see below).
+ (see below), and are also passed to registered trace functions.
.. index::
single: f_back (frame attribute)
@@ -1003,6 +1003,8 @@ Internal types
.. versionadded:: 3.4
+ .. _traceback-objects:
+
Traceback objects
.. index::
object: traceback
@@ -1015,31 +1017,51 @@ Internal types
single: sys.last_traceback
Traceback objects represent a stack trace of an exception. A traceback object
- is created when an exception occurs. When the search for an exception handler
+ is implicitly created when an exception occurs, and may also be explicitly
+ created by calling :class:`types.TracebackType`.
+
+ For implicitly created tracebacks, when the search for an exception handler
unwinds the execution stack, at each unwound level a traceback object is
inserted in front of the current traceback. When an exception handler is
entered, the stack trace is made available to the program. (See section
:ref:`try`.) It is accessible as the third item of the
- tuple returned by ``sys.exc_info()``. When the program contains no suitable
+ tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute
+ of the caught exception.
+
+ When the program contains no suitable
handler, the stack trace is written (nicely formatted) to the standard error
stream; if the interpreter is interactive, it is also made available to the user
as ``sys.last_traceback``.
+ For explicitly created tracebacks, it is up to the creator of the traceback
+ to determine how the ``tb_next`` attributes should be linked to form a
+ full stack trace.
+
.. index::
- single: tb_next (traceback attribute)
single: tb_frame (traceback attribute)
single: tb_lineno (traceback attribute)
single: tb_lasti (traceback attribute)
statement: try
- Special read-only attributes: :attr:`tb_next` is the next level in the stack
- trace (towards the frame where the exception occurred), or ``None`` if there is
- no next level; :attr:`tb_frame` points to the execution frame of the current
- level; :attr:`tb_lineno` gives the line number where the exception occurred;
- :attr:`tb_lasti` indicates the precise instruction. The line number and last
- instruction in the traceback may differ from the line number of its frame object
- if the exception occurred in a :keyword:`try` statement with no matching except
- clause or with a finally clause.
+ Special read-only attributes:
+ :attr:`tb_frame` points to the execution frame of the current level;
+ :attr:`tb_lineno` gives the line number where the exception occurred;
+ :attr:`tb_lasti` indicates the precise instruction.
+ The line number and last instruction in the traceback may differ from the
+ line number of its frame object if the exception occurred in a
+ :keyword:`try` statement with no matching except clause or with a
+ finally clause.
+
+ .. index::
+ single: tb_next (traceback attribute)
+
+ Special writable attribute: :attr:`tb_next` is the next level in the stack
+ trace (towards the frame where the exception occurred), or ``None`` if
+ there is no next level.
+
+ .. versionchanged:: 3.7
+ Traceback objects can now be explicitly instantiated from Python code,
+ and the ``tb_next`` attribute of existing instances can be updated.
Slice objects
.. index:: builtin: slice
@@ -1463,10 +1485,12 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances.
.. method:: object.__getattr__(self, name)
- Called when an attribute lookup has not found the attribute in the usual places
- (i.e. it is not an instance attribute nor is it found in the class tree for
- ``self``). ``name`` is the attribute name. This method should return the
- (computed) attribute value or raise an :exc:`AttributeError` exception.
+ Called when the default attribute access fails with an :exc:`AttributeError`
+ (either :meth:`__getattribute__` raises an :exc:`AttributeError` because
+ *name* is not an instance attribute or an attribute in the class tree
+ for ``self``; or :meth:`__get__` of a *name* property raises
+ :exc:`AttributeError`). This method should either return the (computed)
+ attribute value or raise an :exc:`AttributeError` exception.
Note that if the attribute is found through the normal mechanism,
:meth:`__getattr__` is not called. (This is an intentional asymmetry between
@@ -1833,11 +1857,27 @@ passed through to all metaclass operations described below.
When a class definition is executed, the following steps occur:
+* MRO entries are resolved
* the appropriate metaclass is determined
* the class namespace is prepared
* the class body is executed
* the class object is created
+
+Resolving MRO entries
+^^^^^^^^^^^^^^^^^^^^^
+
+If a base that appears in class definition is not an instance of :class:`type`,
+then an ``__mro_entries__`` method is searched on it. If found, it is called
+with the original bases tuple. This method must return a tuple of classes that
+will be used instead of this base. The tuple may be empty, in such case
+the original base is ignored.
+
+.. seealso::
+
+ :pep:`560` - Core support for typing module and generic types
+
+
Determining the appropriate metaclass
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. index::
@@ -1928,7 +1968,7 @@ current call is identified based on the first argument passed to the method.
be propagated up to the ``type.__new__`` call in order for the class to be
initialised correctly.
Failing to do so will result in a :exc:`DeprecationWarning` in Python 3.6,
- and a :exc:`RuntimeWarning` in the future.
+ and a :exc:`RuntimeError` in Python 3.8.
When using the default metaclass :class:`type`, or any metaclass that ultimately
calls ``type.__new__``, the following additional customisation steps are
@@ -2037,6 +2077,27 @@ case the instance is itself a class.
module) to the language.
+Emulating generic types
+-----------------------
+
+One can implement the generic class syntax as specified by :pep:`484`
+(for example ``List[int]``) by defining a special method
+
+.. classmethod:: object.__class_getitem__(cls, key)
+
+ Return an object representing the specialization of a generic class
+ by type arguments found in *key*.
+
+This method is looked up on the class object itself, and when defined in
+the class body, this method is implicitly a class method. Note, this
+mechanism is primarily reserved for use with static type hints, other usage
+is discouraged.
+
+.. seealso::
+
+ :pep:`560` - Core support for typing module and generic types
+
+
.. _callable-types:
Emulating callable objects
@@ -2340,16 +2401,14 @@ left undefined.
.. method:: object.__complex__(self)
object.__int__(self)
object.__float__(self)
- object.__round__(self, [,n])
.. index::
builtin: complex
builtin: int
builtin: float
- builtin: round
Called to implement the built-in functions :func:`complex`,
- :func:`int`, :func:`float` and :func:`round`. Should return a value
+ :func:`int` and :func:`float`. Should return a value
of the appropriate type.
@@ -2368,6 +2427,23 @@ left undefined.
the same value.
+.. method:: object.__round__(self, [,ndigits])
+ object.__trunc__(self)
+ object.__floor__(self)
+ object.__ceil__(self)
+
+ .. index:: builtin: round
+
+ Called to implement the built-in function :func:`round` and :mod:`math`
+ functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`.
+ Unless *ndigits* is passed to :meth:`!__round__` all these methods should
+ return the value of the object truncated to an :class:`~numbers.Integral`
+ (typically an :class:`int`).
+
+ If :meth:`__int__` is not defined then the built-in function :func:`int`
+ falls back to :meth:`__trunc__`.
+
+
.. _context-managers:
With Statement Context Managers
diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index fb92ad0f07c21e..ad00a837b68a25 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -172,7 +172,7 @@ Common syntax elements for comprehensions are:
.. productionlist::
comprehension: `expression` `comp_for`
- comp_for: [ASYNC] "for" `target_list` "in" `or_test` [`comp_iter`]
+ comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`]
comp_iter: `comp_for` | `comp_if`
comp_if: "if" `expression_nocond` [`comp_iter`]
@@ -777,7 +777,7 @@ whose value is one of the keys of the mapping, and the subscription selects the
value in the mapping that corresponds to that key. (The expression list is a
tuple except if it has exactly one item.)
-If the primary is a sequence, the expression (list) must evaluate to an integer
+If the primary is a sequence, the expression list must evaluate to an integer
or a slice (as discussed in the following section).
The formal syntax makes no special provision for negative indices in
@@ -1060,7 +1060,7 @@ The power operator binds more tightly than unary operators on its left; it binds
less tightly than unary operators on its right. The syntax is:
.. productionlist::
- power: ( `await_expr` | `primary` ) ["**" `u_expr`]
+ power: (`await_expr` | `primary`) ["**" `u_expr`]
Thus, in an unparenthesized sequence of power and unary operators, the operators
are evaluated from right to left (this does not constrain the evaluation order
@@ -1132,7 +1132,7 @@ operators and one for additive operators:
.. productionlist::
m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` |
- : `m_expr` "//" `u_expr`| `m_expr` "/" `u_expr` |
+ : `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` |
: `m_expr` "%" `u_expr`
a_expr: `m_expr` | `a_expr` "+" `m_expr` | `a_expr` "-" `m_expr`
@@ -1144,7 +1144,9 @@ the other must be a sequence. In the former case, the numbers are converted to a
common type and then multiplied together. In the latter case, sequence
repetition is performed; a negative repetition factor yields an empty sequence.
-.. index:: single: matrix multiplication
+.. index::
+ single: matrix multiplication
+ operator: @
The ``@`` (at) operator is intended to be used for matrix multiplication. No
builtin Python types implement this operator.
@@ -1210,7 +1212,7 @@ Shifting operations
The shifting operations have lower priority than the arithmetic operations:
.. productionlist::
- shift_expr: `a_expr` | `shift_expr` ( "<<" | ">>" ) `a_expr`
+ shift_expr: `a_expr` | `shift_expr` ("<<" | ">>") `a_expr`
These operators accept integers as arguments. They shift the first argument to
the left or right by the number of bits given by the second argument.
@@ -1270,7 +1272,7 @@ C, expressions like ``a < b < c`` have the interpretation that is conventional
in mathematics:
.. productionlist::
- comparison: `or_expr` ( `comp_operator` `or_expr` )*
+ comparison: `or_expr` (`comp_operator` `or_expr`)*
comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!="
: | "is" ["not"] | ["not"] "in"
@@ -1613,12 +1615,12 @@ Lambdas
lambda_expr_nocond: "lambda" [`parameter_list`]: `expression_nocond`
Lambda expressions (sometimes called lambda forms) are used to create anonymous
-functions. The expression ``lambda arguments: expression`` yields a function
+functions. The expression ``lambda parameters: expression`` yields a function
object. The unnamed object behaves like a function object defined with:
.. code-block:: none
- def (arguments):
+ def (parameters):
return expression
See section :ref:`function` for the syntax of parameter lists. Note that
@@ -1634,9 +1636,9 @@ Expression lists
.. index:: pair: expression; list
.. productionlist::
- expression_list: `expression` ( "," `expression` )* [","]
- starred_list: `starred_item` ( "," `starred_item` )* [","]
- starred_expression: `expression` | ( `starred_item` "," )* [`starred_item`]
+ expression_list: `expression` ("," `expression`)* [","]
+ starred_list: `starred_item` ("," `starred_item`)* [","]
+ starred_expression: `expression` | (`starred_item` ",")* [`starred_item`]
starred_item: `expression` | "*" `or_expr`
.. index:: object: tuple
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
index 45d417295d06d0..44b5b818aa8233 100644
--- a/Doc/reference/import.rst
+++ b/Doc/reference/import.rst
@@ -616,8 +616,7 @@ the module.
module.__path__
---------------
-By definition, if a module has a ``__path__`` attribute, it is a package,
-regardless of its value.
+By definition, if a module has a ``__path__`` attribute, it is a package.
A package's ``__path__`` attribute is used during imports of its subpackages.
Within the import machinery, it functions much the same as :data:`sys.path`,
diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst
index 95b0f53a83e604..84e8c783838b64 100644
--- a/Doc/reference/lexical_analysis.rst
+++ b/Doc/reference/lexical_analysis.rst
@@ -47,11 +47,12 @@ Physical lines
--------------
A physical line is a sequence of characters terminated by an end-of-line
-sequence. In source files, any of the standard platform line termination
-sequences can be used - the Unix form using ASCII LF (linefeed), the Windows
-form using the ASCII sequence CR LF (return followed by linefeed), or the old
-Macintosh form using the ASCII CR (return) character. All of these forms can be
-used equally, regardless of platform.
+sequence. In source files and strings, any of the standard platform line
+termination sequences can be used - the Unix form using ASCII LF (linefeed),
+the Windows form using the ASCII sequence CR LF (return followed by linefeed),
+or the old Macintosh form using the ASCII CR (return) character. All of these
+forms can be used equally, regardless of platform. The end of input also serves
+as an implicit terminator for the final physical line.
When embedding Python, source code strings should be passed to Python APIs using
the standard C conventions for newline characters (the ``\n`` character,
@@ -313,7 +314,7 @@ The Unicode category codes mentioned above stand for:
* *Nd* - decimal numbers
* *Pc* - connector punctuations
* *Other_ID_Start* - explicit list of characters in `PropList.txt
- `_ to support backwards
+ `_ to support backwards
compatibility
* *Other_ID_Continue* - likewise
@@ -676,7 +677,7 @@ Some examples of formatted string literals::
>>> f"result: {value:{width}.{precision}}" # nested fields
'result: 12.35'
>>> today = datetime(year=2017, month=1, day=27)
- >>> f"{today:%b %d, %Y}" # using date format specifier
+ >>> f"{today:%B %d, %Y}" # using date format specifier
'January 27, 2017'
>>> number = 1024
>>> f"{number:#0x}" # using integer format specifier
@@ -876,4 +877,4 @@ occurrence outside string literals and comments is an unconditional error:
.. rubric:: Footnotes
-.. [#] http://www.unicode.org/Public/10.0.0/ucd/NameAliases.txt
+.. [#] http://www.unicode.org/Public/11.0.0/ucd/NameAliases.txt
diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst
index ef9a5f0dc85422..47f7c7bfffb051 100644
--- a/Doc/reference/simple_stmts.rst
+++ b/Doc/reference/simple_stmts.rst
@@ -708,15 +708,14 @@ The :keyword:`import` statement
keyword: from
.. productionlist::
- import_stmt: "import" `module` ["as" `name`] ( "," `module` ["as" `name`] )*
- : | "from" `relative_module` "import" `identifier` ["as" `name`]
- : ( "," `identifier` ["as" `name`] )*
- : | "from" `relative_module` "import" "(" `identifier` ["as" `name`]
- : ( "," `identifier` ["as" `name`] )* [","] ")"
+ import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])*
+ : | "from" `relative_module` "import" `identifier` ["as" `identifier`]
+ : ("," `identifier` ["as" `identifier`])*
+ : | "from" `relative_module` "import" "(" `identifier` ["as" `identifier`]
+ : ("," `identifier` ["as" `identifier`])* [","] ")"
: | "from" `module` "import" "*"
module: (`identifier` ".")* `identifier`
relative_module: "."* `module` | "."+
- name: `identifier`
The basic import statement (no :keyword:`from` clause) is executed in two
steps:
@@ -838,12 +837,11 @@ features on a per-module basis before the release in which the feature becomes
standard.
.. productionlist:: *
- future_statement: "from" "__future__" "import" feature ["as" name]
- : ("," feature ["as" name])*
- : | "from" "__future__" "import" "(" feature ["as" name]
- : ("," feature ["as" name])* [","] ")"
- feature: identifier
- name: identifier
+ future_stmt: "from" "__future__" "import" `feature` ["as" `identifier`]
+ : ("," `feature` ["as" `identifier`])*
+ : | "from" "__future__" "import" "(" `feature` ["as" `identifier`]
+ : ("," `feature` ["as" `identifier`])* [","] ")"
+ feature: `identifier`
A future statement must appear near the top of the module. The only lines that
can appear before a future statement are:
diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst
index e1687ff04d3286..d5ffb37b2e58cd 100644
--- a/Doc/reference/toplevel_components.rst
+++ b/Doc/reference/toplevel_components.rst
@@ -48,14 +48,15 @@ a complete program; each statement is executed in the namespace of
.. index::
single: UNIX
+ single: Windows
single: command line
single: standard input
-Under Unix, a complete program can be passed to the interpreter in three forms:
-with the :option:`-c` *string* command line option, as a file passed as the
-first command line argument, or as standard input. If the file or standard
-input is a tty device, the interpreter enters interactive mode; otherwise, it
-executes the file as a complete program.
+A complete program can be passed to the interpreter
+in three forms: with the :option:`-c` *string* command line option, as a file
+passed as the first command line argument, or as standard input. If the file
+or standard input is a tty device, the interpreter enters interactive mode;
+otherwise, it executes the file as a complete program.
.. _file-input:
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 00acd4f55b8b95..92506e7b32706f 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -23,9 +23,10 @@
from sphinx import addnodes
from sphinx.builders import Builder
from sphinx.locale import translators
+from sphinx.util import status_iterator
from sphinx.util.nodes import split_explicit_title
from sphinx.writers.html import HTMLTranslator
-from sphinx.writers.text import TextWriter
+from sphinx.writers.text import TextWriter, TextTranslator
from sphinx.writers.latex import LaTeXTranslator
from sphinx.domains.python import PyModulelevel, PyClassmember
@@ -35,7 +36,7 @@
ISSUE_URI = 'https://bugs.python.org/issue%s'
-SOURCE_URI = 'https://github.com/python/cpython/tree/master/%s'
+SOURCE_URI = 'https://github.com/python/cpython/tree/3.7/%s'
# monkey-patch reST parser to disable alphabetic and roman enumerated lists
from docutils.parsers.rst.states import Body
@@ -196,7 +197,7 @@ class DeprecatedRemoved(Directive):
final_argument_whitespace = True
option_spec = {}
- _label = 'Deprecated since version %s, will be removed in version %s'
+ _label = 'Deprecated since version {deprecated}, will be removed in version {removed}'
def run(self):
node = addnodes.versionmodified()
@@ -204,11 +205,12 @@ def run(self):
node['type'] = 'deprecated-removed'
version = (self.arguments[0], self.arguments[1])
node['version'] = version
- text = self._label % version
+ label = translators['sphinx'].gettext(self._label)
+ text = label.format(deprecated=self.arguments[0], removed=self.arguments[1])
if len(self.arguments) == 3:
inodes, messages = self.state.inline_text(self.arguments[2],
self.lineno+1)
- para = nodes.paragraph(self.arguments[2], '', *inodes)
+ para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False)
node.append(para)
else:
messages = []
@@ -220,13 +222,14 @@ def run(self):
content.source = node[0].source
content.line = node[0].line
content += node[0].children
- node[0].replace_self(nodes.paragraph('', '', content))
+ node[0].replace_self(nodes.paragraph('', '', content, translatable=False))
node[0].insert(0, nodes.inline('', '%s: ' % text,
classes=['versionmodified']))
else:
para = nodes.paragraph('', '',
nodes.inline('', '%s.' % text,
- classes=['versionmodified']))
+ classes=['versionmodified']),
+ translatable=False)
node.append(para)
env = self.state.document.settings.env
env.note_versionchange('deprecated', version[0], node, self.lineno)
@@ -293,8 +296,11 @@ def run(self):
class PydocTopicsBuilder(Builder):
name = 'pydoc-topics'
+ default_translator_class = TextTranslator
+
def init(self):
self.topics = {}
+ self.secnumbers = {}
def get_outdated_docs(self):
return 'all pydoc topics'
@@ -304,9 +310,9 @@ def get_target_uri(self, docname, typ=None):
def write(self, *ignored):
writer = TextWriter(self)
- for label in self.status_iterator(pydoc_topic_labels,
- 'building topics... ',
- length=len(pydoc_topic_labels)):
+ for label in status_iterator(pydoc_topic_labels,
+ 'building topics... ',
+ length=len(pydoc_topic_labels)):
if label not in self.env.domaindata['std']['labels']:
self.warn('label %r not in documentation' % label)
continue
diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js
index c450f5eafffc89..d885ff2bbf2166 100644
--- a/Doc/tools/static/switchers.js
+++ b/Doc/tools/static/switchers.js
@@ -10,7 +10,8 @@
'(?:release/\\d.\\d[\\x\\d\\.]*)'];
var all_versions = {
- '3.7': 'dev (3.7)',
+ '3.8': 'dev (3.8)',
+ '3.7': '3.7',
'3.6': '3.6',
'3.5': '3.5',
'2.7': '2.7',
@@ -20,6 +21,7 @@
'en': 'English',
'fr': 'French',
'ja': 'Japanese',
+ 'ko': 'Korean',
};
function build_version_select(current_version, current_release) {
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index cfdd5266c51c04..33cd48f3d81ef4 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -187,6 +187,8 @@ library/profile,,:lineno,filename:lineno(function)
library/pyexpat,,:elem1,
library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">"
library/random,,:len,new_diff = mean(combined[:len(drug)]) - mean(combined[len(drug):])
+library/readline,,:bind,"python:bind -v"
+library/readline,,:bind,"python:bind ^I rl_complete"
library/smtplib,,:port,method must support that as well as a regular host:port
library/socket,,::,'5aef:2b::8'
library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])"
diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html
index fca44e9163cac7..21af621efce95b 100644
--- a/Doc/tools/templates/customsourcelink.html
+++ b/Doc/tools/templates/customsourcelink.html
@@ -4,7 +4,7 @@ {{ _('This Page') }}