Merge branch 'main' into typing-subst-unpacked-vat-tuple

Author: JelleZijlstra

Doc/library/dis.rst

@@ -6,6 +6,12 @@
 
 **Source code:** :source:`Lib/dis.py`
 
+.. testsetup::
+
+   import dis
+   def myfunc(alist):
+       return len(alist)
+
 --------------
 
 The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by
@@ -37,16 +43,17 @@ Example: Given the function :func:`myfunc`::
        return len(alist)
 
 the following command can be used to display the disassembly of
-:func:`myfunc`::
+:func:`myfunc`:
 
-   >>> dis.dis(myfunc)
-     1           0 RESUME                   0
+.. doctest::
 
-     2           2 PUSH_NULL
-                 4 LOAD_GLOBAL              1 (NULL + len)
-                 6 LOAD_FAST                0 (alist)
-                 8 CALL                     1
-                18 RETURN_VALUE
+   >>> dis.dis(myfunc)
+     2           0 RESUME                   0
+   <BLANKLINE>
+     3           2 LOAD_GLOBAL              1 (NULL + len)
+                14 LOAD_FAST                0 (alist)
+                16 CALL                     1
+                26 RETURN_VALUE
 
 (The "2" is a line number).
 
@@ -108,14 +115,15 @@ code.
    .. versionchanged:: 3.11
       Added the ``show_caches`` parameter.
 
-Example::
+Example:
+
+.. doctest::
 
     >>> bytecode = dis.Bytecode(myfunc)
     >>> for instr in bytecode:
     ...     print(instr.opname)
     ...
     RESUME
-    PUSH_NULL
     LOAD_GLOBAL
     LOAD_FAST
     CALL

Doc/library/mailcap.rst

@@ -60,6 +60,18 @@ standard.  However, mailcap files are supported on most Unix systems.
    use) to determine whether or not the mailcap line applies.  :func:`findmatch`
    will automatically check such conditions and skip the entry if the check fails.
 
+   .. versionchanged:: 3.11
+
+      To prevent security issues with shell metacharacters (symbols that have
+      special effects in a shell command line), ``findmatch`` will refuse
+      to inject ASCII characters other than alphanumerics and ``@+=:,./-_``
+      into the returned command line.
+
+      If a disallowed character appears in *filename*, ``findmatch`` will always
+      return ``(None, None)`` as if no entry was found.
+      If such a character appears elsewhere (a value in *plist* or in *MIMEtype*),
+      ``findmatch`` will ignore all mailcap entries which use that value.
+      A :mod:`warning <warnings>` will be raised in either case.
 
 .. function:: getcaps()
 

Doc/using/cmdline.rst

@@ -183,15 +183,37 @@ automatically enabled, if available on your platform (see
    Automatic enabling of tab-completion and history editing.
 
 
+.. _using-on-generic-options:
+
 Generic options
 ~~~~~~~~~~~~~~~
 
 .. cmdoption:: -?
                -h
                --help
 
-   Print a short description of all command line options.
+   Print a short description of all command line options and corresponding
+   environment variables and exit.
+
+.. cmdoption:: --help-env
+
+   Print a short description of Python-specific environment variables
+   and exit.
+
+   .. versionadded:: 3.11
+
+.. cmdoption:: --help-xoptions
 
+   Print a description of implementation-specific :option:`-X` options
+   and exit.
+
+   .. versionadded:: 3.11
+
+.. cmdoption:: --help-all
+
+   Print complete usage information and exit.
+
+   .. versionadded:: 3.11
 
 .. cmdoption:: -V
                --version
@@ -212,6 +234,7 @@ Generic options
    .. versionadded:: 3.6
       The ``-VV`` option.
 
+
 .. _using-on-misc-options:
 
 Miscellaneous options
@@ -460,6 +483,7 @@ Miscellaneous options
    See :ref:`warning-filter` and :ref:`describing-warning-filters` for more
    details.
 
+
 .. cmdoption:: -x
 
    Skip the first line of the source, allowing use of non-Unix forms of
@@ -553,6 +577,7 @@ Miscellaneous options
       The ``-X frozen_modules`` option.
 
 
+
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 

Include/boolobject.h

@@ -19,8 +19,8 @@ PyAPI_DATA(PyLongObject) _Py_FalseStruct;
 PyAPI_DATA(PyLongObject) _Py_TrueStruct;
 
 /* Use these macros */
-#define Py_False ((PyObject *) &_Py_FalseStruct)
-#define Py_True ((PyObject *) &_Py_TrueStruct)
+#define Py_False _PyObject_CAST(&_Py_FalseStruct)
+#define Py_True _PyObject_CAST(&_Py_TrueStruct)
 
 // Test if an object is the True singleton, the same as "x is True" in Python.
 PyAPI_FUNC(int) Py_IsTrue(PyObject *x);

Include/ceval.h

@@ -31,7 +31,7 @@ Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyEval_CallObjectWithKeywords(
 
 /* Deprecated since PyEval_CallObjectWithKeywords is deprecated */
 #define PyEval_CallObject(callable, arg) \
-    PyEval_CallObjectWithKeywords(callable, arg, (PyObject *)NULL)
+    PyEval_CallObjectWithKeywords(callable, arg, _PyObject_CAST(_Py_NULL))
 
 Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyEval_CallFunction(
     PyObject *callable, const char *format, ...);

Include/cpython/code.h

@@ -88,6 +88,7 @@ typedef uint16_t _Py_CODEUNIT;
     PyObject *co_qualname;        /* unicode (qualname, for reference) */      \
     PyObject *co_linetable;       /* bytes object that holds location info */  \
     PyObject *co_weakreflist;     /* to support weakrefs to code objects */    \
+    void *_co_code;               /* cached co_code object/attribute */        \
     /* Scratch space for extra data relating to the code object.               \
        Type is a void* to keep the format private in codeobject.c to force     \
        people to go through the proper APIs. */                                \

Include/pyerrors.h

@@ -62,7 +62,7 @@ PyAPI_FUNC(void) PyException_SetContext(PyObject *, PyObject *);
 
 PyAPI_FUNC(const char *) PyExceptionClass_Name(PyObject *);
 
-#define PyExceptionInstance_Class(x) ((PyObject*)Py_TYPE(x))
+#define PyExceptionInstance_Class(x) _PyObject_CAST(Py_TYPE(x))
 
 #define _PyBaseExceptionGroup_Check(x)                   \
     PyObject_TypeCheck(x, (PyTypeObject *)PyExc_BaseExceptionGroup)

Lib/dis.py

@@ -592,7 +592,7 @@ def _unpack_opargs(code):
         caches = _inline_cache_entries[deop]
         if deop >= HAVE_ARGUMENT:
             arg = code[i+1] | extended_arg
-            extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
+            extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0
             # The oparg is stored as a signed integer
             # If the value exceeds its upper limit, it will overflow and wrap
             # to a negative integer

Lib/mailcap.py

@@ -2,6 +2,7 @@
 
 import os
 import warnings
+import re
 
 __all__ = ["getcaps","findmatch"]
 
@@ -19,6 +20,11 @@ def lineno_sort_key(entry):
     else:
         return 1, 0
 
+_find_unsafe = re.compile(r'[^\xa1-\U0010FFFF\w@+=:,./-]').search
+
+class UnsafeMailcapInput(Warning):
+    """Warning raised when refusing unsafe input"""
+
 
 # Part 1: top-level interface.
 
@@ -171,15 +177,22 @@ def findmatch(caps, MIMEtype, key='view', filename="/dev/null", plist=[]):
     entry to use.
 
     """
+    if _find_unsafe(filename):
+        msg = "Refusing to use mailcap with filename %r. Use a safe temporary filename." % (filename,)
+        warnings.warn(msg, UnsafeMailcapInput)
+        return None, None
     entries = lookup(caps, MIMEtype, key)
     # XXX This code should somehow check for the needsterminal flag.
     for e in entries:
         if 'test' in e:
             test = subst(e['test'], filename, plist)
+            if test is None:
+                continue
             if test and os.system(test) != 0:
                 continue
         command = subst(e[key], MIMEtype, filename, plist)
-        return command, e
+        if command is not None:
+            return command, e
     return None, None
 
 def lookup(caps, MIMEtype, key=None):
@@ -212,14 +225,23 @@ def subst(field, MIMEtype, filename, plist=[]):
             elif c == 's':
                 res = res + filename
             elif c == 't':
+                if _find_unsafe(MIMEtype):
+                    msg = "Refusing to substitute MIME type %r into a shell command." % (MIMEtype,)
+                    warnings.warn(msg, UnsafeMailcapInput)
+                    return None
                 res = res + MIMEtype
             elif c == '{':
                 start = i
                 while i < n and field[i] != '}':
                     i = i+1
                 name = field[start:i]
                 i = i+1
-                res = res + findparam(name, plist)
+                param = findparam(name, plist)
+                if _find_unsafe(param):
+                    msg = "Refusing to substitute parameter %r (%s) into a shell command" % (param, name)
+                    warnings.warn(msg, UnsafeMailcapInput)
+                    return None
+                res = res + param
             # XXX To do:
             # %n == number of parts if type is multipart/*
             # %F == list of alternating type and filename for parts

Lib/pathlib.py

@@ -443,6 +443,8 @@ def __getitem__(self, idx):
 
         if idx >= len(self) or idx < -len(self):
             raise IndexError(idx)
+        if idx < 0:
+            idx += len(self)
         return self._pathcls._from_parsed_parts(self._drv, self._root,
                                                 self._parts[:-idx - 1])