bpo-40059: Add tomllib (PEP-680) (pythonGH-31498)

This adds a new standard library module, `tomllib`, for parsing TOML.

The implementation is based on Tomli (https://github.com/hukkin/tomli).

## Steps taken (converting `tomli` to `tomllib`)

- Move everything in `tomli:src/tomli` to `Lib/tomllib`. Exclude `py.typed`.
- Remove `__version__ = ...` line from `Lib/tomllib/__init__.py`
- Move everything in `tomli:tests` to `Lib/test/test_tomllib`. Exclude the following test data dirs recursively:
  - `tomli:tests/data/invalid/_external/`
  - `tomli:tests/data/valid/_external/`
- Create `Lib/test/test_tomllib/__main__.py`:

  ```python
  import unittest

  from . import load_tests


  unittest.main()
  ```


- Add the following to `Lib/test/test_tomllib/__init__.py`:

  ```python
  import os
  from test.support import load_package_tests

  def load_tests(*args):
      return load_package_tests(os.path.dirname(__file__), *args)
  ```

  Also change `import tomli as tomllib` to `import tomllib`.

- In `cpython/Lib/tomllib/_parser.py` replace `__fp` with `fp` and `__s` with
  `s`. Add the `/` to `load` and `loads` function signatures.

- Run `make regen-stdlib-module-names`

- Create `Doc/library/tomllib.rst` and reference it in `Doc/library/fileformats.rst`

Author: hukkin

.github/CODEOWNERS

@@ -141,6 +141,8 @@ Lib/ast.py                    @isidentical
 **/*cgi*                      @ethanfurman
 **/*tarfile*                  @ethanfurman
 
+**/*tomllib*                  @encukou
+
 # macOS
 /Mac/                         @python/macos-team
 **/*osx_support*              @python/macos-team

Doc/library/fileformats.rst

@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+# Licensed to PSF under a Contributor Agreement.
+
+__all__ = ("tomllib",)
+
+# By changing this one line, we can run the tests against
+# a different module name.
+import tomllib
+
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Doc/library/tomllib.rst

@@ -0,0 +1,6 @@
+import unittest
+
+from . import load_tests
+
+
+unittest.main()

Doc/whatsnew/3.11.rst

@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+# Licensed to PSF under a Contributor Agreement.
+
+"""Utilities for tests that are in the "burntsushi" format."""
+
+import datetime
+from typing import Any
+
+# Aliases for converting TOML compliance format [1] to BurntSushi format [2]
+# [1] https://github.com/toml-lang/compliance/blob/db7c3211fda30ff9ddb10292f4aeda7e2e10abc4/docs/json-encoding.md  # noqa: E501
+# [2] https://github.com/BurntSushi/toml-test/blob/4634fdf3a6ecd6aaea5f4cdcd98b2733c2694993/README.md  # noqa: E501
+_aliases = {
+    "boolean": "bool",
+    "offset datetime": "datetime",
+    "local datetime": "datetime-local",
+    "local date": "date-local",
+    "local time": "time-local",
+}
+
+
+def convert(obj):  # noqa: C901
+    if isinstance(obj, str):
+        return {"type": "string", "value": obj}
+    elif isinstance(obj, bool):
+        return {"type": "bool", "value": str(obj).lower()}
+    elif isinstance(obj, int):
+        return {"type": "integer", "value": str(obj)}
+    elif isinstance(obj, float):
+        return {"type": "float", "value": _normalize_float_str(str(obj))}
+    elif isinstance(obj, datetime.datetime):
+        val = _normalize_datetime_str(obj.isoformat())
+        if obj.tzinfo:
+            return {"type": "datetime", "value": val}
+        return {"type": "datetime-local", "value": val}
+    elif isinstance(obj, datetime.time):
+        return {
+            "type": "time-local",
+            "value": _normalize_localtime_str(str(obj)),
+        }
+    elif isinstance(obj, datetime.date):
+        return {
+            "type": "date-local",
+            "value": str(obj),
+        }
+    elif isinstance(obj, list):
+        return [convert(i) for i in obj]
+    elif isinstance(obj, dict):
+        return {k: convert(v) for k, v in obj.items()}
+    raise Exception("unsupported type")
+
+
+def normalize(obj: Any) -> Any:
+    """Normalize test objects.
+
+    This normalizes primitive values (e.g. floats), and also converts from
+    TOML compliance format [1] to BurntSushi format [2].
+
+    [1] https://github.com/toml-lang/compliance/blob/db7c3211fda30ff9ddb10292f4aeda7e2e10abc4/docs/json-encoding.md  # noqa: E501
+    [2] https://github.com/BurntSushi/toml-test/blob/4634fdf3a6ecd6aaea5f4cdcd98b2733c2694993/README.md  # noqa: E501
+    """
+    if isinstance(obj, list):
+        return [normalize(item) for item in obj]
+    if isinstance(obj, dict):
+        if "type" in obj and "value" in obj:
+            type_ = obj["type"]
+            norm_type = _aliases.get(type_, type_)
+            value = obj["value"]
+            if norm_type == "float":
+                norm_value = _normalize_float_str(value)
+            elif norm_type in {"datetime", "datetime-local"}:
+                norm_value = _normalize_datetime_str(value)
+            elif norm_type == "time-local":
+                norm_value = _normalize_localtime_str(value)
+            else:
+                norm_value = value
+
+            if norm_type == "array":
+                return [normalize(item) for item in value]
+            return {"type": norm_type, "value": norm_value}
+        return {k: normalize(v) for k, v in obj.items()}
+    raise AssertionError("Burntsushi fixtures should be dicts/lists only")
+
+
+def _normalize_datetime_str(dt_str: str) -> str:
+    if dt_str[-1].lower() == "z":
+        dt_str = dt_str[:-1] + "+00:00"
+
+    date = dt_str[:10]
+    rest = dt_str[11:]
+
+    if "+" in rest:
+        sign = "+"
+    elif "-" in rest:
+        sign = "-"
+    else:
+        sign = ""
+
+    if sign:
+        time, _, offset = rest.partition(sign)
+    else:
+        time = rest
+        offset = ""
+
+    time = time.rstrip("0") if "." in time else time
+    return date + "T" + time + sign + offset
+
+
+def _normalize_localtime_str(lt_str: str) -> str:
+    return lt_str.rstrip("0") if "." in lt_str else lt_str
+
+
+def _normalize_float_str(float_str: str) -> str:
+    as_float = float(float_str)
+
+    # Normalize "-0.0" and "+0.0"
+    if as_float == 0:
+        return "0"
+
+    return str(as_float)

Lib/test/test_tomllib/__init__.py

@@ -0,0 +1 @@
+arrr = [true false]

Lib/test/test_tomllib/__main__.py

@@ -0,0 +1,4 @@
+[[parent-table.arr]]
+[parent-table]
+not-arr = 1
+arr = 2

Lib/test/test_tomllib/burntsushi.py

@@ -0,0 +1,2 @@
+a=true
+[[a]]

Lib/test/test_tomllib/data/invalid/array-missing-comma.toml

@@ -0,0 +1 @@
+a=[1

Lib/test/test_tomllib/data/invalid/array-of-tables/overwrite-array-in-parent.toml

@@ -0,0 +1 @@
+v=[1,

Lib/test/test_tomllib/data/invalid/array-of-tables/overwrite-bool-with-aot.toml

@@ -0,0 +1 @@
+v=[

Lib/test/test_tomllib/data/invalid/array/file-end-after-val.toml

@@ -0,0 +1 @@
+"backslash is the last char\

Lib/test/test_tomllib/data/invalid/array/unclosed-after-item.toml

@@ -0,0 +1 @@
+val=falsE

Lib/test/test_tomllib/data/invalid/array/unclosed-empty.toml

@@ -0,0 +1 @@
+val=trUe

Lib/test/test_tomllib/data/invalid/basic-str-ends-in-escape.toml

@@ -0,0 +1 @@
+"only 28 or 29 days in february" = 1988-02-30

Lib/test/test_tomllib/data/invalid/boolean/invalid-false-casing.toml

@@ -0,0 +1,2 @@
+a = false
+a.b = true

Lib/test/test_tomllib/data/invalid/boolean/invalid-true-casing.toml

@@ -0,0 +1,3 @@
+[[tab.arr]]
+[tab]
+arr.val1=1

Lib/test/test_tomllib/data/invalid/dates-and-times/invalid-day.toml

@@ -0,0 +1,4 @@
+[a.b.c.d]
+  z = 9
+[a]
+  b.c.d.k.t = 8

Lib/test/test_tomllib/data/invalid/dotted-keys/access-non-table.toml

@@ -0,0 +1,4 @@
+[a.b.c]
+  z = 9
+[a]
+  b.c.t = 9

Lib/test/test_tomllib/data/invalid/dotted-keys/extend-defined-aot.toml

@@ -0,0 +1 @@
+arrr = { comma-missing = true valid-toml = false }

Lib/test/test_tomllib/data/invalid/dotted-keys/extend-defined-table-with-subtable.toml

@@ -0,0 +1 @@
+table1 = { table2.dupe = 1, table2.dupe = 2 }

Lib/test/test_tomllib/data/invalid/dotted-keys/extend-defined-table.toml

@@ -0,0 +1 @@
+table = { dupe = 1, dupe = 2 }

Lib/test/test_tomllib/data/invalid/inline-table-missing-comma.toml

@@ -0,0 +1 @@
+a={b=1

Lib/test/test_tomllib/data/invalid/inline-table/define-twice-in-subtable.toml

@@ -0,0 +1,2 @@
+a = { b = 1 }
+a.b = 2

Lib/test/test_tomllib/data/invalid/inline-table/define-twice.toml

@@ -0,0 +1,5 @@
+[tab.nested]
+inline-t = { nest = {} }
+
+[tab]
+nested.inline-t.nest = 2

Lib/test/test_tomllib/data/invalid/inline-table/file-end-after-key-val.toml

@@ -0,0 +1,3 @@
+inline-t = { nest = {} }
+
+[[inline-t.nest]]

Lib/test/test_tomllib/data/invalid/inline-table/mutate.toml

@@ -0,0 +1,3 @@
+inline-t = { nest = {} }
+
+[inline-t.nest]

Lib/test/test_tomllib/data/invalid/inline-table/override-val-in-table.toml

@@ -0,0 +1 @@
+a = { b = 1, b.c = 2 }

Lib/test/test_tomllib/data/invalid/inline-table/override-val-with-array.toml

@@ -0,0 +1 @@
+tab = { inner.table = [{}], inner.table.val = "bad" }

Lib/test/test_tomllib/data/invalid/inline-table/override-val-with-table.toml

@@ -0,0 +1 @@
+tab = { inner = { dog = "best" }, inner.cat = "worst" }

Lib/test/test_tomllib/data/invalid/inline-table/overwrite-implicitly.toml

@@ -0,0 +1 @@
+a={

Lib/test/test_tomllib/data/invalid/inline-table/overwrite-value-in-inner-array.toml

@@ -0,0 +1 @@
+# form feed () not allowed in comments

Lib/test/test_tomllib/data/invalid/inline-table/overwrite-value-in-inner-table.toml

@@ -0,0 +1 @@
+escaped-unicode = "\uabag"

Lib/test/test_tomllib/data/invalid/inline-table/unclosed-empty.toml

@@ -0,0 +1 @@
+hex = 0xgabba00f1

Lib/test/test_tomllib/data/invalid/invalid-comment-char.toml

@@ -0,0 +1 @@
+[fwfw.wafw

Lib/test/test_tomllib/data/invalid/invalid-escaped-unicode.toml

@@ -0,0 +1 @@
+fs.fw

Lib/test/test_tomllib/data/invalid/invalid-hex.toml

@@ -0,0 +1 @@
+why-no-value=

Lib/test/test_tomllib/data/invalid/keys-and-vals/ends-early-table-def.toml

@@ -0,0 +1 @@
+fs.

Lib/test/test_tomllib/data/invalid/keys-and-vals/ends-early.toml

@@ -0,0 +1,2 @@
+a=1
+[a.b.c.d]