Merge remote-tracking branch 'upstream/main' into main

Author: jamesadevine

.github/workflows/build.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Install deps
       run: |
         sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y gettext librsvg2-bin mingw-w64
+        sudo eatmydata apt-get install -y gettext librsvg2-bin mingw-w64 latexmk texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
         pip install requests sh click setuptools cpp-coveralls "Sphinx<4" sphinx-rtd-theme recommonmark sphinx-autoapi sphinxcontrib-svg2pdfconverter polib pyyaml astroid isort black awscli
     - name: Versions
       run: |
@@ -73,17 +73,26 @@ jobs:
       with:
         name: stubs
         path: circuitpython-stubs*
-    - name: Docs
+    - name: Test Documentation Build (HTML)
       run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html
     - uses: actions/upload-artifact@v2
       with:
         name: docs
         path: _build/html
+    - name: Test Documentation Build (LaTeX/PDF)
+      run: |
+        make latexpdf
+    - uses: actions/upload-artifact@v2
+      with:
+        name: docs
+        path: _build/latex
     - name: Translations
       run: make check-translate
     - name: New boards check
       run: python3 -u ci_new_boards_check.py
       working-directory: tools
+    - name: Duplicate USB VID/PID Check
+      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Build mpy-cross.static-raspbian
       run: make -C mpy-cross -j2 -f Makefile.static-raspbian
     - uses: actions/upload-artifact@v2
@@ -170,7 +179,7 @@ jobs:
         - "arduino_nano_33_iot"
         - "arduino_zero"
         - "bast_pro_mini_m0"
-        - "bdmicro_vina_m0"
+        - "bdmicro_vina_d21"
         - "bless_dev_board_multi_sensor"
         - "blm_badge"
         - "capablerobot_usbhub"
@@ -201,6 +210,7 @@ jobs:
         - "feather_m0_rfm69"
         - "feather_m0_rfm9x"
         - "feather_m0_supersized"
+        - "feather_m4_can"
         - "feather_m4_express"
         - "feather_m7_1011"
         - "feather_mimxrt1011"
@@ -227,6 +237,7 @@ jobs:
         - "makerdiary_nrf52840_m2_devkit"
         - "makerdiary_nrf52840_mdk"
         - "makerdiary_nrf52840_mdk_usb_dongle"
+        - "matrixportal_m4"
         - "meowbit_v121"
         - "meowmeow"
         - "metro_m0_express"
@@ -253,6 +264,7 @@ jobs:
         - "pca10100"
         - "pewpew10"
         - "pewpew_m4"
+        - "picoplanet"
         - "pirkey_m0"
         - "pitaya_go"
         - "pyb_nano_v2"
@@ -266,6 +278,8 @@ jobs:
         - "pyportal"
         - "pyportal_titano"
         - "pyruler"
+        - "qtpy_m0"
+        - "qtpy_m0_haxpress"
         - "raytac_mdbt50q-db-40"
         - "robohatmm1_m4"
         - "sam32"
@@ -400,9 +414,15 @@ jobs:
       fail-fast: false
       matrix:
         board:
+        - "adafruit_metro_esp32s2"
+        - "electroniccats_bastwifi"
+        - "espressif_kaluga_1"
         - "espressif_saola_1_wroom"
         - "espressif_saola_1_wrover"
+        - "microdev_micro_s2"
+        - "muselab_nanoesp32_s2"
         - "unexpectedmaker_feathers2"
+        - "unexpectedmaker_feathers2_prerelease"
 
     steps:
     - name: Set up Python 3.8
@@ -422,6 +442,11 @@ jobs:
       with:
         path: ${{ github.workspace }}/.idf_tools
         key: ${{ runner.os }}-idf-tools-${{ hashFiles('.git/modules/ports/esp32s2/esp-idf/HEAD') }}-20200801
+    - name: Clone IDF submodules
+      run: |
+        (cd $IDF_PATH && git submodule update --init)
+      env:
+        IDF_PATH: ${{ github.workspace }}/ports/esp32s2/esp-idf
     - name: Install IDF tools
       run: |
         $IDF_PATH/tools/idf_tools.py --non-interactive install required

.gitignore

@@ -57,7 +57,7 @@ _build
 ######################
 genrst/
 /autoapi/
-/shared-bindings/**/*.rst
+/shared-bindings/*/**/*.rst
 
 # ctags and similar
 ###################
@@ -80,5 +80,9 @@ TAGS
 *.mo
 
 .vscode
-
 .idea
+
+# Python Virtual Environments
+####################
+.venv
+.env

.gitmodules

@@ -147,3 +147,9 @@
 [submodule "ports/esp32s2/esp-idf"]
 	path = ports/esp32s2/esp-idf
 	url = https://github.com/tannewt/esp-idf.git
+[submodule "frozen/Adafruit_CircuitPython_RFM9x"]
+	path = frozen/Adafruit_CircuitPython_RFM9x
+	url = https://github.com/adafruit/Adafruit_CircuitPython_RFM9x.git
+[submodule "frozen/Adafruit_CircuitPython_RFM69"]
+	path = frozen/Adafruit_CircuitPython_RFM69
+	url = https://github.com/adafruit/Adafruit_CircuitPython_RFM69.git

.pre-commit-config.yaml

@@ -8,6 +8,6 @@ repos:
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
-        exclude: '^(tests/.*\.exp|tests/cmdline/.*|tests/.*/data/.*)'
+        exclude: '^(tests/.*\.exp|tests/cmdline/.*|tests/.*/data/.*|ports/esp32s2/esp-idf-config/.*|ports/esp32s2/boards/.*/sdkconfig)'
     -   id: trailing-whitespace
         exclude: '^(tests/.*\.exp|tests/cmdline/.*|tests/.*/data/.*)'

.readthedocs.yml

@@ -8,6 +8,13 @@
 
 version: 2
 
+submodules:
+    include:
+        - extmod/ulab
+
+formats:
+  - pdf
+
 python:
     version: 3
     install:

Makefile

@@ -40,7 +40,19 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(BASEOPTS)
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(BASEOPTS)
 
-TRANSLATE_SOURCES = extmod lib main.c ports/atmel-samd ports/cxd56 ports/mimxrt10xx ports/nrf ports/stm py shared-bindings shared-module supervisor
+TRANSLATE_SOURCES = extmod lib main.c ports/atmel-samd ports/cxd56 ports/esp32s2 ports/mimxrt10xx ports/nrf ports/stm py shared-bindings shared-module supervisor
+# Paths to exclude from TRANSLATE_SOURCES
+# Each must be preceded by "-path"; if any wildcards, enclose in quotes.
+# Separate by "-o" (Find's "or" operand)
+TRANSLATE_SOURCES_EXC = -path "ports/*/build-*" \
+	-o -path "ports/*/build" \
+	-o -path ports/esp32s2/esp-idf \
+	-o -path ports/cxd56/spresense-exported-sdk \
+	-o -path ports/stm/st_driver \
+	-o -path ports/atmel-samd/asf4 \
+	-o -path ports/mimxrt10xx/sdk \
+	-o -path lib/tinyusb \
+	-o -path lib/lwip \
 
 .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext stubs
 
@@ -210,7 +222,7 @@ pseudoxml:
 all-source:
 
 locale/circuitpython.pot: all-source
-	find $(TRANSLATE_SOURCES) -iname "*.c" -print | (LC_ALL=C sort) | xgettext -f- -L C -s --add-location=file --keyword=translate -o circuitpython.pot -p locale
+	find $(TRANSLATE_SOURCES) -type d \( $(TRANSLATE_SOURCES_EXC) \) -prune -o -type f \( -iname "*.c" -o -iname "*.h" \) -print | (LC_ALL=C sort) | xgettext -f- -L C -s --add-location=file --keyword=translate -o circuitpython.pot -p locale
 
 # Historically, `make translate` updated the .pot file and ran msgmerge.
 # However, this was a frequent source of merge conflicts.  Weblate can perform
@@ -235,7 +247,7 @@ merge-translate:
 
 .PHONY: check-translate
 check-translate:
-	find $(TRANSLATE_SOURCES) -iname "*.c" -print | (LC_ALL=C sort) | xgettext -f- -L C -s --add-location=file --keyword=translate -o circuitpython.pot.tmp -p locale
+	find $(TRANSLATE_SOURCES) -type d \( $(TRANSLATE_SOURCES_EXC) \) -prune -o -type f \( -iname "*.c" -o -iname "*.h" \) -print | (LC_ALL=C sort) | xgettext -f- -L C -s --add-location=file --keyword=translate -o circuitpython.pot.tmp -p locale
 	$(PYTHON) tools/check_translations.py locale/circuitpython.pot.tmp locale/circuitpython.pot; status=$$?; rm -f locale/circuitpython.pot.tmp; exit $$status
 
 stubs:
@@ -245,6 +257,10 @@ stubs:
 	@$(PYTHON) tools/extract_pyi.py ports/atmel-samd/bindings $(STUBDIR)
 	@$(PYTHON) setup.py -q sdist
 
+.PHONY: check-stubs
+check-stubs: stubs
+	MYPYPATH=$(STUBDIR) mypy --strict $(STUBDIR)
+
 update-frozen-libraries:
 	@echo "Updating all frozen libraries to latest tagged version."
 	cd frozen; for library in *; do cd $$library; ../../tools/git-checkout-latest-tag.sh; cd ..; done

conf.py

@@ -17,14 +17,17 @@
 #
 # SPDX-License-Identifier: MIT
 
-import json
 import logging
 import os
+import re
 import subprocess
 import sys
 import urllib.parse
 
 import recommonmark
+from sphinx.transforms import SphinxTransform
+from docutils import nodes
+from sphinx import addnodes
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -39,6 +42,9 @@
 # Grab the JSON values to use while building the module support matrix
 # in 'shared-bindings/index.rst'
 
+# The stubs must be built before we calculate the shared bindings matrix
+subprocess.check_output(["make", "stubs"])
+
 #modules_support_matrix = shared_bindings_matrix.support_matrix_excluded_boards()
 modules_support_matrix = shared_bindings_matrix.support_matrix_by_board()
 
@@ -74,7 +80,6 @@
     '.md': 'markdown',
 }
 
-subprocess.check_output(["make", "stubs"])
 extensions.append('autoapi.extension')
 
 autoapi_type = 'python'
@@ -84,6 +89,7 @@
 autoapi_add_toctree_entry = False
 autoapi_options = ['members', 'undoc-members', 'private-members', 'show-inheritance', 'special-members', 'show-module-summary']
 autoapi_template_dir = 'docs/autoapi/templates'
+autoapi_python_class_content = "both"
 autoapi_python_use_implicit_namespaces = True
 autoapi_root = "shared-bindings"
 
@@ -106,7 +112,25 @@
 #
 # We don't follow "The short X.Y version" vs "The full version, including alpha/beta/rc tags"
 # breakdown, so use the same version identifier for both to avoid confusion.
-version = release = '0.0.0'
+
+final_version = ""
+git_describe = subprocess.run(
+    ["git", "describe", "--dirty", "--tags"],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT,
+    encoding="utf-8"
+)
+if git_describe.returncode == 0:
+    git_version = re.search(
+        r"^\d(?:\.\d){0,2}(?:\-(?:alpha|beta|rc)\.\d+){0,1}",
+        str(git_describe.stdout)
+    )
+    if git_version:
+        final_version = git_version[0]
+else:
+    print("Failed to retrieve git version:", git_describe.stdout)
+
+version = release = final_version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -122,6 +146,7 @@
 # directories to ignore when looking for source files.
 exclude_patterns = ["**/build*",
                     ".git",
+                    ".env",
                     ".venv",
                     ".direnv",
                     "docs/autoapi",
@@ -423,7 +448,38 @@ def generate_redirects(app):
             with open(redirected_filename, 'w') as f:
                 f.write(TEMPLATE % urllib.parse.quote(to_path, '#/'))
 
+
+class CoreModuleTransform(SphinxTransform):
+    default_priority = 870
+
+    def _convert_first_paragraph_into_title(self):
+        title = self.document.next_node(nodes.title)
+        paragraph = self.document.next_node(nodes.paragraph)
+        if not title or not paragraph:
+            return
+        if isinstance(paragraph[0], nodes.paragraph):
+            paragraph = paragraph[0]
+        if all(isinstance(child, nodes.Text) for child in paragraph.children):
+            for child in paragraph.children:
+                title.append(nodes.Text(" \u2013 "))
+                title.append(child)
+            paragraph.parent.remove(paragraph)
+
+    def _enable_linking_to_nonclass_targets(self):
+        for desc in self.document.traverse(addnodes.desc):
+            for xref in desc.traverse(addnodes.pending_xref):
+                if xref.attributes.get("reftype") == "class":
+                    xref.attributes.pop("refspecific", None)
+
+    def apply(self, **kwargs):
+        docname = self.env.docname
+        if docname.startswith(autoapi_root) and docname.endswith("/index"):
+            self._convert_first_paragraph_into_title()
+            self._enable_linking_to_nonclass_targets()
+
+
 def setup(app):
     app.add_css_file("customstyle.css")
     app.add_config_value('redirects_file', 'redirects', 'env')
     app.connect('builder-inited', generate_redirects)
+    app.add_transform(CoreModuleTransform)