diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml new file mode 100644 index 000000000..7aea86d67 --- /dev/null +++ b/.github/workflows/build-binaries.yml @@ -0,0 +1,72 @@ +name: Build Binaries +on: + push: + branches: + - main + - "releases/*" + +jobs: + # Compile the binaries and upload artifacts + compile-binaries: + strategy: + fail-fast: true + matrix: + include: + - os: ubuntu-latest + package-suffix: linux-amd64 + - os: macos-latest + package-suffix: macos-amd64 + - os: windows-latest + package-suffix: windows-amd64 + - os: ubuntu-arm + package-suffix: linux-aarch64 + # Need the 8 CPU version that has 12GB of RAM, the 4 CPU version + # only has 6 GB. + runsOn: buildjet-8vcpu-ubuntu-2204-arm + runs-on: ${{ matrix.runsOn || matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + # actions/setup-python doesn't yet support ARM + - if: ${{ !endsWith(matrix.os, '-arm') }} + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - if: ${{ matrix.os == 'ubuntu-arm' }} + uses: deadsnakes/action@v2.1.1 + with: + python-version: "3.11" + + # Install Rust locally for non-Linux (Linux uses an internal docker + # command to build with cibuildwheel which uses rustup install defined + # in pyproject.toml) + - if: ${{ runner.os != 'Linux' }} + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - if: ${{ runner.os != 'Linux' }} + uses: Swatinem/rust-cache@v1 + with: + working-directory: temporalio/bridge + + # Prepare + - run: python -m pip install --upgrade wheel poetry poethepoet + - run: poetry install --no-root -E opentelemetry + + # Add the source dist only for Linux x64 for now + - if: ${{ matrix.package-suffix == 'linux-amd64' }} + run: poetry build --format sdist + + # Build and fix the wheel + - run: poetry run cibuildwheel --output-dir dist + - run: poe fix-wheel + + # Simple test + - run: poe test-dist-single + + # Upload dist + - uses: actions/upload-artifact@v2 + with: + name: packages-${{ matrix.package-suffix }} + path: dist diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7321d94f0..63d549951 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: Continuous Integration -on: # rebuild any PRs and main branch changes +on: pull_request: push: branches: @@ -12,15 +12,16 @@ jobs: strategy: fail-fast: true matrix: - # TODO(cretz): 3.10.8 is breaking Windows Rust build - python: ["3.7", "3.10.7"] - os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.7", "3.11"] + os: [ubuntu-latest, macos-latest, windows-latest, ubuntu-arm] include: - os: ubuntu-latest - python: 3.10.7 + python: "3.11" docsTarget: true protoCheckTarget: true - runs-on: ${{ matrix.os }} + - os: ubuntu-arm + runsOn: buildjet-4vcpu-ubuntu-2204-arm + runs-on: ${{ matrix.runsOn || matrix.os }} steps: - name: Print build information run: "echo head_ref: ${{ github.head_ref }}, ref: ${{ github.ref }}, os: ${{ matrix.os }}, python: ${{ matrix.python }}" @@ -33,7 +34,13 @@ jobs: - uses: Swatinem/rust-cache@v1 with: working-directory: temporalio/bridge - - uses: actions/setup-python@v4 + # actions/setup-python doesn't yet support ARM + - if: ${{ !endsWith(matrix.os, '-arm') }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - if: ${{ matrix.os == 'ubuntu-arm' }} + uses: deadsnakes/action@v2.1.1 with: python-version: ${{ matrix.python }} - run: python -m pip install --upgrade wheel poetry poethepoet @@ -41,7 +48,9 @@ jobs: - run: poe lint - run: poe build-develop - run: poe test -s -o log_cli_level=DEBUG - - run: poe test -s -o log_cli_level=DEBUG --workflow-environment time-skipping + # Time skipping doesn't yet support ARM + - if: ${{ !endsWith(matrix.os, '-arm') }} + run: poe test -s -o log_cli_level=DEBUG --workflow-environment time-skipping # Confirm protos are already generated properly - name: Check generated protos @@ -58,103 +67,3 @@ jobs: - name: Deploy prod API docs if: ${{ github.ref == 'refs/heads/main' && matrix.docsTarget }} run: npx vercel deploy build/apidocs -t ${{ secrets.VERCEL_TOKEN }} --name python --scope temporal --prod --yes - - # Compile the binaries and upload artifacts - compile-binaries: - strategy: - fail-fast: true - matrix: - include: - - os: ubuntu-latest - package-suffix: linux-amd64 - ci-arch: auto - - os: macos-latest - package-suffix: macos-amd64 - ci-arch: auto - # TODO(cretz): Disabling macOS arm because cibuildwheel is still - # generating an x64 wheel name even for arm64 - #- os: macos-latest - # package-suffix: macos-arm64 - # ci-arch: arm64 - rust-add-target: aarch64-apple-darwin - - os: windows-latest - package-suffix: windows-amd64 - ci-arch: auto - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - uses: actions/setup-python@v4 - with: - python-version: "3.10.7" - - # Install Rust locally for non-Linux (Linux uses an internal docker - # command to build with cibuildwheel which uses rustup install defined - # in pyproject.toml) - - if: ${{ runner.os != 'Linux' }} - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - target: ${{ matrix.rust-add-target }} - - if: ${{ runner.os != 'Linux' }} - uses: Swatinem/rust-cache@v1 - with: - working-directory: temporalio/bridge - - # Prepare - - run: python -m pip install --upgrade wheel poetry poethepoet - - run: poetry install --no-root -E opentelemetry - - # Add the source dist only for Linux x64 for now - - if: ${{ matrix.package-suffix == 'linux-amd64' }} - run: poetry build --format sdist - - # Build and fix the wheel - - run: poetry run cibuildwheel --output-dir dist --arch ${{ matrix.ci-arch }} - - run: poe fix-wheel - - # Do test only for ci-arch auto (i.e. local machine) - - if: ${{ matrix.ci-arch == 'auto' }} - run: poe test-dist-single - - # Upload dist - - uses: actions/upload-artifact@v2 - with: - name: packages-${{ matrix.package-suffix }} - path: dist - - # We separate out Linux aarch64 so we can choose not to run it during PR since - # it is so slow in cibuildwheel (uses QEMU emulation). We can put this back in - # the above matrix when Linux ARM runners are available. - compile-binaries-linux-aarch64: - # Skip compiling Linux aarch64 on PR - if: ${{ github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - uses: actions/setup-python@v4 - with: - python-version: "3.10.7" - - # Need QEMU for ARM build on Linux - - uses: docker/setup-qemu-action@v1 - with: - image: tonistiigi/binfmt:latest - platforms: arm64 - - # Prepare - - run: python -m pip install --upgrade wheel poetry poethepoet - - run: poetry install --no-root -E opentelemetry - - # Build and fix the wheel - - run: poetry run cibuildwheel --output-dir dist --arch aarch64 - - run: poe fix-wheel - - # Upload dist - - uses: actions/upload-artifact@v2 - with: - name: packages-linux-aarch64 - path: dist diff --git a/poetry.lock b/poetry.lock index 034f71968..9bd138d36 100644 --- a/poetry.lock +++ b/poetry.lock @@ -147,7 +147,7 @@ unicode_backport = ["unicodedata2"] [[package]] name = "cibuildwheel" -version = "2.10.1" +version = "2.11.2" description = "Build Python wheels on CI with minimal configuration." category = "dev" optional = false @@ -164,9 +164,8 @@ tomli = {version = "*", markers = "python_version < \"3.11\""} typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} [package.extras] -all = ["build", "click", "ghapi", "jinja2", "jinja2 (>=3.1.2)", "mkdocs (==1.3.1)", "mkdocs-include-markdown-plugin (==2.8.0)", "mkdocs-macros-plugin", "mypy (>=0.901)", "packaging (>=21.0)", "pip-tools", "pygithub", "pymdown-extensions", "pytest (>=6)", "pytest-timeout", "pytest-xdist", "pyyaml", "requests", "rich (>=9.6)", "tomli-w", "types-certifi", "types-click", "types-jinja2", "types-pyyaml", "types-requests", "types-toml"] -bin = ["click", "ghapi", "packaging (>=21.0)", "pip-tools", "pygithub", "pyyaml", "requests", "rich (>=9.6)"] -dev = ["build", "click", "ghapi", "jinja2", "mypy (>=0.901)", "packaging (>=21.0)", "pip-tools", "pygithub", "pytest (>=6)", "pytest-timeout", "pytest-xdist", "pyyaml", "requests", "rich (>=9.6)", "tomli-w", "types-certifi", "types-click", "types-jinja2", "types-pyyaml", "types-requests", "types-toml"] +bin = ["click", "packaging (>=21.0)", "pip-tools", "pygithub", "pyyaml", "requests", "rich (>=9.6)"] +dev = ["build", "click", "jinja2", "mypy (>=0.901)", "packaging (>=21.0)", "pip-tools", "pygithub", "pytest (>=6)", "pytest-timeout", "pytest-xdist", "pyyaml", "requests", "rich (>=9.6)", "tomli-w", "types-certifi", "types-click", "types-jinja2", "types-pyyaml", "types-requests", "types-toml"] docs = ["jinja2 (>=3.1.2)", "mkdocs (==1.3.1)", "mkdocs-include-markdown-plugin (==2.8.0)", "mkdocs-macros-plugin", "pymdown-extensions"] mypy = ["mypy (>=0.901)", "types-certifi", "types-click", "types-jinja2", "types-pyyaml", "types-requests", "types-toml"] test = ["build", "jinja2", "pytest (>=6)", "pytest-timeout", "pytest-xdist", "tomli-w"] @@ -1072,7 +1071,7 @@ opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "2de30dab6de94505d43b59178d6abba15ba69898c9c31bbe238eafda50515526" +content-hash = "b7760614decf80c88ae7969affe1d734497ff51249f140bc7bdd6641ab0f39ff" [metadata.files] appdirs = [ @@ -1207,8 +1206,8 @@ charset-normalizer = [ {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, ] cibuildwheel = [ - {file = "cibuildwheel-2.10.1-py3-none-any.whl", hash = "sha256:78a4c4d207fd3f271ddbba7364b961cfafc1fcbc52725a5253310d2c2a7ad67c"}, - {file = "cibuildwheel-2.10.1.tar.gz", hash = "sha256:c10fb19081cab3983bfb5d2ef62ef1521776d6e2dc9be350d697e3123239a1be"}, + {file = "cibuildwheel-2.11.2-py3-none-any.whl", hash = "sha256:392a39c7022da9ac016362b727c3ae39b304742adfcc36dab7ab0a47b90cc461"}, + {file = "cibuildwheel-2.11.2.tar.gz", hash = "sha256:e8dab8e6cf77ff0002f65873bba36b3969b23f522cc014538abcda2cbd24329f"}, ] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, diff --git a/pyproject.toml b/pyproject.toml index f85ac38b3..607cf5446 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ typing-extensions = "^4.2.0" [tool.poetry.dev-dependencies] black = "^22.3.0" -cibuildwheel = "^2.7.0" +cibuildwheel = "^2.11.0" grpcio-tools = "^1.48.0" isort = "^5.10.1" mypy = "^0.971" @@ -110,11 +110,6 @@ build-verbosity = "1" before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y && yum install -y openssl-devel" environment = { PATH = "$PATH:$HOME/.cargo/bin", CARGO_NET_GIT_FETCH_WITH_CLI = "true" } -[[tool.cibuildwheel.overrides]] -# We need the aarch64 target for Rust -before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable --target aarch64-unknown-linux-gnu -y && yum install -y openssl-devel" -select = "*_aarch64" - [tool.isort] profile = "black" skip_gitignore = true diff --git a/temporalio/worker/workflow_instance.py b/temporalio/worker/workflow_instance.py index 7b1e27432..08c1e2ae7 100644 --- a/temporalio/worker/workflow_instance.py +++ b/temporalio/worker/workflow_instance.py @@ -1135,7 +1135,12 @@ def _next_seq(self, type: str) -> int: self._curr_seqs[type] = seq return seq - def _register_task(self, task: asyncio.Task, *, name: Optional[str]) -> None: + def _register_task( + self, + task: asyncio.Task, + *, + name: Optional[str], + ) -> None: # Name not supported on older Python versions if sys.version_info >= (3, 8): # Put the workflow info at the end of the task name @@ -1324,8 +1329,13 @@ def create_task( coro: Union[Awaitable[_T], Generator[Any, None, _T]], *, name: Optional[str] = None, + context: Optional[contextvars.Context] = None, ) -> asyncio.Task[_T]: - task = asyncio.Task(coro, loop=self) + # Context only supported on newer Python versions + if sys.version_info >= (3, 11): + task = asyncio.Task(coro, loop=self, context=context) # type: ignore + else: + task = asyncio.Task(coro, loop=self) self._register_task(task, name=name) return task diff --git a/temporalio/worker/workflow_sandbox/restrictions.py b/temporalio/worker/workflow_sandbox/restrictions.py index 8bd68cd43..83adb05c0 100644 --- a/temporalio/worker/workflow_sandbox/restrictions.py +++ b/temporalio/worker/workflow_sandbox/restrictions.py @@ -158,7 +158,7 @@ def nested_child(path: Sequence[str], child: SandboxMatcher) -> SandboxMatcher: An string containing a single asterisk can be used to match all. """ - children: Mapping[str, SandboxMatcher] = types.MappingProxyType({}) + children: Mapping[str, SandboxMatcher] = dataclasses.field(default_factory=dict) """Immutable mapping of child matchers.""" match_self: bool = False @@ -354,7 +354,7 @@ def with_child_unrestricted(self, *child_path: str) -> SandboxMatcher: ) # sys.stdlib_module_names is only available on 3.10+, so we hardcode here. A -# test will fail if this list doesn't match the 3.10+ Python version it was +# test will fail if this list doesn't match the latest Python version it was # generated against, spitting out the expected list. This is a string instead # of a list of strings due to black wanting to format this to one item each # line in a list. @@ -368,27 +368,27 @@ def with_child_unrestricted(self, *child_path: str) -> SandboxMatcher: "_pickle,_posixshmem,_posixsubprocess,_py_abc,_pydecimal,_pyio,_queue,_random,_scproxy," "_sha1,_sha256,_sha3,_sha512,_signal,_sitebuiltins,_socket,_sqlite3,_sre,_ssl,_stat," "_statistics,_string,_strptime,_struct,_symtable,_thread,_threading_local,_tkinter," - "_tracemalloc,_uuid,_warnings,_weakref,_weakrefset,_winapi,_zoneinfo,abc,aifc,antigravity," - "argparse,array,ast,asynchat,asyncio,asyncore,atexit,audioop,base64,bdb,binascii,binhex," - "bisect,builtins,bz2,cProfile,calendar,cgi,cgitb,chunk,cmath,cmd,code,codecs,codeop," - "collections,colorsys,compileall,concurrent,configparser,contextlib,contextvars,copy," - "copyreg,crypt,csv,ctypes,curses,dataclasses,datetime,dbm,decimal,difflib,dis,distutils," - "doctest,email,encodings,ensurepip,enum,errno,faulthandler,fcntl,filecmp,fileinput," - "fnmatch,fractions,ftplib,functools,gc,genericpath,getopt,getpass,gettext,glob,graphlib," - "grp,gzip,hashlib,heapq,hmac,html,http,idlelib,imaplib,imghdr,imp,importlib,inspect," - "io,ipaddress,itertools,json,keyword,lib2to3,linecache,locale,logging,lzma,mailbox," - "mailcap,marshal,math,mimetypes,mmap,modulefinder,msilib,msvcrt,multiprocessing,netrc," - "nis,nntplib,nt,ntpath,nturl2path,numbers,opcode,operator,optparse,os,ossaudiodev," - "pathlib,pdb,pickle,pickletools,pipes,pkgutil,platform,plistlib,poplib,posix,posixpath," - "pprint,profile,pstats,pty,pwd,py_compile,pyclbr,pydoc,pydoc_data,pyexpat,queue,quopri," - "random,re,readline,reprlib,resource,rlcompleter,runpy,sched,secrets,select,selectors," - "shelve,shlex,shutil,signal,site,smtpd,smtplib,sndhdr,socket,socketserver,spwd,sqlite3," - "sre_compile,sre_constants,sre_parse,ssl,stat,statistics,string,stringprep,struct," - "subprocess,sunau,symtable,sys,sysconfig,syslog,tabnanny,tarfile,telnetlib,tempfile," - "termios,textwrap,this,threading,time,timeit,tkinter,token,tokenize,trace,traceback," - "tracemalloc,tty,turtle,turtledemo,types,typing,unicodedata,unittest,urllib,uu,uuid," - "venv,warnings,wave,weakref,webbrowser,winreg,winsound,wsgiref,xdrlib,xml,xmlrpc,zipapp," - "zipfile,zipimport,zlib,zoneinfo" + "_tokenize,_tracemalloc,_typing,_uuid,_warnings,_weakref,_weakrefset,_winapi,_zoneinfo," + "abc,aifc,antigravity,argparse,array,ast,asynchat,asyncio,asyncore,atexit,audioop," + "base64,bdb,binascii,bisect,builtins,bz2,cProfile,calendar,cgi,cgitb,chunk,cmath,cmd," + "code,codecs,codeop,collections,colorsys,compileall,concurrent,configparser,contextlib," + "contextvars,copy,copyreg,crypt,csv,ctypes,curses,dataclasses,datetime,dbm,decimal," + "difflib,dis,distutils,doctest,email,encodings,ensurepip,enum,errno,faulthandler,fcntl," + "filecmp,fileinput,fnmatch,fractions,ftplib,functools,gc,genericpath,getopt,getpass," + "gettext,glob,graphlib,grp,gzip,hashlib,heapq,hmac,html,http,idlelib,imaplib,imghdr," + "imp,importlib,inspect,io,ipaddress,itertools,json,keyword,lib2to3,linecache,locale," + "logging,lzma,mailbox,mailcap,marshal,math,mimetypes,mmap,modulefinder,msilib,msvcrt," + "multiprocessing,netrc,nis,nntplib,nt,ntpath,nturl2path,numbers,opcode,operator,optparse," + "os,ossaudiodev,pathlib,pdb,pickle,pickletools,pipes,pkgutil,platform,plistlib,poplib," + "posix,posixpath,pprint,profile,pstats,pty,pwd,py_compile,pyclbr,pydoc,pydoc_data," + "pyexpat,queue,quopri,random,re,readline,reprlib,resource,rlcompleter,runpy,sched," + "secrets,select,selectors,shelve,shlex,shutil,signal,site,smtpd,smtplib,sndhdr,socket," + "socketserver,spwd,sqlite3,sre_compile,sre_constants,sre_parse,ssl,stat,statistics," + "string,stringprep,struct,subprocess,sunau,symtable,sys,sysconfig,syslog,tabnanny," + "tarfile,telnetlib,tempfile,termios,textwrap,this,threading,time,timeit,tkinter,token," + "tokenize,tomllib,trace,traceback,tracemalloc,tty,turtle,turtledemo,types,typing,unicodedata," + "unittest,urllib,uu,uuid,venv,warnings,wave,weakref,webbrowser,winreg,winsound,wsgiref," + "xdrlib,xml,xmlrpc,zipapp,zipfile,zipimport,zlib,zoneinfo" ) SandboxRestrictions.passthrough_modules_maximum = SandboxRestrictions.passthrough_modules_with_temporal | SandboxMatcher( @@ -648,7 +648,6 @@ def from_proxy(v: _RestrictedProxy) -> _RestrictionState: obj: object context: RestrictionContext matcher: SandboxMatcher - wrap_dict: bool = True def assert_child_not_restricted(self, name: str) -> None: if ( @@ -793,9 +792,9 @@ def __getattribute__(self, __name: str) -> Any: state.assert_child_not_restricted(__name) ret = object.__getattribute__(self, "__getattr__")(__name) - # If there is a child matcher, restrict if we can. As a special case, we - # don't wrap __dict__ if we're asked not to. - if __name != "__dict__" or state.wrap_dict: + # Since Python 3.11, the importer references __spec__ on module, so we + # allow that through + if __name != "__spec__": child_matcher = state.matcher.child_matcher(__name) if child_matcher and _is_restrictable(ret): ret = _RestrictedProxy( diff --git a/tests/testing/test_workflow.py b/tests/testing/test_workflow.py index 75a999dde..fb73bae75 100644 --- a/tests/testing/test_workflow.py +++ b/tests/testing/test_workflow.py @@ -1,4 +1,5 @@ import asyncio +import platform import uuid from datetime import datetime, timedelta, timezone from time import monotonic @@ -35,7 +36,13 @@ async def some_signal(self) -> None: pass +def skip_if_not_x86() -> None: + if platform.machine() not in ("i386", "AMD64", "x86_64"): + pytest.skip("Time skipping server does not run outside x86") + + async def test_workflow_env_time_skipping_basic(): + skip_if_not_x86() async with await WorkflowEnvironment.start_time_skipping() as env: async with new_worker(env.client, ReallySlowWorkflow) as worker: # Check that time is around now @@ -51,6 +58,7 @@ async def test_workflow_env_time_skipping_basic(): async def test_workflow_env_time_skipping_manual(): + skip_if_not_x86() async with await WorkflowEnvironment.start_time_skipping() as env: async with new_worker(env.client, ReallySlowWorkflow) as worker: # Start workflow @@ -101,6 +109,7 @@ async def run(self) -> str: async def test_workflow_env_time_skipping_heartbeat_timeout(): + skip_if_not_x86() async with await WorkflowEnvironment.start_time_skipping() as env: async with new_worker( env.client, @@ -128,6 +137,7 @@ async def run(self) -> str: async def test_workflow_env_time_skipping_disabled(): + skip_if_not_x86() async with await WorkflowEnvironment.start_time_skipping() as env: async with new_worker(env.client, ShortSleepWorkflow) as worker: # Confirm when executing normally it does not sleep for a full 3s diff --git a/tests/worker/workflow_sandbox/test_restrictions.py b/tests/worker/workflow_sandbox/test_restrictions.py index 5e29052d2..3d7050848 100644 --- a/tests/worker/workflow_sandbox/test_restrictions.py +++ b/tests/worker/workflow_sandbox/test_restrictions.py @@ -16,8 +16,8 @@ def test_workflow_sandbox_stdlib_module_names(): - if sys.version_info != (3, 10): - pytest.skip("Test only runs on 3.10") + if sys.version_info[1] != 11: + pytest.skip("Test only runs on 3.11") actual_names = ",".join(sorted(sys.stdlib_module_names)) # Uncomment to print code for generating these code_lines = [""]