diff --git a/.github/workflows/release-dotnet.yml b/.github/workflows/release-dotnet.yml index e43747b5..8a6c8645 100644 --- a/.github/workflows/release-dotnet.yml +++ b/.github/workflows/release-dotnet.yml @@ -18,26 +18,30 @@ jobs: name: Package Nuget runs-on: windows-latest steps: - - name: Checkout - uses: actions/checkout@v2.4.0 - with: - submodules: "true" - - name: Download workflow artifact - uses: dawidd6/action-download-artifact@v2.17.0 - with: - workflow: "build-libs.yml" - path: ./libs + - uses: actions/checkout@master - uses: actions/setup-dotnet@v1 with: dotnet-version: '6.0.x' + - name: Set up Python 3.9 + uses: actions/setup-python@v1 + with: + python-version: 3.9 - uses: trinsic-id/set-version@v0.1 id: setversion with: githubToken: ${{ secrets.API_GITHUB_TOKEN }} overrideVersion: ${{ github.event.inputs.packageVersion }} + - name: release-downloader + uses: robinraju/release-downloader@v1.3 + with: + repository: trinsic-id/okapi + latest: true + token: ${{ secrets.API_GITHUB_TOKEN }} + fileName: "libs.zip" - name: Create Nuget package run: | + python ../../../devops/build_sdks.py --package-version=${{ steps.setversion.outputs.packageVersion }} --language=dotnet dotnet workload install ios dotnet workload install maccatalyst dotnet restore diff --git a/.github/workflows/release-java.yml b/.github/workflows/release-java.yml index 3da6265a..1989bb80 100644 --- a/.github/workflows/release-java.yml +++ b/.github/workflows/release-java.yml @@ -30,17 +30,22 @@ jobs: with: java-version: '15' distribution: 'adopt' - - name: Download workflow artifact - uses: dawidd6/action-download-artifact@v2.17.0 + - name: Set up Python 3.9 + uses: actions/setup-python@v1 with: - workflow: "build-libs.yml" - path: ./libs - branch: main + python-version: 3.9 - uses: trinsic-id/set-version@v0.1 id: setversion with: githubToken: ${{ secrets.API_GITHUB_TOKEN }} overrideVersion: ${{ github.event.inputs.packageVersion }} + - name: release-downloader + uses: robinraju/release-downloader@v1.3 + with: + repository: trinsic-id/okapi + latest: true + token: ${{ secrets.API_GITHUB_TOKEN }} + fileName: "libs.zip" - name: Publish package run: | python ../devops/build_sdks.py --package-version=${{ steps.setversion.outputs.packageVersion }} --language=java diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 8f0f8537..1d04f3f7 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -19,15 +19,26 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@master - - name: Set up Python 3.9 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: python-version: 3.9 + - uses: trinsic-id/set-version@v0.1 + id: setversion + with: + githubToken: ${{ secrets.API_GITHUB_TOKEN }} + overrideVersion: ${{ github.event.inputs.packageVersion }} + - name: release-downloader + uses: robinraju/release-downloader@v1.3 + with: + repository: trinsic-id/okapi + latest: true + token: ${{ secrets.API_GITHUB_TOKEN }} + fileName: "libs.zip" - name: Build, Test, Pack run: | python -m pip install -r requirements.txt - python -m pip install build - python ../devops/build_sdks.py --package-version=${{ github.event.inputs.packageVersion }} --language=python + python ../devops/build_sdks.py --package-version=${{ steps.setversion.outputs.packageVersion }} --language=python python -m build --sdist --wheel --outdir dist/ . shell: pwsh working-directory: python diff --git a/.github/workflows/release-ruby.yml b/.github/workflows/release-ruby.yml index c4558b68..45a84424 100644 --- a/.github/workflows/release-ruby.yml +++ b/.github/workflows/release-ruby.yml @@ -29,17 +29,27 @@ jobs: uses: ruby/setup-ruby@v1.76.0 with: ruby-version: 2.7 - - name: Download workflow artifact - uses: dawidd6/action-download-artifact@v2.17.0 + - name: Set up Python 3.9 + uses: actions/setup-python@v1 with: - workflow: "build-libs.yml" - path: ./libs - branch: main + python-version: 3.9 + - uses: trinsic-id/set-version@v0.1 + id: setversion + with: + githubToken: ${{ secrets.API_GITHUB_TOKEN }} + overrideVersion: ${{ github.event.inputs.packageVersion }} + - name: release-downloader + uses: robinraju/release-downloader@v1.3 + with: + repository: trinsic-id/okapi + latest: true + token: ${{ secrets.API_GITHUB_TOKEN }} + fileName: "libs.zip" - name: Build and run tests run: | gem install bundler bundle install - python ../devops/build_sdks.py --package-version=${{ github.event.inputs.packageVersion }} --language=ruby + python ../devops/build_sdks.py --package-version=${{ steps.setversion.outputs.packageVersion }} --language=ruby mkdir -p $HOME/.gem touch $HOME/.gem/credentials diff --git a/.github/workflows/release-swift.yml b/.github/workflows/release-swift.yml index 3045a7bd..92e5d170 100644 --- a/.github/workflows/release-swift.yml +++ b/.github/workflows/release-swift.yml @@ -21,17 +21,33 @@ jobs: - name: Checkout uses: actions/checkout@v2.4.0 - - name: Download workflow artifact - uses: dawidd6/action-download-artifact@v2.17.0 + - name: Set up Python 3.9 + uses: actions/setup-python@v1 with: - workflow: "build-libs.yml" - path: ./libs + python-version: 3.9 + - uses: trinsic-id/set-version@v0.1 + id: setversion + with: + githubToken: ${{ secrets.API_GITHUB_TOKEN }} + overrideVersion: ${{ github.event.inputs.packageVersion }} + - name: release-downloader + uses: robinraju/release-downloader@v1.3 + with: + repository: trinsic-id/okapi + latest: true + token: ${{ secrets.API_GITHUB_TOKEN }} + fileName: "libs.zip" - uses: trinsic-id/set-version@v0.1 id: setversion with: githubToken: ${{ secrets.API_GITHUB_TOKEN }} overrideVersion: ${{ github.event.inputs.packageVersion }} + - run: | + python ./devops/build_sdks.py --package-version=${{ steps.setversion.outputs.packageVersion }} --language=none + working-directory: ./ + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Create Framework run: | diff --git a/devops/build_sdks.py b/devops/build_sdks.py index 496c97e3..82a3d2c5 100644 --- a/devops/build_sdks.py +++ b/devops/build_sdks.py @@ -7,9 +7,10 @@ import os import platform import shutil -from os.path import join, abspath, dirname, isdir, split import subprocess +from os.path import join, abspath, dirname, isdir, split, exists from typing import Dict +from zipfile import ZipFile try: import requests @@ -22,9 +23,13 @@ def parse_version_tag(): raise NotImplementedError -def get_os_arch_path(extract_dir, windows_path): +def get_libs_dir() -> str: + return abspath(join(dirname(__file__), '..', 'libs')) + + +def get_os_arch_path(windows_path): copy_from = '' - libs_dir = join(extract_dir, 'libs') + libs_dir = get_libs_dir() os_name = platform.system().lower() processor_name = platform.machine().lower() if os_name == 'windows': @@ -41,28 +46,28 @@ def get_os_arch_path(extract_dir, windows_path): return copy_from - def set_env_var(name, value): env_file = os.getenv('GITHUB_ENV') + if env_file is None: + return with open(env_file, "a") as file: file.write(f"{name}={value}") def copy_okapi_libs(copy_to: str, windows_path='windows'): - okapi_dir = abspath(join(dirname(__file__), '..')) - copy_from = get_os_arch_path(okapi_dir, windows_path) + copy_from = get_os_arch_path(windows_path) logging.info(f"Copying okapi libs from: {copy_from}\nto: {copy_to}") for copy_file in glob.glob(join(copy_from, '*.*')): shutil.copy2(copy_file, copy_to) try: - shutil.copy2(join(okapi_dir, 'libs', 'C_header', 'okapi.h'), copy_to) + shutil.copy2(join(get_libs_dir(), 'C_header', 'okapi.h'), copy_to) except FileNotFoundError: pass def deep_copy_okapi_libs(copy_to: str): - copy_from = abspath(join(dirname(__file__), '..','libs')) + copy_from = abspath(join(dirname(__file__), '..', 'libs')) logging.info(f"Copying okapi libs from: {copy_from}\nto: {copy_to}") shutil.rmtree(copy_to, ignore_errors=True) shutil.copytree(copy_from, copy_to) @@ -85,7 +90,6 @@ def clean_dir(language_dir: str) -> None: os.mkdir(language_dir) - def update_line(file_name: str, replace_lines: Dict[str, str]) -> None: with open(file_name, 'r') as fid: file_lines = fid.readlines() @@ -120,31 +124,34 @@ def get_sdk_dir() -> str: def build_python(args) -> None: # Update version in setup.cfg python_dir = get_language_dir('python') - update_line(join(python_dir, 'setup.cfg'), - {'version = ': f'version = {get_package_versions(args)}'}) + update_line(join(python_dir, 'setup.cfg'), {'version = ': f'version = {get_package_versions(args)}'}) # TODO - Support ARM - copy_okapi_file(abspath(join(dirname(__file__), '..','libs','windows','okapi.dll')),abspath(join(python_dir,'libs','windows'))) - copy_okapi_file(abspath(join(dirname(__file__), '..','libs','macos','libokapi.dylib')), abspath(join(python_dir,'libs','macos'))) - copy_okapi_file(abspath(join(dirname(__file__), '..','libs','linux','libokapi.so')), abspath(join(python_dir,'libs','linux'))) + copy_okapi_file(abspath(join(dirname(__file__), '..', 'libs', 'windows', 'okapi.dll')), + abspath(join(python_dir, 'libs', 'windows'))) + copy_okapi_file(abspath(join(dirname(__file__), '..', 'libs', 'macos', 'libokapi.dylib')), + abspath(join(python_dir, 'libs', 'macos'))) + copy_okapi_file(abspath(join(dirname(__file__), '..', 'libs', 'linux', 'libokapi.so')), + abspath(join(python_dir, 'libs', 'linux'))) def build_java(args) -> None: # Update version in setup.cfg java_dir = get_language_dir('java') - update_line(join(java_dir, 'build.gradle'), - {'def jarVersion': f'def jarVersion = "{get_package_versions(args)}"'}) + update_line(join(java_dir, 'build.gradle'), {'def jarVersion': f'def jarVersion = "{get_package_versions(args)}"'}) copy_okapi_libs(abspath(join(java_dir, '..', 'libs'))) def build_ruby(args) -> None: # Update version in setup.cfg ruby_dir = get_language_dir('ruby') - update_line(join(ruby_dir, 'lib', 'version.rb'), - {' VERSION =': f" VERSION = '{get_package_versions(args)}'"}) + update_line(join(ruby_dir, 'lib', 'version.rb'), {' VERSION =': f" VERSION = '{get_package_versions(args)}'"}) # TODO - Support Ruby on ARM - copy_okapi_file(abspath(join(dirname(__file__), '..','libs','windows','okapi.dll')),abspath(join(ruby_dir,'libs','windows'))) - copy_okapi_file(abspath(join(dirname(__file__), '..','libs','macos','libokapi.dylib')), abspath(join(ruby_dir,'libs','macos'))) - copy_okapi_file(abspath(join(dirname(__file__), '..','libs','linux','libokapi.so')), abspath(join(ruby_dir,'libs','linux'))) + copy_okapi_file(abspath(join(dirname(__file__), '..', 'libs', 'windows', 'okapi.dll')), + abspath(join(ruby_dir, 'libs', 'windows'))) + copy_okapi_file(abspath(join(dirname(__file__), '..', 'libs', 'macos', 'libokapi.dylib')), + abspath(join(ruby_dir, 'libs', 'macos'))) + copy_okapi_file(abspath(join(dirname(__file__), '..', 'libs', 'linux', 'libokapi.so')), + abspath(join(ruby_dir, 'libs', 'linux'))) def build_golang(args) -> None: @@ -174,11 +181,8 @@ def build_java_docs(args): # https://github.com/fchastanet/groovydoc-to-markdown # npm install in the root of sdk subprocess.Popen( - [ - 'node', './node_modules/groovydoc-to-markdown/src/doc2md.js', - './java', 'java', './docs/reference/java' - ], cwd=get_sdk_dir() - ).wait() + ['node', './node_modules/groovydoc-to-markdown/src/doc2md.js', './java', 'java', './docs/reference/java'], + cwd=get_sdk_dir()).wait() def build_dotnet_docs(args) -> None: @@ -188,37 +192,28 @@ def build_dotnet_docs(args) -> None: output_doc_folder = './docs/reference/dotnet' clean_dir(abspath(join(get_sdk_dir(), output_doc_folder))) subprocess.Popen( - [ - "defaultdocumentation", - "--AssemblyFilePath", assembly_file, - "--OutputDirectoryPath", output_doc_folder, - "--FileNameMode", "Name", - "--GeneratedPages", "Namespaces", - ], - cwd=get_sdk_dir() - ).wait() + ["defaultdocumentation", "--AssemblyFilePath", assembly_file, "--OutputDirectoryPath", output_doc_folder, + "--FileNameMode", "Name", "--GeneratedPages", "Namespaces", ], cwd=get_sdk_dir()).wait() def build_go_docs(args): # https://github.com/posener/goreadme # go get github.com/posener/goreadme/cmd/goreadme - goreadme_args = ['-recursive', '-functions', '-methods', '-types', '-variabless'] # Yes, that's a duplicated s, it's on purpose. + goreadme_args = ['-recursive', '-functions', '-methods', '-types', + '-variabless'] # Yes, that's a duplicated s, it's on purpose. doc_path = abspath(join(get_language_dir('docs'), 'reference', 'go')) def write_doc_file(input_path: str, output_file: str): logging.info(f"goreadme(input={input_path}, output={output_file})") print(f"goreadme(input={input_path}, output={output_file})") with open(join(doc_path, f'{output_file}.md'), 'w') as output: - subprocess.Popen( - ['goreadme', *goreadme_args], - cwd=input_path, stdout=output - ).wait() + subprocess.Popen(['goreadme', *goreadme_args], cwd=input_path, stdout=output).wait() # Handle the subdirectories for sub_folder in glob.glob(join(input_path, '**')): if isdir(sub_folder): _, folder_name = split(sub_folder) write_doc_file(sub_folder, folder_name) - + write_doc_file(get_language_dir('go'), 'index') @@ -229,24 +224,43 @@ def parse_arguments(): return parser.parse_args() +def extract_libs_zip(): + # Look for a libs.zip file to extract + libs_zip = abspath(join(dirname(__file__), '..', 'libs.zip')) + if exists(libs_zip): + print(f'Found released zip binary, extracting...') + with ZipFile(libs_zip) as zip_ref: + zip_ref.extractall(abspath(join(dirname(__file__), '..'))) + + +def continue_on_error(fcn, args) -> None: + try: + fcn(args) + except Exception as e: + print(e) + + def main(): # Get command line arguments args = parse_arguments() langs_to_build = [lang.lower() for lang in (args.language + ',').split(',')] build_all = 'all' in langs_to_build + + extract_libs_zip() + # Update version information if build_all or 'python' in langs_to_build: - build_python(args) + continue_on_error(build_python, args) if build_all or 'java' in langs_to_build: - build_java(args) + continue_on_error(build_java, args) if build_all or 'ruby' in langs_to_build: - build_ruby(args) + continue_on_error(build_ruby, args) if build_all or 'golang' in langs_to_build: - build_golang(args) + continue_on_error(build_golang, args) if build_all or 'docs' in langs_to_build: - build_java_docs(args) - build_go_docs(args) - build_dotnet_docs(args) + continue_on_error(build_java_docs, args) + continue_on_error(build_go_docs, args) + continue_on_error(build_dotnet_docs, args) if __name__ == "__main__": diff --git a/devops/build_winget.py b/devops/build_winget.py new file mode 100644 index 00000000..6016bb2e --- /dev/null +++ b/devops/build_winget.py @@ -0,0 +1,30 @@ +import os +import shutil +import yaml +import requests +import json +# This only runs if you have the winget-pkgs repository downloaded. +okapi_winget_path = r'C:\work\winget-pkgs\manifests\t\trinsic\okapi' +existing_version = sorted(os.listdir(okapi_winget_path))[-1] + +# Go to github releases, get the latest +data = requests.get(r'https://api.github.com/repos/trinsic-id/okapi/releases') +release_json = json.loads(data.content) +release_version = str(release_json[0]['name']).replace('v', '') +# TODO - Get the provided version + +# Create new version dir +release_manifest = os.path.join(okapi_winget_path, release_version) +shutil.rmtree(release_manifest, ignore_errors=True) +shutil.copytree(os.path.join(okapi_winget_path, existing_version), release_manifest, dirs_exist_ok=True) + +# Update the yaml files +installer_yaml_file = os.path.join(release_manifest, 'trinsic.okapi.installer.yaml') +with open(installer_yaml_file, 'r') as f: + installer_doc = yaml.load(f) + +installer_doc['PackageVersion'] = release_version +installer_doc['Installers'][0]['InstallerUrl'] = [asset['browser_download_url'] for asset in release_json[0]['assets'] if str(asset['browser_download_url']).endswith('.msi')][0] +# Write the installer doc back out +with open(installer_yaml_file, 'w') as f: + yaml.dump(installer_doc, f) diff --git a/devops/requirements.txt b/devops/requirements.txt index cc7e2ec1..13b96f48 100644 --- a/devops/requirements.txt +++ b/devops/requirements.txt @@ -1,2 +1,4 @@ betterproto[compiler] -grpcio-tools \ No newline at end of file +grpcio-tools +requests~=2.27.1 +PyYAML~=6.0 \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt index 8640177d..931d9f48 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -7,4 +7,5 @@ setuptools~=60.5.0 black flake8 pytest -pytest-cov \ No newline at end of file +pytest-cov +build \ No newline at end of file diff --git a/python/setup.cfg b/python/setup.cfg index 16e7fa6e..e94d2fea 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = trinsic-okapi -version = 1.2.1 +version = 1.4.0 author = Scott Phillips author_email = scott.phillips@trinsic.id description = Trinsic okapi SDK bindings @@ -20,7 +20,7 @@ packages = find: python_requires = >=3.7 install_requires = protobuf>=3.17.3 - betterproto>=2.0.0b3 + betterproto>=2.0.0b4 requests>=2.22.0 [options.packages.find] @@ -30,6 +30,7 @@ exclude = tests # Recommend matching the black line length (default 88), # rather than using the flake8 default of 79: max-line-length = 88 +doctests = True extend-ignore = # See https://github.com/PyCQA/pycodestyle/issues/373 E203, diff --git a/python/trinsicokapi/wrapper.py b/python/trinsicokapi/wrapper.py index a73efb7d..ed80fa58 100644 --- a/python/trinsicokapi/wrapper.py +++ b/python/trinsicokapi/wrapper.py @@ -1,6 +1,7 @@ import ctypes import os import platform +import sys import threading from ctypes import CDLL from ctypes.util import find_library @@ -61,6 +62,7 @@ def find_native_lib() -> str: os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'libs')), lib_name, ) + or _check_path(os.path.join(sys.prefix, 'libs'), lib_name) or _check_path(os.getenv('LD_LIBRARY_PATH', ''), lib_name) or find_library(lib_name) )