[android][circle_ci] android build job (#3679)

IvanKobzarev · fmassa · web-flow · commit d6fee5a4c451 · 2021-05-10T12:00:38.000+02:00
Co-authored-by: Francisco Massa &lt;fvsmassa@gmail.com&gt;
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -160,6 +160,14 @@ torchvision_ios_params: &torchvision_ios_params
     IOS_ARCH: << parameters.ios_arch >>
     IOS_PLATFORM: << parameters.ios_platform >>
 
+torchvision_android_params: &torchvision_android_params
+  parameters:
+    build_environment:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: << parameters.build_environment >>
+
 smoke_test_common: &smoke_test_common
   <<: *binary_common
   docker:
@@ -394,6 +402,42 @@ jobs:
           cat "$script"
           source "$script"
 
+  binary_android_build:
+    <<: *torchvision_android_params
+    docker:
+      - image: circleci/android:api-29-ndk
+    resource_class: xlarge
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh"
+          cat "$script"
+          source "$script"
+    - store_artifacts:
+        path: ~/workspace/artifacts
+
+  binary_android_upload:
+    <<: *torchvision_android_params
+    docker:
+      - image: circleci/android:api-29-ndk
+    resource_class: xlarge
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run:
+        name: Upload
+        no_output_timeout: "1h"
+        command: |
+          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh"
+          cat "$script"
+          source "$script"
+
   binary_macos_conda:
     <<: *binary_common
     macos:
@@ -1506,6 +1550,9 @@ workflows:
           ios_arch: arm64
           ios_platform: OS
           name: binary_libtorchvision_ops_ios_12.0.0_arm64
+      - binary_android_build:
+          build_environment: binary-libtorchvision_ops-android
+          name: binary_libtorchvision_ops_android
 
   unittest:
     jobs:
@@ -1681,6 +1728,14 @@ workflows:
           requires:
           - nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64
           - nightly_binary_libtorchvision_ops_ios_12.0.0_arm64
+      - binary_android_upload:
+          build_environment: nightly-binary-libtorchvision_ops-android-upload
+          context: org-member
+          filters:
+            branches:
+              only:
+              - nightly
+          name: nightly_binary_libtorchvision_ops_android_upload
       - binary_linux_wheel:
           conda_docker_image: pytorch/conda-builder:cpu
           cu_version: cpu
diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -160,6 +160,14 @@ torchvision_ios_params: &torchvision_ios_params
     IOS_ARCH: << parameters.ios_arch >>
     IOS_PLATFORM: << parameters.ios_platform >>
 
+torchvision_android_params: &torchvision_android_params
+  parameters:
+    build_environment:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: << parameters.build_environment >>
+
 smoke_test_common: &smoke_test_common
   <<: *binary_common
   docker:
@@ -394,6 +402,42 @@ jobs:
           cat "$script"
           source "$script"
 
+  binary_android_build:
+    <<: *torchvision_android_params
+    docker:
+      - image: circleci/android:api-29-ndk
+    resource_class: xlarge
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh"
+          cat "$script"
+          source "$script"
+    - store_artifacts:
+        path: ~/workspace/artifacts
+
+  binary_android_upload:
+    <<: *torchvision_android_params
+    docker:
+      - image: circleci/android:api-29-ndk
+    resource_class: xlarge
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run:
+        name: Upload
+        no_output_timeout: "1h"
+        command: |
+          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh"
+          cat "$script"
+          source "$script"
+
   binary_macos_conda:
     <<: *binary_common
     macos:
@@ -934,6 +978,7 @@ workflows:
       - torchhub_test
       - torch_onnx_test
       {{ ios_workflows() }}
+      {{ android_workflows() }}
 
   unittest:
     jobs:
@@ -954,6 +999,7 @@ workflows:
       - torchhub_test
       - torch_onnx_test
       {{ ios_workflows(nightly=True) }}
+      {{ android_workflows(nightly=True) }}
       {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
   docker_build:
     triggers:
diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py
@@ -291,6 +291,32 @@ def ios_workflows(indentation=6, nightly=False):
     return indent(indentation, jobs)
 
 
+def android_workflows(indentation=6, nightly=False):
+    jobs = []
+    build_job_names = []
+    name_prefix = "nightly_" if nightly else ""
+    env_prefix = "nightly-" if nightly else ""
+
+    name = f'{name_prefix}binary_libtorchvision_ops_android'
+    build_job_names.append(name)
+    build_job = {
+        'build_environment': f'{env_prefix}binary-libtorchvision_ops-android',
+        'name': name,
+    }
+
+    if nightly:
+        upload_job = {
+            'build_environment': f'{env_prefix}binary-libtorchvision_ops-android-upload',
+            'context': 'org-member',
+            'filters': gen_filter_branch_tree('nightly'),
+            'name': f'{name_prefix}binary_libtorchvision_ops_android_upload'
+        }
+        jobs.append({'binary_android_upload': upload_job})
+    else:
+        jobs.append({'binary_android_build': build_job})
+    return indent(indentation, jobs)
+
+
 if __name__ == "__main__":
     d = os.path.dirname(__file__)
     env = jinja2.Environment(
@@ -306,4 +332,5 @@ def ios_workflows(indentation=6, nightly=False):
             unittest_workflows=unittest_workflows,
             cmake_workflows=cmake_workflows,
             ios_workflows=ios_workflows,
+            android_workflows=android_workflows,
         ))
diff --git a/.circleci/unittest/android/scripts/binary_android_build.sh b/.circleci/unittest/android/scripts/binary_android_build.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+set -ex -o pipefail
+
+echo "DIR: $(pwd)"
+echo "ANDROID_HOME=${ANDROID_HOME}"
+echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}"
+echo "JAVA_HOME=${JAVA_HOME}"
+
+WORKSPACE=/home/circleci/workspace
+VISION_ANDROID=/home/circleci/project/android
+
+. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh
+
+GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties
+rm -f $GRADLE_LOCAL_PROPERTIES
+
+echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES
+echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES
+
+echo "GRADLE_PATH $GRADLE_PATH"
+echo "GRADLE_HOME $GRADLE_HOME"
+
+${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} assemble || true
+
+mkdir -p ~/workspace/artifacts
+find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz
+find . -type f -name *apk -print | xargs tar cfvz ~/workspace/artifacts/artifacts-apks.tgz
diff --git a/.circleci/unittest/android/scripts/binary_android_upload.sh b/.circleci/unittest/android/scripts/binary_android_upload.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -ex -o pipefail
+
+echo "DIR: $(pwd)"
+echo "ANDROID_HOME=${ANDROID_HOME}"
+echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}"
+echo "JAVA_HOME=${JAVA_HOME}"
+
+WORKSPACE=/home/circleci/workspace
+VISION_ANDROID=/home/circleci/project/android
+
+. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh
+
+GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties
+rm -f $GRADLE_LOCAL_PROPERTIES
+GRADLE_PROPERTIES=/home/circleci/project/android/gradle.properties
+
+echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES
+echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES
+
+echo "SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES
+echo "mavenCentralRepositoryUsername=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES
+echo "SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES
+echo "mavenCentralRepositoryPassword=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES
+
+echo "signing.keyId=${ANDROID_SIGN_KEY}" >> $GRADLE_PROPERTIES
+echo "signing.password=${ANDROID_SIGN_PASS}" >> $GRADLE_PROPERTIES
+
+cat /home/circleci/project/android/gradle.properties | grep VERSION
+
+${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} ops:uploadArchives
+
+mkdir -p ~/workspace/artifacts
+find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz
diff --git a/.circleci/unittest/android/scripts/install_gradle.sh b/.circleci/unittest/android/scripts/install_gradle.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -ex
+
+_https_amazon_aws=https://ossci-android.s3.amazonaws.com
+GRADLE_VERSION=6.8.3
+
+_gradle_home=/opt/gradle
+sudo rm -rf $gradle_home
+sudo mkdir -p $_gradle_home
+
+curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip
+
+sudo unzip -q /tmp/gradle.zip -d $_gradle_home
+rm /tmp/gradle.zip
+
+sudo chmod -R 777 $_gradle_home
+
+export GRADLE_HOME=$_gradle_home/gradle-$GRADLE_VERSION
+export GRADLE_PATH=${GRADLE_HOME}/bin/gradle
diff --git a/android/build.gradle b/android/build.gradle
@@ -15,6 +15,7 @@ allprojects {
             androidSupportAppCompatV7Version = "28.0.0"
             fbjniJavaOnlyVersion = "0.0.3"
             soLoaderNativeLoaderVersion = "0.8.0"
+            pytorchAndroidVersion = "1.9.0-SNAPSHOT"
         }
 
         repositories {
diff --git a/android/gradle.properties b/android/gradle.properties
@@ -21,4 +21,4 @@ android.enableJetifier=true
 
 testAppAllVariantsEnabled=false
 
-org.gradle.jvmargs=-Xmx4096m
+org.gradle.jvmargs=-Xmx12g
diff --git a/android/ops/build.gradle b/android/ops/build.gradle
@@ -55,8 +55,7 @@ android {
 dependencies {
     implementation 'com.android.support:appcompat-v7:' + rootProject.androidSupportAppCompatV7Version
 
-    implementation 'org.pytorch:pytorch_android:1.8.0-SNAPSHOT'
-    extractForNativeBuild 'org.pytorch:pytorch_android:1.8.0-SNAPSHOT'
+    extractForNativeBuild "org.pytorch:pytorch_android:$pytorchAndroidVersion"
 
     // For testing: deps on local aar files
     //implementation(name: 'pytorch_android-release', ext: 'aar')
diff --git a/android/test_app/app/build.gradle b/android/test_app/app/build.gradle
@@ -75,6 +75,7 @@ android {
     }
     packagingOptions {
         doNotStrip '**.so'
+        pickFirst '**.so'
     }
 
     // Filtering for CI
@@ -101,8 +102,8 @@ dependencies {
     implementation 'com.facebook.soloader:nativeloader:0.8.0'
     localImplementation project(':ops')
 
-    implementation 'org.pytorch:pytorch_android:1.8.0-SNAPSHOT'
-    implementation 'org.pytorch:pytorch_android_torchvision:1.8.0-SNAPSHOT'
+    implementation "org.pytorch:pytorch_android:$pytorchAndroidVersion"
+    implementation "org.pytorch:pytorch_android_torchvision:$pytorchAndroidVersion"
 
     aarImplementation(name: 'pytorch_android-release', ext: 'aar')
     aarImplementation(name: 'pytorch_android_torchvision-release', ext: 'aar')

Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@ allprojects {`
`15`	`15`	`androidSupportAppCompatV7Version = "28.0.0"`
`16`	`16`	`fbjniJavaOnlyVersion = "0.0.3"`
`17`	`17`	`soLoaderNativeLoaderVersion = "0.8.0"`
	`18`	`+ pytorchAndroidVersion = "1.9.0-SNAPSHOT"`
`18`	`19`	`}`
`19`	`20`
`20`	`21`	`repositories {`
Original file line number	Diff line number	Diff line change
`@@ -21,4 +21,4 @@ android.enableJetifier=true`
`21`	`21`
`22`	`22`	`testAppAllVariantsEnabled=false`
`23`	`23`
`24`		`-org.gradle.jvmargs=-Xmx4096m`
	`24`	`+org.gradle.jvmargs=-Xmx12g`