Add M2 branch for release testing

.github/workflows/example-sanity-check.yml

@@ -3,21 +3,9 @@ on:
     - cron: "0 */12 * * *"
   workflow_dispatch:
 jobs:
-  # Wait for up to a minute for previous run to complete, abort if not done by then
-  pre-ci:
-    runs-on: ubuntu-latest
-    timeout-minutes: 1
-    steps:
-      - name: 'Block Concurrent Executions'
-        uses: softprops/turnstyle@v1
-        with:
-          poll-interval-seconds: 10
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
-  linux-arm64:
-    needs: pre-ci
-    runs-on: ubuntu-16.04
+  linux-x86_64:
+    runs-on: ubuntu-18.04
     steps:
       - uses: AutoModality/action-clean@v1
       - name: Cancel Previous Runs
@@ -29,15 +17,15 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      - name: Build on  linux-arm64
+      - name: Build on  linux-x86_64
         shell: bash
         env:
           DEBIAN_FRONTEND: noninteractive
           GITHUB_TOKEN:  ${{ secrets.GITHUB_TOKEN }}
         run: |
           mvn --version
           for folder in `ls`; do
-             if [[ -d "$folder" ]]; then
+             if  [ -d "$folder" ] && [ "$folder" != "mvn-project-template" ]; then
                cd "$folder"
                if test -f "pom.xml"; then
                   mvn clean test

NOTICE.txt

@@ -0,0 +1,20 @@
+Eclipse Deeplearning4j
+Copyright 2021 Eclipse Deeplearning4j Contributors
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software developed at
+* Skymind Inc  (Apache 2.0). Copyright (C) 2015-2018 Skymind Inc .
+
+This product includes software developed at
+* Konduit KK  (Apache 2.0). Copyright (C) 2020.
+
+
+This product includes software from the Tensorflow Project  (Apache 2.0).
+* Copyright (C) 2015-2018 Tensorflow Authors.
+
+# https://github.com/onnx/onnx
+
+This product includes software from the Onnx Project project (Apache 2.0).
+* Copyright (C)  2020 Onnx Contributors (https://github.com/onnx/onnx)

README.md

@@ -24,12 +24,12 @@ We do not monitor the github issues of this repository very often.
 The **Eclipse Deeplearning4J** (DL4J) ecosystem is a set of projects intended to support all the needs of a JVM based deep learning application. This means starting with the raw data, loading and preprocessing it from wherever and whatever format it is in to building and tuning a wide variety of simple and complex deep learning networks.
 
 The DL4J stack comprises of:
-- **DL4J**: High level API to build MultiLayerNetworks and ComputationGraphs with a variety of layers, including custom ones. Supports importing Keras models from h5, including tf.keras models (as of 1.0.0-M1.1) and also supports distributed training on Apache Spark
+- **DL4J**: High level API to build MultiLayerNetworks and ComputationGraphs with a variety of layers, including custom ones. Supports importing Keras models from h5, including tf.keras models (as of 1.0.0-M2) and also supports distributed training on Apache Spark
 - **ND4J**: General purpose linear algebra library with over 500 mathematical, linear algebra and deep learning operations. ND4J is based on the highly-optimized C++ codebase LibND4J that provides CPU (AVX2/512) and GPU (CUDA) support and acceleration by libraries such as OpenBLAS, OneDNN (MKL-DNN), cuDNN, cuBLAS, etc
 - **SameDiff** : Part of the ND4J library, SameDiff is our automatic differentiation / deep learning framework. SameDiff uses a graph-based (define then run) approach, similar to TensorFlow graph mode. Eager graph (TensorFlow 2.x eager/PyTorch) graph execution is planned. SameDiff supports importing TensorFlow frozen model format .pb (protobuf) models. Import for ONNX, TensorFlow SavedModel and Keras models are planned. Deeplearning4j also has full SameDiff support for easily writing custom layers and loss functions.
 - **DataVec**: ETL for machine learning data in a wide variety of formats and files (HDFS, Spark, Images, Video, Audio, CSV, Excel etc)
 - **Arbiter**: Library for hyperparameter search
-- **LibND4J** : C++ library that underpins everything. For more information on how the JVM acceses native arrays and operations refer to [JavaCPP](https://github.com/bytedeco/javacpp)
+- **LibND4J** : C++ library that underpins everything. For more information on how the JVM accesses native arrays and operations refer to [JavaCPP](https://github.com/bytedeco/javacpp)
 
 All projects in the DL4J ecosystem support Windows, Linux and macOS. Hardware support includes CUDA GPUs (10.0, 10.1, 10.2 except OSX), x86 CPU (x86_64, avx2, avx512), ARM CPU (arm, arm64, armhf) and PowerPC (ppc64le).
 

android-examples/app/build.gradle

@@ -48,7 +48,7 @@ android {
 }
 
 dependencies {
-    def dl4jVersion = '1.0.0-M1.1'
+    def dl4jVersion = '1.0.0-M2'
     def openblasVersion = '0.3.10-1.5.5'
     def opencvVersion = '4.5.1-1.5.5'
     def leptonicaVersion = '1.80.0-1.5.5'

data-pipeline-examples/pom.xml

@@ -27,12 +27,12 @@ information regarding copyright ownership.
 
     <groupId>org.deeplearning4j</groupId>
     <artifactId>data-pipeline-examples</artifactId>
-    <version>1.0.0-M1.1</version>
+    <version>1.0.0-M2</version>
     <name>Building a data pipeline prior to modeling</name>
     <description>Loading raw data and processing it before training</description>
 
     <properties>
-        <dl4j-master.version>1.0.0-M1.1</dl4j-master.version>
+        <dl4j-master.version>1.0.0-M2</dl4j-master.version>
         <!-- Change the nd4j.backend property to nd4j-cuda-X-platform to use CUDA GPUs -->
         <!-- <nd4j.backend>nd4j-cuda-10.2-platform</nd4j.backend> -->
         <nd4j.backend>nd4j-native</nd4j.backend>
@@ -42,7 +42,7 @@ information regarding copyright ownership.
         <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
         <maven-shade-plugin.version>2.4.3</maven-shade-plugin.version>
         <logback.version>1.1.7</logback.version>
-        <scala.binary.version>2.11</scala.binary.version>
+        <scala.binary.version>2.12</scala.binary.version>
         <spark.version>2.4.8</spark.version>
         <tablesaw.version>0.38.5</tablesaw.version>
         <!-- Note: tablesaw needs a newer version of guava -->
@@ -62,6 +62,27 @@ information regarding copyright ownership.
                 <updatePolicy>daily</updatePolicy>  <!-- Optional, update daily -->
             </snapshots>
         </repository>
+         <repository>
+            <id>M2</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1019</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+<repository>
+            <id>M2-1</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1032</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+   <repository>
+            <id>M2-2</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1034</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
     </repositories>
 
 
@@ -76,6 +97,13 @@ information regarding copyright ownership.
     </dependencyManagement>
 
     <dependencies>
+        <dependency>
+            <groupId>org.deeplearning4j</groupId>
+            <artifactId>resources</artifactId>
+            <version>${dl4j-master.version}</version>
+        </dependency>
+
+
         <dependency>
             <groupId>org.nd4j</groupId>
             <artifactId>${nd4j.backend}</artifactId>

dl4j-distributed-training-examples/pom.xml

@@ -27,18 +27,18 @@ information regarding copyright ownership.
 
     <groupId>org.deeplearning4j</groupId>
     <artifactId>dl4j-distributed-training-examples</artifactId>
-    <version>1.0.0-M1.1</version>
+    <version>1.0.0-M2</version>
     <name>Introduction to Distributed Training with DL4J</name>
     <description>A set of examples introducing distributed training with the DL4J framework</description>
 
     <properties>
-        <dl4j-master.version>1.0.0-M1.1</dl4j-master.version>
+        <dl4j-master.version>1.0.0-M2</dl4j-master.version>
         <!-- Change the nd4j.backend property to nd4j-cuda-X-platform to use CUDA GPUs -->
         <!-- <nd4j.backend>nd4j-cuda-10.2-platform</nd4j.backend> -->
         <nd4j.backend>nd4j-native</nd4j.backend>
         <java.version>1.8</java.version>
         <shadedClassifier>bin</shadedClassifier>
-        <scala.binary.version>2.11</scala.binary.version>
+        <scala.binary.version>2.12</scala.binary.version>
         <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
         <maven.minimum.version>3.3.1</maven.minimum.version>
         <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
@@ -49,7 +49,7 @@ information regarding copyright ownership.
         <jcommander.version>1.27</jcommander.version>
         <azure.hadoop.version>2.7.4</azure.hadoop.version>
         <azure.storage.version>2.0.0</azure.storage.version>
-        <spark.version>2.1.0</spark.version>
+        <spark.version>2.4.8</spark.version>
         <aws.sdk.version>1.11.109</aws.sdk.version>
         <jackson.version>2.5.1</jackson.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -68,11 +68,38 @@ information regarding copyright ownership.
                 <updatePolicy>daily</updatePolicy>  <!-- Optional, update daily -->
             </snapshots>
         </repository>
+         <repository>
+            <id>M2</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1019</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+<repository>
+            <id>M2-1</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1032</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+   <repository>
+            <id>M2-2</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1034</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
     </repositories>
 
 
 
     <dependencies>
+        <dependency>
+            <groupId>org.deeplearning4j</groupId>
+            <artifactId>resources</artifactId>
+            <version>${dl4j-master.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_${scala.binary.version}</artifactId>

dl4j-distributed-training-examples/scripts/patentExamplePreproc.sh

@@ -36,7 +36,7 @@ MINIBATCH=32                                    #Minibatch size for preprocessed
 
 #Other variables. Don't modify these
 SCRIPTDIR=$(dirname "$0")
-JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M1.1-bin.jar
+JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M2-bin.jar
 AZURE_ACCT=fs.azure.account.key.${AZURE_STORAGE_ACCT}.blob.core.windows.net
 
 

dl4j-distributed-training-examples/scripts/patentExampleTrain.sh

@@ -44,7 +44,7 @@ AERON_BUFFER=33554432
 
 #Other variables. Don't modify these
 SCRIPTDIR=$(dirname "$0")
-JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M1.1-bin.jar
+JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M2-bin.jar
 AZURE_ACCT=fs.azure.account.key.${AZURE_STORAGE_ACCT}.blob.core.windows.net
 
 

dl4j-distributed-training-examples/scripts/tinyImagenetTrain.sh

@@ -36,7 +36,7 @@ EXECUTOR_MEMORY=12G
 MASTER_PORT=7077                                # Port for the spark master. Default is 7077
 MINIBATCH=32                                    # Minibatch size for preprocessed datasets
 SCRIPTDIR=$(dirname "$0")
-JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M1.1-bin.jar
+JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M2-bin.jar
 
 #Memory
 OFFHEAP_MEM_JAVACPP=20G         # Maximum amount of off-heap memory

dl4j-examples/pom.xml

@@ -32,7 +32,7 @@ information regarding copyright ownership.
     <description>A set of examples introducing the DL4J framework</description>
 
     <properties>
-        <dl4j-master.version>1.0.0-M1.1</dl4j-master.version>
+        <dl4j-master.version>1.0.0-M2</dl4j-master.version>
         <!-- Change the nd4j.backend property to nd4j-cuda-X-platform to use CUDA GPUs -->
         <!-- <nd4j.backend>nd4j-cuda-10.2-platform</nd4j.backend> -->
         <nd4j.backend>nd4j-native</nd4j.backend>
@@ -64,6 +64,27 @@ information regarding copyright ownership.
                 <updatePolicy>daily</updatePolicy>  <!-- Optional, update daily -->
             </snapshots>
         </repository>
+         <repository>
+            <id>M2</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1019</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+<repository>
+            <id>M2-1</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1032</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+   <repository>
+            <id>M2-2</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/orgdeeplearning4j-1034</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
     </repositories>
 
     <dependencies>
@@ -99,6 +120,13 @@ information regarding copyright ownership.
             <artifactId>deeplearning4j-core</artifactId>
             <version>${dl4j-master.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.deeplearning4j</groupId>
+            <artifactId>resources</artifactId>
+            <version>${dl4j-master.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.deeplearning4j</groupId>
             <artifactId>deeplearning4j-ui</artifactId>

dl4j-examples/src/main/java/org/deeplearning4j/examples/advanced/features/transferlearning/editlastlayer/presave/FitFromFeaturized.java

@@ -23,8 +23,6 @@
 import org.deeplearning4j.nn.conf.distribution.NormalDistribution;
 import org.deeplearning4j.nn.conf.layers.OutputLayer;
 import org.deeplearning4j.nn.graph.ComputationGraph;
-import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
-import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration;
 import org.deeplearning4j.nn.transferlearning.TransferLearning;
 import org.deeplearning4j.nn.transferlearning.TransferLearningHelper;
@@ -61,7 +59,7 @@ public class FitFromFeaturized {
     protected static final int numClasses = 5;
     protected static final int nEpochs = 3;
 
-    public static void main(String [] args) throws IOException, InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
+    public static void main(String [] args) throws IOException {
 
         //Import vgg
         //Note that the model imported does not have an output layer (check printed summary)

dl4j-examples/src/main/java/org/deeplearning4j/examples/advanced/features/transferlearning/iterators/FlowerDataSetIteratorFeaturized.java

@@ -20,8 +20,6 @@
 package org.deeplearning4j.examples.advanced.features.transferlearning.iterators;
 
 import org.deeplearning4j.examples.advanced.features.transferlearning.editlastlayer.presave.FeaturizedPreSave;
-import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
-import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 import org.nd4j.linalg.dataset.AsyncDataSetIterator;
 import org.nd4j.linalg.dataset.ExistingMiniBatchDataSetIterator;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
@@ -39,7 +37,7 @@ public class FlowerDataSetIteratorFeaturized {
 
     private static String featureExtractorLayer = FeaturizedPreSave.featurizeExtractionLayer;
 
-    public static DataSetIterator trainIterator() throws UnsupportedKerasConfigurationException, IOException, InvalidKerasConfigurationException {
+    public static DataSetIterator trainIterator() throws  IOException {
         runFeaturize();
         DataSetIterator existingTrainingData = new ExistingMiniBatchDataSetIterator(new File("trainFolder"),"flowers-"+featureExtractorLayer+"-train-%d.bin");
         DataSetIterator asyncTrainIter = new AsyncDataSetIterator(existingTrainingData);
@@ -51,7 +49,7 @@ public static DataSetIterator testIterator() {
         return asyncTestIter;
     }
 
-    private static void runFeaturize() throws InvalidKerasConfigurationException, IOException, UnsupportedKerasConfigurationException {
+    private static void runFeaturize() throws IOException {
         File trainDir = new File("trainFolder","flowers-"+featureExtractorLayer+"-train-0.bin");
         if (!trainDir.isFile()) {
             log.info("\n\tFEATURIZED DATA NOT FOUND. \n\t\tRUNNING \"FeaturizedPreSave\" first to do presave of featurized data");