Merge branch 'lite_build' of https://github.com/RedisAI/RedisAI into lite_build

Author: DvirDukhan

.circleci/config.yml

@@ -19,6 +19,17 @@ commands:
               circleci step halt
             fi
 
+  abort_for_noci:
+    steps:
+      - run:
+          name: Ignore CI for specific branches
+          command: |
+            if [[ $CIRCLE_BRANCH == *noci ]]; then
+              echo "Identifies as actively ignoring CI, no testing required."
+              circleci step halt
+            fi
+
+
   early_return_for_forked_pull_requests:
     description: >-
       If this build is from a fork, stop executing the current job and return success.
@@ -40,8 +51,8 @@ commands:
             sudo apt-get -qq update
             sudo apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
             bash <(curl -fsSL https://get.docker.com)
-      - setup_remote_docker:
-          docker_layer_caching: true
+#      - setup_remote_docker:
+#          docker_layer_caching: true
 
   relocate-docker-storage:
     steps:
@@ -84,6 +95,7 @@ commands:
         type: string
     steps:
       - abort_for_docs
+      - abort_for_noci
       - checkout-all
       - restore_cache:
           keys:
@@ -129,18 +141,22 @@ commands:
   platforms-build-steps:
     steps:
       - abort_for_docs
+      - abort_for_noci
+      - early_return_for_forked_pull_requests
       - checkout-all
       - relocate-docker-storage
       - setup-automation
       - run:
           name: Build for platform
           command: |
             pushd opt/build/docker
+            docker login -u redisfab -p $DOCKER_REDISFAB_PWD
             for osnick in bionic xenial; do
-                make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build
-                make CPU=1 LITE=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build
-                make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build
-                make GPU=1 LITE=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build
+                make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build publish
+                make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build publish
+                make CPU=1 LITE=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build publish
+                make GPU=1 LITE=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build publish
+                docker image prune -f
             done
             popd > /dev/null
             logstar=bin/artifacts/tests-logs-cpu.tgz
@@ -149,18 +165,6 @@ commands:
             if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi
             (cd bin/artifacts; tar -cf snapshots.tar snapshots/)
           no_output_timeout: 40m
-      - early_return_for_forked_pull_requests
-      - run:
-          name: Build for platform (publish)
-          command: |
-            docker login -u redisfab -p $DOCKER_REDISFAB_PWD
-            cd opt/build/docker
-            for osnick in bionic xenial; do
-                make CPU=1 OSNICK=$osnick VERBOSE=1 publish
-                make CPU=1 LITE=1 OSNICK=$osnick VERBOSE=1 publish
-                make GPU=1 OSNICK=$osnick VERBOSE=1 publish
-                make GPU=1 LITE=1 OSNICK=$osnick VERBOSE=1 publish
-            done
       - persist_to_workspace:
           root: bin/
           paths:
@@ -176,6 +180,7 @@ commands:
         type: string
     steps:
       - abort_for_docs
+      - abort_for_noci
       - early_return_for_forked_pull_requests
       - run:
           name: Deploy to S3
@@ -188,6 +193,8 @@ jobs:
     docker:
       - image: redislabsmodules/llvm-toolset:latest
     steps:
+      - abort_for_docs
+      - abort_for_noci
       - checkout-all
       - setup-build-system
       - run:
@@ -202,12 +209,14 @@ jobs:
       - build-steps:
           platform: debian
 
+  # this build runs on a fixed machine, due to a storage need
+  # nothing about it necessitates the machine itself, other than the need for more disk.
   platforms-build:
     machine:
       enabled: true
       docker_layer_caching: true
-      resource_class: gpu.nvidia.small
-      image: ubuntu-1604-cuda-11.1:202012-01
+      resource_class: medium
+      image: ubuntu-2004:202101-01
     steps:
       - platforms-build-steps
 
@@ -216,6 +225,7 @@ jobs:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
       - abort_for_docs
+      - abort_for_noci
       - checkout-all
       - restore_cache:
           keys:
@@ -247,6 +257,7 @@ jobs:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
       - abort_for_docs
+      - abort_for_noci
       - checkout-all
       - restore_cache:
           keys:
@@ -276,8 +287,9 @@ jobs:
     docker:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
-      - early_return_for_forked_pull_requests
       - abort_for_docs
+      - abort_for_noci
+      - early_return_for_forked_pull_requests
       - checkout-all
       - restore_cache:
           keys:
@@ -347,6 +359,7 @@ jobs:
 
     steps:
       - abort_for_docs
+      - abort_for_noci
       - checkout-all
       - run:
           name: Relocate docker overlay2 dir
@@ -378,6 +391,7 @@ jobs:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
       - abort_for_docs
+      - abort_for_noci
       - early_return_for_forked_pull_requests
       - attach_workspace:
           at: workspace
@@ -395,6 +409,7 @@ jobs:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
       - abort_for_docs
+      - abort_for_noci
       - early_return_for_forked_pull_requests
       - attach_workspace:
           at: workspace
@@ -418,6 +433,7 @@ jobs:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
       - abort_for_docs
+      - abort_for_noci
       - early_return_for_forked_pull_requests
       - attach_workspace:
           at: workspace

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve RedisAI
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. ...
+2. ....
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (please complete the following information):**
+ - OS: [e.g. ubuntu 20.04]
+ - Version [e.g. 1.2.2]
+ - Platfrom [e.g. x86, Jetson, ARM]
+ - Runtime [e.g. CPU, CUDA]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for pushing RedisAI further
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/trigger-build-and-test-gpu.yaml

@@ -5,6 +5,8 @@ on:
   pull_request:
     types:
       - labeled
+      - synchronize
+      - assigned
 
 jobs:
   valgrind_general:

.github/workflows/trigger-valgrind.yaml

@@ -5,6 +5,8 @@ on:
   pull_request:
     types:
       - labeled
+      - synchronize
+      - assigned
 
 jobs:
   valgrind_general:

Dockerfile

@@ -29,7 +29,7 @@ WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
-COPY ./tests/flow/test_requirements.txt tests/flow/
+ADD ./tests/flow/ tests/flow/
 
 RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
@@ -67,7 +67,7 @@ ARG REDIS_VER
 ARG PACK
 
 RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
-RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi 
+RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
 
 ENV REDIS_MODULES /usr/lib/redis/modules
 ENV LD_LIBRARY_PATH $REDIS_MODULES

Dockerfile.arm

@@ -31,7 +31,7 @@ WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
-COPY ./tests/flow/test_requirements.txt tests/flow
+COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow
 
 RUN ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py

Dockerfile.gpu

@@ -18,7 +18,6 @@ ARG TEST=0
 
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
-FROM nvidia/cuda:10.2-cudnn8-devel-${OS} AS cuda_10.2
 FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
 
 ARG OSNICK
@@ -35,13 +34,10 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
-COPY --from=cuda_10.2 /usr/local/cuda-10.2 /usr/local/cuda-10.2
-COPY --from=cuda_10.2 /usr/lib/x86_64-linux-gnu/libcu* /usr/lib/x86_64-linux-gnu/
-
-RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64:/usr/local/cuda-10.2/lib64:$LD_LIBRARY_PATH > /etc/profile.d/cuda.sh
+RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64:$LD_LIBRARY_PATH > /etc/profile.d/cuda.sh
 
 COPY ./opt/ opt/
-COPY ./tests/flow/test_requirements.txt tests/flow/
+ADD ./tests/flow/ tests/flow/
 
 RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
@@ -75,13 +71,12 @@ FROM nvidia/cuda:${CUDA_VER}-runtime-${OS}
 ARG OS
 
 RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
-RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi 
+RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
 
 ENV REDIS_MODULES /usr/lib/redis/modules
 RUN mkdir -p $REDIS_MODULES/
 
 COPY --from=redis /usr/local/ /usr/local/
-COPY --from=builder /usr/local/cuda-10.2 /usr/local/cuda-10.2
 COPY --from=builder /usr/lib/x86_64-linux-gnu/libcu* /usr/lib/x86_64-linux-gnu/
 COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
 

Dockerfile.gpu-test

@@ -17,7 +17,6 @@ ARG PACK=1
 
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
-FROM nvidia/cuda:10.2-cudnn8-devel-${OS} AS cuda_10.2
 FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
 
 SHELL ["/bin/bash", "-c"]
@@ -28,14 +27,11 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
-COPY --from=cuda_10.2 /usr/local/cuda-10.2 /usr/local/cuda-10.2
-COPY --from=cuda_10.2 /usr/lib/x86_64-linux-gnu/libcu* /usr/lib/x86_64-linux-gnu/
-
-RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64:/usr/local/cuda-10.2/lib64:$LD_LIBRARY_PATH > /etc/profile.d/cuda.sh
+RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64:$LD_LIBRARY_PATH > /etc/profile.d/cuda.sh
 
 COPY ./opt/ opt/
-COPY ./tests/flow/test_requirements.txt tests/flow/
-COPY ./tests/flow/Install_RedisGears.sh tests/flow/
+COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow/tests_setup/
+COPY ./tests/flow/tests_setup/Install_RedisGears.sh tests/flow/tests_setup/
 
 RUN VENV=venv FORCE=1 ./opt/readies/bin/getpy3
 

Dockerfile.jetson

@@ -38,7 +38,7 @@ WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
-COPY ./tests/flow/test_requirements.txt tests/flow/
+COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow/
 
 RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py