Makefile: added pack and test targets

.circleci/config.yml

@@ -1,8 +1,8 @@
-version: 2
+version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/python:3.6.8
+      - image: circleci/python:3.7.4-buster
 
     steps:
       - checkout
@@ -17,16 +17,9 @@ jobs:
       - run:
           name: install dependencies
           command: |
-            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
-            sudo apt-get install -y git-lfs patchelf
-            git lfs install
-            curl -o cmake-3.12.4-Linux-x86_64.sh https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh
-            mkdir /home/circleci/project/cmake-bin
-            sh cmake-3.12.4-Linux-x86_64.sh --prefix=/home/circleci/project/cmake-bin --skip-license
-            sudo ln -s /home/circleci/project/cmake-bin/bin/cmake /usr/local/bin/cmake
-            echo `cmake --version`
-            rm -rf /home/circleci/project/deps
-            bash get_deps.sh cpu
+            sudo ./deps/readies/bin/getpy
+            sudo ./system-setup.py
+            make deps
             git clone git://github.com/antirez/redis.git --branch 5.0.3
             (cd redis && make malloc=libc -j4 && sudo make install)
 
@@ -38,32 +31,19 @@ jobs:
       - run:
           name: Build
           command: |
-            rm -rf build
-            mkdir build
-            cd build
-            cmake -DDEPS_PATH=../deps/install ..
-            make && make install
-            cd ..
-            make rlec_runpath_fix
+            make all
+            make pack
 
       - run:
           name: Test
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip -q uninstall -y RLTest || true
-            pip -q install git+https://github.com/RedisLabsModules/RLTest@py3
-            pip -q install -r test/test_requirements.txt
-            git lfs pull
-            python3 -m RLTest --test test/basic_tests.py --module install/redisai.so
+          command: make test
 
       - run:
           name: Persist Artifacts
           command: |
             mkdir -p ~/workspace/build
-            cp install/ramp/$MODULE_ARTIFACT ~/workspace/
-            # cp deps/install/lib/*.so*  ~/workspace/
-            cp -r install/backends ~/workspace/
+            cp install-cpu/$MODULE_ARTIFACT ~/workspace/
+            cp -r install-cpu/backends ~/workspace/
             cp ramp.yml ~/workspace/
       - persist_to_workspace:
           root: ~/workspace

.dockerignore

@@ -1,4 +1,9 @@
-deps/*
-install/*
+/deps/*
+!/deps/readies/
+!/deps/*.py
+/build/
+/install/
 **/*.o
-**/*.so
+**/*.so
+/.venv/
+/venv/

.gitignore

@@ -7,11 +7,18 @@ __pycache__
 *.pyc
 
 # Dirs
-install/
-deps/
-build/
+/install/
+/deps/*
+!/deps/readies/
+!/deps/*.py
+/build/
 examples/js/node_modules/
 examples/js/package-lock.json
+/test/venv/
+/test/logs/
+/1/
+/venv/
+/.venv/
 
 # Eclipse (if that's your jam...)
 .project

CMakeLists.txt

@@ -18,6 +18,8 @@ GET_FILENAME_COMPONENT(depsAbs
     "${DEPS_PATH}" REALPATH
     BASE_DIR "${CMAKE_BINARY_DIR}")
 
+# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-9.0)
+
 INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include)
 INCLUDE_DIRECTORIES(${depsAbs}/libtensorflow/include)
 INCLUDE_DIRECTORIES(${depsAbs}/libtorch/include)
@@ -68,7 +70,7 @@ IF (APPLE)
         LINK_FLAGS "-undefined dynamic_lookup")
 ENDIF()
 
-SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install)
+SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${ENGINE})
 
 INSTALL(TARGETS redisai LIBRARY DESTINATION .)
 

Dockerfile

@@ -1,46 +1,42 @@
-FROM redis AS builder
+ARG OS=debian:buster
 
-ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf"
+#----------------------------------------------------------------------------------------------
+FROM redis AS redis
+FROM ${OS} AS builder
 
-# install latest cmake
-ADD https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh /cmake-3.12.4-Linux-x86_64.sh
-RUN mkdir /opt/cmake
-RUN sh /cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
-RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
-RUN cmake --version
+ARG PACK=0
+ARG TEST=0
 
-# Set up a build environment
-RUN set -ex;\
-    deps="$DEPS";\
-    apt-get update;\
-    apt-get install -y --no-install-recommends $deps
-
-# Get the dependencies
 WORKDIR /redisai
+COPY --from=redis /usr/local/ /usr/local/
+
+COPY ./deps/readies/ deps/readies/
+COPY ./system-setup.py .
+COPY ./test/test_requirements.txt test/
+
+RUN ./deps/readies/bin/getpy
+RUN ./system-setup.py
+
+COPY ./get_deps.sh .
+RUN ./get_deps.sh cpu
+
 ADD ./ /redisai
-RUN set -ex;\
-    mkdir -p deps;\
-    DEPS_DIRECTORY=deps bash ./get_deps.sh cpu
-
-# Build the source
-RUN set -ex;\
-    rm -rf build;\
-    mkdir -p build;\
-    cd build;\
-    cmake -DDEPS_PATH=../deps/install ..;\
-    make && make install;\
-    cd ..
-
-# Package the runner
+RUN make all
+RUN if [ "$PACK" = "1" ]; then make pack; fi
+RUN if [ "$TEST" = "1" ]; then make test; fi
+
+#----------------------------------------------------------------------------------------------
 FROM redis
 
 RUN set -e; apt-get -qq update; apt-get install -y libgomp1
 
-RUN set -ex;\
-    mkdir -p /usr/lib/redis/modules/;
+RUN mkdir -p /usr/lib/redis/modules/
 
-COPY --from=builder /redisai/install/ /usr/lib/redis/modules/
+COPY --from=builder /redisai/install-cpu/ /usr/lib/redis/modules/
 
 WORKDIR /data
 EXPOSE 6379
-CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"]
+CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so", \
+        "TF", "/usr/lib/redis/modules//backends/redisai_tensorflow/redisai_tensorflow.so", \
+        "TORCH", "/usr/lib/redis/modules//backends/redisai_torch/redisai_torch.so", \
+        "ONNX", "/usr/lib/redis/modules//backends/redisai_onnxruntime/redisai_onnxruntime.so"]

Makefile

@@ -1,8 +1,14 @@
+override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1))
 
-ifeq ($(CUDA),1)
-DEPS_FLAGS=
+ifeq ($(GPU),1)
+ifeq ($(CPU),1)
+$(error CPU=1 and GPU=1 (or CUDA=1) are conflicting)
+endif
+DEPS_FLAGS=gpu
+ENGINE=gpu
 else
 DEPS_FLAGS=cpu
+ENGINE=cpu
 endif
 
 export REDIS_ENT_LIB_PATH=/opt/redislabs/lib
@@ -16,52 +22,70 @@ else
 PACK_VER:=$(VERSION)
 endif
 
-BINDIR=$(PWD)/install
+BINDIR=$(PWD)/install-$(ENGINE)
+
+BACKENDS_PATH ?= $(BINDIR)/backends
+
+#----------------------------------------------------------------------------------------------
+
+setup:
+	@echo Setting up system...
+	$(SHOW)./deps/readies/bin/getpy
+	$(SHOW)./system-setup.py
+
+.PHONY: all setup build clean deps pack pack_ramp pack_deps test
+
+all: build
 
-.PHONY: all clean deps pack rlec_runpath_fix
+ifeq ($(DEBUG),1)
+CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
+endif
 
-all:
-ifeq ($(wildcard build/.),)
+build:
+ifeq ($(wildcard build/Makefile),)
 	mkdir -p build
 	cd build; \
-	cmake -DDEPS_PATH=../deps/install ..
+	cmake -DENGINE=$(ENGINE) -DDEPS_PATH=../deps/install-$(ENGINE) $(CMAKE_FLAGS) ..
 endif
-	$(MAKE) -C build && $(MAKE) -C build install
+	$(MAKE) -C build
+	$(MAKE) -C build install
 
 clean:
 ifeq ($(ALL),1)
-	rm -rf build deps
+	rm -rf build install deps/dlpack deps/install-$(ENGINE) deps/*.tar.gz deps/*.zip deps/*.tgz
 else
 	$(MAKE) -C build clean
 endif
 
-deps:
+deps fetch:
 	@echo Fetching dependencies...
 	@./get_deps.sh $(DEPS_FLAGS)
 
-# in pack: create ramp/redisai.so with RUNPATH set to /opt/redislabs/lib for RLEC compliance
-rlec_runpath_fix: 
-	@echo Fixing RLEC RUNPATH...
-	@mkdir -p $(BINDIR)/ramp
-	@cp -f $(BINDIR)/redisai.so $(BINDIR)/ramp/
-	# @patchelf --set-rpath $(REDIS_ENT_LIB_PATH) $(BINDIR)/ramp/redisai.so
+pack: pack_ramp pack_deps
 
-pack: rlec_runpath_fix
+pack_ramp:
 	@[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; }
-	@[ ! -e $(REDIS_ENT_LIB_PATH) ] || { echo "$(REDIS_ENT_LIB_PATH) exists - aborting."; exit 1; }
-ifeq ($(wildcard build/pyenv/.),)
-	@virtualenv build/pyenv ;\
-	. ./build/pyenv/bin/activate ;\
-	pip install git+https://github.com/RedisLabs/RAMP
-endif
 	@echo "Building RAMP file ..."
 	@set -e ;\
-	. ./build/pyenv/bin/activate ;\
-	ln -fs $(PWD)/deps/install/lib/ $(REDIS_ENT_LIB_PATH) ;\
-	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.${PACK_VER}.zip" $(BINDIR)/ramp/redisai.so 2>&1 > /dev/null ;\
-	rm $(REDIS_ENT_LIB_PATH)
-	@echo Done.
-	@echo "Building dependencies file redisai-dependencies.${PACK_VER}.tgz ..."
-	@cd deps/install/lib; \
-	tar pczf ../../../build/redisai-dependencies.${PACK_VER}.tgz *.so*
-	@echo Done.
+	RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\
+	LD_LIBRARY_PATH=$(PWD)/deps/install-$(ENGINE)/lib \
+	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(BINDIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
+	tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\
+	rm -f $RAMPOUT ;\
+	echo "Done."
+
+pack_deps: pack_ramp
+	@echo "Building dependencies file ..."
+	@set -e ;\
+	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
+	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
+	cd install-$(ENGINE) ;\
+	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER-$(ENGINE).tgz ;\
+	echo "Done."
+
+test:
+	@git lfs install
+	@git lfs pull
+	@set -e ;\
+	cd test ;\
+	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(BINDIR)/redisai.so

deps/readies/.gitignore

@@ -0,0 +1,36 @@
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+*.lib
+*.a
+*.la
+*.lo
+*.dll
+*.so
+*.so.*
+*.dylib
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# C/C++
+*.gch
+*.pch
+*.d
+
+# Python
+*.pyc
+
+# CMake
+CMakeFiles/
+*.cmake
+CMakeCache.txt
+
+# Eclipse
+.cproject
+.project

deps/readies/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Redis Labs Modules
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

deps/readies/README.md

@@ -0,0 +1,9 @@
+# Library cluster of common Redis Modules code
+
+Cetara: C/C++
+
+Paella: Python
+
+Shibumi: Bash
+
+mk: GNU Make