diff --git a/.circleci/config.yml b/.circleci/config.yml index ba0497c2c..edb39d88e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,16 +15,15 @@ commands: - run: name: Install dependencies command: | - ./opt/readies/bin/getpy + ./opt/readies/bin/getpy3 BREW_NO_UPDATE=1 ./opt/system-setup.py - git clone git://github.com/antirez/redis.git --branch 5.0.7 - (cd redis && make malloc=libc -j $(nproc) && make install) - redis-server --version + git clone git://github.com/antirez/redis.git --branch 5.0.7; (cd redis; make malloc=libc -j $(nproc); make install); redis-server --version ./get_deps.sh cpu - save_cache: paths: - deps - key: v1-dependencies-{{ checksum "get_deps.sh" }} + - /usr/local + key: build-dependencies-{{ checksum "get_deps.sh" }} - run: name: Set up workspace command: | @@ -38,7 +37,6 @@ commands: command: | mkdir -p ~/workspace/tests make -C opt test SHOW=1 VERBOSE=1 - cp test/logs/* ~/workspace/tests - run: name: Package command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 @@ -49,8 +47,9 @@ commands: - 'packages/release/*.tgz' - 'packages/branch/*.zip' - 'packages/branch/*.tgz' - - store_test_results: - path: ~/workspace/tests + - store_artifacts: + path: test/logs + deploy: parameters: from: @@ -68,9 +67,37 @@ jobs: steps: - ci_steps: platform: debian + + coverage: + docker: + - image: redisfab/rmbuilder:x64-buster + steps: + - checkout + - run: + name: Install dependencies + command: | + ./opt/readies/bin/getpy3 + ./opt/system-setup.py + # git clone git://github.com/antirez/redis.git --branch 5.0.7; cd redis; make malloc=libc -j $(nproc); make install; redis-server --version + - restore_cache: + keys: + - build-dependencies-{{ checksum "get_deps.sh" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + - run: + name: Build + command: | + # make fetch SHOW=1 + make -C opt build COV=1 SHOW=1 + - run: + name: Test with coverage + command: | + make -C opt test SHOW=1 COV=1 + make -C opt cov-upload + build-macos: macos: - xcode: 10.2.1 + xcode: 11.1.0 steps: - run: name: Fix macOS Python installation @@ -78,6 +105,7 @@ jobs: brew reinstall -f python2 - ci_steps: platform: macosx + build-multiarch-docker: machine: enabled: true @@ -102,6 +130,7 @@ jobs: sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD make -C opt/build/docker build sudo make -C opt/build/docker publish + build-gpu: machine: enabled: true @@ -110,13 +139,6 @@ jobs: image: ubuntu-1604-cuda-10.1:201909-23 steps: - checkout -# - run: -# name: Checkout LFS -# command: | -# curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash -# sudo apt-get install -y git-lfs -# git lfs install -# git lfs pull - run: name: Build command: | @@ -127,8 +149,9 @@ jobs: mkdir -p ~/workspace/tests docker run --gpus all -v $HOME/workspace/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test no_output_timeout: 30m - - store_test_results: + - store_artifacts: path: ~/workspace/tests + deploy_package: parameters: package: @@ -164,6 +187,14 @@ workflows: filters: tags: only: /.*/ + - coverage: + requires: + - build + filters: + branches: + only: /.*/ + tags: + only: /.*/ - build-macos: filters: branches: diff --git a/.dockerignore b/.dockerignore index 3805aedef..9e415bc14 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,4 @@ venv*/ *.zip *.tgz *.tar.gz +/VARAINT diff --git a/.gitignore b/.gitignore index be9f4c530..dce0540c4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ venv*/ *.zip *.tgz *.tar.gz +/VARIANT # Misc .DS_Store diff --git a/CMakeLists.txt b/CMakeLists.txt index 69f8e7f7e..d838245f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,16 +16,29 @@ IF(NOT CMAKE_BUILD_TYPE) "Debug" "Release" "MinSizeRel" "RelWithDebInfo") ENDIF() -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") +#---------------------------------------------------------------------------------------------- + +SET(CMAKE_CC_COMMON_FLAGS "-fPIC") + +IF (USE_COVERAGE) + IF (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + MESSAGE(FATAL_ERROR "Build type must be DEBUG for coverage") + ENDIF() + SET(CMAKE_CC_COMMON_FLAGS "${CMAKE_CC_COMMON_FLAGS} -coverage") +ENDIF() + +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CC_COMMON_FLAGS}") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CC_COMMON_FLAGS}") # For adding specific Release flags set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") # Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces -set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer") -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer") +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND") + +#---------------------------------------------------------------------------------------------- option(BUILD_TF "Build the TensorFlow backend" ON) option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON) diff --git a/Dockerfile b/Dockerfile index cfa539476..47893ebf8 100755 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.arm b/Dockerfile.arm index 4c4865066..2cf2f91e1 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -19,7 +19,7 @@ WORKDIR /build COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 3e7f94c9c..216b45431 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -22,7 +22,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 6dbc1ffa7..ecc5e93eb 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -12,6 +12,8 @@ ARG ARCH=x64 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 AS builder +SHELL ["/bin/bash", "-c"] + ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility @@ -21,21 +23,37 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py +RUN set -e ;\ + python3 -m virtualenv venv --system-site-packages;\ + . venv/bin/activate ;\ + ./opt/system-setup.py ARG DEPS_ARGS="" COPY ./get_deps.sh . -RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi +RUN set -e ;\ + . venv/bin/activate ;\ + if [[ -z $DEPS_ARGS ]]; then \ + ./get_deps.sh gpu ;\ + else \ + env $DEPS_ARGS ./get_deps.sh gpu ;\ + fi ARG BUILD_ARGS="" ADD ./ /build -RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1 +RUN set -e ;\ + . venv/bin/activate ;\ + make -C opt build GPU=1 $BUILD_ARGS SHOW=1 ARG PACK=1 -RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi +RUN set -e ;\ + if [[ $PACK == 1 ]]; then \ + . venv/bin/activate ;\ + make -C opt pack GPU=1 VERBOSE=1 ;\ + fi RUN git remote set-url origin https://github.com/RedisAI/RedisAI -CMD ["bash", "-c", "make -C opt test GPU=1 SHOW=1"] +CMD ["bash", "-c", ". ./venv/bin/activate; make -C opt test GPU=1 SHOW=1"] diff --git a/README.md b/README.md index 8c59b051e..d47c9f75d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redisai/redisai.svg)](https://hub.docker.com/r/redisai/redisai/builds/) [![Mailing List](https://img.shields.io/badge/Mailing%20List-RedisAI-blue)](https://groups.google.com/forum/#!forum/redisai) [![Gitter](https://badges.gitter.im/RedisLabs/RedisAI.svg)](https://gitter.im/RedisLabs/RedisAI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![codecov](https://codecov.io/gh/RedisAI/RedisAI/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisAI/RedisAI) # RedisAI diff --git a/opt/Makefile b/opt/Makefile index fb4b4dfca..da9f0f9d3 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -2,26 +2,55 @@ ROOT:=.. include readies/mk/main -MK_CMAKE:=1 -MK_CMAKE_INSTALL:=1 +MK.cmake:=1 +MK_CUSTOM_CMAKE_BUILD:=1 +MK_CUSTOM_CLEAN:=1 + +MK_ALL_TARGETS=bindirs fetch build pack + +ifneq ($(VGD),) +VALGRIND=$(VGD) +endif + +ifeq ($(VALGRIND),1) +DEBUG ?= 1 +endif + +ifeq ($(COV),1) +DEBUG ?= 1 +endif + +#---------------------------------------------------------------------------------------------- define HELP -make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) -make fetch # download and prepare dependant modules -make build # compile and link -make clean # remove build artifacts - ALL=1 # remove entire artifacts directory -make test # run tests - TEST=test # run only test `test` with Redis output - TEST_ARGS=args # add extra RLTest `args` - VERBOSE=1 # verbose tests output - TEST_COVERAGE=off # add coverage -make pack # create installation packages - PACK_DEPS=0 # do not pack dependencies - INTO=dir # place artifacts in `dir` - BRANCH=name # use `name` as branch name -make deploy # copy packages to S3 -make release # release a version +make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) +make fetch # download and prepare dependant modules + +make build # compile and link + DEBUG=1 # build for debugging + COV=1 # build for coverage analysis (implies DEBUG=1) + VARIANT=name # build variant `name` + WHY=1 # explain CMake decisions (into /tmp/cmake.why) +make clean # remove build artifacts + ALL=1 # remove entire artifacts directory +make install # create ready-to-run scheme (module and engines) + +make test # run tests + TEST=test # run only test `test` with Redis output + TEST_ARGS=args # add extra RLTest `args` + VERBOSE=1 # verbose tests output + COV=1 # perform coverage analysis + VALGRIND|VGD=1 # test with Valgrind (implies DEBUG=1) + CALLGRIND|CGD=1 # test with Callgrind (implies DEBUG=1) +make cov-upload # upload coverage data to codecov.io (requires CODECOV_TOKEN) + +make pack # create installation packages + PACK_DEPS=0 # do not pack dependencies + INTO=dir # place artifacts in `dir` + BRANCH=name # use `name` as branch name +make deploy # copy packages to S3 +make release # release a version +make docker # build docker image fetch and build options: WITH_TF=0 # SKip TensofFlow @@ -50,30 +79,35 @@ DEPS_FLAGS=cpu DEVICE=cpu endif +#---------------------------------------------------------------------------------------------- + SRCDIR=.. BINDIR=$(BINROOT)/src -# INSTALL_DIR=$(ROOT)/install-$(DEVICE) DEPS_DIR=$(ROOT)/deps/$(OS)-$(ARCH)-$(DEVICE) -INSTALL_DIR=$(ROOT)/bin/$(OS)-$(ARCH)-$(DEVICE)/install -REDIS_VALGRID_SUPRESS=$(ROOT)/opt/redis_valgrind.sup +INSTALL_DIR=$(BINROOT)/install-$(DEVICE) + TARGET=$(BINDIR)/redisai.so +INSTALLED_TARGET=$(INSTALL_DIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends +#---------------------------------------------------------------------------------------------- + CMAKE_FILES += \ $(SRCDIR)/CMakeLists.txt \ $(SRCDIR)/src/CMakeLists.txt \ $(SRCDIR)/libtorch_c/CMakeLists.txt -WITH_COVERAGE=off -ifeq ($(TEST_COVERAGE),on) -WITH_COVERAGE=on +ifeq ($(COV),1) +USE_COVERAGE=on +else +USE_COVERAGE=off endif CMAKE_FLAGS += \ -DDEPS_PATH=$(abspath $(DEPS_DIR)) \ -DINSTALL_PATH=$(abspath $(INSTALL_DIR)) \ - -DENABLE_CODECOVERAGE=$(WITH_COVERAGE) \ + -DUSE_COVERAGE=$(USE_COVERAGE) \ -DDEVICE=$(DEVICE) ifeq ($(WITH_TF),0) @@ -96,23 +130,21 @@ include $(MK)/defs #---------------------------------------------------------------------------------------------- -.PHONY: deps fetch pack pack_ramp pack_deps test +.PHONY: fetch deps pack pack_ramp pack_deps test include $(MK)/rules #---------------------------------------------------------------------------------------------- -prebuild: - $(SHOW)if [ ! -d $(DEPS_DIR) ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi +#prebuild: +# $(SHOW)if [ ! -d $(DEPS_DIR) ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi -$(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) - $(SHOW)mkdir -p $(INSTALL_DIR) +cmake-build $(TARGET): $(MK_MAKEFILES) $(SHOW)$(MAKE) -C $(BINDIR) + $(SHOW)mkdir -p $(INSTALL_DIR) $(SHOW)$(MAKE) -C $(BINDIR) install -# $(SHOW)cd $(ROOT) ;\ -# if [ ! -e install ]; then ln -sf install-$(DEVICE) install; fi -install: +install $(INSTALLED_TARGET): $(TARGET) $(SHOW)mkdir -p $(INSTALL_DIR) $(SHOW)$(MAKE) -C $(BINDIR) install @@ -127,13 +159,18 @@ endif #---------------------------------------------------------------------------------------------- +setup: + @echo Setting up system... + $(SHOW)$(ROOT)/opt/readies/bin/getpy3 + $(SHOW)$(ROOT)/opt/system-setup.py + fetch deps: @echo Fetching dependencies... $(SHOW)VERBOSE=$(_SHOW) $(ROOT)/get_deps.sh $(DEPS_FLAGS) #---------------------------------------------------------------------------------------------- -pack: +pack: $(INSTALLED_TARGET) ifneq ($(PACK_DEPS),0) $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=1 ./pack.sh else @@ -142,76 +179,27 @@ endif #---------------------------------------------------------------------------------------------- -TEST_REPORT_DIR ?= $(PWD) -ifeq ($(VERBOSE),1) -TEST_ARGS += -v -endif -ifeq ($(TEST),) -TEST= -PYDEBUG= -else -TEST_ARGS += -s --test $(TEST) -PYDEBUG=1 -endif +export GEN ?= 1 +export SLAVES ?= 1 +export AOF ?= 1 -GEN ?= 1 -SLAVES ?= 1 -AOF ?= 1 +test: + $(COVERAGE_RESET) + $(SHOW)\ + DEVICE=$(DEVICE) \ + MODULE=$(INSTALLED_TARGET) \ + GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + VALGRIND=$(VALGRIND) \ + $(ROOT)/test/tests.sh + $(COVERAGE_COLLECT_REPORT) -TEST_PREFIX=set -e; cd $(ROOT)/test -# TODO: --errors-for-leak-kinds=definite -VALGRIND_OPTIONS="--leak-check=full -q --show-reachable=no --show-possibly-lost=no" -TEST_CMD= DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so -VALGRIND_TEST_CMD= DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so --no-output-catch --use-valgrind --vg-no-fail-on-errors --vg-verbose --vg-options $(VALGRIND_OPTIONS) --vg-suppressions $(realpath $(REDIS_VALGRID_SUPRESS)) +valgrind: + $(SHOW)$(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET)) -test: -ifneq ($(NO_LFS),1) - $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi - $(SHOW)cd $(ROOT); git lfs pull -endif -ifeq ($(GEN),1) - $(SHOW)$(TEST_PREFIX); $(TEST_CMD) -endif -ifeq ($(AOF),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-aof:\n\n" ;\ - $(TEST_CMD) --use-aof -endif -ifeq ($(SLAVES),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-slaves:\n\n" ;\ - $(TEST_CMD) --use-slaves -endif -ifeq ($(VALGRIND),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with valgrind:\n\n" ;\ - $(VALGRIND_TEST_CMD) -endif +callgrind: + $(SHOW)CALLGRIND=1 $(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET)) #---------------------------------------------------------------------------------------------- docker: $(SHOW)docker build -t redisai --build-arg TEST=1 --build-arg PACK=1 .. - -#---------------------------------------------------------------------------------------------- -# Currently only testing for leaks using TF -MODULE_ARGS=\ - BACKENDSPATH $(realpath $(BINDIR)) \ - TF redisai_tensorflow.so - -VALGRIND_ARGS=\ - $(VALGRIND_OPTIONS) \ - --suppressions=$(realpath $(REDIS_VALGRID_SUPRESS)) \ - -v redis-server --protected-mode no --save "" --appendonly no - -valgrind: $(TARGET) - $(SHOW)valgrind $(VALGRIND_ARGS) --loadmodule $(realpath $(TARGET)) $(MODULE_ARGS) - -CALLGRIND_ARGS=\ - --tool=callgrind \ - --dump-instr=yes \ - --simulate-cache=no \ - --collect-jumps=no \ - --collect-atstart=yes \ - --instr-atstart=yes \ - -v redis-server --protected-mode no --save "" --appendonly no - -callgrind: $(TARGET) - $(SHOW)valgrind $(CALLGRIND_ARGS) --loadmodule $(realpath $(TARGET)) $(MODULE_ARGS) \ No newline at end of file diff --git a/opt/readies/bin/filter-colors b/opt/readies/bin/filter-colors new file mode 100644 index 000000000..b51496e17 --- /dev/null +++ b/opt/readies/bin/filter-colors @@ -0,0 +1,4 @@ +#!/bin/bash + +sed 's/\x1b\[[0-9;]*m//g' + diff --git a/opt/readies/bin/getgosu b/opt/readies/bin/getgosu new file mode 100644 index 000000000..b1fa736bd --- /dev/null +++ b/opt/readies/bin/getgosu @@ -0,0 +1,36 @@ +#!/usr/bin/env python2 + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +import paella + +platform = paella.Platform() + +gosu_arch={ + "x64": "amd64", + "arm64v8": "arm64", + "arm32v7": "armhf" +} + +version = os.environ.get('GOSU_VERSION', '1.11') + +gosu = "https://github.com/tianon/gosu/releases/download/{}/gosu-{}".format(version, gosu_arch[platform.arch]) +# paella.wget(gosu, dest='/usr/local/bin/gosu') +# the former fails with: +# urllib2.URLError: +# thus falling back to wget +sh("wget -q -O /usr/local/bin/gosu {}".format(gosu)) + +# paella.wget(gosu + ".asc", dest='/usr/local/bin/gosu.asc') + +## note: gpg sometimes fails due to network problems, disabling ipv6 helps +# export GNUPGHOME="$(mktemp -d)" +# echo "disable-ipv6" >> $GNUPGHOME/dirmngr.conf +# gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 +# gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu +# gpgconf --kill all +# rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc + +os.chmod('/usr/local/bin/gosu', 0755) diff --git a/opt/readies/bin/getpy b/opt/readies/bin/getpy3 similarity index 100% rename from opt/readies/bin/getpy rename to opt/readies/bin/getpy3 diff --git a/opt/readies/bin/sep1 b/opt/readies/bin/sep0 similarity index 100% rename from opt/readies/bin/sep1 rename to opt/readies/bin/sep0 diff --git a/opt/readies/bin/system-setup.py b/opt/readies/bin/system-setup.py new file mode 100644 index 000000000..5acb059b1 --- /dev/null +++ b/opt/readies/bin/system-setup.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python2 + +import sys +import os +import argparse + +HERE=os.path.join(os.path.dirname(__file__) +sys.path.insert(0, HERE, "..")) +import paella + +#---------------------------------------------------------------------------------------------- + +class SystemSetup(paella.Setup): + def __init__(self, nop=False): + paella.Setup.__init__(self, nop) + + def common_first(self): + # self.install("") + # self.group_install("") + # self.setup_pip() + # self.pip_install("") + pass + + def debian_compat(self): + pass + + def redhat_compat(self): + pass + + def fedora(self): + pass + + def macosx(self): + pass + + def common_last(self): + pass + +#---------------------------------------------------------------------------------------------- + +parser = argparse.ArgumentParser(description='Set up system for build.') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +# parser.add_argument('--bool', action="store_true", help="flag") +# parser.add_argument('--int', type=int, default=1, help='number') +# parser.add_argument('--str', type=str, default='str', help='number') +args = parser.parse_args() + +SystemSetup(nop = args.nop).setup() diff --git a/opt/readies/cetara/README.md b/opt/readies/cetara/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/opt/readies/cetara/diag/gdb.c b/opt/readies/cetara/diag/gdb.c index 8afa1697d..d4eb44ca6 100755 --- a/opt/readies/cetara/diag/gdb.c +++ b/opt/readies/cetara/diag/gdb.c @@ -4,21 +4,53 @@ #include "readies/cetara/diag/gdb.h" +#include #include #include +#include #include #include +#include #include #include +#include /////////////////////////////////////////////////////////////////////////////////////////////// #ifdef __linux__ -#if 1 +static inline bool _via_gdb_simple() +{ + const int status_fd = open("/proc/self/status", O_RDONLY); + if (status_fd == -1) + return false; + + char buf[4096]; + const ssize_t num_read = read(status_fd, buf, sizeof(buf) - 1); + if (num_read <= 0) + return false; + + buf[num_read] = '\0'; + char tracer_pid[] = "TracerPid:"; + const char *tracer_pid_p = strstr(buf, tracer_pid); + if (!tracer_pid_p) + return false; + + for (const char *p = tracer_pid_p + sizeof(tracer_pid) - 1; p <= buf + num_read; ++p) + { + if (isspace(*p)) + continue; + return isdigit(*p) && *p != '0'; + } + + return false; +} static inline bool _via_gdb() { + if (_via_gdb_simple()) + return true; + int pid; int from_child[2] = {-1, -1}; @@ -78,37 +110,6 @@ static inline bool _via_gdb() } } -#else - -static inline bool _via_gdb() -{ - const int status_fd = open("/proc/self/status", O_RDONLY); - if (status_fd == -1) - return false; - - char buf[4096]; - const ssize_t num_read = read(status_fd, buf, sizeof(buf) - 1); - if (num_read <= 0) - return false; - - buf[num_read] = '\0'; - constexpr char tracer_pid[] = "TracerPid:"; - const auto tracer_pid_p = strstr(buf, tracer_pid); - if (!tracer_pid_p) - return false; - - for (const char *p = tracer_pid_p + sizeof(tracer_pid) - 1; p <= buf + num_read; ++p) - { - if (isspace(*p)) - continue; - return isdigit(*p) && *p != '0'; - } - - return false; -} - -#endif // 1 - #elif defined(__APPLE__) static inline bool _via_gdb() diff --git a/opt/readies/mk/README.md b/opt/readies/mk/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/opt/readies/mk/bindirs.defs b/opt/readies/mk/bindirs.defs index d84ba40c8..48fadc04e 100755 --- a/opt/readies/mk/bindirs.defs +++ b/opt/readies/mk/bindirs.defs @@ -1,5 +1,7 @@ +ifndef BINROOT BINROOT=$(ROOT)/bin/$(FULL_VARIANT) +endif BINROOT.release=$(ROOT)/bin/$(FULL_VARIANT.release) BIN_DIRS=$(sort $(patsubst %/,%,$(BINDIR) $(dir $(OBJECTS))) $(BINDIRS)) diff --git a/opt/readies/mk/bindirs.rules b/opt/readies/mk/bindirs.rules index 4b57e8d49..bf2d5fe87 100755 --- a/opt/readies/mk/bindirs.rules +++ b/opt/readies/mk/bindirs.rules @@ -1,4 +1,6 @@ +ifneq ($(MK.nobindir),1) + ifeq ($(BINDIR),) $(error BINDIR is undefined) endif @@ -7,4 +9,14 @@ endif bindirs: $(BIN_DIRS) +ifeq ($(DIAG),1) +$(info *** BIN_DIRS=$(BIN_DIRS)) +endif + $(foreach DIR,$(BIN_DIRS),$(eval $(call mkdir_rule,$(DIR)))) + +else + +bindirs: ; + +endif # MK.nobindir diff --git a/opt/readies/mk/build.defs b/opt/readies/mk/build.defs index e69de29bb..78ac8dd8d 100755 --- a/opt/readies/mk/build.defs +++ b/opt/readies/mk/build.defs @@ -0,0 +1,3 @@ + +MK_CLEAN_DIR=$(SRCDIR) +MK_CLEAN_ALL_DIRS += $(BINDIR) diff --git a/opt/readies/mk/build.rules b/opt/readies/mk/build.rules index 1a3c31e2b..02862f8db 100755 --- a/opt/readies/mk/build.rules +++ b/opt/readies/mk/build.rules @@ -1,11 +1,20 @@ .PHONY: build clean -build: $(MK_MAKEFILES) $(TARGET) - -#clean: -#ifeq ($(ALL),1) -# $(SHOW)rm -rf $(BINDIR) -#else -# $(SHOW)$(MAKE) clean -C $(BUILD_DIR) -#endif \ No newline at end of file +ifeq ($(DIAG),1) +$(info *** DEFAULT_TARGETS=$(DEFAULT_TARGETS)) +$(info *** MK_MAKEFILES=$(MK_MAKEFILES)) +endif + +build: $(DEFAULT_TARGETS) $(MK_MAKEFILES) $(TARGET) + +ifneq ($(MK_CUSTOM_CLEAN),1) + +clean: +ifeq ($(ALL),1) + $(SHOW)if [ ! -z "$(MK_CLEAN_ALL_DIRS)" ]; then rm -rf $(sort $(MK_CLEAN_ALL_DIRS)); fi +else + $(SHOW)$(MAKE) clean -C $(MK_CLEAN_DIR) +endif + +endif # MK_CUSTOM_CLEAN diff --git a/opt/readies/mk/cmake.defs b/opt/readies/mk/cmake.defs index 0ed2ecad0..7c1d6d693 100755 --- a/opt/readies/mk/cmake.defs +++ b/opt/readies/mk/cmake.defs @@ -1,7 +1,10 @@ -ifneq ($(MK_CMAKE),) +ifneq ($(MK.cmake),) + +MK_CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt MK_MAKEFILES += $(BINDIR)/Makefile +DEFAULT_TARGETS += cmake-build ifeq ($(DEBUG),1) CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug @@ -9,6 +12,7 @@ endif ifeq ($(WHY),1) CMAKE_WHY=--trace-expand +CMAKE_SINK= >/tmp/cmake.why 2>&1 endif -endif # MK_CMAKE +endif # MK.cmake diff --git a/opt/readies/mk/cmake.rules b/opt/readies/mk/cmake.rules index 686fae8f1..e553c7b63 100755 --- a/opt/readies/mk/cmake.rules +++ b/opt/readies/mk/cmake.rules @@ -1,14 +1,15 @@ -ifneq ($(MK_CMAKE),) +ifneq ($(MK.cmake),) -CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt - -$(BINDIR)/Makefile : bindirs $(MK_CMAKE_FILES) +$(BINDIR)/Makefile : $(MK_CMAKE_FILES) $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi - $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) + $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) $(CMAKE_SINK) + +ifneq ($(MK_CUSTOM_CMAKE_BUILD),1) + +cmake-build $(TARGET): $(MK_MAKEFILES) + $(SHOW)$(MAKE) -C $(BINDIR) -# $(SHOW)cd $(BINDIR); \ -# CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \ -# cmake $(CMAKE_FLAGS) $$CMAKE_REL_ROOT +endif -endif # MK_CMAKE +endif # MK.cmake diff --git a/opt/readies/mk/common.defs b/opt/readies/mk/common.defs index 1735cbceb..976aedd61 100755 --- a/opt/readies/mk/common.defs +++ b/opt/readies/mk/common.defs @@ -15,3 +15,7 @@ MAKEFLAGS += --no-builtin-rules --no-print-directory define __SEP import os; rows, cols = os.popen('stty size', 'r').read().split(); print('-' * (int(cols) - 1) + \"\n\") endef + +ifneq ($(filter cov-upload,$(MAKECMDGOALS)),) +COV=1 +endif diff --git a/opt/readies/mk/common.rules b/opt/readies/mk/common.rules index f84f87be2..633cf2f6b 100755 --- a/opt/readies/mk/common.rules +++ b/opt/readies/mk/common.rules @@ -1,5 +1,6 @@ .PHONY: __sep -__sep: - @python -c "$(__SEP)" +__sep: ; +# @python -c "$(__SEP)" + diff --git a/opt/readies/mk/configure.defs b/opt/readies/mk/configure.defs index da576887b..6b4f5fa20 100755 --- a/opt/readies/mk/configure.defs +++ b/opt/readies/mk/configure.defs @@ -1,2 +1,13 @@ -CONFIGURE_FLAGS= +ifneq (,$(MK.configure)) + +CONFIGURE_FLAGS = $(CONFIGURE_FLAGS.$(OS)) + +CONFIGURE_BUILD_DIR ?= $(BINDIR) + +CONFIGURE_TARGET ?= $(TARGET) + +MK_CLEAN_DIR=$(CONFIGURE_BUILD_DIR) +MK_CLEAN_ALL_DIRS += $(CONFIGURE_BUILD_DIR) + +endif # MK.configure diff --git a/opt/readies/mk/configure.rules b/opt/readies/mk/configure.rules index 1ef0cbc51..8378279d9 100755 --- a/opt/readies/mk/configure.rules +++ b/opt/readies/mk/configure.rules @@ -1,15 +1,12 @@ -.PHONY: build clean - -build: $(MK_MAKEFILES) $(TARGET) -ifeq (,$(wildcard $(BUILD_DIR)/Makefile)) - $(SHOW)cd $(BUILD_DIR); $(abspath $(SRCDIR))/configure $(CONFIGURE_FLAGS) -endif - @make -C $(BUILD_DIR) - -clean: -ifeq ($(ALL),1) - $(SHOW)rm -rf $(BINDIR) -else - $(SHOW)$(MAKE) clean -C $(BUILD_DIR) -endif +ifneq (,$(MK.configure)) + +$(CONFIGURE_BUILD_DIR)/Makefile: $(SRCDIR)/configure + $(SHOW)set -e ;\ + cd $(CONFIGURE_BUILD_DIR) ;\ + $(abspath $(SRCDIR))/configure $(CONFIGURE_FLAGS) + +$(CONFIGURE_TARGET): $(CONFIGURE_BUILD_DIR)/Makefile + $(SHOW)make -C $(CONFIGURE_BUILD_DIR) -j $(NPROC) + +endif # MK.configure diff --git a/opt/readies/mk/coverage.defs b/opt/readies/mk/coverage.defs new file mode 100644 index 000000000..f38f8d9ba --- /dev/null +++ b/opt/readies/mk/coverage.defs @@ -0,0 +1,51 @@ + +ifneq ($(filter cov-upload,$(MAKECMDGOALS)),) +COV=1 +endif + +ifeq ($(COV),1) + +CC_FLAGS.coverage += --coverage +LD_FLAGS.coverage += --coverage + +COV_INFO=$(BINROOT)/cov.info +COV_DIR=$(BINROOT)/cov + +COV_EXCLUDE += \ + '/Applications/*' \ + '/usr/*' + +define COVERAGE_RESET +$(SHOW)set -e ;\ +echo "Starting coverage analysys." ;\ +mkdir -p $(COV_DIR) ;\ +lcov --directory $(BINROOT) --base-directory $(SRCDIR) -z > /dev/null 2>&1 +endef + +define COVERAGE_COLLECT +$(SHOW)set -e ;\ +echo "Collecting coverage data ..." ;\ +lcov --capture --directory $(BINROOT) --base-directory $(SRCDIR) --output-file $(COV_INFO) > /dev/null 2>&1 ;\ +lcov -o $(COV_INFO) -r $(COV_INFO) $(COV_EXCLUDE) > /dev/null 2>&1 +endef + +define COVERAGE_REPORT +$(SHOW)set -e ;\ +lcov -l $(COV_INFO) ;\ +genhtml --legend -o $(COV_DIR) $(COV_INFO) > /dev/null 2>&1 ;\ +echo "Coverage info at $$(realpath $(COV_DIR))/index.html" +endef + +define COVERAGE_COLLECT_REPORT +$(COVERAGE_COLLECT) +$(COVERAGE_REPORT) +endef + +else + +COVERAGE_RESET= +COVERAGE_COLLECT= +COVERAGE_REPORT= +COVERAGE_COLLECT_REPORT= + +endif # COV diff --git a/opt/readies/mk/coverage.rules b/opt/readies/mk/coverage.rules new file mode 100644 index 000000000..6f3ee0671 --- /dev/null +++ b/opt/readies/mk/coverage.rules @@ -0,0 +1,11 @@ + +ifeq ($(COV),1) + +cov-upload: +ifneq ($(CODECOV_TOKEN),) + $(SHOW)bash <(curl -s https://codecov.io/bash) -f $(COV_INFO) +else + @echo "To upload to codecov.io, please define CODECOV_TOKEN for this module." +endif + +endif # COV diff --git a/opt/readies/mk/defs b/opt/readies/mk/defs index c4f0d4016..7ba0de577 100755 --- a/opt/readies/mk/defs +++ b/opt/readies/mk/defs @@ -1,4 +1,15 @@ include $(MK)/help.defs include $(MK)/build.defs + +ifneq ($(MK.cmake),) include $(MK)/cmake.defs +endif + +ifneq ($(MK.configure),) +include $(MK)/configure.defs +endif + +ifeq ($(COV),1) +include $(MK)/coverage.defs +endif diff --git a/opt/readies/mk/docker b/opt/readies/mk/docker new file mode 100644 index 000000000..1679e95d4 --- /dev/null +++ b/opt/readies/mk/docker @@ -0,0 +1,123 @@ + +.NOTPARALLEL: + +MK.nobindir=1 + +MK_ALL_TARGETS=build publish + +ifeq ($(VERSION),) +VERSION:=$(patsubst v%,%,$(shell git describe --tags `git rev-list --tags --max-count=1`)) +endif +ifeq ($(VERSION),) +$(error Cannot determine version. Aborting.) +endif + +BUILD_OPT=--rm +# --squash + +DOCKERFILE_STEM ?= $(ROOT)/Dockerfile + +#---------------------------------------------------------------------------------------------- + +define HELP +make build # build Docker images + X86=1 # build x64 Docker image + ARM7=1 # build arm32v7 Docker image + ARM8=1 # build arm64v8 Docker image +make publish # build and push multi-arch Docker manifest + X86=1 # push x64 Docker image and multi-arch manifest + ARM7=1 # push arm32v7 Docker image and multi-arch manifest + ARM8=1 # push arm64v8 Docker image and multi-arch manifest + PUSH=0 # only push multi-arch manifest +make show # display registry information from Docker repo + +Other arguments: + VERSION=x.y.z # build and publish version x.y.z + OSNICK=nick # nick=buster|stretch|bionic +endef + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUSH,push)) + +#---------------------------------------------------------------------------------------------- + +define build_x64 # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-x64 -f $(DOCKERFILE_STEM) \ + $(foreach A,$(DOCKER_BUILD_ARGS),--build-arg $(A)) \ + $(ROOT) + +.PHONY: build_$(1) +endef + +define build_arm # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-$(1) -f $(DOCKERFILE_STEM).arm \ + $(foreach A,$(DOCKER_BUILD_ARGS),--build-arg $(A)) \ + --build-arg ARCH=$(1) \ + $(ROOT) + +.PHONY: build_$(1) +endef + +define push # (1=arch) +push_$(1): + @docker push $(STEM)-$(OSNICK):$(VERSION)-$(1) + +.PHONY: push_$(1) +endef + +define create_manifest # (1=version) +@docker manifest create -a $(STEM)-$(OSNICK):$(1) \ + -a $(STEM)-$(OSNICK):$(VERSION)-x64 \ + -a $(STEM)-$(OSNICK):$(VERSION)-arm64v8 \ + -a $(STEM)-$(OSNICK):$(VERSION)-arm32v7 +@docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm32v7 --os linux --arch arm --variant v7 +@docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm64v8 --os linux --arch arm64 --variant v8 +@docker manifest push -p $(STEM)-$(OSNICK):$(1) +endef + +#---------------------------------------------------------------------------------------------- + +DEFAULT_TARGETS += $(BUILD_TARGETS) + +include $(MK)/defs + +include $(MK)/rules + +$(eval $(call build_x64,x64)) +$(eval $(call build_arm,arm64v8)) +$(eval $(call build_arm,arm32v7)) + +$(eval $(call push,x64)) +$(eval $(call push,arm64v8)) +$(eval $(call push,arm32v7)) + +ifneq ($(PUSH),0) +publish: $(PUSH_TARGETS) +else +publish: +endif + $(call create_manifest,$(VERSION)) + $(call create_manifest,latest) + +show: + @echo "$(STEM)-$(OSNICK):" +ifeq ($(INSPECT),1) + @docker manifest inspect $(STEM)-$(OSNICK):$(VERSION) | jq +else + @curl -s -X "GET" https://cloud.docker.com/v2/repositories/$(STEM)-$(OSNICK)/tags/ | jq +endif + +.PHONY: build publish show diff --git a/opt/readies/mk/git.defs b/opt/readies/mk/git.defs new file mode 100644 index 000000000..6a09915a9 --- /dev/null +++ b/opt/readies/mk/git.defs @@ -0,0 +1,5 @@ + +ifeq ($(shell { [ -d .git ] || git rev-parse --git-dir >/dev/null 2>&1; echo -n $$?; }),0) +GIT_SHA := $(shell git rev-parse HEAD) +GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+" 2>/dev/null || git rev-parse --short HEAD) +endif diff --git a/opt/readies/mk/macosx.defs b/opt/readies/mk/macosx.defs new file mode 100644 index 000000000..65486aa58 --- /dev/null +++ b/opt/readies/mk/macosx.defs @@ -0,0 +1,42 @@ + +ifeq ($(OS),macosx) + +#---------------------------------------------------------------------------------------------- + +# pip install --user puts packages here: +export PATH:=$(PATH):$(HOME)/Library/Python/2.7/bin + +#---------------------------------------------------------------------------------------------- + +ZLIB_PREFIX:=$(shell brew --prefix zlib) +LIBSSL_PREFIX:=$(shell brew --prefix openssl) +READLINE_PREFIX:=$(shell brew --prefix readline) +SDK_PREFIX:=$(shell xcrun --show-sdk-path) + +export CPPFLAGS:=\ + -I$(ZLIB_PREFIX)/include \ + -I$(READLINE_PREFIX)/include \ + -I$(LIBSSL_PREFIX)/include \ + -I$(SDK_PREFIX)/usr/include + +export LDFLAGS:=\ + -L$(ZLIB_PREFIX)/lib \ + -L$(LIBSSL_PREFIX)/lib \ + -L$(READLINE_PREFIX)/lib \ + -L$(SDK_PREFIX)/usr/lib + +PKG_CONFIG_DIRS += \ + $(ZLIB_PREFIX) \ + $(LIBSSL_PREFIX) \ + $(READLINE_PREFIX) + +export PKG_CONFIG_PATH:=$(subst $(__SPACE),:,$(strip $(addsuffix /lib/pkgconfig,$(PKG_CONFIG_DIRS)))):$(PKG_CONFIG_PATH) + +CONFIGURE_FLAGS.macosx += \ + CPPFLAGS='$(CPPFLAGS)' \ + LDFLAGS='$(LDFLAGS)' \ + --with-openssl=$(LIBSSL_PREFIX) + +#---------------------------------------------------------------------------------------------- + +endif # macosx diff --git a/opt/readies/mk/main b/opt/readies/mk/main index 05fa4297b..cd590fce6 100755 --- a/opt/readies/mk/main +++ b/opt/readies/mk/main @@ -7,23 +7,44 @@ endif ifneq ($(wildcard $(ROOT)/deps/readies),) READIES:=$(ROOT)/deps/readies -else -ifneq ($(wildcard $(ROOT)/opt/readies),) +else ifneq ($(wildcard $(ROOT)/opt/readies),) READIES:=$(ROOT)/opt/readies else $(error Cannot find readies root) endif -endif MK:=$(READIES)/mk -ifneq ($(shell { CHECK=1 $(READIES)/bin/getpy; echo $$?; }),0) -$(error It seems prerequisites have not been installed: please run 'make setup'.) +MK.pyver ?= 3 +MK.getpy=getpy$(MK.pyver) + +ifneq ($(shell { CHECK=1 $(READIES)/bin/$(MK.getpy); echo -n $$?; }),0) +ifneq ($(filter setup,$(MAKECMDGOALS)),) +$(error It seems build prerequisites have not been installed: please run 'sudo make setup'.) +endif +__NO_PYTHON=1 endif MK_ALL_TARGETS:=bindirs build +DEFAULT_TARGETS:=bindirs + +SUDO:=$(shell [ $$( command -v sudo >/dev/null 2>&1; echo $$? ) = 0 ] && echo sudo) +USER:=$(shell whoami) include $(MK)/functions include $(MK)/common.defs + +ifneq ($(__NO_PYTHON),1) + +include $(MK)/platform.defs + +ifeq ($(OS),macosx) +include $(MK)/macosx.defs +endif + +include $(MK)/git.defs include $(MK)/variant.defs include $(MK)/bindirs.defs +include $(MK)/nproc.defs + +endif # __NO_PYTHON \ No newline at end of file diff --git a/opt/readies/mk/nproc.defs b/opt/readies/mk/nproc.defs new file mode 100644 index 000000000..95f6d3b1f --- /dev/null +++ b/opt/readies/mk/nproc.defs @@ -0,0 +1,6 @@ + +ifeq ($(OS),linux) +NPROC:=$(shell nproc) +else ifeq ($(OS),macosx) +NPROC:=$(shell sysctl -n hw.physicalcpu) +endif diff --git a/opt/readies/mk/platform.defs b/opt/readies/mk/platform.defs new file mode 100644 index 000000000..50896184e --- /dev/null +++ b/opt/readies/mk/platform.defs @@ -0,0 +1,7 @@ + +OS:=$(shell $(READIES)/bin/platform --os) +# ifeq ($(OS),linux) +# OS:=$(shell $(READIES)/bin/platform --dist) +# endif + +ARCH=$(shell $(READIES)/bin/platform --arch) diff --git a/opt/readies/mk/rules b/opt/readies/mk/rules index e433c92b2..77bd717fa 100755 --- a/opt/readies/mk/rules +++ b/opt/readies/mk/rules @@ -1,11 +1,21 @@ -.PHONY: all - -all: $(MK_ALL_TARGETS) +default: build include $(MK)/common.rules include $(MK)/variant.rules include $(MK)/bindirs.rules include $(MK)/help.rules include $(MK)/build.rules +include $(MK)/coverage.rules + +ifneq ($(MK.cmake),) include $(MK)/cmake.rules +endif + +ifneq ($(MK.configure),) +include $(MK)/configure.rules +endif + +all: $(MK_ALL_TARGETS) + +.PHONY: all default diff --git a/opt/readies/mk/variant.defs b/opt/readies/mk/variant.defs index eccb47d8a..4277263fa 100755 --- a/opt/readies/mk/variant.defs +++ b/opt/readies/mk/variant.defs @@ -1,20 +1,8 @@ -OS:=$(shell $(READIES)/bin/platform --os) -# ifeq ($(OS),linux) -# OS:=$(shell $(READIES)/bin/platform --dist) -# endif - -ARCH=$(shell $(READIES)/bin/platform --arch) - -#---------------------------------------------------------------------------------------------- - -ifeq ($(shell { [ -d .git ] || git rev-parse --git-dir >/dev/null 2>&1; echo $?; }),0) -GIT_SHA := $(shell git rev-parse HEAD) -GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+") +ifeq ($(COV),1) +export DEBUG ?= 1 endif -#---------------------------------------------------------------------------------------------- - ifeq ($(DEBUG),1) FLAVOR=debug else @@ -23,13 +11,18 @@ endif #---------------------------------------------------------------------------------------------- +ifeq ($(COV),1) +VARIANT.list:=$(sort $(VARIANT.list) cov) +endif + VARIANT.file:=$(shell if [ -f $(ROOT)/VARIANT ]; then cat $(ROOT)/VARIANT; fi) # if VARIANT not specified and we're the not in submake, use one from file, if present ifeq ($(origin VARIANT),undefined) - ifneq ($(VARIANT.primary),) + ifeq ($(VARIANT.primary),) ifneq ($(VARIANT.file),) VARIANT:=$(VARIANT.file) + $(info VARIANT is $(VARIANT)) endif endif else # VARIANT specified diff --git a/opt/readies/paella/README.md b/opt/readies/paella/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/opt/readies/paella/debug.py b/opt/readies/paella/debug.py index bf6ee7bbc..c615a09c5 100755 --- a/opt/readies/paella/debug.py +++ b/opt/readies/paella/debug.py @@ -4,13 +4,17 @@ #---------------------------------------------------------------------------------------------- pydebug = os.environ.get('PYDEBUG', '') -if pydebug == '1' or pydebug == 'pudb': +if pydebug == '1': try: - from pudb import set_trace as bb + from ipdb import set_trace as bb except ImportError: from pdb import set_trace as bb +elif pydebug == 'pudb': + from pudb import set_trace as bb elif pydebug == 'pdb': from pdb import set_trace as bb +elif pydebug == 'ipdb': + from ipdb import set_trace as bb else: def bb(): pass diff --git a/opt/readies/paella/platform.py b/opt/readies/paella/platform.py index e91ab6ecb..9105089fb 100755 --- a/opt/readies/paella/platform.py +++ b/opt/readies/paella/platform.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import platform +import os import re #---------------------------------------------------------------------------------------------- @@ -35,6 +36,13 @@ def osnick(self): def __init__(self, strict=False): self.os = self.dist = self.os_ver = self.full_os_ver = self.osnick = self.arch = '?' + # self.os = os.getenv("READIES_PLATFORM_OS", '?') + # self.dist = os.getenv("READIES_PLATFORM_DIST", '?') + # self.os_ver = os.getenv("READIES_PLATFORM_OS_VER", '?') + # self.full_os_ver = os.getenv("READIES_PLATFORM_FULL_OS_VER", '?') + # self.osnick = os.getenv("READIES_PLATFORM_OSNICK", '?') + # self.arch = os.getenv("READIES_PLATFORM_ARCH", '?') + self.os = platform.system().lower() if self.os == 'linux': if False: @@ -189,6 +197,8 @@ def invoke(self): assert(False), "Cannot determine installer" elif os == 'macosx': self.macosx() + elif os == 'freebsd': + self.freebsd() self.common_last() diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index be36d0456..b10af35ed 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -2,6 +2,7 @@ import os import sys import tempfile +import textwrap from .platform import OnPlatform, Platform #---------------------------------------------------------------------------------------------- @@ -11,6 +12,8 @@ def __init__(self, nop=False): self.nop = nop def run(self, cmd, output_on_error=False, _try=False): + if cmd.find('\n') > -1: + cmd = str.lstrip(textwrap.dedent(cmd)).replace('\n', '; ') print(cmd) sys.stdout.flush() if self.nop: @@ -216,8 +219,17 @@ def install_downloaders(self, _try=False): self.install("curl wget", _try=_try) def install_git_lfs_on_linux(self, _try=False): - self.run("set -e; wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O /tmp/git-lfs.tar.gz") - self.run("cd /tmp; tar xzf git-lfs.tar.gz; ./install.sh") + self.run(""" + set -e + d=$(mktemp -d /tmp/git-lfs.XXXXXX) + mkdir -p $d + wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O $d/git-lfs.tar.gz + cd $d + tar xf git-lfs.tar.gz + cd - + $d/install.sh + rm -rf $d + """) # cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash" # if self.platform.is_redhat_compat(): @@ -227,7 +239,12 @@ def install_git_lfs_on_linux(self, _try=False): # self.install("git-lfs", _try=_try) def install_gnu_utils(self, _try=False): - self.install("make findutils gnu-sed gnu-tar") + packs = "" + if self.os == 'macosx': + packs= "make findutils gnu-sed gnu-tar gawk" + elif self.os == 'freebsd': + packs = "gmake findutils gsed gtar gawk" + self.install(packs) for x in ['make', 'find', 'sed', 'tar']: p = "/usr/local/bin/{}".format(x) if not os.path.exists(p): diff --git a/opt/readies/shibumi/README.md b/opt/readies/shibumi/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/opt/system-setup.py b/opt/system-setup.py index 910865d66..07939b1f6 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -17,7 +17,7 @@ def __init__(self, nop=False): def common_first(self): self.install_downloaders() self.setup_pip() - self.pip3_install("wheel") + self.pip3_install("wheel virtualenv") self.pip3_install("setuptools --upgrade") if self.os == 'linux': @@ -41,7 +41,7 @@ def redhat_compat(self): else: self.run("amazon-linux-extras install epel", output_on_error=True) self.install("python3 python3-devel") - self.pip_install("psutil") + self.pip3_install("psutil") self.install_git_lfs_on_linux() @@ -56,22 +56,22 @@ def macosx(self): if out.splitlines() == []: fatal("Xcode tools are not installed. Please run xcode-select --install.") - # workaround for ssl issue, needed in CircleCI - #if os.environ.get('MACOS_PYTHON_SSL_FIX') == '1': - # self.run("brew unlink python@2") - # self.run("brew reinstall python3") - self.install_gnu_utils() self.install("git-lfs") self.install("redis") def common_last(self): - if not self.has_command("RLTest"): - self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") - if not self.has_command("ramp"): - self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") + # this is due to rmbuilder older versions. should be removed once fixed. + self.run("python3 -m pip uninstall -y -q redis redis-py-cluster ramp-packer RLTest rmtest semantic-version || true") + # redis-py-cluster should be installed from git due to redis-py dependency + self.pip3_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master") + # the following can be probably installed from pypi + self.pip3_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master") + self.pip3_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master") + root = os.path.join(os.path.dirname(__file__), "..") self.pip3_install("-r {}/test/test_requirements.txt".format(root)) + #---------------------------------------------------------------------------------------------- diff --git a/test/basic_tests.py b/test/basic_tests.py deleted file mode 100644 index b8f4be3e6..000000000 --- a/test/basic_tests.py +++ /dev/null @@ -1,1158 +0,0 @@ -from multiprocessing import Pool, Process -import redis - -import numpy as np -from skimage.io import imread -from skimage.transform import resize -import random -import time -import json -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../opt/readies")) -import paella - -TEST_TF = os.environ.get("TEST_TF") != "0" and os.environ.get("WITH_TF") != "0" -TEST_TFLITE = os.environ.get("TEST_TFLITE") != "0" and os.environ.get("WITH_TFLITE") != "0" -TEST_PT = os.environ.get("TEST_PT") != "0" and os.environ.get("WITH_PT") != "0" -TEST_ONNX = os.environ.get("TEST_ONNX") != "0" and os.environ.get("WITH_ORT") != "0" - - -''' -python -m RLTest --test basic_tests.py --module path/to/redisai.so -''' - -DEVICE = os.environ.get('DEVICE', 'CPU').upper() -print(f"Running tests on {DEVICE}\n") - - -def check_cuda(): - return os.system('which nvcc') - - -def info_to_dict(info): - info = [el.decode('ascii') if type(el) is bytes else el for el in info] - return dict(zip(info[::2], info[1::2])) - - -def run_test_multiproc(env, n_procs, fn, args=tuple()): - procs = [] - - def tmpfn(): - con = env.getConnection() - fn(con, *args) - return 1 - - for _ in range(n_procs): - p = Process(target=tmpfn) - p.start() - procs.append(p) - - [p.join() for p in procs] - - -def example_multiproc_fn(env): - env.execute_command('set', 'x', 1) - - -def test_example_multiproc(env): - run_test_multiproc(env, 10, lambda x: x.execute_command('set', 'x', 1)) - r = env.cmd('get', 'x') - env.assertEqual(r, b'1') - - -def test_set_tensor(env): - con = env.getConnection() - con.execute_command('AI.TENSORSET', 'x', 'FLOAT', 2, 'VALUES', 2, 3) - tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'2', b'3']) - con.execute_command('AI.TENSORSET', 'x', 'INT32', 2, 'VALUES', 2, 3) - tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [2, 3]) - - # ERR unsupported data format - try: - con.execute_command('AI.TENSORSET', 'z', 'INT32', 2, 'unsupported', 2, 3) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "invalid argument found in tensor shape") - - # ERR invalid value - try: - con.execute_command('AI.TENSORSET', 'z', 'FLOAT', 2, 'VALUES', 2, 'A') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "invalid value") - - # ERR invalid value - try: - con.execute_command('AI.TENSORSET', 'z', 'INT32', 2, 'VALUES', 2, 'A') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "invalid value") - - try: - con.execute_command('AI.TENSORSET', 1) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', '2') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES', 1) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES', '1') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - time.sleep(0.1) - - for _ in env.reloadingIterator(): - env.assertExists('x') - - -def test_get_tensor(env): - con = env.getConnection() - con.execute_command('AI.TENSORSET', 't_FLOAT', 'FLOAT', 2, 'VALUES', 2, 3) - con.execute_command('AI.TENSORSET', 't_INT8', 'INT8', 2, 'VALUES', 1, 1) - con.execute_command('AI.TENSORSET', 't_INT16', 'INT8', 2, 'VALUES', 1, 1) - con.execute_command('AI.TENSORSET', 't_INT32', 'INT8', 2, 'VALUES', 1, 1) - con.execute_command('AI.TENSORSET', 't_INT64', 'INT8', 2, 'VALUES', 1, 1) - - tensor = con.execute_command('AI.TENSORGET', 't_FLOAT', 'BLOB') - values = tensor[-1] - - tensor = con.execute_command('AI.TENSORGET', 't_INT8', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [1,1]) - - tensor = con.execute_command('AI.TENSORGET', 't_INT16', 'VALUES') - values = tensor[-1] - env.assertEqual(values,[1,1]) - - tensor = con.execute_command('AI.TENSORGET', 't_INT32', 'VALUES') - values = tensor[-1] - env.assertEqual(values,[1,1]) - - - tensor = con.execute_command('AI.TENSORGET', 't_INT64', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [1,1]) - - - tensor = con.execute_command('AI.TENSORGET', 't_INT32', 'META') - values = tensor[-1] - env.assertEqual(values, [2]) - - # ERR unsupported data format - try: - con.execute_command('AI.TENSORGET', 't_FLOAT', 'unsupported') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "unsupported data format") - - -def test_del_tf_model(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'graph.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - env.assertEqual(ret, b'OK') - - con.execute_command('AI.MODELDEL', 'm') - env.assertFalse(env.execute_command('EXISTS', 'm')) - - # ERR no model at key - try: - con.execute_command('AI.MODELDEL', 'm') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("no model at key",exception.__str__()) - - # ERR wrong type - try: - con.execute_command('SET', 'NOT_MODEL', 'BAR') - con.execute_command('AI.MODELDEL', 'NOT_MODEL') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value",exception.__str__()) - - -def test_run_tf_model(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'graph.pb') - wrong_model_filename = os.path.join(test_data_path, 'pt-minimal.pt') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - env.assertEqual(len(ret), 3) - # TODO: enable me - # env.assertEqual(ret[0], b'TF') - # env.assertEqual(ret[1], b'CPU') - - # ERR WrongArity - try: - con.execute_command('AI.MODELGET') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("wrong number of arguments for 'AI.MODELGET' command", exception.__str__() ) - - # ERR WRONGTYPE - con.execute_command('SET', 'NOT_MODEL', 'BAR') - try: - con.execute_command('AI.MODELGET', 'NOT_MODEL') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - # cleanup - con.execute_command('DEL', 'NOT_MODEL') - - # ERR cannot get model from empty key - con.execute_command('DEL', 'DONT_EXIST') - try: - con.execute_command('AI.MODELGET', 'DONT_EXIST') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("cannot get model from empty key", exception.__str__()) - - try: - ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', wrong_model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'TF', - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_2', 'PORCH', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_3', 'TORCH', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_4', 'TF', - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_5', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'c', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_6', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mult', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_7', 'TF', DEVICE, model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a_', 'b', 'OUTPUTS', 'mul') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul_') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - # ERR Invalid GraphDef - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "Invalid GraphDef") - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - - - con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - con.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'm') - info_dict_0 = info_to_dict(info) - - env.assertEqual(info_dict_0['KEY'], 'm') - env.assertEqual(info_dict_0['TYPE'], 'MODEL') - env.assertEqual(info_dict_0['BACKEND'], 'TF') - env.assertTrue(info_dict_0['DURATION'] > 0) - env.assertEqual(info_dict_0['SAMPLES'], 2) - env.assertEqual(info_dict_0['CALLS'], 1) - env.assertEqual(info_dict_0['ERRORS'], 0) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'm') - info_dict_1 = info_to_dict(info) - - env.assertTrue(info_dict_1['DURATION'] > info_dict_0['DURATION']) - env.assertEqual(info_dict_1['SAMPLES'], 4) - env.assertEqual(info_dict_1['CALLS'], 2) - env.assertEqual(info_dict_1['ERRORS'], 0) - - ret = con.execute_command('AI.INFO', 'm', 'RESETSTAT') - env.assertEqual(ret, b'OK') - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - info = con.execute_command('AI.INFO', 'm') - info_dict_2 = info_to_dict(info) - - env.assertTrue(info_dict_2['DURATION'] < info_dict_1['DURATION']) - env.assertEqual(info_dict_2['SAMPLES'], 2) - env.assertEqual(info_dict_2['CALLS'], 1) - env.assertEqual(info_dict_2['ERRORS'], 0) - - tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'4', b'9', b'4', b'9']) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') - - con.execute_command('AI.MODELDEL', 'm') - env.assertFalse(env.execute_command('EXISTS', 'm')) - - -def test_run_torch_model(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'pt-minimal.pt') - wrong_model_filename = os.path.join(test_data_path, 'graph.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'TORCH', DEVICE, model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - # TODO: enable me - # env.assertEqual(ret[0], b'TORCH') - # env.assertEqual(ret[1], b'CPU') - - try: - con.execute_command('AI.MODELSET', 'm', 'TORCH', DEVICE, wrong_model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'TORCH', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_2', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - env.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - env.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'b', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_3', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c', 'd') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'4', b'6', b'4', b'6']) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') - - -def test_run_onnx_model(env): - if not TEST_ONNX: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'mnist.onnx') - wrong_model_filename = os.path.join(test_data_path, 'graph.pb') - sample_filename = os.path.join(test_data_path, 'one.raw') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - - with open(sample_filename, 'rb') as f: - sample_raw = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - env.assertEqual(len(ret), 3) - # TODO: enable me - # env.assertEqual(ret[0], b'ONNX') - # env.assertEqual(ret[1], b'CPU') - - try: - con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, wrong_model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'ONNX', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_2', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 1, 1, 28, 28, 'BLOB', sample_raw) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_3', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b') - - tensor = con.execute_command('AI.TENSORGET', 'b', 'VALUES') - values = tensor[-1] - argmax = max(range(len(values)), key=lambda i: values[i]) - - env.assertEqual(argmax, 1) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'b', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - - -def test_run_onnxml_model(env): - if not TEST_ONNX: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - linear_model_filename = os.path.join(test_data_path, 'linear_iris.onnx') - logreg_model_filename = os.path.join(test_data_path, 'logreg_iris.onnx') - - with open(linear_model_filename, 'rb') as f: - linear_model = f.read() - - with open(logreg_model_filename, 'rb') as f: - logreg_model = f.read() - - ret = con.execute_command('AI.MODELSET', 'linear', 'ONNX', DEVICE, linear_model) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELSET', 'logreg', 'ONNX', DEVICE, logreg_model) - env.assertEqual(ret, b'OK') - - con.execute_command('AI.TENSORSET', 'features', 'FLOAT', 1, 4, 'VALUES', 5.1, 3.5, 1.4, 0.2) - - con.execute_command('AI.MODELRUN', 'linear', 'INPUTS', 'features', 'OUTPUTS', 'linear_out') - con.execute_command('AI.MODELRUN', 'logreg', 'INPUTS', 'features', 'OUTPUTS', 'logreg_out', 'logreg_probs') - - linear_out = con.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') - logreg_out = con.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') - - env.assertEqual(float(linear_out[2][0]), -0.090524077415466309) - env.assertEqual(logreg_out[2][0], 0) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - linear_out2 = con2.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') - logreg_out2 = con2.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') - env.assertEqual(linear_out, linear_out2) - env.assertEqual(logreg_out, logreg_out2) - - for _ in env.reloadingIterator(): - env.assertExists('linear') - env.assertExists('logreg') - - -def test_run_tflite_model(env): - if not TEST_TFLITE: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'mnist_model_quant.tflite') - wrong_model_filename = os.path.join(test_data_path, 'graph.pb') - sample_filename = os.path.join(test_data_path, 'one.raw') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(model_filename, 'rb') as f: - model_pb2 = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - - with open(sample_filename, 'rb') as f: - sample_raw = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'TFLITE', 'CPU', model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - env.assertEqual(len(ret), 3) - # TODO: enable me - # env.assertEqual(ret[0], b'TFLITE') - # env.assertEqual(ret[1], b'CPU') - - # try: - # con.execute_command('AI.MODELSET', 'm_1', 'TFLITE', 'CPU', wrong_model_pb) - # except Exception as e: - # exception = e - # env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'TFLITE', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - ret = con.execute_command('AI.MODELSET', 'm_2', 'TFLITE', 'CPU', model_pb2) - - try: - con.execute_command('AI.MODELSET', 'm_2', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 1, 1, 28, 28, 'BLOB', sample_raw) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b', 'c') - - tensor = con.execute_command('AI.TENSORGET', 'b', 'VALUES') - value = tensor[-1][0] - - env.assertEqual(value, 1) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') - - -def test_set_tensor_multiproc(env): - run_test_multiproc(env, 10, - lambda env: env.execute_command('AI.TENSORSET', 'x', 'FLOAT', 2, 'VALUES', 2, 3)) - - con = env.getConnection() - - tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'2', b'3']) - - -def load_mobilenet_test_data(): - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - labels_filename = os.path.join(test_data_path, 'imagenet_class_index.json') - image_filename = os.path.join(test_data_path, 'panda.jpg') - model_filename = os.path.join(test_data_path, 'mobilenet_v2_1.4_224_frozen.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(labels_filename, 'r') as f: - labels = json.load(f) - - img_height, img_width = 224, 224 - - img = imread(image_filename) - img = resize(img, (img_height, img_width), mode='constant', anti_aliasing=True) - img = img.astype(np.float32) - - return model_pb, labels, img - - -def test_run_mobilenet(env): - if not TEST_TF: - return - - con = env.getConnection() - - input_var = 'input' - output_var = 'MobilenetV2/Predictions/Reshape_1' - - model_pb, labels, img = load_mobilenet_test_data() - - con.execute_command('AI.MODELSET', 'mobilenet', 'TF', DEVICE, - 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) - - con.execute_command('AI.TENSORSET', 'input', - 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], - 'BLOB', img.tobytes()) - - con.execute_command('AI.MODELRUN', 'mobilenet', - 'INPUTS', 'input', 'OUTPUTS', 'output') - - dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') - - dtype_map = {b'FLOAT': np.float32} - tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) - label_id = np.argmax(tensor) - 1 - - _, label = labels[str(label_id)] - - env.assertEqual(label, 'giant_panda') - - -def run_mobilenet(con, img, input_var, output_var): - time.sleep(0.5 * random.randint(0, 10)) - con.execute_command('AI.TENSORSET', 'input', - 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], - 'BLOB', img.tobytes()) - - con.execute_command('AI.MODELRUN', 'mobilenet', - 'INPUTS', 'input', 'OUTPUTS', 'output') - - # env.execute_command('DEL', 'input') - - -def test_run_mobilenet_multiproc(env): - if not TEST_TF: - return - - con = env.getConnection() - - input_var = 'input' - output_var = 'MobilenetV2/Predictions/Reshape_1' - - model_pb, labels, img = load_mobilenet_test_data() - con.execute_command('AI.MODELSET', 'mobilenet', 'TF', DEVICE, - 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) - - run_test_multiproc(env, 30, run_mobilenet, (img, input_var, output_var)) - - dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') - - dtype_map = {b'FLOAT': np.float32} - tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) - label_id = np.argmax(tensor) - 1 - - _, label = labels[str(label_id)] - - env.assertEqual( - label, 'giant_panda' - ) - - #@@@ possible workaround for side-effect test failure - # env.restartAndReload() - -def test_set_script(env): - if not TEST_PT: - return - - con = env.getConnection() - - try: - con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, 'return 1') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTSET', 'nope') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTSET', 'more', DEVICE) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - script_filename = os.path.join(test_data_path, 'script.txt') - - with open(script_filename, 'rb') as f: - script = f.read() - - con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) - - for _ in env.reloadingIterator(): - env.assertExists('ket') - - - - -def test_del_script(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - script_filename = os.path.join(test_data_path, 'script.txt') - - with open(script_filename, 'rb') as f: - script = f.read() - - ret = con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.SCRIPTDEL', 'ket') - env.assertFalse(con.execute_command('EXISTS', 'ket')) - - # ERR no script at key from SCRIPTDEL - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTDEL', 'EMPTY') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("no script at key", exception.__str__()) - - # ERR wrong type from SCRIPTDEL - try: - con.execute_command('SET', 'NOT_SCRIPT', 'BAR') - con.execute_command('AI.SCRIPTDEL', 'NOT_SCRIPT') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - -def test_run_script(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - script_filename = os.path.join(test_data_path, 'script.txt') - - with open(script_filename, 'rb') as f: - script = f.read() - - ret = con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - env.assertEqual(ret, b'OK') - ret = con.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - env.assertEqual(ret, b'OK') - - # TODO: enable me ( this is hanging CI ) - # ret = con.execute_command('AI.SCRIPTGET', 'ket') - # TODO: enable me - # env.assertEqual([b'CPU',script],ret) - - # ERR no script at key from SCRIPTGET - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTGET', 'EMPTY') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("cannot get script from empty key", exception.__str__()) - - # ERR wrong type from SCRIPTGET - try: - con.execute_command('SET', 'NOT_SCRIPT', 'BAR') - con.execute_command('AI.SCRIPTGET', 'NOT_SCRIPT') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - # ERR no script at key from SCRIPTRUN - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTRUN', 'EMPTY', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("script key is empty", exception.__str__()) - - # ERR wrong type from SCRIPTRUN - try: - con.execute_command('SET', 'NOT_SCRIPT', 'BAR') - con.execute_command('AI.SCRIPTRUN', 'NOT_SCRIPT', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - # ERR Input key is empty - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'EMPTY', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("Input key is empty", exception.__str__()) - - # ERR Input key not tensor - try: - con.execute_command('SET', 'NOT_TENSOR', 'BAR') - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'NOT_TENSOR', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'b', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'ket') - info_dict_0 = info_to_dict(info) - - env.assertEqual(info_dict_0['KEY'], 'ket') - env.assertEqual(info_dict_0['TYPE'], 'SCRIPT') - env.assertEqual(info_dict_0['BACKEND'], 'TORCH') - env.assertTrue(info_dict_0['DURATION'] > 0) - env.assertEqual(info_dict_0['SAMPLES'], -1) - env.assertEqual(info_dict_0['CALLS'], 4) - env.assertEqual(info_dict_0['ERRORS'], 3) - - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'ket') - info_dict_1 = info_to_dict(info) - - env.assertTrue(info_dict_1['DURATION'] > info_dict_0['DURATION']) - env.assertEqual(info_dict_1['SAMPLES'], -1) - env.assertEqual(info_dict_1['CALLS'], 5) - env.assertEqual(info_dict_1['ERRORS'], 3) - - ret = con.execute_command('AI.INFO', 'ket', 'RESETSTAT') - env.assertEqual(ret, b'OK') - - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'ket') - info_dict_2 = info_to_dict(info) - - env.assertTrue(info_dict_2['DURATION'] < info_dict_1['DURATION']) - env.assertEqual(info_dict_2['SAMPLES'], -1) - env.assertEqual(info_dict_2['CALLS'], 1) - env.assertEqual(info_dict_2['ERRORS'], 0) - - tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'4', b'6', b'4', b'6']) - - time.sleep(0.1) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('ket') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') diff --git a/test/double-panda.py b/test/double-panda.py deleted file mode 100755 index ab040da8f..000000000 --- a/test/double-panda.py +++ /dev/null @@ -1,94 +0,0 @@ -from RLTest import Env - -from multiprocessing import Pool, Process -import redis - -import numpy as np -from skimage.io import imread -from skimage.transform import resize -import random -import time -import json -import os -import sys - - -def run_test_multiproc(env, n_procs, fn, args=tuple()): - procs = [] - - def tmpfn(): - e = env.getConnection() - fn(e, *args) - return 1 - - for _ in range(n_procs): - p = Process(target=tmpfn) - p.start() - procs.append(p) - - [p.join() for p in procs] - - -def load_mobilenet_test_data(): - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - labels_filename = os.path.join(test_data_path, 'imagenet_class_index.json') - image_filename = os.path.join(test_data_path, 'panda.jpg') - model_filename = os.path.join(test_data_path, 'mobilenet_v2_1.4_224_frozen.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(labels_filename, 'r') as f: - labels = json.load(f) - - img_height, img_width = 224, 224 - - img = imread(image_filename) - img = resize(img, (img_height, img_width), mode='constant', anti_aliasing=True) - img = img.astype(np.float32) - #@@@ this one instead of the above will not blow up, but the test will obviously fail: - # img = np.zeros([224, 224, 3], dtype=np.float32) - - return model_pb, labels, img - - -def test_1_run_mobilenet_multiproc(env): - input_var = 'input' - output_var = 'MobilenetV2/Predictions/Reshape_1' - - con = env - - model_pb, labels, img = load_mobilenet_test_data() - con.execute_command('AI.MODELSET', 'mobilenet', 'TF', 'CPU', - 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) - - run_test_multiproc(env, 30, run_mobilenet, (img, input_var, output_var)) - - dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') - - dtype_map = {b'FLOAT': np.float32} - tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) - label_id = np.argmax(tensor) - 1 - - _, label = labels[str(label_id)] - - env.assertEqual( - label, 'giant_panda' - ) - - #@@@ this one also works as a workaround: - # env.restartAndReload() - - -def run_mobilenet(con, img, input_var, output_var): - time.sleep(0.5 * random.randint(0, 10)) - con.execute_command('AI.TENSORSET', 'input', - 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], - 'BLOB', img.tobytes()) - - con.execute_command('AI.MODELRUN', 'mobilenet', 'INPUTS', 'input', 'OUTPUTS', 'output') - # con.execute_command('DEL', 'input') - - -def test_2_run_mobilenet_multiproc(env): - test_1_run_mobilenet_multiproc(env) diff --git a/test/tests.sh b/test/tests.sh new file mode 100755 index 000000000..4d265b650 --- /dev/null +++ b/test/tests.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +[[ $VERBOSE == 1 ]] && set -x +[[ $IGNERR == 1 ]] || set -e + +error() { + echo "There are errors:" + gawk 'NR>L-4 && NR>> ":""),$0 }' L=$1 $0 + exit 1 +} + +[[ -z $_Dbg_DEBUGGER_LEVEL ]] && trap 'error $LINENO' ERR + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. $HERE/../opt/readies/shibumi/functions + +export ROOT=$(realpath $HERE/..) + +#---------------------------------------------------------------------------------------------- + +help() { + cat <<-END + Run Python tests. + + [ARGVARS...] tests.sh [--help|help] [] + + Argument variables: + VERBOSE=1 Print commands + IGNERR=1 Do not abort on error + + DEVICE=cpu|gpu Device for testing + GEN=0|1 General tests + AOF=0|1 Tests with --test-aof + SLAVES=0|1 Tests with --test-slaves + + TEST=test Run specific test (e.g. test.py:test_name) + VALGRIND|VGD=1 Run with Valgrind + CALLGRIND|CGD=1 Run with Callgrind + + END +} + +#---------------------------------------------------------------------------------------------- + +install_git_lfs() { + [[ $NO_LFS == 1 ]] && return + [[ $(git lfs env > /dev/null 2>&1 ; echo $?) != 0 ]] && git lfs install + git lfs pull +} + +#---------------------------------------------------------------------------------------------- + +check_redis_server() { + if ! command -v redis-server > /dev/null; then + echo "Cannot find redis-server. Aborting." + exit 1 + fi +} + +#---------------------------------------------------------------------------------------------- + +valgrind_config() { + export VG_OPTIONS=" + -q \ + --leak-check=full \ + --show-reachable=no \ + --show-possibly-lost=no" + + VALGRIND_SUPRESSIONS=$ROOT/opt/redis_valgrind.sup + + RLTEST_ARGS+="\ + --no-output-catch \ + --use-valgrind \ + --vg-no-fail-on-errors \ + --vg-verbose \ + --vg-suppressions $VALGRIND_SUPRESSIONS" +} + +#---------------------------------------------------------------------------------------------- + +run_tests() { + local title="$1" + [[ ! -z $title ]] && { $ROOT/opt/readies/bin/sep0; printf "Tests with $title:\n\n"; } + $OP python3 -m RLTest --clear-logs --module $MODULE $RLTEST_ARGS +} + +#---------------------------------------------------------------------------------------------- + +[[ $1 == --help || $1 == help ]] && { help; exit 0; } + +DEVICE=${DEVICE:-cpu} + +GEN=${GEN:-1} +SLAVES=${SLAVES:-0} +AOF=${AOF:-0} + +GDB=${GDB:-0} + +OP="" +[[ $NOP == 1 ]] && OP="echo" + +MODULE=${MODULE:-$1} +[[ -z $MODULE || ! -f $MODULE ]] && { echo "Module not found. Aborting."; exit 1; } + +RLTEST_ARGS="" + +[[ $VALGRIND == 1 || $VGD == 1 ]] && valgrind_config + +if [[ ! -z $TEST ]]; then + RLTEST_ARGS+=" -s --test $TEST" + export PYDEBUG=${PYDEBUG:-1} +fi + +[[ $VERBOSE == 1 ]] && RLTEST_ARGS+=" -v" + +#---------------------------------------------------------------------------------------------- + +cd $ROOT/test + +install_git_lfs +check_redis_server + +[[ $GEN == 1 ]] && run_tests +[[ $SLAVES == 1 ]] && RLTEST_ARGS+=" --use-slaves" run_tests "--use-slaves" +[[ $AOF == 1 ]] && RLTEST_ARGS+=" --use-aof" run_tests "--use-aof" diff --git a/test/valgrind.sh b/test/valgrind.sh new file mode 100755 index 000000000..7e205b52e --- /dev/null +++ b/test/valgrind.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +[[ $VERBOSE == 1 ]] && set -x +[[ $IGNERR == 1 ]] || set -e + +error() { + echo "There are errors." + exit 1 +} + +# trap error ERR + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. $HERE/../opt/readies/shibumi/functions + +export ROOT=$(realpath $HERE/..) + +#---------------------------------------------------------------------------------------------- + +help() { + cat <<-END + Run Valgrind/Callgrind on RedisAI. + + [ARGVARS...] valgrind.sh [--help|help] [] + + Argument variables: + VERBOSE=1 Print commands + IGNERR=1 Do not abort on error + + CALLGRIND|CGD=1 Run with Callgrind + + END +} + +#---------------------------------------------------------------------------------------------- + +check_redis_server() { + if ! command -v redis-server > /dev/null; then + echo "Cannot find redis-server. Aborting." + exit 1 + fi +} + +#---------------------------------------------------------------------------------------------- + +valgrind_config() { + VALGRIND_SUPRESSIONS=$ROOT/opt/redis_valgrind.sup + + if [[ $CALLGRIND == 1 ]]; then + + VALGRIND_OPTIONS+="\ + --tool=callgrind \ + --dump-instr=yes \ + --simulate-cache=no \ + --collect-jumps=no \ + --collect-atstart=yes \ + --instr-atstart=yes \ + --callgrind-out-file=$MODULE.call" + else + VALGRIND_OPTIONS+="\ + -q \ + --leak-check=full \ + --show-reachable=no \ + --show-possibly-lost=no \ + --show-leak-kinds=all" + fi + + VALGRIND_OPTIONS+=" -v" +} + +#---------------------------------------------------------------------------------------------- + +[[ $1 == --help || $1 == help ]] && { help; exit 0; } + +OP="" +[[ $NOP == 1 ]] && OP="echo" + +MODULE=${MODULE:-$1} +[[ -z $MODULE || ! -f $MODULE ]] && { echo "Module not found. Aborting."; exit 1; } + +VALGRIND_OPTIONS="" +valgrind_config + +#---------------------------------------------------------------------------------------------- + +cd $ROOT/test + +check_redis_server +$OP valgrind $(echo "$VALGRIND_OPTIONS") redis-server --protected-mode no --save '' --appendonly no --loadmodule $MODULE