diff --git a/examples/int8/training/vgg16/README.md b/examples/training/vgg16/README.md similarity index 100% rename from examples/int8/training/vgg16/README.md rename to examples/training/vgg16/README.md diff --git a/examples/int8/training/vgg16/export_ckpt.py b/examples/training/vgg16/export_ckpt.py similarity index 100% rename from examples/int8/training/vgg16/export_ckpt.py rename to examples/training/vgg16/export_ckpt.py diff --git a/examples/int8/training/vgg16/export_qat.py b/examples/training/vgg16/export_qat.py similarity index 100% rename from examples/int8/training/vgg16/export_qat.py rename to examples/training/vgg16/export_qat.py diff --git a/examples/int8/training/vgg16/finetune_qat.py b/examples/training/vgg16/finetune_qat.py similarity index 100% rename from examples/int8/training/vgg16/finetune_qat.py rename to examples/training/vgg16/finetune_qat.py diff --git a/examples/int8/training/vgg16/main.py b/examples/training/vgg16/main.py similarity index 100% rename from examples/int8/training/vgg16/main.py rename to examples/training/vgg16/main.py diff --git a/examples/int8/training/vgg16/requirements.txt b/examples/training/vgg16/requirements.txt similarity index 100% rename from examples/int8/training/vgg16/requirements.txt rename to examples/training/vgg16/requirements.txt diff --git a/examples/int8/training/vgg16/test_qat.py b/examples/training/vgg16/test_qat.py similarity index 100% rename from examples/int8/training/vgg16/test_qat.py rename to examples/training/vgg16/test_qat.py diff --git a/examples/int8/training/vgg16/vgg16.py b/examples/training/vgg16/vgg16.py similarity index 100% rename from examples/int8/training/vgg16/vgg16.py rename to examples/training/vgg16/vgg16.py diff --git a/examples/benchmark/BUILD b/tools/cpp_benchmark/BUILD similarity index 89% rename from examples/benchmark/BUILD rename to tools/cpp_benchmark/BUILD index 7ee021ea21..8aa36b0e90 100644 --- a/examples/benchmark/BUILD +++ b/tools/cpp_benchmark/BUILD @@ -1,7 +1,7 @@ package(default_visibility = ["//visibility:public"]) cc_binary( - name = "benchmark", + name = "cpp_benchmark", srcs = [ "main.cpp", "timer.h", diff --git a/examples/benchmark/README.md b/tools/cpp_benchmark/README.md similarity index 76% rename from examples/benchmark/README.md rename to tools/cpp_benchmark/README.md index 3a264c9544..a40698f670 100644 --- a/examples/benchmark/README.md +++ b/tools/cpp_benchmark/README.md @@ -9,13 +9,13 @@ Run with bazel: > Note: Make sure libtorch and TensorRT are in your LD_LIBRARY_PATH before running, if you need a location you can `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:[WORKSPACE ROOT]/bazel-Torch-TensorRT/external/libtorch/lib:[WORKSPACE ROOT]/bazel-Torch-TensorRT/external/tensorrt/lib` ``` sh -bazel run //cpp/benchmark --cxxopt="-DNDEBUG" --cxxopt="-DJIT" --cxxopt="-DTRT" -- [PATH TO JIT MODULE FILE] [INPUT SIZE (explicit batch)] +bazel run //tools/cpp_benchmark/int8/benchmark --cxxopt="-DNDEBUG" --cxxopt="-DJIT" --cxxopt="-DTRT" -- [PATH TO JIT MODULE FILE] [INPUT SIZE (explicit batch)] ``` For example: ``` shell -bazel run //cpp/benchmark --cxxopt="-DNDEBUG" --cxxopt="-DJIT" --cxxopt="-DTRT" -- $(realpath /tests/models/resnet50.jit.pt) "(32 3 224 224)" +bazel run //tools/cpp_benchmark/int8/benchmark --cxxopt="-DNDEBUG" --cxxopt="-DJIT" --cxxopt="-DTRT" -- $(realpath /tests/models/resnet50.jit.pt) "(32 3 224 224)" ``` ### Options diff --git a/examples/int8/benchmark/BUILD b/tools/cpp_benchmark/int8/benchmark/BUILD similarity index 100% rename from examples/int8/benchmark/BUILD rename to tools/cpp_benchmark/int8/benchmark/BUILD diff --git a/examples/int8/benchmark/benchmark.cpp b/tools/cpp_benchmark/int8/benchmark/benchmark.cpp similarity index 100% rename from examples/int8/benchmark/benchmark.cpp rename to tools/cpp_benchmark/int8/benchmark/benchmark.cpp diff --git a/examples/int8/benchmark/benchmark.h b/tools/cpp_benchmark/int8/benchmark/benchmark.h similarity index 100% rename from examples/int8/benchmark/benchmark.h rename to tools/cpp_benchmark/int8/benchmark/benchmark.h diff --git a/examples/benchmark/timer.h b/tools/cpp_benchmark/int8/benchmark/timer.h similarity index 100% rename from examples/benchmark/timer.h rename to tools/cpp_benchmark/int8/benchmark/timer.h diff --git a/examples/int8/datasets/BUILD b/tools/cpp_benchmark/int8/datasets/BUILD similarity index 100% rename from examples/int8/datasets/BUILD rename to tools/cpp_benchmark/int8/datasets/BUILD diff --git a/examples/int8/datasets/cifar10.cpp b/tools/cpp_benchmark/int8/datasets/cifar10.cpp similarity index 98% rename from examples/int8/datasets/cifar10.cpp rename to tools/cpp_benchmark/int8/datasets/cifar10.cpp index dff4716dac..135b878230 100644 --- a/examples/int8/datasets/cifar10.cpp +++ b/tools/cpp_benchmark/int8/datasets/cifar10.cpp @@ -1,4 +1,4 @@ -#include "examples/int8/datasets/cifar10.h" +#include "tools/cpp_benchmark/int8/datasets/cifar10.h" #include "torch/data/example.h" #include "torch/torch.h" #include "torch/types.h" diff --git a/examples/int8/datasets/cifar10.h b/tools/cpp_benchmark/int8/datasets/cifar10.h similarity index 100% rename from examples/int8/datasets/cifar10.h rename to tools/cpp_benchmark/int8/datasets/cifar10.h diff --git a/examples/int8/ptq/BUILD b/tools/cpp_benchmark/int8/ptq/BUILD similarity index 77% rename from examples/int8/ptq/BUILD rename to tools/cpp_benchmark/int8/ptq/BUILD index 2e26656633..f45d81b3c8 100644 --- a/examples/int8/ptq/BUILD +++ b/tools/cpp_benchmark/int8/ptq/BUILD @@ -13,8 +13,8 @@ cc_binary( ], deps = [ "//cpp:torch_tensorrt", - "//examples/int8/benchmark", - "//examples/int8/datasets:cifar10", + "//tools/cpp_benchmark/int8/benchmark", + "//tools/cpp_benchmark/int8/datasets:cifar10", "@libtorch", "@libtorch//:caffe2", "@tensorrt//:nvinfer", diff --git a/examples/int8/ptq/Makefile b/tools/cpp_benchmark/int8/ptq/Makefile similarity index 100% rename from examples/int8/ptq/Makefile rename to tools/cpp_benchmark/int8/ptq/Makefile diff --git a/examples/int8/ptq/README.md b/tools/cpp_benchmark/int8/ptq/README.md similarity index 95% rename from examples/int8/ptq/README.md rename to tools/cpp_benchmark/int8/ptq/README.md index 7878c88b15..4ee6618504 100644 --- a/examples/int8/ptq/README.md +++ b/tools/cpp_benchmark/int8/ptq/README.md @@ -13,7 +13,7 @@ LibTorch provides a `Dataloader` and `Dataset` API which steamlines preprocessin Here is an example interface of a `torch::Dataset` class for CIFAR10: ```C++ -//examples/int8/ptq/datasets/cifar10.h +//tools/cpp_benchmark/int8/ptq/datasets/cifar10.h #pragma once #include "torch/data/datasets/base.h" @@ -120,19 +120,19 @@ This is a short example application that shows how to use Torch-TensorRT to perf ## Prerequisites 1. Download CIFAR10 Dataset Binary version ([https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz](https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz)) -2. Train a network on CIFAR10 (see `examples/int8/training/vgg16/README.md` for a VGG16 recipe) +2. Train a network on CIFAR10 (see `examples/training/vgg16/README.md` for a VGG16 recipe) 3. Export model to torchscript ## Compilation using bazel ``` shell -bazel run //examples/int8/ptq --compilation_mode=opt +bazel run //tools/cpp_benchmark/int8/ptq --compilation_mode=opt ``` If you want insight into what is going under the hood or need debug symbols ``` shell -bazel run //examples/int8/ptq --compilation_mode=dbg +bazel run //tools/cpp_benchmark/int8/ptq --compilation_mode=dbg ``` This will build a binary named `ptq` in `bazel-out/k8-/bin/cpp/int8/ptq/` directory. Optionally you can add this to `$PATH` environment variable to run `ptq` from anywhere on your system. @@ -166,7 +166,7 @@ We import header files `cifar10.h` and `benchmark.h` from `ROOT_DIR`. `ROOT_DIR` By default it is set to `../../../`. If your Torch-TensorRT directory structure is different, please set `ROOT_DIR` accordingly. ```sh -cd examples/int8/ptq +cd tools/cpp_benchmark/int8/ptq # This will generate a ptq binary make ROOT_DIR= CUDA_VERSION=11.1 ./ptq diff --git a/examples/benchmark/py/models/.gitkeep b/tools/cpp_benchmark/int8/ptq/deps/.gitkeep similarity index 100% rename from examples/benchmark/py/models/.gitkeep rename to tools/cpp_benchmark/int8/ptq/deps/.gitkeep diff --git a/examples/int8/ptq/main.cpp b/tools/cpp_benchmark/int8/ptq/main.cpp similarity index 98% rename from examples/int8/ptq/main.cpp rename to tools/cpp_benchmark/int8/ptq/main.cpp index fed2b2337a..517d9e7697 100644 --- a/examples/int8/ptq/main.cpp +++ b/tools/cpp_benchmark/int8/ptq/main.cpp @@ -3,8 +3,8 @@ #include "torch_tensorrt/ptq.h" #include "torch_tensorrt/torch_tensorrt.h" -#include "examples/int8/benchmark/benchmark.h" -#include "examples/int8/datasets/cifar10.h" +#include "tools/cpp_benchmark/int8/benchmark/benchmark.h" +#include "tools/cpp_benchmark//int8/datasets/cifar10.h" #include #include diff --git a/examples/int8/qat/BUILD b/tools/cpp_benchmark/int8/qat/BUILD similarity index 77% rename from examples/int8/qat/BUILD rename to tools/cpp_benchmark/int8/qat/BUILD index d312f9a08a..a6b4b513ea 100644 --- a/examples/int8/qat/BUILD +++ b/tools/cpp_benchmark/int8/qat/BUILD @@ -13,8 +13,8 @@ cc_binary( ], deps = [ "//cpp:torch_tensorrt", - "//examples/int8/benchmark", - "//examples/int8/datasets:cifar10", + "//tools/cpp_benchmark/int8/benchmark", + "//tools/cpp_benchmark/int8/datasets:cifar10", "@libtorch", "@libtorch//:caffe2", "@tensorrt//:nvinfer", diff --git a/examples/int8/qat/Makefile b/tools/cpp_benchmark/int8/qat/Makefile similarity index 100% rename from examples/int8/qat/Makefile rename to tools/cpp_benchmark/int8/qat/Makefile diff --git a/examples/int8/qat/README.md b/tools/cpp_benchmark/int8/qat/README.md similarity index 92% rename from examples/int8/qat/README.md rename to tools/cpp_benchmark/int8/qat/README.md index f0c5773600..7a2630dd9d 100644 --- a/examples/int8/qat/README.md +++ b/tools/cpp_benchmark/int8/qat/README.md @@ -12,7 +12,7 @@ This is a short example application that shows how to use Torch-TensorRT to perf ## Prerequisites 1. Download CIFAR10 Dataset Binary version ([https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz](https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz)) -2. Train a network on CIFAR10 and perform quantization aware training on it. Refer to `examples/int8/training/vgg16/README.md` for detailed instructions. +2. Train a network on CIFAR10 and perform quantization aware training on it. Refer to `examples/training/vgg16/README.md` for detailed instructions. Export the QAT model to Torchscript. 3. Install NVIDIA's pytorch quantization toolkit 4. TensorRT 8.0.1.6 or above @@ -20,13 +20,13 @@ This is a short example application that shows how to use Torch-TensorRT to perf ## Compilation using bazel ``` shell -bazel run //examples/int8/qat --compilation_mode=opt +bazel run //tools/cpp_benchmark/int8/qat --compilation_mode=opt ``` If you want insight into what is going under the hood or need debug symbols ``` shell -bazel run //examples/int8/qat --compilation_mode=dbg +bazel run //tools/cpp_benchmark/int8/qat --compilation_mode=dbg ``` This will build a binary named `qat` in `bazel-out/k8-/bin/cpp/int8/qat/` directory. Optionally you can add this to `$PATH` environment variable to run `qat` from anywhere on your system. @@ -60,7 +60,7 @@ We import header files `cifar10.h` and `benchmark.h` from `ROOT_DIR`. `ROOT_DIR` By default it is set to `../../../`. If your Torch-TensorRT directory structure is different, please set `ROOT_DIR` accordingly. ```sh -cd examples/int8/qat +cd tools/cpp_benchmark/int8/qat # This will generate a ptq binary make ROOT_DIR= CUDA_VERSION=11.1 ./qat diff --git a/examples/int8/ptq/deps/.gitkeep b/tools/cpp_benchmark/int8/qat/deps/.gitkeep similarity index 100% rename from examples/int8/ptq/deps/.gitkeep rename to tools/cpp_benchmark/int8/qat/deps/.gitkeep diff --git a/examples/int8/qat/main.cpp b/tools/cpp_benchmark/int8/qat/main.cpp similarity index 97% rename from examples/int8/qat/main.cpp rename to tools/cpp_benchmark/int8/qat/main.cpp index b1bec3e6a2..8c9a0d2283 100644 --- a/examples/int8/qat/main.cpp +++ b/tools/cpp_benchmark/int8/qat/main.cpp @@ -2,8 +2,8 @@ #include "torch/torch.h" #include "torch_tensorrt/torch_tensorrt.h" -#include "examples/int8/benchmark/benchmark.h" -#include "examples/int8/datasets/cifar10.h" +#include "tools/cpp_benchmark/int8/benchmark/benchmark.h" +#include "tools/cpp_benchmark/int8/datasets/cifar10.h" #include #include diff --git a/examples/benchmark/main.cpp b/tools/cpp_benchmark/main.cpp similarity index 100% rename from examples/benchmark/main.cpp rename to tools/cpp_benchmark/main.cpp diff --git a/examples/int8/benchmark/timer.h b/tools/cpp_benchmark/timer.h similarity index 100% rename from examples/int8/benchmark/timer.h rename to tools/cpp_benchmark/timer.h diff --git a/examples/benchmark/py/README.md b/tools/perf/README.md similarity index 100% rename from examples/benchmark/py/README.md rename to tools/perf/README.md diff --git a/examples/benchmark/py/config/vgg16.yml b/tools/perf/config/vgg16.yml similarity index 100% rename from examples/benchmark/py/config/vgg16.yml rename to tools/perf/config/vgg16.yml diff --git a/examples/benchmark/py/config/vgg16_trt.yml b/tools/perf/config/vgg16_trt.yml similarity index 100% rename from examples/benchmark/py/config/vgg16_trt.yml rename to tools/perf/config/vgg16_trt.yml diff --git a/examples/int8/qat/deps/.gitkeep b/tools/perf/models/.gitkeep similarity index 100% rename from examples/int8/qat/deps/.gitkeep rename to tools/perf/models/.gitkeep diff --git a/examples/benchmark/py/perf_run.py b/tools/perf/perf_run.py similarity index 100% rename from examples/benchmark/py/perf_run.py rename to tools/perf/perf_run.py diff --git a/examples/benchmark/py/requirements.txt b/tools/perf/requirements.txt similarity index 100% rename from examples/benchmark/py/requirements.txt rename to tools/perf/requirements.txt