diff --git a/.github/actions/build-cpp/Dockerfile b/.github/actions/build-cpp/Dockerfile index 44e1a8354..9ae312f13 100644 --- a/.github/actions/build-cpp/Dockerfile +++ b/.github/actions/build-cpp/Dockerfile @@ -15,7 +15,7 @@ WORKDIR /pktvisor-src RUN apt-get update && \ apt-get upgrade --yes --force-yes && \ apt-get install --yes --force-yes --no-install-recommends ${BUILD_DEPS} && \ - pip3 install 'conan==1.59.0' --force-reinstall + pip3 install 'conan==1.61.0' --force-reinstall RUN chmod +x /entrypoint.sh diff --git a/.github/actions/build-go/Dockerfile b/.github/actions/build-go/Dockerfile index d213b8ace..9d78003b9 100644 --- a/.github/actions/build-go/Dockerfile +++ b/.github/actions/build-go/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:latest +FROM golang:bullseye LABEL author="Everton Haise Taques " LABEL maintainer="netboxlabs" diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index cefa6703d..f341dbdd2 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -24,56 +24,58 @@ jobs: # cross-platform coverage. # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - runs-on: macos-11 + runs-on: macos-latest steps: - uses: actions/checkout@v3 - - name: install gcc 12.2 - run: | - gcc --version - brew search gcc - brew install gcc@12 - gcc --version - + - name: Create Build Environment # Some projects don't allow in-source building, so create a separate build directory # We'll use this as our working directory for all subsequent commands run: cmake -E make_directory ${{github.workspace}}/build + - name: Python Setup + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Get Conan id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 - -# - name: Setup Conan Cache -# uses: actions/cache@v3 -# with: -# path: ${{github.workspace}}/build/conan_home/ -# key: conan-${{ runner.os }}-${{ hashFiles('conanfile.txt', '*/conanfile.txt') }} -# restore-keys: conan-${{ runner.os }}- - -# - name: Configure CMake -# # Use a bash shell so we can use the same syntax for environment variable -# # access regardless of the host operating system -# shell: bash -# working-directory: ${{github.workspace}}/build -# # Note the current convention is to use the -S and -B options here to specify source -# # and build directories, but this is only available with CMake 3.13 and higher. -# # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 -# run: PKG_CONFIG_PATH=${{github.workspace}}/local/lib/pkgconfig cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE - -# - name: Build -# working-directory: ${{github.workspace}}/build -# shell: bash -# # Execute the build. You can specify a specific target with "--target " -# run: cmake --build . --config $BUILD_TYPE -- -j 2 - -# - name: Test -# working-directory: ${{github.workspace}}/build -# shell: bash -# # Execute tests defined by the CMake configuration. -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: ctest -C $BUILD_TYPE + version: 1.61.0 + + - name: Setup OSX Environment + run: echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV + + - name: Setup Conan Cache + uses: actions/cache@v3 + with: + path: ${{github.workspace}}/build/conan_home/ + key: conan-${{ runner.os }}-${{ hashFiles('conanfile.txt', '*/conanfile.txt') }} + restore-keys: conan-${{ runner.os }}- + + - name: Configure CMake + # Use a bash shell so we can use the same syntax for environment variable + # access regardless of the host operating system + shell: bash + working-directory: ${{github.workspace}}/build + # Note the current convention is to use the -S and -B options here to specify source + # and build directories, but this is only available with CMake 3.13 and higher. + # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 + run: PKG_CONFIG_PATH=${{github.workspace}}/local/lib/pkgconfig cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE + + - name: Build + working-directory: ${{github.workspace}}/build + shell: bash + # Execute the build. You can specify a specific target with "--target " + run: cmake --build . --config $BUILD_TYPE -- -j 2 + + - name: Test + working-directory: ${{github.workspace}}/build + shell: bash + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: ctest -C $BUILD_TYPE unit-tests-linux: # The CMake configure and build commands are platform agnostic and should work equally @@ -99,7 +101,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: linux package install run: | @@ -166,7 +168,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: Remove libpcap from conanfile shell: bash @@ -396,7 +398,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: Configure CMake to generate VERSION shell: bash @@ -848,6 +850,7 @@ jobs: with: context: "." file: "./Dockerfile" + goarch: "arm64" - name: Debug artifacts run: ls -lha . diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 44c6d756f..539cfdccd 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -32,7 +32,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: Setup Conan Cache uses: actions/cache@v3 @@ -84,7 +84,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: Configure CMake to generate VERSION shell: bash @@ -134,7 +134,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: Configure CMake to generate VERSION shell: bash @@ -279,7 +279,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: Remove libpcap from conanfile shell: bash @@ -423,6 +423,7 @@ jobs: uses: ./.github/actions/build-go with: context: "." + goarch: arm64 file: "./Dockerfile" - name: Debug artifacts diff --git a/.github/workflows/build_cross.yml b/.github/workflows/build_cross.yml index 195a2454f..c27514eb9 100644 --- a/.github/workflows/build_cross.yml +++ b/.github/workflows/build_cross.yml @@ -69,7 +69,7 @@ jobs: curl -L "${{matrix.toolchain}}" | tar -C toolchain -xz --strip-components=1 - name: Install Conan - run: pip install --no-cache-dir 'conan==1.59.0' --force-reinstall + run: pip install --no-cache-dir 'conan==1.61.0' --force-reinstall - name: Create Conan configuration run: | diff --git a/.github/workflows/build_debug.yml b/.github/workflows/build_debug.yml index e8320436e..573d2eaf1 100644 --- a/.github/workflows/build_debug.yml +++ b/.github/workflows/build_debug.yml @@ -33,7 +33,7 @@ jobs: id: conan uses: turtlebrowser/get-conan@main with: - version: 1.59.0 + version: 1.61.0 - name: Setup Conan Cache uses: actions/cache@v3 diff --git a/.github/workflows/code-ql.yml b/.github/workflows/code-ql.yml index 460cefa3c..42a34cadb 100644 --- a/.github/workflows/code-ql.yml +++ b/.github/workflows/code-ql.yml @@ -60,7 +60,7 @@ jobs: - run: | # Run Build - set up dependencies, env vars, compile, and make test #install conan - pip install --no-cache-dir 'conan==1.59.0' --force-reinstall + pip install --no-cache-dir 'conan==1.61.0' --force-reinstall # create conan config CONAN_V2_MODE=1 conan config init conan config set general.revisions_enabled=1 diff --git a/.gitignore b/.gitignore index 3dac48a13..83447f79a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ docs/internals/html appimage/*.AppImage /test-config*.yaml localconfig/ +.cache/ +.vscode/ +build/ diff --git a/3rd/datasketches/common/CMakeLists.txt b/3rd/datasketches/common/CMakeLists.txt index 73f0ba26c..add757425 100644 --- a/3rd/datasketches/common/CMakeLists.txt +++ b/3rd/datasketches/common/CMakeLists.txt @@ -37,4 +37,9 @@ target_sources(common ${CMAKE_CURRENT_SOURCE_DIR}/include/conditional_back_inserter.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/conditional_forward.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/ceiling_power_of_2.hpp - ) \ No newline at end of file + ${CMAKE_CURRENT_SOURCE_DIR}/include/kolmogorov_smirnov.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/kolmogorov_smirnov_impl.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/quantiles_sorted_view.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/quantiles_sorted_view_impl.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/version.hpp.in + ) diff --git a/3rd/datasketches/common/include/MurmurHash3.h b/3rd/datasketches/common/include/MurmurHash3.h index 610ce4b20..ed9962a1c 100644 --- a/3rd/datasketches/common/include/MurmurHash3.h +++ b/3rd/datasketches/common/include/MurmurHash3.h @@ -29,14 +29,13 @@ typedef unsigned char uint8_t; typedef unsigned int uint32_t; typedef unsigned __int64 uint64_t; -#define FORCE_INLINE __forceinline +#define MURMUR3_FORCE_INLINE __forceinline #include -#define ROTL32(x,y) _rotl(x,y) -#define ROTL64(x,y) _rotl64(x,y) +#define MURMUR3_ROTL64(x,y) _rotl64(x,y) -#define BIG_CONSTANT(x) (x) +#define MURMUR3_BIG_CONSTANT(x) (x) // Other compilers @@ -44,22 +43,16 @@ typedef unsigned __int64 uint64_t; #include -#define FORCE_INLINE inline __attribute__((always_inline)) - -inline uint32_t rotl32 ( uint32_t x, int8_t r ) -{ - return (x << r) | (x >> (32 - r)); -} +#define MURMUR3_FORCE_INLINE inline __attribute__((always_inline)) inline uint64_t rotl64 ( uint64_t x, int8_t r ) { return (x << r) | (x >> (64 - r)); } -#define ROTL32(x,y) rotl32(x,y) -#define ROTL64(x,y) rotl64(x,y) +#define MURMUR3_ROTL64(x,y) rotl64(x,y) -#define BIG_CONSTANT(x) (x##LLU) +#define MURMUR3_BIG_CONSTANT(x) (x##LLU) #endif // !defined(_MSC_VER) @@ -78,7 +71,7 @@ typedef struct { // Block read - if your platform needs to do endian-swapping or can only // handle aligned reads, do the conversion here -FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, size_t i ) +MURMUR3_FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, size_t i ) { uint64_t res; memcpy(&res, p + i, sizeof(res)); @@ -88,20 +81,21 @@ FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, size_t i ) //----------------------------------------------------------------------------- // Finalization mix - force all bits of a hash block to avalanche -FORCE_INLINE uint64_t fmix64 ( uint64_t k ) +MURMUR3_FORCE_INLINE uint64_t fmix64 ( uint64_t k ) { k ^= k >> 33; - k *= BIG_CONSTANT(0xff51afd7ed558ccd); + k *= MURMUR3_BIG_CONSTANT(0xff51afd7ed558ccd); k ^= k >> 33; - k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k *= MURMUR3_BIG_CONSTANT(0xc4ceb9fe1a85ec53); k ^= k >> 33; return k; } -FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t seed, HashState& out) { - static const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); - static const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); +MURMUR3_FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, + uint64_t seed, HashState& out) { + static const uint64_t c1 = MURMUR3_BIG_CONSTANT(0x87c37b91114253d5); + static const uint64_t c2 = MURMUR3_BIG_CONSTANT(0x4cf5ad432745937f); const uint8_t* data = (const uint8_t*)key; @@ -118,13 +112,13 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t uint64_t k1 = getblock64(blocks, i * 2 + 0); uint64_t k2 = getblock64(blocks, i * 2 + 1); - k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; out.h1 ^= k1; - out.h1 = ROTL64(out.h1,27); + k1 *= c1; k1 = MURMUR3_ROTL64(k1,31); k1 *= c2; out.h1 ^= k1; + out.h1 = MURMUR3_ROTL64(out.h1,27); out.h1 += out.h2; out.h1 = out.h1*5+0x52dce729; - k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; out.h2 ^= k2; - out.h2 = ROTL64(out.h2,31); + k2 *= c2; k2 = MURMUR3_ROTL64(k2,33); k2 *= c1; out.h2 ^= k2; + out.h2 = MURMUR3_ROTL64(out.h2,31); out.h2 += out.h1; out.h2 = out.h2*5+0x38495ab5; } @@ -144,7 +138,7 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t case 11: k2 ^= ((uint64_t)tail[10]) << 16; // falls through case 10: k2 ^= ((uint64_t)tail[ 9]) << 8; // falls through case 9: k2 ^= ((uint64_t)tail[ 8]) << 0; - k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; out.h2 ^= k2; + k2 *= c2; k2 = MURMUR3_ROTL64(k2,33); k2 *= c1; out.h2 ^= k2; // falls through case 8: k1 ^= ((uint64_t)tail[ 7]) << 56; // falls through case 7: k1 ^= ((uint64_t)tail[ 6]) << 48; // falls through @@ -154,7 +148,7 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t case 3: k1 ^= ((uint64_t)tail[ 2]) << 16; // falls through case 2: k1 ^= ((uint64_t)tail[ 1]) << 8; // falls through case 1: k1 ^= ((uint64_t)tail[ 0]) << 0; - k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; out.h1 ^= k1; + k1 *= c1; k1 = MURMUR3_ROTL64(k1,31); k1 *= c2; out.h1 ^= k1; }; //---------- @@ -175,10 +169,14 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t //----------------------------------------------------------------------------- -FORCE_INLINE uint16_t compute_seed_hash(uint64_t seed) { +MURMUR3_FORCE_INLINE uint16_t compute_seed_hash(uint64_t seed) { HashState hashes; MurmurHash3_x64_128(&seed, sizeof(seed), 0, hashes); return static_cast(hashes.h1 & 0xffff); } +#undef MURMUR3_FORCE_INLINE +#undef MURMUR3_ROTL64 +#undef MURMUR3_BIG_CONSTANT + #endif // _MURMURHASH3_H_ diff --git a/3rd/datasketches/common/include/binomial_bounds.hpp b/3rd/datasketches/common/include/binomial_bounds.hpp index c1243e510..3b73535be 100644 --- a/3rd/datasketches/common/include/binomial_bounds.hpp +++ b/3rd/datasketches/common/include/binomial_bounds.hpp @@ -22,6 +22,7 @@ #include #include +#include /* * This class enables the estimation of error bounds given a sample set size, the sampling diff --git a/3rd/datasketches/common/include/common_defs.hpp b/3rd/datasketches/common/include/common_defs.hpp index dadcaac02..fc80fe022 100644 --- a/3rd/datasketches/common/include/common_defs.hpp +++ b/3rd/datasketches/common/include/common_defs.hpp @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include namespace datasketches { @@ -34,6 +37,19 @@ enum resize_factor { X1 = 0, X2, X4, X8 }; template using AllocChar = typename std::allocator_traits::template rebind_alloc; template using string = std::basic_string, AllocChar>; +// thread-safe random bit +static thread_local std::independent_bits_engine + random_bit(static_cast(std::chrono::system_clock::now().time_since_epoch().count() + + std::hash{}(std::this_thread::get_id()))); + +// common random declarations +namespace random_utils { + static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2 + static thread_local std::mt19937_64 rand(rd()); + static thread_local std::uniform_real_distribution<> next_double(0.0, 1.0); +} + + // utility function to hide unused compiler warning // usually has no additional cost template void unused(T&&...) {} @@ -63,7 +79,7 @@ static inline void read(std::istream& is, T* ptr, size_t size_bytes) { } template -static inline void write(std::ostream& os, T& value) { +static inline void write(std::ostream& os, T value) { os.write(reinterpret_cast(&value), sizeof(T)); } @@ -72,6 +88,16 @@ static inline void write(std::ostream& os, const T* ptr, size_t size_bytes) { os.write(reinterpret_cast(ptr), size_bytes); } +// wrapper for iterators to implement operator-> returning temporary value +template +class return_value_holder { +public: + return_value_holder(T value): value_(value) {} + const T* operator->() const { return std::addressof(value_); } +private: + T value_; +}; + } // namespace #endif // _COMMON_DEFS_HPP_ diff --git a/3rd/datasketches/common/include/count_zeros.hpp b/3rd/datasketches/common/include/count_zeros.hpp index cdd994026..51cbc0c90 100644 --- a/3rd/datasketches/common/include/count_zeros.hpp +++ b/3rd/datasketches/common/include/count_zeros.hpp @@ -91,6 +91,17 @@ static inline uint8_t count_leading_zeros_in_u64(uint64_t input) { return 56 + byte_leading_zeros_table[(input ) & FCLZ_MASK_08]; } +static inline uint8_t count_leading_zeros_in_u32(uint32_t input) { + if (input > FCLZ_MASK_24) + return byte_leading_zeros_table[(input >> 24) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_16) + return 8 + byte_leading_zeros_table[(input >> 16) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_08) + return 16 + byte_leading_zeros_table[(input >> 8) & FCLZ_MASK_08]; + if (true) + return 24 + byte_leading_zeros_table[(input ) & FCLZ_MASK_08]; +} + static inline uint8_t count_trailing_zeros_in_u32(uint32_t input) { for (int i = 0; i < 4; i++) { const int byte = input & 0xff; diff --git a/3rd/datasketches/kll/include/kolmogorov_smirnov.hpp b/3rd/datasketches/common/include/kolmogorov_smirnov.hpp similarity index 83% rename from 3rd/datasketches/kll/include/kolmogorov_smirnov.hpp rename to 3rd/datasketches/common/include/kolmogorov_smirnov.hpp index 90685225f..d00853de2 100644 --- a/3rd/datasketches/kll/include/kolmogorov_smirnov.hpp +++ b/3rd/datasketches/common/include/kolmogorov_smirnov.hpp @@ -25,7 +25,8 @@ namespace datasketches { class kolmogorov_smirnov { public: /** - * Computes the raw delta area between two KLL quantile sketches for the Kolmogorov-Smirnov Test. + * Computes the raw delta area between two quantile sketches for the Kolmogorov-Smirnov Test. + * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T. * @param sketch1 KLL sketch 1 * @param sketch2 KLL sketch 2 * @return the raw delta between two KLL quantile sketches @@ -37,6 +38,7 @@ class kolmogorov_smirnov { * Computes the adjusted delta area threshold for the Kolmogorov-Smirnov Test. * Adjusts the computed threshold by the error epsilons of the two given sketches. * See Kolmogorov–Smirnov Test + * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T. * @param sketch1 KLL sketch 1 * @param sketch2 KLL sketch 2 * @param p Target p-value. Typically .001 to .1, e.g., .05. @@ -46,7 +48,8 @@ class kolmogorov_smirnov { static double threshold(const Sketch& sketch1, const Sketch& sketch2, double p); /** - * Performs the Kolmogorov-Smirnov Test between two KLL quantiles sketches. + * Performs the Kolmogorov-Smirnov Test between two quantile sketches. + * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T. * Note: if the given sketches have insufficient data or if the sketch sizes are too small, * this will return false. * @param sketch1 KLL sketch 1 @@ -57,7 +60,6 @@ class kolmogorov_smirnov { */ template static bool test(const Sketch& sketch1, const Sketch& sketch2, double p); - }; } /* namespace datasketches */ diff --git a/3rd/datasketches/kll/include/kolmogorov_smirnov_impl.hpp b/3rd/datasketches/common/include/kolmogorov_smirnov_impl.hpp similarity index 69% rename from 3rd/datasketches/kll/include/kolmogorov_smirnov_impl.hpp rename to 3rd/datasketches/common/include/kolmogorov_smirnov_impl.hpp index 17eaf0d29..8cfb979de 100644 --- a/3rd/datasketches/kll/include/kolmogorov_smirnov_impl.hpp +++ b/3rd/datasketches/common/include/kolmogorov_smirnov_impl.hpp @@ -20,39 +20,36 @@ #ifndef KOLMOGOROV_SMIRNOV_IMPL_HPP_ #define KOLMOGOROV_SMIRNOV_IMPL_HPP_ -namespace datasketches { +#include +#include -// type resolver -template -kll_quantile_calculator make_quantile_calculator(const kll_sketch& sketch) { - return kll_quantile_calculator(sketch); -} +namespace datasketches { template double kolmogorov_smirnov::delta(const Sketch& sketch1, const Sketch& sketch2) { - using Comparator = typename Sketch::comparator; - auto calc1 = make_quantile_calculator(sketch1); - auto calc2 = make_quantile_calculator(sketch2); - auto it1 = calc1.begin(); - auto it2 = calc2.begin(); + auto comparator = sketch1.get_comparator(); // assuming the same comparator in sketch2 + auto view1 = sketch1.get_sorted_view(); + auto view2 = sketch2.get_sorted_view(); + auto it1 = view1.begin(); + auto it2 = view2.begin(); const auto n1 = sketch1.get_n(); const auto n2 = sketch2.get_n(); double delta = 0; - while (it1 != calc1.end() && it2 != calc2.end()) { - const double norm_cum_wt1 = static_cast((*it1).second) / n1; - const double norm_cum_wt2 = static_cast((*it2).second) / n2; + while (it1 != view1.end() && it2 != view2.end()) { + const double norm_cum_wt1 = static_cast(it1.get_cumulative_weight(false)) / n1; + const double norm_cum_wt2 = static_cast(it2.get_cumulative_weight(false)) / n2; delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2)); - if (Comparator()((*it1).first, (*it2).first)) { + if (comparator((*it1).first, (*it2).first)) { ++it1; - } else if (Comparator()((*it2).first, (*it1).first)) { + } else if (comparator((*it2).first, (*it1).first)) { ++it2; } else { ++it1; ++it2; } } - const double norm_cum_wt1 = it1 == calc1.end() ? 1 : static_cast((*it1).second) / n1; - const double norm_cum_wt2 = it2 == calc2.end() ? 1 : static_cast((*it2).second) / n2; + const double norm_cum_wt1 = it1 == view1.end() ? 1 : static_cast(it1.get_cumulative_weight(false)) / n1; + const double norm_cum_wt2 = it2 == view2.end() ? 1 : static_cast(it2.get_cumulative_weight(false)) / n2; delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2)); return delta; } diff --git a/3rd/datasketches/common/include/memory_operations.hpp b/3rd/datasketches/common/include/memory_operations.hpp index 986b2b066..e5cc058d9 100644 --- a/3rd/datasketches/common/include/memory_operations.hpp +++ b/3rd/datasketches/common/include/memory_operations.hpp @@ -23,6 +23,8 @@ #include #include #include +#include +#include namespace datasketches { @@ -53,14 +55,14 @@ static inline size_t copy_to_mem(const void* src, void* dst, size_t size) { } template -static inline size_t copy_to_mem(const T& item, void* dst) { - memcpy(dst, &item, sizeof(T)); +static inline size_t copy_from_mem(const void* src, T& item) { + memcpy(&item, src, sizeof(T)); return sizeof(T); } template -static inline size_t copy_from_mem(const void* src, T& item) { - memcpy(&item, src, sizeof(T)); +static inline size_t copy_to_mem(T item, void* dst) { + memcpy(dst, &item, sizeof(T)); return sizeof(T); } diff --git a/3rd/datasketches/common/include/quantiles_sorted_view.hpp b/3rd/datasketches/common/include/quantiles_sorted_view.hpp new file mode 100755 index 000000000..e965cb179 --- /dev/null +++ b/3rd/datasketches/common/include/quantiles_sorted_view.hpp @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef QUANTILES_SORTED_VIEW_HPP_ +#define QUANTILES_SORTED_VIEW_HPP_ + +#include +#include + +#include "common_defs.hpp" + +namespace datasketches { + +template< + typename T, + typename Comparator, // strict weak ordering function (see C++ named requirements: Compare) + typename Allocator +> +class quantiles_sorted_view { +public: + using Entry = typename std::conditional::value, std::pair, std::pair>::type; + using AllocEntry = typename std::allocator_traits::template rebind_alloc; + using Container = std::vector; + + quantiles_sorted_view(uint32_t num, const Comparator& comparator, const Allocator& allocator); + + template + void add(Iterator begin, Iterator end, uint64_t weight); + + void convert_to_cummulative(); + + class const_iterator; + const_iterator begin() const; + const_iterator end() const; + + size_t size() const; + + double get_rank(const T& item, bool inclusive = true) const; + + using quantile_return_type = typename std::conditional::value, T, const T&>::type; + quantile_return_type get_quantile(double rank, bool inclusive = true) const; + + using vector_double = std::vector::template rebind_alloc>; + vector_double get_CDF(const T* split_points, uint32_t size, bool inclusive = true) const; + vector_double get_PMF(const T* split_points, uint32_t size, bool inclusive = true) const; + +private: + Comparator comparator_; + uint64_t total_weight_; + Container entries_; + + static inline const T& deref_helper(const T* t) { return *t; } + static inline T deref_helper(T t) { return t; } + + struct compare_pairs_by_first { + explicit compare_pairs_by_first(const Comparator& comparator): comparator_(comparator) {} + bool operator()(const Entry& a, const Entry& b) const { + return comparator_(deref_helper(a.first), deref_helper(b.first)); + } + Comparator comparator_; + }; + + struct compare_pairs_by_second { + bool operator()(const Entry& a, const Entry& b) const { + return a.second < b.second; + } + }; + + template::value, int>::type = 0> + static inline T ref_helper(const T& t) { return t; } + + template::value, int>::type = 0> + static inline const T* ref_helper(const T& t) { return std::addressof(t); } + + template::value, int>::type = 0> + static inline Entry make_dummy_entry(uint64_t weight) { return Entry(0, weight); } + + template::value, int>::type = 0> + static inline Entry make_dummy_entry(uint64_t weight) { return Entry(nullptr, weight); } + + template::value, int>::type = 0> + static inline void check_split_points(const T* items, uint32_t size) { + for (uint32_t i = 0; i < size ; i++) { + if (std::isnan(items[i])) { + throw std::invalid_argument("Values must not be NaN"); + } + if ((i < (size - 1)) && !(Comparator()(items[i], items[i + 1]))) { + throw std::invalid_argument("Values must be unique and monotonically increasing"); + } + } + } + + template::value, int>::type = 0> + static inline void check_split_points(const T* items, uint32_t size) { + for (uint32_t i = 0; i < size ; i++) { + if ((i < (size - 1)) && !(Comparator()(items[i], items[i + 1]))) { + throw std::invalid_argument("Items must be unique and monotonically increasing"); + } + } + } +}; + +template +class quantiles_sorted_view::const_iterator: public quantiles_sorted_view::Container::const_iterator { +public: + using Base = typename quantiles_sorted_view::Container::const_iterator; + using value_type = typename std::conditional::value, typename Base::value_type, std::pair>::type; + + const_iterator(const Base& it, const Base& begin): Base(it), begin(begin) {} + + template::value, int>::type = 0> + const value_type operator*() const { return Base::operator*(); } + + template::value, int>::type = 0> + const value_type operator*() const { return value_type(*(Base::operator*().first), Base::operator*().second); } + + template::value, int>::type = 0> + const value_type* operator->() const { return Base::operator->(); } + + template::value, int>::type = 0> + const return_value_holder operator->() const { return **this; } + + uint64_t get_weight() const { + if (*this == begin) return Base::operator*().second; + return Base::operator*().second - (*this - 1).operator*().second; + } + + uint64_t get_cumulative_weight(bool inclusive = true) const { + return inclusive ? Base::operator*().second : Base::operator*().second - get_weight(); + } + +private: + Base begin; +}; + +} /* namespace datasketches */ + +#include "quantiles_sorted_view_impl.hpp" + +#endif diff --git a/3rd/datasketches/common/include/quantiles_sorted_view_impl.hpp b/3rd/datasketches/common/include/quantiles_sorted_view_impl.hpp new file mode 100755 index 000000000..326301e0e --- /dev/null +++ b/3rd/datasketches/common/include/quantiles_sorted_view_impl.hpp @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef QUANTILES_SORTED_VIEW_IMPL_HPP_ +#define QUANTILES_SORTED_VIEW_IMPL_HPP_ + +#include +#include +#include + +namespace datasketches { + +template +quantiles_sorted_view::quantiles_sorted_view(uint32_t num, const C& comparator, const A& allocator): +comparator_(comparator), +total_weight_(0), +entries_(allocator) +{ + entries_.reserve(num); +} + +template +template +void quantiles_sorted_view::add(Iterator first, Iterator last, uint64_t weight) { + const size_t size_before = entries_.size(); + for (auto it = first; it != last; ++it) entries_.push_back(Entry(ref_helper(*it), weight)); + if (size_before > 0) { + Container tmp(entries_.get_allocator()); + tmp.reserve(entries_.capacity()); + std::merge( + entries_.begin(), entries_.begin() + size_before, + entries_.begin() + size_before, entries_.end(), + std::back_inserter(tmp), compare_pairs_by_first(comparator_) + ); + std::swap(tmp, entries_); + } +} + +template +void quantiles_sorted_view::convert_to_cummulative() { + for (auto& entry: entries_) { + total_weight_ += entry.second; + entry.second = total_weight_; + } +} + +template +double quantiles_sorted_view::get_rank(const T& item, bool inclusive) const { + if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + auto it = inclusive ? + std::upper_bound(entries_.begin(), entries_.end(), Entry(ref_helper(item), 0), compare_pairs_by_first(comparator_)) + : std::lower_bound(entries_.begin(), entries_.end(), Entry(ref_helper(item), 0), compare_pairs_by_first(comparator_)); + // we need item just before + if (it == entries_.begin()) return 0; + --it; + return static_cast(it->second) / total_weight_; +} + +template +auto quantiles_sorted_view::get_quantile(double rank, bool inclusive) const -> quantile_return_type { + if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + uint64_t weight = inclusive ? std::ceil(rank * total_weight_) : rank * total_weight_; + auto it = inclusive ? + std::lower_bound(entries_.begin(), entries_.end(), make_dummy_entry(weight), compare_pairs_by_second()) + : std::upper_bound(entries_.begin(), entries_.end(), make_dummy_entry(weight), compare_pairs_by_second()); + if (it == entries_.end()) return deref_helper(entries_[entries_.size() - 1].first); + return deref_helper(it->first); +} + +template +auto quantiles_sorted_view::get_CDF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double { + if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + vector_double buckets(entries_.get_allocator()); + if (entries_.size() == 0) return buckets; + check_split_points(split_points, size); + buckets.reserve(size + 1); + for (uint32_t i = 0; i < size; ++i) buckets.push_back(get_rank(split_points[i], inclusive)); + buckets.push_back(1); + return buckets; +} + +template +auto quantiles_sorted_view::get_PMF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double { + auto buckets = get_CDF(split_points, size, inclusive); + if (buckets.size() == 0) return buckets; + for (uint32_t i = size; i > 0; --i) { + buckets[i] -= buckets[i - 1]; + } + return buckets; +} + +template +auto quantiles_sorted_view::begin() const -> const_iterator { + return const_iterator(entries_.begin(), entries_.begin()); +} + +template +auto quantiles_sorted_view::end() const -> const_iterator { + return const_iterator(entries_.end(), entries_.begin()); +} + +template +size_t quantiles_sorted_view::size() const { + return entries_.size(); +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/common/include/version.hpp.in b/3rd/datasketches/common/include/version.hpp.in new file mode 100644 index 000000000..04c1eb39c --- /dev/null +++ b/3rd/datasketches/common/include/version.hpp.in @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _VERSION_HPP_ +#define _VERSION_HPP_ + +namespace datasketches { + +// the configured options and settings for DataSketches +constexpr int VERSION_MAJOR {@DataSketches_VERSION_MAJOR@}; +constexpr int VERSION_MINOR {@DataSketches_VERSION_MINOR@}; +constexpr int VERSION_PATCH {@DataSketches_VERSION_PATCH@}; +constexpr int VERSION_TWEAK {@DataSketches_VERSION_TWEAK@}; + +constexpr auto VERSION_STR = "@DataSketches_VERSION@"; +constexpr auto SOURCE_URL = "https://github.com/apache/datasketches-cpp"; + +} + +#endif // _VERSION_HPP_ diff --git a/3rd/datasketches/cpc/include/cpc_compressor_impl.hpp b/3rd/datasketches/cpc/include/cpc_compressor_impl.hpp index c49bf0a12..7f323be7c 100644 --- a/3rd/datasketches/cpc/include/cpc_compressor_impl.hpp +++ b/3rd/datasketches/cpc/include/cpc_compressor_impl.hpp @@ -23,6 +23,7 @@ #define CPC_COMPRESSOR_IMPL_HPP_ #include +#include #include "compression_data.hpp" #include "cpc_util.hpp" @@ -296,6 +297,7 @@ void cpc_compressor::compress_sliding_flavor(const cpc_sketch_alloc& sourc // changes the implied ordering of the pairs, so we must do it before sorting. const uint8_t pseudo_phase = determine_pseudo_phase(source.get_lg_k(), source.get_num_coupons()); + if (pseudo_phase >= 16) throw std::logic_error("unexpected pseudo phase for sliding flavor"); const uint8_t* permutation = column_permutations_for_encoding[pseudo_phase]; const uint8_t offset = source.window_offset; @@ -332,7 +334,7 @@ void cpc_compressor::uncompress_sliding_flavor(const compressed_state& sou lg_k, source.table_data.get_allocator()); const uint8_t pseudo_phase = determine_pseudo_phase(lg_k, num_coupons); - if (pseudo_phase >= 16) throw std::logic_error("pseudo phase >= 16"); + if (pseudo_phase >= 16) throw std::logic_error("unexpected pseudo phase for sliding flavor"); const uint8_t* permutation = column_permutations_for_decoding[pseudo_phase]; uint8_t offset = cpc_sketch_alloc::determine_correct_offset(lg_k, num_coupons); @@ -447,7 +449,7 @@ uint8_t cpc_compressor::determine_pseudo_phase(uint8_t lg_k, uint32_t c) { if (lg_k < 4) throw std::logic_error("lgK < 4"); const size_t tmp = c >> (lg_k - 4); const uint8_t phase = tmp & 15; - if (phase < 0 || phase >= 16) throw std::out_of_range("wrong phase"); + if (phase >= 16) throw std::out_of_range("wrong phase"); return phase; } } diff --git a/3rd/datasketches/cpc/include/cpc_confidence.hpp b/3rd/datasketches/cpc/include/cpc_confidence.hpp index e5670575f..f77dfe56f 100644 --- a/3rd/datasketches/cpc/include/cpc_confidence.hpp +++ b/3rd/datasketches/cpc/include/cpc_confidence.hpp @@ -23,13 +23,14 @@ #define CPC_CONFIDENCE_HPP_ #include +#include #include "cpc_sketch.hpp" namespace datasketches { // ln 2.0 -static const double ICON_ERROT_CONSTANT = 0.693147180559945286; +static const double ICON_ERROR_CONSTANT = 0.693147180559945286; // 1, 2, 3, // kappa static const int16_t ICON_LOW_SIDE_DATA [33] = { // Empirically measured at N = 1000 * K. @@ -101,7 +102,7 @@ double get_icon_confidence_lb(const cpc_sketch_alloc& sketch, int kappa) { const long k = 1 << lg_k; if (lg_k < 4) throw std::logic_error("lgk < 4"); if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3"); - double x = ICON_ERROT_CONSTANT; + double x = ICON_ERROR_CONSTANT; if (lg_k <= 14) x = ((double) ICON_HIGH_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0; const double rel = x / sqrt(k); const double eps = kappa * rel; @@ -119,7 +120,7 @@ double get_icon_confidence_ub(const cpc_sketch_alloc& sketch, int kappa) { const long k = 1 << lg_k; if (lg_k < 4) throw std::logic_error("lgk < 4"); if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3"); - double x = ICON_ERROT_CONSTANT; + double x = ICON_ERROR_CONSTANT; if (lg_k <= 14) x = ((double) ICON_LOW_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0; const double rel = x / sqrt(k); const double eps = kappa * rel; diff --git a/3rd/datasketches/cpc/include/cpc_sketch_impl.hpp b/3rd/datasketches/cpc/include/cpc_sketch_impl.hpp index 0881b7035..c5f467e88 100644 --- a/3rd/datasketches/cpc/include/cpc_sketch_impl.hpp +++ b/3rd/datasketches/cpc/include/cpc_sketch_impl.hpp @@ -359,7 +359,7 @@ void cpc_sketch_alloc::refresh_kxp(const uint64_t* bit_matrix) { // for improved numerical accuracy, we separately sum the bytes of the U64's double byte_sums[8]; // allocating on the stack - std::fill(byte_sums, &byte_sums[8], 0); + std::fill(byte_sums, byte_sums + 8, 0); for (size_t i = 0; i < k; i++) { uint64_t word = bit_matrix[i]; diff --git a/3rd/datasketches/cpc/include/cpc_union_impl.hpp b/3rd/datasketches/cpc/include/cpc_union_impl.hpp index 324d258a8..5865cf515 100644 --- a/3rd/datasketches/cpc/include/cpc_union_impl.hpp +++ b/3rd/datasketches/cpc/include/cpc_union_impl.hpp @@ -22,6 +22,8 @@ #include "count_zeros.hpp" +#include + namespace datasketches { template diff --git a/3rd/datasketches/cpc/include/cpc_util.hpp b/3rd/datasketches/cpc/include/cpc_util.hpp index 9bf8aa38e..e5664951a 100644 --- a/3rd/datasketches/cpc/include/cpc_util.hpp +++ b/3rd/datasketches/cpc/include/cpc_util.hpp @@ -89,7 +89,13 @@ static inline uint32_t warren_count_bits_set_in_matrix(const uint64_t* array, ui // This code is Figure 5-9 in "Hacker's Delight" by Henry S. Warren. -#define CSA(h,l,a,b,c) {uint64_t u = a ^ b; uint64_t v = c; h = (a & b) | (u & v); l = u ^ v;} +#define DATASKETCHES_CSA(h, l, a, b, c) \ + { \ + uint64_t u = a ^ b; \ + uint64_t v = c; \ + h = (a & b) | (u & v); \ + l = u ^ v; \ + } static inline uint32_t count_bits_set_in_matrix(const uint64_t* a, uint32_t length) { if ((length & 0x7) != 0) throw std::invalid_argument("the length of the array must be a multiple of 8"); @@ -98,15 +104,15 @@ static inline uint32_t count_bits_set_in_matrix(const uint64_t* a, uint32_t leng fours = twos = ones = 0; for (uint32_t i = 0; i <= length - 8; i += 8) { - CSA(twos_a, ones, ones, a[i+0], a[i+1]); - CSA(twos_b, ones, ones, a[i+2], a[i+3]); - CSA(fours_a, twos, twos, twos_a, twos_b); + DATASKETCHES_CSA(twos_a, ones, ones, a[i+0], a[i+1]); + DATASKETCHES_CSA(twos_b, ones, ones, a[i+2], a[i+3]); + DATASKETCHES_CSA(fours_a, twos, twos, twos_a, twos_b); - CSA(twos_a, ones, ones, a[i+4], a[i+5]); - CSA(twos_b, ones, ones, a[i+6], a[i+7]); - CSA(fours_b, twos, twos, twos_a, twos_b); + DATASKETCHES_CSA(twos_a, ones, ones, a[i+4], a[i+5]); + DATASKETCHES_CSA(twos_b, ones, ones, a[i+6], a[i+7]); + DATASKETCHES_CSA(fours_b, twos, twos, twos_a, twos_b); - CSA(eights, fours, fours, fours_a, fours_b); + DATASKETCHES_CSA(eights, fours, fours, fours_a, fours_b); total += warren_bit_count(eights); } @@ -119,6 +125,8 @@ static inline uint32_t count_bits_set_in_matrix(const uint64_t* a, uint32_t leng return total; } +#undef DATASKETCHES_CSA + // Here are some timings made with quickTestMerge.c // for the "5 5" case: diff --git a/3rd/datasketches/fi/include/frequent_items_sketch.hpp b/3rd/datasketches/fi/include/frequent_items_sketch.hpp index 6efe2b906..02a08540b 100644 --- a/3rd/datasketches/fi/include/frequent_items_sketch.hpp +++ b/3rd/datasketches/fi/include/frequent_items_sketch.hpp @@ -34,7 +34,7 @@ namespace datasketches { /* * Based on Java implementation here: - * https://github.com/DataSketches/sketches-core/blob/master/src/main/java/com/yahoo/sketches/frequencies/ItemsSketch.java + * https://github.com/apache/datasketches-java/blob/master/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java * author Alexander Saydakov */ @@ -46,7 +46,6 @@ template< typename W = uint64_t, typename H = std::hash, typename E = std::equal_to, - typename S = serde, typename A = std::allocator > class frequent_items_sketch { @@ -60,11 +59,13 @@ class frequent_items_sketch { * @param lg_max_map_size Log2 of the physical size of the internal hash map managed by this * sketch. The maximum capacity of this internal hash map is 0.75 times 2^lg_max_map_size. * Both the ultimate accuracy and size of this sketch are functions of lg_max_map_size. - * * @param lg_start_map_size Log2 of the starting physical size of the internal hash * map managed by this sketch. + * @param equal instance of Equality operator + * @param allocator instance of an Allocator */ - explicit frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size = LG_MIN_MAP_SIZE, const A& allocator = A()); + explicit frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size = LG_MIN_MAP_SIZE, + const E& equal = E(), const A& allocator = A()); /** * Update this sketch with an item and a positive weight (frequency count). @@ -158,7 +159,7 @@ class frequent_items_sketch { /** * Returns epsilon used to compute a priori error. * This is just the value 3.5 / maxMapSize. - * @param maxMapSize the planned map size to be used when constructing this sketch. + * @param lg_max_map_size the planned map size to be used when constructing this sketch. * @return epsilon used to compute a priori error. */ static double get_epsilon(uint8_t lg_max_map_size); @@ -167,13 +168,13 @@ class frequent_items_sketch { * Returns the estimated a priori error given the max_map_size for the sketch and the * estimated_total_stream_weight. * @param lg_max_map_size the planned map size to be used when constructing this sketch. - * @param estimated_total_stream_weight the estimated total stream weight. + * @param estimated_total_weight the estimated total stream weight. * @return the estimated a priori error. */ static double get_apriori_error(uint8_t lg_max_map_size, W estimated_total_weight); class row; - typedef typename std::vector::template rebind_alloc> vector_row; // alias for users + using vector_row = typename std::vector::template rebind_alloc>; /** * Returns an array of rows that include frequent items, estimates, upper and lower bounds @@ -225,45 +226,60 @@ class frequent_items_sketch { /** * Computes size needed to serialize the current state of the sketch. * This can be expensive since every item needs to be looked at. + * @param sd instance of a SerDe * @return size in bytes needed to serialize this sketch */ - size_t get_serialized_size_bytes() const; + template> + size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const; /** * This method serializes the sketch into a given stream in a binary form * @param os output stream + * @param sd instance of a SerDe */ - void serialize(std::ostream& os) const; + template> + void serialize(std::ostream& os, const SerDe& sd = SerDe()) const; // This is a convenience alias for users // The type returned by the following serialize method using vector_bytes = std::vector::template rebind_alloc>; - /** * This method serializes the sketch as a vector of bytes. * An optional header can be reserved in front of the sketch. * It is a blank space of a given size. * This header is used in Datasketches PostgreSQL extension. * @param header_size_bytes space to reserve in front of the sketch + * @param sd instance of a SerDe * @return serialized sketch as a vector of bytes */ - vector_bytes serialize(unsigned header_size_bytes = 0) const; + template> + vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& sd = SerDe()) const; /** * This method deserializes a sketch from a given stream. * @param is input stream + * @param sd instance of a SerDe + * @param equal instance of Equality operator + * @param allocator instance of an Allocator * @return an instance of the sketch */ - static frequent_items_sketch deserialize(std::istream& is, const A& allocator = A()); + template> + static frequent_items_sketch deserialize(std::istream& is, const SerDe& sd = SerDe(), + const E& equal = E(), const A& allocator = A()); /** * This method deserializes a sketch from a given array of bytes. * @param bytes pointer to the array of bytes * @param size the size of the array + * @param sd instance of a SerDe + * @param equal instance of Equality operator + * @param allocator instance of an Allocator * @return an instance of the sketch */ - static frequent_items_sketch deserialize(const void* bytes, size_t size, const A& allocator = A()); + template> + static frequent_items_sketch deserialize(const void* bytes, size_t size, const SerDe& sd = SerDe(), + const E& equal = E(), const A& allocator = A()); /** * Returns a human readable summary of this sketch @@ -302,8 +318,8 @@ class frequent_items_sketch { class items_deleter; }; -template -class frequent_items_sketch::row { +template +class frequent_items_sketch::row { public: row(const T* item, W weight, W offset): item(item), weight(weight), offset(offset) {} diff --git a/3rd/datasketches/fi/include/frequent_items_sketch_impl.hpp b/3rd/datasketches/fi/include/frequent_items_sketch_impl.hpp index 602dfc627..d72f46248 100644 --- a/3rd/datasketches/fi/include/frequent_items_sketch_impl.hpp +++ b/3rd/datasketches/fi/include/frequent_items_sketch_impl.hpp @@ -23,46 +23,49 @@ #include #include #include +#include #include "memory_operations.hpp" namespace datasketches { // clang++ seems to require this declaration for CMAKE_BUILD_TYPE='Debug" -template -const uint8_t frequent_items_sketch::LG_MIN_MAP_SIZE; +template +const uint8_t frequent_items_sketch::LG_MIN_MAP_SIZE; -template -frequent_items_sketch::frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size, const A& allocator): +template +frequent_items_sketch::frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size, + const E& equal, const A& allocator): total_weight(0), offset(0), map( std::max(lg_start_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE), std::max(lg_max_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE), + equal, allocator ) { if (lg_start_map_size > lg_max_map_size) throw std::invalid_argument("starting size must not be greater than maximum size"); } -template -void frequent_items_sketch::update(const T& item, W weight) { +template +void frequent_items_sketch::update(const T& item, W weight) { check_weight(weight); if (weight == 0) return; total_weight += weight; offset += map.adjust_or_insert(item, weight); } -template -void frequent_items_sketch::update(T&& item, W weight) { +template +void frequent_items_sketch::update(T&& item, W weight) { check_weight(weight); if (weight == 0) return; total_weight += weight; offset += map.adjust_or_insert(std::move(item), weight); } -template -void frequent_items_sketch::merge(const frequent_items_sketch& other) { +template +void frequent_items_sketch::merge(const frequent_items_sketch& other) { if (other.is_empty()) return; const W merged_total_weight = total_weight + other.get_total_weight(); // for correction at the end for (auto it: other.map) { @@ -72,8 +75,8 @@ void frequent_items_sketch::merge(const frequent_items_sketch& total_weight = merged_total_weight; } -template -void frequent_items_sketch::merge(frequent_items_sketch&& other) { +template +void frequent_items_sketch::merge(frequent_items_sketch&& other) { if (other.is_empty()) return; const W merged_total_weight = total_weight + other.get_total_weight(); // for correction at the end for (auto it: other.map) { @@ -83,69 +86,67 @@ void frequent_items_sketch::merge(frequent_items_sketch&& othe total_weight = merged_total_weight; } -template -bool frequent_items_sketch::is_empty() const { +template +bool frequent_items_sketch::is_empty() const { return map.get_num_active() == 0; } -template -uint32_t frequent_items_sketch::get_num_active_items() const { +template +uint32_t frequent_items_sketch::get_num_active_items() const { return map.get_num_active(); } -template -W frequent_items_sketch::get_total_weight() const { +template +W frequent_items_sketch::get_total_weight() const { return total_weight; } -template -W frequent_items_sketch::get_estimate(const T& item) const { +template +W frequent_items_sketch::get_estimate(const T& item) const { // if item is tracked estimate = weight + offset, otherwise 0 const W weight = map.get(item); if (weight > 0) return weight + offset; return 0; } -template -W frequent_items_sketch::get_lower_bound(const T& item) const { +template +W frequent_items_sketch::get_lower_bound(const T& item) const { return map.get(item); } -template -W frequent_items_sketch::get_upper_bound(const T& item) const { +template +W frequent_items_sketch::get_upper_bound(const T& item) const { return map.get(item) + offset; } -template -W frequent_items_sketch::get_maximum_error() const { +template +W frequent_items_sketch::get_maximum_error() const { return offset; } -template -double frequent_items_sketch::get_epsilon() const { +template +double frequent_items_sketch::get_epsilon() const { return EPSILON_FACTOR / (1 << map.get_lg_max_size()); } -template -double frequent_items_sketch::get_epsilon(uint8_t lg_max_map_size) { +template +double frequent_items_sketch::get_epsilon(uint8_t lg_max_map_size) { return EPSILON_FACTOR / (1 << lg_max_map_size); } -template -double frequent_items_sketch::get_apriori_error(uint8_t lg_max_map_size, W estimated_total_weight) { +template +double frequent_items_sketch::get_apriori_error(uint8_t lg_max_map_size, W estimated_total_weight) { return get_epsilon(lg_max_map_size) * estimated_total_weight; } -template -typename frequent_items_sketch::vector_row -frequent_items_sketch::get_frequent_items(frequent_items_error_type err_type) const { +template +auto frequent_items_sketch::get_frequent_items(frequent_items_error_type err_type) const -> vector_row { return get_frequent_items(err_type, get_maximum_error()); } -template -typename frequent_items_sketch::vector_row -frequent_items_sketch::get_frequent_items(frequent_items_error_type err_type, W threshold) const { +template +auto frequent_items_sketch::get_frequent_items(frequent_items_error_type err_type, W threshold) const -> vector_row { vector_row items(map.get_allocator()); for (auto it: map) { const W lb = it.second; @@ -159,8 +160,9 @@ frequent_items_sketch::get_frequent_items(frequent_items_error return items; } -template -void frequent_items_sketch::serialize(std::ostream& os) const { +template +template +void frequent_items_sketch::serialize(std::ostream& os, const SerDe& sd) const { const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY; write(os, preamble_longs); const uint8_t serial_version = SERIAL_VERSION; @@ -198,23 +200,25 @@ void frequent_items_sketch::serialize(std::ostream& os) const } write(os, weights, sizeof(W) * num_items); aw.deallocate(weights, num_items); - S().serialize(os, items, num_items); + sd.serialize(os, items, num_items); for (i = 0; i < num_items; i++) items[i].~T(); alloc.deallocate(items, num_items); } } -template -size_t frequent_items_sketch::get_serialized_size_bytes() const { +template +template +size_t frequent_items_sketch::get_serialized_size_bytes(const SerDe& sd) const { if (is_empty()) return PREAMBLE_LONGS_EMPTY * sizeof(uint64_t); size_t size = PREAMBLE_LONGS_NONEMPTY * sizeof(uint64_t) + map.get_num_active() * sizeof(W); - for (auto it: map) size += S().size_of_item(it.first); + for (auto it: map) size += sd.size_of_item(it.first); return size; } -template -auto frequent_items_sketch::serialize(unsigned header_size_bytes) const -> vector_bytes { - const size_t size = header_size_bytes + get_serialized_size_bytes(); +template +template +auto frequent_items_sketch::serialize(unsigned header_size_bytes, const SerDe& sd) const -> vector_bytes { + const size_t size = header_size_bytes + get_serialized_size_bytes(sd); vector_bytes bytes(size, 0, map.get_allocator()); uint8_t* ptr = bytes.data() + header_size_bytes; uint8_t* end_ptr = ptr + size; @@ -255,15 +259,15 @@ auto frequent_items_sketch::serialize(unsigned header_size_byt ptr += copy_to_mem(weights, ptr, sizeof(W) * num_items); aw.deallocate(weights, num_items); const size_t bytes_remaining = end_ptr - ptr; - ptr += S().serialize(ptr, bytes_remaining, items, num_items); + ptr += sd.serialize(ptr, bytes_remaining, items, num_items); for (i = 0; i < num_items; i++) items[i].~T(); alloc.deallocate(items, num_items); } return bytes; } -template -class frequent_items_sketch::items_deleter { +template +class frequent_items_sketch::items_deleter { public: items_deleter(uint32_t num, bool destroy, const A& allocator): allocator_(allocator), num_(num), destroy_(destroy) {} @@ -282,8 +286,10 @@ class frequent_items_sketch::items_deleter { bool destroy_; }; -template -frequent_items_sketch frequent_items_sketch::deserialize(std::istream& is, const A& allocator) { +template +template +frequent_items_sketch frequent_items_sketch::deserialize(std::istream& is, + const SerDe& sd, const E& equal, const A& allocator) { const auto preamble_longs = read(is); const auto serial_version = read(is); const auto family_id = read(is); @@ -299,7 +305,7 @@ frequent_items_sketch frequent_items_sketch: check_family_id(family_id); check_size(lg_cur_size, lg_max_size); - frequent_items_sketch sketch(lg_max_size, lg_cur_size, allocator); + frequent_items_sketch sketch(lg_max_size, lg_cur_size, equal, allocator); if (!is_empty) { const auto num_items = read(is); read(is); // unused @@ -312,7 +318,7 @@ frequent_items_sketch frequent_items_sketch: read(is, weights.data(), sizeof(W) * num_items); A alloc(allocator); std::unique_ptr items(alloc.allocate(num_items), items_deleter(num_items, false, alloc)); - S().deserialize(is, items.get(), num_items); + sd.deserialize(is, items.get(), num_items); items.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed for (uint32_t i = 0; i < num_items; i++) { sketch.update(std::move(items.get()[i]), weights[i]); @@ -325,8 +331,10 @@ frequent_items_sketch frequent_items_sketch: return sketch; } -template -frequent_items_sketch frequent_items_sketch::deserialize(const void* bytes, size_t size, const A& allocator) { +template +template +frequent_items_sketch frequent_items_sketch::deserialize(const void* bytes, size_t size, + const SerDe& sd, const E& equal, const A& allocator) { ensure_minimum_memory(size, 8); const char* ptr = static_cast(bytes); const char* base = static_cast(bytes); @@ -352,7 +360,7 @@ frequent_items_sketch frequent_items_sketch: check_size(lg_cur_size, lg_max_size); ensure_minimum_memory(size, preamble_longs * sizeof(uint64_t)); - frequent_items_sketch sketch(lg_max_size, lg_cur_size, allocator); + frequent_items_sketch sketch(lg_max_size, lg_cur_size, equal, allocator); if (!is_empty) { uint32_t num_items; ptr += copy_from_mem(ptr, num_items); @@ -370,7 +378,7 @@ frequent_items_sketch frequent_items_sketch: A alloc(allocator); std::unique_ptr items(alloc.allocate(num_items), items_deleter(num_items, false, alloc)); const size_t bytes_remaining = size - (ptr - base); - ptr += S().deserialize(ptr, bytes_remaining, items.get(), num_items); + ptr += sd.deserialize(ptr, bytes_remaining, items.get(), num_items); items.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed for (uint32_t i = 0; i < num_items; i++) { sketch.update(std::move(items.get()[i]), weights[i]); @@ -382,8 +390,8 @@ frequent_items_sketch frequent_items_sketch: return sketch; } -template -void frequent_items_sketch::check_preamble_longs(uint8_t preamble_longs, bool is_empty) { +template +void frequent_items_sketch::check_preamble_longs(uint8_t preamble_longs, bool is_empty) { if (is_empty) { if (preamble_longs != PREAMBLE_LONGS_EMPTY) { throw std::invalid_argument("Possible corruption: preamble longs of an empty sketch must be " + std::to_string(PREAMBLE_LONGS_EMPTY) + ": " + std::to_string(preamble_longs)); @@ -395,22 +403,22 @@ void frequent_items_sketch::check_preamble_longs(uint8_t pream } } -template -void frequent_items_sketch::check_serial_version(uint8_t serial_version) { +template +void frequent_items_sketch::check_serial_version(uint8_t serial_version) { if (serial_version != SERIAL_VERSION) { throw std::invalid_argument("Possible corruption: serial version must be " + std::to_string(SERIAL_VERSION) + ": " + std::to_string(serial_version)); } } -template -void frequent_items_sketch::check_family_id(uint8_t family_id) { +template +void frequent_items_sketch::check_family_id(uint8_t family_id) { if (family_id != FAMILY_ID) { throw std::invalid_argument("Possible corruption: family ID must be " + std::to_string(FAMILY_ID) + ": " + std::to_string(family_id)); } } -template -void frequent_items_sketch::check_size(uint8_t lg_cur_size, uint8_t lg_max_size) { +template +void frequent_items_sketch::check_size(uint8_t lg_cur_size, uint8_t lg_max_size) { if (lg_cur_size > lg_max_size) { throw std::invalid_argument("Possible corruption: expected lg_cur_size <= lg_max_size: " + std::to_string(lg_cur_size) + " <= " + std::to_string(lg_max_size)); } @@ -419,8 +427,8 @@ void frequent_items_sketch::check_size(uint8_t lg_cur_size, ui } } -template -string frequent_items_sketch::to_string(bool print_items) const { +template +string frequent_items_sketch::to_string(bool print_items) const { // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements. // The stream does not support passing an allocator instance, and alternatives are complicated. std::ostringstream os; @@ -450,23 +458,23 @@ string frequent_items_sketch::to_string(bool print_items) c } // version for integral signed type -template +template template::value && std::is_signed::value, int>::type> -void frequent_items_sketch::check_weight(WW weight) { +void frequent_items_sketch::check_weight(WW weight) { if (weight < 0) { throw std::invalid_argument("weight must be non-negative"); } } // version for integral unsigned type - no-op -template +template template::value && std::is_unsigned::value, int>::type> -void frequent_items_sketch::check_weight(WW) {} +void frequent_items_sketch::check_weight(WW) {} // version for floating point type -template +template template::value, int>::type> -void frequent_items_sketch::check_weight(WW weight) { +void frequent_items_sketch::check_weight(WW weight) { if (weight < 0) { throw std::invalid_argument("weight must be non-negative"); } diff --git a/3rd/datasketches/fi/include/reverse_purge_hash_map.hpp b/3rd/datasketches/fi/include/reverse_purge_hash_map.hpp index fc4cd83b5..b75abc43c 100644 --- a/3rd/datasketches/fi/include/reverse_purge_hash_map.hpp +++ b/3rd/datasketches/fi/include/reverse_purge_hash_map.hpp @@ -29,21 +29,27 @@ namespace datasketches { * This is a specialized linear-probing hash map with a reverse purge operation * that removes all entries in the map with values that are less than zero. * Based on Java implementation here: - * https://github.com/DataSketches/sketches-core/blob/master/src/main/java/com/yahoo/sketches/frequencies/ReversePurgeItemHashMap.java + * https://github.com/apache/datasketches-java/blob/master/src/main/java/org/apache/datasketches/frequencies/ReversePurgeItemHashMap.java * author Alexander Saydakov */ -template, typename E = std::equal_to, typename A = std::allocator> +template< + typename K, + typename V = uint64_t, + typename H = std::hash, + typename E = std::equal_to, + typename A = std::allocator +> class reverse_purge_hash_map { public: using AllocV = typename std::allocator_traits::template rebind_alloc; using AllocU16 = typename std::allocator_traits::template rebind_alloc; - reverse_purge_hash_map(uint8_t lg_size, uint8_t lg_max_size, const A& allocator); + reverse_purge_hash_map(uint8_t lg_size, uint8_t lg_max_size, const E& equal, const A& allocator); reverse_purge_hash_map(const reverse_purge_hash_map& other); reverse_purge_hash_map(reverse_purge_hash_map&& other) noexcept; ~reverse_purge_hash_map(); - reverse_purge_hash_map& operator=(reverse_purge_hash_map other); + reverse_purge_hash_map& operator=(const reverse_purge_hash_map& other); reverse_purge_hash_map& operator=(reverse_purge_hash_map&& other); template @@ -65,6 +71,7 @@ class reverse_purge_hash_map { static constexpr uint16_t DRIFT_LIMIT = 1024; // used only for stress testing static constexpr uint32_t MAX_SAMPLE_SIZE = 1024; // number of samples to compute approximate median during purge + E equal_; A allocator_; uint8_t lg_cur_size_; uint8_t lg_max_size_; @@ -84,8 +91,14 @@ class reverse_purge_hash_map { // This iterator uses strides based on golden ratio to avoid clustering during merge template -class reverse_purge_hash_map::iterator: public std::iterator { +class reverse_purge_hash_map::iterator { public: + using iterator_category = std::input_iterator_tag; + using value_type = std::pair; + using difference_type = void; + using pointer = void; + using reference = const value_type; + friend class reverse_purge_hash_map; iterator& operator++() { ++count; @@ -100,8 +113,8 @@ class reverse_purge_hash_map::iterator: public std::iterator operator*() const { - return std::pair(map->keys_[index], map->values_[index]); + reference operator*() const { + return value_type(map->keys_[index], map->values_[index]); } private: static constexpr double GOLDEN_RATIO_RECIPROCAL = 0.6180339887498949; // = (sqrt(5) - 1) / 2 diff --git a/3rd/datasketches/fi/include/reverse_purge_hash_map_impl.hpp b/3rd/datasketches/fi/include/reverse_purge_hash_map_impl.hpp index 0b05d8996..fa2ad8242 100644 --- a/3rd/datasketches/fi/include/reverse_purge_hash_map_impl.hpp +++ b/3rd/datasketches/fi/include/reverse_purge_hash_map_impl.hpp @@ -34,7 +34,9 @@ template constexpr uint32_t reverse_purge_hash_map::MAX_SAMPLE_SIZE; template -reverse_purge_hash_map::reverse_purge_hash_map(uint8_t lg_cur_size, uint8_t lg_max_size, const A& allocator): +reverse_purge_hash_map::reverse_purge_hash_map(uint8_t lg_cur_size, uint8_t lg_max_size, + const E& equal, const A& allocator): +equal_(equal), allocator_(allocator), lg_cur_size_(lg_cur_size), lg_max_size_(lg_max_size), @@ -52,6 +54,7 @@ states_(nullptr) template reverse_purge_hash_map::reverse_purge_hash_map(const reverse_purge_hash_map& other): +equal_(other.equal_), allocator_(other.allocator_), lg_cur_size_(other.lg_cur_size_), lg_max_size_(other.lg_max_size_), @@ -71,8 +74,8 @@ states_(nullptr) if (other.states_[i] > 0) { new (&keys_[i]) K(other.keys_[i]); values_[i] = other.values_[i]; + if (--num == 0) break; } - if (--num == 0) break; } } std::copy(other.states_, other.states_ + size, states_); @@ -80,6 +83,7 @@ states_(nullptr) template reverse_purge_hash_map::reverse_purge_hash_map(reverse_purge_hash_map&& other) noexcept: +equal_(std::move(other.equal_)), allocator_(std::move(other.allocator_)), lg_cur_size_(other.lg_cur_size_), lg_max_size_(other.lg_max_size_), @@ -119,19 +123,22 @@ reverse_purge_hash_map::~reverse_purge_hash_map() { } template -reverse_purge_hash_map& reverse_purge_hash_map::operator=(reverse_purge_hash_map other) { - std::swap(allocator_, other.allocator_); - std::swap(lg_cur_size_, other.lg_cur_size_); - std::swap(lg_max_size_, other.lg_max_size_); - std::swap(num_active_, other.num_active_); - std::swap(keys_, other.keys_); - std::swap(values_, other.values_); - std::swap(states_, other.states_); +reverse_purge_hash_map& reverse_purge_hash_map::operator=(const reverse_purge_hash_map& other) { + reverse_purge_hash_map copy(other); + std::swap(equal_, copy.equal_); + std::swap(allocator_, copy.allocator_); + std::swap(lg_cur_size_, copy.lg_cur_size_); + std::swap(lg_max_size_, copy.lg_max_size_); + std::swap(num_active_, copy.num_active_); + std::swap(keys_, copy.keys_); + std::swap(values_, copy.values_); + std::swap(states_, copy.states_); return *this; } template reverse_purge_hash_map& reverse_purge_hash_map::operator=(reverse_purge_hash_map&& other) { + std::swap(equal_, other.equal_); std::swap(allocator_, other.allocator_); std::swap(lg_cur_size_, other.lg_cur_size_); std::swap(lg_max_size_, other.lg_max_size_); diff --git a/3rd/datasketches/kll/CMakeLists.txt b/3rd/datasketches/kll/CMakeLists.txt index d33b529b1..4f3b49175 100644 --- a/3rd/datasketches/kll/CMakeLists.txt +++ b/3rd/datasketches/kll/CMakeLists.txt @@ -33,10 +33,6 @@ list(APPEND kll_HEADERS "include/kll_sketch.hpp") list(APPEND kll_HEADERS "include/kll_sketch_impl.hpp") list(APPEND kll_HEADERS "include/kll_helper.hpp") list(APPEND kll_HEADERS "include/kll_helper_impl.hpp") -list(APPEND kll_HEADERS "include/kll_quantile_calculator.hpp") -list(APPEND kll_HEADERS "include/kll_quantile_calculator_impl.hpp") -list(APPEND kll_HEADERS "include/kolmogorov_smirnov.hpp") -list(APPEND kll_HEADERS "include/kolmogorov_smirnov_impl.hpp") install(TARGETS kll EXPORT ${PROJECT_NAME} @@ -51,8 +47,4 @@ target_sources(kll ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_helper_impl.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_sketch.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_sketch_impl.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_quantile_calculator.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_quantile_calculator_impl.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/kolmogorov_smirnov.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/kolmogorov_smirnov_impl.hpp ) \ No newline at end of file diff --git a/3rd/datasketches/kll/include/kll_helper.hpp b/3rd/datasketches/kll/include/kll_helper.hpp index c026538ff..ff7a5fe09 100644 --- a/3rd/datasketches/kll/include/kll_helper.hpp +++ b/3rd/datasketches/kll/include/kll_helper.hpp @@ -20,15 +20,10 @@ #ifndef KLL_HELPER_HPP_ #define KLL_HELPER_HPP_ -#include #include -#include namespace datasketches { -static std::independent_bits_engine - random_bit(static_cast(std::chrono::system_clock::now().time_since_epoch().count())); - #ifdef KLL_VALIDATION extern uint32_t kll_next_offset; #endif @@ -52,38 +47,6 @@ class kll_helper { static inline uint16_t int_cap_aux_aux(uint16_t k, uint8_t depth); static inline uint64_t sum_the_sample_weights(uint8_t num_levels, const uint32_t* levels); - /* - * This version is for floating point types - * Checks the sequential validity of the given array of values. - * They must be unique, monotonically increasing and not NaN. - */ - template - static typename std::enable_if::value, void>::type - validate_values(const T* values, uint32_t size) { - for (uint32_t i = 0; i < size ; i++) { - if (std::isnan(values[i])) { - throw std::invalid_argument("Values must not be NaN"); - } - if ((i < (size - 1)) && !(C()(values[i], values[i + 1]))) { - throw std::invalid_argument("Values must be unique and monotonically increasing"); - } - } - } - /* - * This version is for non-floating point types - * Checks the sequential validity of the given array of values. - * They must be unique and monotonically increasing. - */ - template - static typename std::enable_if::value, void>::type - validate_values(const T* values, uint32_t size) { - for (uint32_t i = 0; i < size ; i++) { - if ((i < (size - 1)) && !(C()(values[i], values[i + 1]))) { - throw std::invalid_argument("Values must be unique and monotonically increasing"); - } - } - } - template static void randomly_halve_down(T* buf, uint32_t start, uint32_t length); diff --git a/3rd/datasketches/kll/include/kll_helper_impl.hpp b/3rd/datasketches/kll/include/kll_helper_impl.hpp index caa611b23..321761bb5 100644 --- a/3rd/datasketches/kll/include/kll_helper_impl.hpp +++ b/3rd/datasketches/kll/include/kll_helper_impl.hpp @@ -21,6 +21,9 @@ #define KLL_HELPER_IMPL_HPP_ #include +#include + +#include "common_defs.hpp" namespace datasketches { @@ -227,7 +230,7 @@ kll_helper::compress_result kll_helper::general_compress(uint16_t k, uint8_t m, // move level over as is // make sure we are not moving data upwards if (raw_beg < out_levels[current_level]) throw std::logic_error("wrong move"); - std::move(&items[raw_beg], &items[raw_lim], &items[out_levels[current_level]]); + std::move(items + raw_beg, items + raw_lim, items + out_levels[current_level]); out_levels[current_level + 1] = out_levels[current_level] + raw_pop; } else { // The sketch is too full AND this level is too full, so we compact it @@ -248,7 +251,7 @@ kll_helper::compress_result kll_helper::general_compress(uint16_t k, uint8_t m, // level zero might not be sorted, so we must sort it if we wish to compact it if ((current_level == 0) && !is_level_zero_sorted) { - std::sort(&items[adj_beg], &items[adj_beg + adj_pop], C()); + std::sort(items + adj_beg, items + adj_beg + adj_pop, C()); } if (pop_above == 0) { // Level above is empty, so halve up diff --git a/3rd/datasketches/kll/include/kll_quantile_calculator.hpp b/3rd/datasketches/kll/include/kll_quantile_calculator.hpp deleted file mode 100644 index a329ff264..000000000 --- a/3rd/datasketches/kll/include/kll_quantile_calculator.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifndef KLL_QUANTILE_CALCULATOR_HPP_ -#define KLL_QUANTILE_CALCULATOR_HPP_ - -#include - -namespace datasketches { - -// forward declaration -template class kll_sketch; - -template -class kll_quantile_calculator { - public: - using Entry = std::pair; - using AllocEntry = typename std::allocator_traits::template rebind_alloc; - using Container = std::vector; - using const_iterator = typename Container::const_iterator; - - template - kll_quantile_calculator(const kll_sketch& sketch); - - T get_quantile(double fraction) const; - const_iterator begin() const; - const_iterator end() const; - - private: - using AllocU32 = typename std::allocator_traits::template rebind_alloc; - using vector_u32 = std::vector; - uint64_t n_; - vector_u32 levels_; - Container entries_; - - void populate_from_sketch(const T* items, const uint32_t* levels, uint8_t num_levels); - T approximately_answer_positional_query(uint64_t pos) const; - void convert_to_preceding_cummulative(); - uint32_t chunk_containing_pos(uint64_t pos) const; - uint32_t search_for_chunk_containing_pos(uint64_t pos, uint64_t l, uint64_t r) const; - static void merge_sorted_blocks(Container& entries, const uint32_t* levels, uint8_t num_levels, uint32_t num_items); - static void merge_sorted_blocks_direct(Container& orig, Container& temp, const uint32_t* levels, uint8_t starting_level, uint8_t num_levels); - static void merge_sorted_blocks_reversed(Container& orig, Container& temp, const uint32_t* levels, uint8_t starting_level, uint8_t num_levels); - static uint64_t pos_of_phi(double phi, uint64_t n); - - template - struct compare_pair_by_first { - template - bool operator()(Entry1&& a, Entry2&& b) const { - return Comparator()(std::forward(a).first, std::forward(b).first); - } - }; -}; - -} /* namespace datasketches */ - -#include "kll_quantile_calculator_impl.hpp" - -#endif // KLL_QUANTILE_CALCULATOR_HPP_ diff --git a/3rd/datasketches/kll/include/kll_quantile_calculator_impl.hpp b/3rd/datasketches/kll/include/kll_quantile_calculator_impl.hpp deleted file mode 100644 index 4d3d118e2..000000000 --- a/3rd/datasketches/kll/include/kll_quantile_calculator_impl.hpp +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifndef KLL_QUANTILE_CALCULATOR_IMPL_HPP_ -#define KLL_QUANTILE_CALCULATOR_IMPL_HPP_ - -#include -#include -#include - -#include "kll_helper.hpp" - -namespace datasketches { - -template -template -kll_quantile_calculator::kll_quantile_calculator(const kll_sketch& sketch): -n_(sketch.n_), levels_(sketch.num_levels_ + 1, 0, sketch.allocator_), entries_(sketch.allocator_) -{ - const uint32_t num_items = sketch.levels_[sketch.num_levels_] - sketch.levels_[0]; - if (num_items > 0) { - entries_.reserve(num_items); - populate_from_sketch(sketch.items_, sketch.levels_.data(), sketch.num_levels_); - if (!sketch.is_level_zero_sorted_) std::sort(entries_.begin(), entries_.begin() + levels_[1], compare_pair_by_first()); - merge_sorted_blocks(entries_, levels_.data(), static_cast(levels_.size()) - 1, num_items); - if (!is_sorted(entries_.begin(), entries_.end(), compare_pair_by_first())) throw std::logic_error("entries must be sorted"); - convert_to_preceding_cummulative(); - } -} - -template -T kll_quantile_calculator::get_quantile(double fraction) const { - return approximately_answer_positional_query(pos_of_phi(fraction, n_)); -} - -template -auto kll_quantile_calculator::begin() const -> const_iterator { - return entries_.begin(); -} - -template -auto kll_quantile_calculator::end() const -> const_iterator { - return entries_.end(); -} - -template -void kll_quantile_calculator::populate_from_sketch(const T* items, const uint32_t* levels, uint8_t num_levels) { - size_t src_level = 0; - size_t dst_level = 0; - uint64_t weight = 1; - uint32_t offset = levels[0]; - while (src_level < num_levels) { - const uint32_t from_index(levels[src_level] - offset); - const uint32_t to_index(levels[src_level + 1] - offset); // exclusive - if (from_index < to_index) { // skip empty levels - for (uint32_t i = from_index; i < to_index; ++i) { - entries_.push_back(Entry(items[i + offset], weight)); - } - levels_[dst_level] = from_index; - levels_[dst_level + 1] = to_index; - dst_level++; - } - src_level++; - weight *= 2; - } - if (levels_.size() > static_cast(dst_level + 1)) levels_.resize(dst_level + 1); -} - -template -T kll_quantile_calculator::approximately_answer_positional_query(uint64_t pos) const { - if (pos >= n_) throw std::logic_error("position out of range"); - const uint32_t num_items = levels_[levels_.size() - 1]; - if (pos > entries_[num_items - 1].second) return entries_[num_items - 1].first; - const uint32_t index = chunk_containing_pos(pos); - return entries_[index].first; -} - -template -void kll_quantile_calculator::convert_to_preceding_cummulative() { - uint64_t subtotal = 0; - for (auto& entry: entries_) { - const uint64_t new_subtotal = subtotal + entry.second; - entry.second = subtotal; - subtotal = new_subtotal; - } -} - -template -uint64_t kll_quantile_calculator::pos_of_phi(double phi, uint64_t n) { - const uint64_t pos = static_cast(std::floor(phi * n)); - return (pos == n) ? n - 1 : pos; -} - -template -uint32_t kll_quantile_calculator::chunk_containing_pos(uint64_t pos) const { - if (entries_.size() < 1) throw std::logic_error("array too short"); - if (pos < entries_[0].second) throw std::logic_error("position too small"); - if (pos > entries_[entries_.size() - 1].second) throw std::logic_error("position too large"); - return search_for_chunk_containing_pos(pos, 0, entries_.size()); -} - -template -uint32_t kll_quantile_calculator::search_for_chunk_containing_pos(uint64_t pos, uint64_t l, uint64_t r) const { - if (l + 1 == r) { - return static_cast(l); - } - const uint64_t m = l + (r - l) / 2; - if (entries_[m].second <= pos) { - return search_for_chunk_containing_pos(pos, m, r); - } - return search_for_chunk_containing_pos(pos, l, m); -} - -template -void kll_quantile_calculator::merge_sorted_blocks(Container& entries, const uint32_t* levels, uint8_t num_levels, uint32_t num_items) { - if (num_levels == 1) return; - Container temporary(entries.get_allocator()); - temporary.reserve(num_items); - merge_sorted_blocks_direct(entries, temporary, levels, 0, num_levels); -} - -template -void kll_quantile_calculator::merge_sorted_blocks_direct(Container& orig, Container& temp, const uint32_t* levels, - uint8_t starting_level, uint8_t num_levels) { - if (num_levels == 1) return; - const uint8_t num_levels_1 = num_levels / 2; - const uint8_t num_levels_2 = num_levels - num_levels_1; - const uint8_t starting_level_1 = starting_level; - const uint8_t starting_level_2 = starting_level + num_levels_1; - const auto initial_size = temp.size(); - merge_sorted_blocks_reversed(orig, temp, levels, starting_level_1, num_levels_1); - merge_sorted_blocks_reversed(orig, temp, levels, starting_level_2, num_levels_2); - const uint32_t num_items_1 = levels[starting_level_1 + num_levels_1] - levels[starting_level_1]; - const auto chunk_begin = temp.begin() + initial_size; - std::merge( - std::make_move_iterator(chunk_begin), std::make_move_iterator(chunk_begin + num_items_1), - std::make_move_iterator(chunk_begin + num_items_1), std::make_move_iterator(temp.end()), - orig.begin() + levels[starting_level], compare_pair_by_first() - ); - temp.erase(chunk_begin, temp.end()); -} - -template -void kll_quantile_calculator::merge_sorted_blocks_reversed(Container& orig, Container& temp, const uint32_t* levels, - uint8_t starting_level, uint8_t num_levels) { - if (num_levels == 1) { - std::move(orig.begin() + levels[starting_level], orig.begin() + levels[starting_level + 1], std::back_inserter(temp)); - return; - } - const uint8_t num_levels_1 = num_levels / 2; - const uint8_t num_levels_2 = num_levels - num_levels_1; - const uint8_t starting_level_1 = starting_level; - const uint8_t starting_level_2 = starting_level + num_levels_1; - merge_sorted_blocks_direct(orig, temp, levels, starting_level_1, num_levels_1); - merge_sorted_blocks_direct(orig, temp, levels, starting_level_2, num_levels_2); - std::merge( - std::make_move_iterator(orig.begin() + levels[starting_level_1]), - std::make_move_iterator(orig.begin() + levels[starting_level_1 + num_levels_1]), - std::make_move_iterator(orig.begin() + levels[starting_level_2]), - std::make_move_iterator(orig.begin() + levels[starting_level_2 + num_levels_2]), - std::back_inserter(temp), - compare_pair_by_first() - ); -} - -} /* namespace datasketches */ - -#endif // KLL_QUANTILE_CALCULATOR_IMPL_HPP_ diff --git a/3rd/datasketches/kll/include/kll_sketch.hpp b/3rd/datasketches/kll/include/kll_sketch.hpp index fa5552025..560f0975e 100644 --- a/3rd/datasketches/kll/include/kll_sketch.hpp +++ b/3rd/datasketches/kll/include/kll_sketch.hpp @@ -20,13 +20,12 @@ #ifndef KLL_SKETCH_HPP_ #define KLL_SKETCH_HPP_ -#include #include #include -#include "kll_quantile_calculator.hpp" #include "common_defs.hpp" #include "serde.hpp" +#include "quantiles_sorted_view.hpp" namespace datasketches { @@ -35,10 +34,10 @@ namespace datasketches { * and nearly optimal accuracy per retained item. * See Optimal Quantile Approximation in Streams. * - *

This is a stochastic streaming sketch that enables near-real time analysis of the - * approximate distribution of values from a very large stream in a single pass, requiring only - * that the values are comparable. - * The analysis is obtained using get_quantile() or get_quantiles() functions or the + *

This is a stochastic streaming sketch that enables near real-time analysis of the + * approximate distribution of items from a very large stream in a single pass, requiring only + * that the items are comparable. + * The analysis is obtained using get_quantile() function or the * inverse functions get_rank(), get_PMF() (Probability Mass Function), and get_CDF() * (Cumulative Distribution Function). * @@ -46,14 +45,15 @@ namespace datasketches { * with the equivalent Java implementation only when template parameter T = float * (32-bit single precision values). * - *

Given an input stream of N numeric values, the absolute rank of any specific - * value is defined as its index (0 to N-1) in the hypothetical sorted stream of all - * N input values. + *

Given an input stream of N items, the natural rank of any specific + * item is defined as its index (1 to N) in inclusive mode + * or (0 to N-1) in exclusive mode + * in the hypothetical sorted stream of all N input items. * - *

The normalized rank (rank) of any specific value is defined as its - * absolute rank divided by N. - * Thus, the normalized rank is a value between zero and one. - * In the documentation for this sketch absolute rank is never used so any + *

The normalized rank (rank) of any specific item is defined as its + * natural rank divided by N. + * Thus, the normalized rank is between zero and one. + * In the documentation for this sketch natural rank is never used so any * reference to just rank should be interpreted to mean normalized rank. * *

This sketch is configured with a parameter k, which affects the size of the sketch @@ -62,18 +62,18 @@ namespace datasketches { *

The estimation error is commonly called epsilon (or eps) and is a fraction * between zero and one. Larger values of k result in smaller values of epsilon. * Epsilon is always with respect to the rank and cannot be applied to the - * corresponding values. + * corresponding items. * - *

The relationship between the normalized rank and the corresponding values can be viewed + *

The relationship between the normalized rank and the corresponding items can be viewed * as a two dimensional monotonic plot with the normalized rank on one axis and the - * corresponding values on the other axis. If the y-axis is specified as the value-axis and + * corresponding items on the other axis. If the y-axis is specified as the item-axis and * the x-axis as the normalized rank, then y = get_quantile(x) is a monotonically * increasing function. * - *

The functions get_quantile(rank) and get_quantiles(...) translate ranks into - * corresponding values. The functions get_rank(value), + *

The function get_quantile(rank) translates ranks into + * corresponding quantiles. The functions get_rank(item), * get_CDF(...) (Cumulative Distribution Function), and get_PMF(...) - * (Probability Mass Function) perform the opposite operation and translate values into ranks. + * (Probability Mass Function) perform the opposite operation and translate items into ranks. * *

The getPMF(...) function has about 13 to 47% worse rank error (depending * on k) than the other queries because the mass of each "bin" of the PMF has @@ -85,60 +85,60 @@ namespace datasketches { * *

A get_quantile(rank) query has the following guarantees: *

    - *
  • Let v = get_quantile(r) where r is the rank between zero and one.
    • - *
  • The value v will be a value from the input stream.
    • - *
  • Let trueRank be the true rank of v derived from the hypothetical sorted - * stream of all N values.
    • + *
  • Let q = get_quantile(r) where r is the rank between zero and one.
    • + *
  • The quantile q will be an item from the input stream.
    • + *
  • Let trueRank be the true rank of q derived from the hypothetical sorted + * stream of all N items.
    • *
  • Let eps = get_normalized_rank_error(false).
    • *
  • Then r - eps ≤ trueRank ≤ r + eps with a confidence of 99%. Note that the - * error is on the rank, not the value.
    • + * error is on the rank, not the quantile. *
* - *

A get_rank(value) query has the following guarantees: + *

A get_rank(item) query has the following guarantees: *

    - *
  • Let r = get_rank(v) where v is a value between the min and max values of + *
  • Let r = get_rank(i) where i is an item between the min and max items of * the input stream.
    • - *
  • Let true_rank be the true rank of v derived from the hypothetical sorted - * stream of all N values.
    • + *
  • Let true_rank be the true rank of i derived from the hypothetical sorted + * stream of all N items.
    • *
  • Let eps = get_normalized_rank_error(false).
    • *
  • Then r - eps ≤ trueRank ≤ r + eps with a confidence of 99%.
    • *
* *

A get_PMF() query has the following guarantees: *

    - *
  • Let {r1, r2, ..., r(m+1)} = get_PMF(v1, v2, ..., vm) where v1, v2 are values - * between the min and max values of the input stream. - *
  • Let massi = estimated mass between vi and vi+1.
    • - *
  • Let trueMass be the true mass between the values of vi, - * vi+1 derived from the hypothetical sorted stream of all N values.
    • + *
  • Let {r1, r2, ..., r(m+1)} = get_PMF(s1, s2, ..., sm) where s1, s2 are + * split points (items from the input domain) between the min and max items of the input stream. + *
  • Let massi = estimated mass between si and si+1.
    • + *
  • Let trueMass be the true mass between the items of si, + * si+1 derived from the hypothetical sorted stream of all N items.
    • *
  • Let eps = get_normalized_rank_error(true).
    • *
  • then mass - eps ≤ trueMass ≤ mass + eps with a confidence of 99%.
    • - *
  • r(m+1) includes the mass of all points larger than vm.
    • + *
  • r(m+1) includes the mass of all points larger than sm.
    • *
* *

A get_CDF(...) query has the following guarantees; *

    - *
  • Let {r1, r2, ..., r(m+1)} = get_CDF(v1, v2, ..., vm) where v1, v2 are values - * between the min and max values of the input stream. + *
  • Let {r1, r2, ..., r(m+1)} = get_CDF(s1, s2, ..., sm) where s1, s2, ... are + * split points (items from the input domain) between the min and max items of the input stream. *
  • Let massi = ri+1 - ri.
    • - *
  • Let trueMass be the true mass between the true ranks of vi, - * vi+1 derived from the hypothetical sorted stream of all N values.
    • + *
  • Let trueMass be the true mass between the true ranks of si, + * si+1 derived from the hypothetical sorted stream of all N items.
    • *
  • Let eps = get_normalized_rank_error(true).
    • *
  • then mass - eps ≤ trueMass ≤ mass + eps with a confidence of 99%.
    • - *
  • 1 - r(m+1) includes the mass of all points larger than vm.
    • + *
  • 1 - r(m+1) includes the mass of all points larger than sm.
    • *
* *

From the above, it might seem like we could make some estimates to bound the - * value returned from a call to get_quantile(). The sketch, however, does not - * let us derive error bounds or confidences around values. Because errors are independent, we + * item returned from a call to get_quantile(). The sketch, however, does not + * let us derive error bounds or confidences around items. Because errors are independent, we * can approximately bracket a value as shown below, but there are no error estimates available. * Additionally, the interval may be quite large for certain distributions. *

    - *
  • Let v = get_quantile(r), the estimated quantile value of rank r.
    • + *
  • Let q = get_quantile(r), the estimated quantile of rank r.
    • *
  • Let eps = get_normalized_rank_error(false).
    • - *
  • Let vlo = estimated quantile value of rank (r - eps).
    • - *
  • Let vhi = estimated quantile value of rank (r + eps).
    • - *
  • Then vlo ≤ v ≤ vhi, with 99% confidence.
    • + *
  • Let qlo = estimated quantile of rank (r - eps).
    • + *
  • Let qhi = estimated quantile of rank (r + eps).
    • + *
  • Then qlo ≤ q ≤ qhi, with 99% confidence.
    • *
* * author Kevin Lang @@ -146,63 +146,54 @@ namespace datasketches { * author Lee Rhodes */ -template using AllocU8 = typename std::allocator_traits::template rebind_alloc; -template using vector_u8 = std::vector>; -template using AllocU32 = typename std::allocator_traits::template rebind_alloc; -template using vector_u32 = std::vector>; -template using AllocD = typename std::allocator_traits::template rebind_alloc; -template using vector_d = std::vector>; - namespace kll_constants { const uint16_t DEFAULT_K = 200; } -template , typename S = serde, typename A = std::allocator> +template < + typename T, + typename C = std::less, // strict weak ordering function (see C++ named requirements: Compare) + typename A = std::allocator +> class kll_sketch { public: using value_type = T; using comparator = C; + using vector_u32 = std::vector::template rebind_alloc>; static const uint8_t DEFAULT_M = 8; - // TODO: Redundant and deprecated. Will be remove din next major version. - static const uint16_t DEFAULT_K = kll_constants::DEFAULT_K; static const uint16_t MIN_K = DEFAULT_M; static const uint16_t MAX_K = (1 << 16) - 1; - explicit kll_sketch(uint16_t k = kll_constants::DEFAULT_K, const A& allocator = A()); + explicit kll_sketch(uint16_t k = kll_constants::DEFAULT_K, const C& comparator = C(), const A& allocator = A()); kll_sketch(const kll_sketch& other); kll_sketch(kll_sketch&& other) noexcept; ~kll_sketch(); kll_sketch& operator=(const kll_sketch& other); kll_sketch& operator=(kll_sketch&& other); - /** - * Updates this sketch with the given data item. - * This method takes lvalue. - * @param value an item from a stream of items + /* + * Type converting constructor. + * @param other sketch of a different type + * @param comparator instance of a Comparator + * @param allocator instance of an Allocator */ - void update(const T& value); + template + explicit kll_sketch(const kll_sketch& other, const C& comparator = C(), const A& allocator = A()); /** * Updates this sketch with the given data item. - * This method takes rvalue. - * @param value an item from a stream of items + * @param item from a stream of items */ - void update(T&& value); + template + void update(FwdT&& item); /** * Merges another sketch into this one. - * This method takes lvalue. * @param other sketch to merge into this one */ - void merge(const kll_sketch& other); - - /** - * Merges another sketch into this one. - * This method takes rvalue. - * @param other sketch to merge into this one - */ - void merge(kll_sketch&& other); + template + void merge(FwdSk&& other); /** * Returns true if this sketch is empty. @@ -235,136 +226,148 @@ class kll_sketch { bool is_estimation_mode() const; /** - * Returns the min value of the stream. - * For floating point types: if the sketch is empty this returns NaN. - * For other types: if the sketch is empty this throws runtime_error. - * @return the min value of the stream + * Returns the min item of the stream. + * If the sketch is empty this throws std::runtime_error. + * @return the min item of the stream */ - T get_min_value() const; + T get_min_item() const; /** - * Returns the max value of the stream. - * For floating point types: if the sketch is empty this returns NaN. - * For other types: if the sketch is empty this throws runtime_error. - * @return the max value of the stream + * Returns the max item of the stream. + * If the sketch is empty this throws std::runtime_error. + * @return the max item of the stream */ - T get_max_value() const; + T get_max_item() const; /** - * Returns an approximation to the value of the data item - * that would be preceded by the given fraction of a hypothetical sorted - * version of the input stream so far. - *

- * Note that this method has a fairly large overhead (microseconds instead of nanoseconds) - * so it should not be called multiple times to get different quantiles from the same - * sketch. Instead use get_quantiles(), which pays the overhead only once. - *

- * For floating point types: if the sketch is empty this returns NaN. - * For other types: if the sketch is empty this throws runtime_error. + * Returns an instance of the comparator for this sketch. + * @return comparator + */ + C get_comparator() const; + + /** + * Returns an instance of the allocator for this sketch. + * @return allocator + */ + A get_allocator() const; + + /** + * Returns an item from the sketch that is the best approximation to an item + * from the original stream with the given rank. + * + *

If the sketch is empty this throws std::runtime_error. * - * @param fraction the specified fractional position in the hypothetical sorted stream. - * These are also called normalized ranks or fractional ranks. - * If fraction = 0.0, the true minimum value of the stream is returned. - * If fraction = 1.0, the true maximum value of the stream is returned. + * @param rank of an item in the hypothetical sorted stream. + * @param inclusive if true, the given rank is considered inclusive (includes weight of an item) * - * @return the approximation to the value at the given fraction + * @return approximate quantile associated with the given rank */ - T get_quantile(double fraction) const; + using quantile_return_type = typename quantiles_sorted_view::quantile_return_type; + quantile_return_type get_quantile(double rank, bool inclusive = true) const; /** - * This is a more efficient multiple-query version of get_quantile(). - *

* This returns an array that could have been generated by using get_quantile() for each - * fractional rank separately, but would be very inefficient. - * This method incurs the internal set-up overhead once and obtains multiple quantile values in - * a single query. It is strongly recommend that this method be used instead of multiple calls - * to get_quantile(). + * rank separately. * - *

If the sketch is empty this returns an empty vector. + *

If the sketch is empty this throws std::runtime_error. * - * @param fractions given array of fractional positions in the hypothetical sorted stream. - * These are also called normalized ranks or fractional ranks. - * These fractions must be in the interval [0.0, 1.0], inclusive. + * @param ranks given array of ranks in the hypothetical sorted stream. + * These ranks must be in the interval [0.0, 1.0]. + * @param size the number of ranks in the array + * @param inclusive if true, the given ranks are considered inclusive (include weights of items) * - * @return array of approximations to the given fractions in the same order as given fractions - * in the input array. + * @return array of approximate quantiles corresponding to the given ranks in the same order. + * + * Deprecated. Will be removed in the next major version. Use get_quantile() instead. */ - std::vector get_quantiles(const double* fractions, uint32_t size) const; + std::vector get_quantiles(const double* ranks, uint32_t size, bool inclusive = true) const; /** * This is a multiple-query version of get_quantile() that allows the caller to - * specify the number of evenly-spaced fractional ranks. + * specify the number of evenly-spaced ranks. + * + *

If the sketch is empty this throws std::runtime_error. * - *

If the sketch is empty this returns an empty vector. + * @param num an integer that specifies the number of evenly-spaced ranks. + * This must be an integer greater than 0. A value of 1 will return the quantile of rank 0. + * A value of 2 will return quantiles of ranks 0 and 1. A value of 3 will return quantiles of ranks 0, + * 0.5 (median) and 1, etc. + * @param inclusive if true, the ranks are considered inclusive (include weights of items) * - * @param num an integer that specifies the number of evenly-spaced fractional ranks. - * This must be an integer greater than 0. A value of 1 will return the min value. - * A value of 2 will return the min and the max value. A value of 3 will return the min, - * the median and the max value, etc. + * @return array of approximate quantiles corresponding to the given number of evenly-spaced ranks. * - * @return array of approximations to the given number of evenly-spaced fractional ranks. + * Deprecated. Will be removed in the next major version. Use get_quantile() instead. */ - std::vector get_quantiles(uint32_t num) const; + std::vector get_quantiles(uint32_t num, bool inclusive = true) const; /** - * Returns an approximation to the normalized (fractional) rank of the given value from 0 to 1, - * inclusive. + * Returns an approximation to the normalized rank of the given item from 0 to 1, inclusive. * *

The resulting approximation has a probabilistic guarantee that can be obtained from the * get_normalized_rank_error(false) function. * - *

If the sketch is empty this returns NaN. + *

If the sketch is empty this throws std::runtime_error. + * + * @param item to be ranked. + * @param inclusive if true the weight of the given item is included into the rank. + * Otherwise the rank equals the sum of the weights of all items that are less than the given item + * according to the comparator C. * - * @param value to be ranked - * @return an approximate rank of the given value + * @return an approximate rank of the given item */ - double get_rank(const T& value) const; + double get_rank(const T& item, bool inclusive = true) const; /** * Returns an approximation to the Probability Mass Function (PMF) of the input stream - * given a set of split points (values). + * given a set of split points (items). * *

The resulting approximations have a probabilistic guarantee that can be obtained from the * get_normalized_rank_error(true) function. * - *

If the sketch is empty this returns an empty vector. + *

If the sketch is empty this throws std::runtime_error. * - * @param split_points an array of m unique, monotonically increasing values - * that divide the input domain into m+1 consecutive disjoint intervals. - * The definition of an "interval" is inclusive of the left split point (or minimum value) and - * exclusive of the right split point, with the exception that the last interval will include - * the maximum value. - * It is not necessary to include either the min or max values in these split points. + * @param split_points an array of m unique, monotonically increasing items + * that divide the input domain into m+1 consecutive disjoint intervals (bins). + * + * @param size the number of split points in the array + * + * @param inclusive if true the rank of an item includes its own weight, and therefore + * if the sketch contains items equal to a slit point, then in PMF such items are + * included into the interval to the left of split point. Otherwise they are included into the interval + * to the right of split point. * * @return an array of m+1 doubles each of which is an approximation - * to the fraction of the input stream values (the mass) that fall into one of those intervals. - * The definition of an "interval" is inclusive of the left split point and exclusive of the right - * split point, with the exception that the last interval will include maximum value. + * to the fraction of the input stream items (the mass) that fall into one of those intervals. */ - vector_d get_PMF(const T* split_points, uint32_t size) const; + using vector_double = typename quantiles_sorted_view::vector_double; + vector_double get_PMF(const T* split_points, uint32_t size, bool inclusive = true) const; /** * Returns an approximation to the Cumulative Distribution Function (CDF), which is the - * cumulative analog of the PMF, of the input stream given a set of split points (values). + * cumulative analog of the PMF, of the input stream given a set of split points (items). * *

The resulting approximations have a probabilistic guarantee that can be obtained from the * get_normalized_rank_error(false) function. * - *

If the sketch is empty this returns an empty vector. + *

If the sketch is empty this throws std::runtime_error. * - * @param split_points an array of m unique, monotonically increasing values + * @param split_points an array of m unique, monotonically increasing items * that divide the input domain into m+1 consecutive disjoint intervals. - * The definition of an "interval" is inclusive of the left split point (or minimum value) and - * exclusive of the right split point, with the exception that the last interval will include - * the maximum value. - * It is not necessary to include either the min or max values in these split points. * - * @return an array of m+1 double values, which are a consecutive approximation to the CDF + * @param size the number of split points in the array + * + * @param inclusive if true the rank of an item includes its own weight, and therefore + * if the sketch contains items equal to a slit point, then in CDF such items are + * included into the interval to the left of split point. Otherwise they are included into + * the interval to the right of split point. + * + * @return an array of m+1 doubles, which are a consecutive approximation to the CDF * of the input stream given the split_points. The value at array position j of the returned * CDF array is the sum of the returned values in positions 0 through j of the returned PMF - * array. + * array. This can be viewed as array of ranks of the given split points plus one more value + * that is always 1. */ - vector_d get_CDF(const T* split_points, uint32_t size) const; + vector_double get_CDF(const T* split_points, uint32_t size, bool inclusive = true) const; /** * Gets the approximate rank error of this sketch normalized as a fraction between zero and one. @@ -378,18 +381,20 @@ class kll_sketch { /** * Computes size needed to serialize the current state of the sketch. * This version is for fixed-size arithmetic types (integral and floating point). + * @param sd instance of a SerDe * @return size in bytes needed to serialize this sketch */ - template::value, int>::type = 0> - size_t get_serialized_size_bytes() const; + template, typename std::enable_if::value, int>::type = 0> + size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const; /** * Computes size needed to serialize the current state of the sketch. * This version is for all other types and can be expensive since every item needs to be looked at. + * @param sd instance of a SerDe * @return size in bytes needed to serialize this sketch */ - template::value, int>::type = 0> - size_t get_serialized_size_bytes() const; + template, typename std::enable_if::value, int>::type = 0> + size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const; /** * Returns upper bound on the serialized size of a sketch given a parameter k and stream @@ -421,12 +426,14 @@ class kll_sketch { /** * This method serializes the sketch into a given stream in a binary form * @param os output stream + * @param sd instance of a SerDe */ - void serialize(std::ostream& os) const; + template> + void serialize(std::ostream& os, const SerDe& sd = SerDe()) const; // This is a convenience alias for users // The type returned by the following serialize method - using vector_bytes = vector_u8; + using vector_bytes = std::vector::template rebind_alloc>; /** * This method serializes the sketch as a vector of bytes. @@ -434,23 +441,36 @@ class kll_sketch { * It is a blank space of a given size. * This header is used in Datasketches PostgreSQL extension. * @param header_size_bytes space to reserve in front of the sketch + * @param sd instance of a SerDe + * @return serialized sketch as a vector of bytes */ - vector_bytes serialize(unsigned header_size_bytes = 0) const; + template> + vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& sd = SerDe()) const; /** * This method deserializes a sketch from a given stream. * @param is input stream + * @param sd instance of a SerDe + * @param comparator instance of a Comparator + * @param allocator instance of an Allocator * @return an instance of a sketch */ - static kll_sketch deserialize(std::istream& is, const A& allocator = A()); + template> + static kll_sketch deserialize(std::istream& is, const SerDe& sd = SerDe(), + const C& comparator = C(), const A& allocator = A()); /** * This method deserializes a sketch from a given array of bytes. * @param bytes pointer to the array of bytes * @param size the size of the array + * @param sd instance of a SerDe + * @param comparator instance of a Comparator + * @param allocator instance of an Allocator * @return an instance of a sketch */ - static kll_sketch deserialize(const void* bytes, size_t size, const A& allocator = A()); + template> + static kll_sketch deserialize(const void* bytes, size_t size, const SerDe& sd = SerDe(), + const C& comparator = C(), const A& allocator = A()); /* * Gets the normalized rank error given k and pmf. @@ -472,15 +492,11 @@ class kll_sketch { const_iterator begin() const; const_iterator end() const; - #ifdef KLL_VALIDATION - uint8_t get_num_levels() { return num_levels_; } - uint32_t* get_levels() { return levels_; } - T* get_items() { return items_; } - #endif + quantiles_sorted_view get_sorted_view() const; private: /* Serialized sketch layout: - * Adr: + * Addr: * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | * 0 || unused | M |--------K--------| Flags | FamID | SerVer | PreambleInts | * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 | @@ -502,30 +518,30 @@ class kll_sketch { static const uint8_t PREAMBLE_INTS_SHORT = 2; // for empty and single item static const uint8_t PREAMBLE_INTS_FULL = 5; + C comparator_; A allocator_; uint16_t k_; uint8_t m_; // minimum buffer "width" uint16_t min_k_; // for error estimation after merging with different k - uint64_t n_; uint8_t num_levels_; - vector_u32 levels_; + bool is_level_zero_sorted_; + uint64_t n_; + vector_u32 levels_; T* items_; uint32_t items_size_; - T* min_value_; - T* max_value_; - bool is_level_zero_sorted_; - - friend class kll_quantile_calculator; + T* min_item_; + T* max_item_; + mutable quantiles_sorted_view* sorted_view_; // for deserialization class item_deleter; class items_deleter; - kll_sketch(uint16_t k, uint16_t min_k, uint64_t n, uint8_t num_levels, vector_u32&& levels, - std::unique_ptr items, uint32_t items_size, std::unique_ptr min_value, - std::unique_ptr max_value, bool is_level_zero_sorted); + kll_sketch(uint16_t k, uint16_t min_k, uint64_t n, uint8_t num_levels, vector_u32&& levels, + std::unique_ptr items, uint32_t items_size, std::unique_ptr min_item, + std::unique_ptr max_item, bool is_level_zero_sorted, const C& comparator); // common update code - inline void update_min_max(const T& value); + inline void update_min_max(const T& item); inline uint32_t internal_update(); // The following code is only valid in the special case of exactly reaching capacity while updating. @@ -535,15 +551,12 @@ class kll_sketch { uint8_t find_level_to_compact() const; void add_empty_top_level_to_completely_full_sketch(); void sort_level_zero(); - std::unique_ptr, std::function*)>> get_quantile_calculator(); - vector_d get_PMF_or_CDF(const T* split_points, uint32_t size, bool is_CDF) const; - void increment_buckets_unsorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, - const T* split_points, uint32_t size, double* buckets) const; - void increment_buckets_sorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, - const T* split_points, uint32_t size, double* buckets) const; + template void merge_higher_levels(O&& other, uint64_t final_n); - void populate_work_arrays(const kll_sketch& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels); - void populate_work_arrays(kll_sketch&& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels); + + template + void populate_work_arrays(FwdSk&& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels); + void assert_correct_total_weight() const; uint32_t safe_level_size(uint8_t level) const; uint32_t get_num_retained_above_level_zero() const; @@ -553,39 +566,41 @@ class kll_sketch { static void check_serial_version(uint8_t serial_version); static void check_family_id(uint8_t family_id); - // implementations for floating point types - template::value, int>::type = 0> - static TT get_invalid_value() { - return std::numeric_limits::quiet_NaN(); - } + void check_sorting() const; template::value, int>::type = 0> - static inline bool check_update_value(TT value) { - return !std::isnan(value); + static inline bool check_update_item(TT item) { + return !std::isnan(item); } - // implementations for all other types template::value, int>::type = 0> - static TT get_invalid_value() { - throw std::runtime_error("getting quantiles from empty sketch is not supported for this type of values"); - } - - template::value, int>::type = 0> - static inline bool check_update_value(TT) { + static inline bool check_update_item(TT) { return true; } + // for type converting constructor + template friend class kll_sketch; + + void setup_sorted_view() const; // modifies mutable state + void reset_sorted_view(); }; -template -class kll_sketch::const_iterator: public std::iterator { +template +class kll_sketch::const_iterator { public: - friend class kll_sketch; + using iterator_category = std::input_iterator_tag; + using value_type = std::pair; + using difference_type = void; + using pointer = const return_value_holder; + using reference = const value_type; + + friend class kll_sketch; const_iterator& operator++(); const_iterator& operator++(int); bool operator==(const const_iterator& other) const; bool operator!=(const const_iterator& other) const; - const std::pair operator*() const; + reference operator*() const; + pointer operator->() const; private: const T* items; const uint32_t* levels; diff --git a/3rd/datasketches/kll/include/kll_sketch_impl.hpp b/3rd/datasketches/kll/include/kll_sketch_impl.hpp index a0a280c34..cced0caa1 100644 --- a/3rd/datasketches/kll/include/kll_sketch_impl.hpp +++ b/3rd/datasketches/kll/include/kll_sketch_impl.hpp @@ -23,26 +23,31 @@ #include #include #include +#include +#include "conditional_forward.hpp" +#include "count_zeros.hpp" #include "memory_operations.hpp" #include "kll_helper.hpp" namespace datasketches { -template -kll_sketch::kll_sketch(uint16_t k, const A& allocator): +template +kll_sketch::kll_sketch(uint16_t k, const C& comparator, const A& allocator): +comparator_(comparator), allocator_(allocator), k_(k), m_(DEFAULT_M), min_k_(k), -n_(0), num_levels_(1), +is_level_zero_sorted_(false), +n_(0), levels_(2, 0, allocator), items_(nullptr), items_size_(k_), -min_value_(nullptr), -max_value_(nullptr), -is_level_zero_sorted_(false) +min_item_(nullptr), +max_item_(nullptr), +sorted_view_(nullptr) { if (k < MIN_K || k > MAX_K) { throw std::invalid_argument("K must be >= " + std::to_string(MIN_K) + " and <= " + std::to_string(MAX_K) + ": " + std::to_string(k)); @@ -51,315 +56,317 @@ is_level_zero_sorted_(false) items_ = allocator_.allocate(items_size_); } -template -kll_sketch::kll_sketch(const kll_sketch& other): +template +kll_sketch::kll_sketch(const kll_sketch& other): +comparator_(other.comparator_), allocator_(other.allocator_), k_(other.k_), m_(other.m_), min_k_(other.min_k_), -n_(other.n_), num_levels_(other.num_levels_), +is_level_zero_sorted_(other.is_level_zero_sorted_), +n_(other.n_), levels_(other.levels_), items_(nullptr), items_size_(other.items_size_), -min_value_(nullptr), -max_value_(nullptr), -is_level_zero_sorted_(other.is_level_zero_sorted_) +min_item_(nullptr), +max_item_(nullptr), +sorted_view_(nullptr) { items_ = allocator_.allocate(items_size_); - std::copy(&other.items_[levels_[0]], &other.items_[levels_[num_levels_]], &items_[levels_[0]]); - if (other.min_value_ != nullptr) min_value_ = new (allocator_.allocate(1)) T(*other.min_value_); - if (other.max_value_ != nullptr) max_value_ = new (allocator_.allocate(1)) T(*other.max_value_); + for (auto i = levels_[0]; i < levels_[num_levels_]; ++i) new (&items_[i]) T(other.items_[i]); + if (other.min_item_ != nullptr) min_item_ = new (allocator_.allocate(1)) T(*other.min_item_); + if (other.max_item_ != nullptr) max_item_ = new (allocator_.allocate(1)) T(*other.max_item_); } -template -kll_sketch::kll_sketch(kll_sketch&& other) noexcept: +template +kll_sketch::kll_sketch(kll_sketch&& other) noexcept: +comparator_(std::move(other.comparator_)), allocator_(std::move(other.allocator_)), k_(other.k_), m_(other.m_), min_k_(other.min_k_), -n_(other.n_), num_levels_(other.num_levels_), +is_level_zero_sorted_(other.is_level_zero_sorted_), +n_(other.n_), levels_(std::move(other.levels_)), items_(other.items_), items_size_(other.items_size_), -min_value_(other.min_value_), -max_value_(other.max_value_), -is_level_zero_sorted_(other.is_level_zero_sorted_) +min_item_(other.min_item_), +max_item_(other.max_item_), +sorted_view_(nullptr) { other.items_ = nullptr; - other.min_value_ = nullptr; - other.max_value_ = nullptr; + other.min_item_ = nullptr; + other.max_item_ = nullptr; } -template -kll_sketch& kll_sketch::operator=(const kll_sketch& other) { - kll_sketch copy(other); +template +kll_sketch& kll_sketch::operator=(const kll_sketch& other) { + kll_sketch copy(other); + std::swap(comparator_, copy.comparator_); std::swap(allocator_, copy.allocator_); std::swap(k_, copy.k_); std::swap(m_, copy.m_); std::swap(min_k_, copy.min_k_); - std::swap(n_, copy.n_); std::swap(num_levels_, copy.num_levels_); + std::swap(is_level_zero_sorted_, copy.is_level_zero_sorted_); + std::swap(n_, copy.n_); std::swap(levels_, copy.levels_); std::swap(items_, copy.items_); std::swap(items_size_, copy.items_size_); - std::swap(min_value_, copy.min_value_); - std::swap(max_value_, copy.max_value_); - std::swap(is_level_zero_sorted_, copy.is_level_zero_sorted_); + std::swap(min_item_, copy.min_item_); + std::swap(max_item_, copy.max_item_); + reset_sorted_view(); return *this; } -template -kll_sketch& kll_sketch::operator=(kll_sketch&& other) { +template +kll_sketch& kll_sketch::operator=(kll_sketch&& other) { + std::swap(comparator_, other.comparator_); std::swap(allocator_, other.allocator_); std::swap(k_, other.k_); std::swap(m_, other.m_); std::swap(min_k_, other.min_k_); - std::swap(n_, other.n_); std::swap(num_levels_, other.num_levels_); + std::swap(is_level_zero_sorted_, other.is_level_zero_sorted_); + std::swap(n_, other.n_); std::swap(levels_, other.levels_); std::swap(items_, other.items_); std::swap(items_size_, other.items_size_); - std::swap(min_value_, other.min_value_); - std::swap(max_value_, other.max_value_); - std::swap(is_level_zero_sorted_, other.is_level_zero_sorted_); + std::swap(min_item_, other.min_item_); + std::swap(max_item_, other.max_item_); + reset_sorted_view(); return *this; } -template -kll_sketch::~kll_sketch() { +template +kll_sketch::~kll_sketch() { if (items_ != nullptr) { const uint32_t begin = levels_[0]; const uint32_t end = levels_[num_levels_]; for (uint32_t i = begin; i < end; i++) items_[i].~T(); allocator_.deallocate(items_, items_size_); } - if (min_value_ != nullptr) { - min_value_->~T(); - allocator_.deallocate(min_value_, 1); + if (min_item_ != nullptr) { + min_item_->~T(); + allocator_.deallocate(min_item_, 1); } - if (max_value_ != nullptr) { - max_value_->~T(); - allocator_.deallocate(max_value_, 1); + if (max_item_ != nullptr) { + max_item_->~T(); + allocator_.deallocate(max_item_, 1); } + reset_sorted_view(); } -template -void kll_sketch::update(const T& value) { - if (!check_update_value(value)) { return; } - update_min_max(value); - const uint32_t index = internal_update(); - new (&items_[index]) T(value); +template +template +kll_sketch::kll_sketch(const kll_sketch& other, const C& comparator, const A& allocator): +comparator_(comparator), +allocator_(allocator), +k_(other.k_), +m_(other.m_), +min_k_(other.min_k_), +num_levels_(other.num_levels_), +is_level_zero_sorted_(other.is_level_zero_sorted_), +n_(other.n_), +levels_(other.levels_, allocator_), +items_(nullptr), +items_size_(other.items_size_), +min_item_(nullptr), +max_item_(nullptr), +sorted_view_(nullptr) +{ + static_assert( + std::is_constructible::value, + "Type converting constructor requires new type to be constructible from existing type" + ); + items_ = allocator_.allocate(items_size_); + for (auto i = levels_[0]; i < levels_[num_levels_]; ++i) new (&items_[i]) T(other.items_[i]); + if (other.min_item_ != nullptr) min_item_ = new (allocator_.allocate(1)) T(*other.min_item_); + if (other.max_item_ != nullptr) max_item_ = new (allocator_.allocate(1)) T(*other.max_item_); + check_sorting(); } -template -void kll_sketch::update(T&& value) { - if (!check_update_value(value)) { return; } - update_min_max(value); +template +template +void kll_sketch::update(FwdT&& item) { + if (!check_update_item(item)) { return; } + update_min_max(item); const uint32_t index = internal_update(); - new (&items_[index]) T(std::move(value)); + new (&items_[index]) T(std::forward(item)); + reset_sorted_view(); } -template -void kll_sketch::update_min_max(const T& value) { +template +void kll_sketch::update_min_max(const T& item) { if (is_empty()) { - min_value_ = new (allocator_.allocate(1)) T(value); - max_value_ = new (allocator_.allocate(1)) T(value); + min_item_ = new (allocator_.allocate(1)) T(item); + max_item_ = new (allocator_.allocate(1)) T(item); } else { - if (C()(value, *min_value_)) *min_value_ = value; - if (C()(*max_value_, value)) *max_value_ = value; + if (comparator_(item, *min_item_)) *min_item_ = item; + if (comparator_(*max_item_, item)) *max_item_ = item; } } -template -uint32_t kll_sketch::internal_update() { +template +uint32_t kll_sketch::internal_update() { if (levels_[0] == 0) compress_while_updating(); n_++; is_level_zero_sorted_ = false; return --levels_[0]; } -template -void kll_sketch::merge(const kll_sketch& other) { +template +template +void kll_sketch::merge(FwdSk&& other) { if (other.is_empty()) return; if (m_ != other.m_) { throw std::invalid_argument("incompatible M: " + std::to_string(m_) + " and " + std::to_string(other.m_)); } if (is_empty()) { - min_value_ = new (allocator_.allocate(1)) T(*other.min_value_); - max_value_ = new (allocator_.allocate(1)) T(*other.max_value_); + min_item_ = new (allocator_.allocate(1)) T(conditional_forward(*other.min_item_)); + max_item_ = new (allocator_.allocate(1)) T(conditional_forward(*other.max_item_)); } else { - if (C()(*other.min_value_, *min_value_)) *min_value_ = *other.min_value_; - if (C()(*max_value_, *other.max_value_)) *max_value_ = *other.max_value_; + if (comparator_(*other.min_item_, *min_item_)) *min_item_ = conditional_forward(*other.min_item_); + if (comparator_(*max_item_, *other.max_item_)) *max_item_ = conditional_forward(*other.max_item_); } const uint64_t final_n = n_ + other.n_; for (uint32_t i = other.levels_[0]; i < other.levels_[1]; i++) { const uint32_t index = internal_update(); - new (&items_[index]) T(other.items_[i]); + new (&items_[index]) T(conditional_forward(other.items_[i])); } if (other.num_levels_ >= 2) merge_higher_levels(other, final_n); n_ = final_n; if (other.is_estimation_mode()) min_k_ = std::min(min_k_, other.min_k_); assert_correct_total_weight(); + reset_sorted_view(); } -template -void kll_sketch::merge(kll_sketch&& other) { - if (other.is_empty()) return; - if (m_ != other.m_) { - throw std::invalid_argument("incompatible M: " + std::to_string(m_) + " and " + std::to_string(other.m_)); - } - if (is_empty()) { - min_value_ = new (allocator_.allocate(1)) T(std::move(*other.min_value_)); - max_value_ = new (allocator_.allocate(1)) T(std::move(*other.max_value_)); - } else { - if (C()(*other.min_value_, *min_value_)) *min_value_ = std::move(*other.min_value_); - if (C()(*max_value_, *other.max_value_)) *max_value_ = std::move(*other.max_value_); - } - const uint64_t final_n = n_ + other.n_; - for (uint32_t i = other.levels_[0]; i < other.levels_[1]; i++) { - const uint32_t index = internal_update(); - new (&items_[index]) T(std::move(other.items_[i])); - } - if (other.num_levels_ >= 2) merge_higher_levels(std::forward(other), final_n); - n_ = final_n; - if (other.is_estimation_mode()) min_k_ = std::min(min_k_, other.min_k_); - assert_correct_total_weight(); -} - -template -bool kll_sketch::is_empty() const { +template +bool kll_sketch::is_empty() const { return n_ == 0; } -template -uint16_t kll_sketch::get_k() const { +template +uint16_t kll_sketch::get_k() const { return k_; } -template -uint64_t kll_sketch::get_n() const { +template +uint64_t kll_sketch::get_n() const { return n_; } -template -uint32_t kll_sketch::get_num_retained() const { +template +uint32_t kll_sketch::get_num_retained() const { return levels_[num_levels_] - levels_[0]; } -template -bool kll_sketch::is_estimation_mode() const { +template +bool kll_sketch::is_estimation_mode() const { return num_levels_ > 1; } -template -T kll_sketch::get_min_value() const { - if (is_empty()) return get_invalid_value(); - return *min_value_; +template +T kll_sketch::get_min_item() const { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + return *min_item_; +} + +template +T kll_sketch::get_max_item() const { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + return *max_item_; +} + +template +C kll_sketch::get_comparator() const { + return comparator_; } -template -T kll_sketch::get_max_value() const { - if (is_empty()) return get_invalid_value(); - return *max_value_; +template +A kll_sketch::get_allocator() const { + return allocator_; } -template -T kll_sketch::get_quantile(double fraction) const { - if (is_empty()) return get_invalid_value(); - if (fraction == 0.0) return *min_value_; - if (fraction == 1.0) return *max_value_; - if ((fraction < 0.0) || (fraction > 1.0)) { - throw std::invalid_argument("Fraction cannot be less than zero or greater than 1.0"); +template +double kll_sketch::get_rank(const T& item, bool inclusive) const { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + setup_sorted_view(); + return sorted_view_->get_rank(item, inclusive); +} + +template +auto kll_sketch::get_PMF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + setup_sorted_view(); + return sorted_view_->get_PMF(split_points, size, inclusive); +} + +template +auto kll_sketch::get_CDF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + setup_sorted_view(); + return sorted_view_->get_CDF(split_points, size, inclusive); +} + +template +auto kll_sketch::get_quantile(double rank, bool inclusive) const -> quantile_return_type { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); + if ((rank < 0.0) || (rank > 1.0)) { + throw std::invalid_argument("normalized rank cannot be less than zero or greater than 1.0"); } - // has side effect of sorting level zero if needed - auto quantile_calculator(const_cast(this)->get_quantile_calculator()); - return quantile_calculator->get_quantile(fraction); + // may have a side effect of sorting level zero if needed + setup_sorted_view(); + return sorted_view_->get_quantile(rank, inclusive); } -template -std::vector kll_sketch::get_quantiles(const double* fractions, uint32_t size) const { +template +std::vector kll_sketch::get_quantiles(const double* ranks, uint32_t size, bool inclusive) const { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); std::vector quantiles(allocator_); - if (is_empty()) return quantiles; - std::unique_ptr, std::function*)>> quantile_calculator; quantiles.reserve(size); + + // may have a side effect of sorting level zero if needed + setup_sorted_view(); + for (uint32_t i = 0; i < size; i++) { - const double fraction = fractions[i]; - if ((fraction < 0.0) || (fraction > 1.0)) { - throw std::invalid_argument("Fraction cannot be less than zero or greater than 1.0"); - } - if (fraction == 0.0) quantiles.push_back(*min_value_); - else if (fraction == 1.0) quantiles.push_back(*max_value_); - else { - if (!quantile_calculator) { - // has side effect of sorting level zero if needed - quantile_calculator = const_cast(this)->get_quantile_calculator(); - } - quantiles.push_back(quantile_calculator->get_quantile(fraction)); + const double rank = ranks[i]; + if ((rank < 0.0) || (rank > 1.0)) { + throw std::invalid_argument("normalized rank cannot be less than 0 or greater than 1"); } + quantiles.push_back(sorted_view_->get_quantile(rank, inclusive)); } return quantiles; } -template -std::vector kll_sketch::get_quantiles(uint32_t num) const { - if (is_empty()) return std::vector(allocator_); +template +std::vector kll_sketch::get_quantiles(uint32_t num, bool inclusive) const { + if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch"); if (num == 0) { throw std::invalid_argument("num must be > 0"); } - vector_d fractions(num, 0, allocator_); - fractions[0] = 0.0; + vector_double ranks(num, 0, allocator_); + ranks[0] = 0.0; for (size_t i = 1; i < num; i++) { - fractions[i] = static_cast(i) / (num - 1); + ranks[i] = static_cast(i) / (num - 1); } if (num > 1) { - fractions[num - 1] = 1.0; + ranks[num - 1] = 1.0; } - return get_quantiles(fractions.data(), num); -} - -template -double kll_sketch::get_rank(const T& value) const { - if (is_empty()) return std::numeric_limits::quiet_NaN(); - uint8_t level = 0; - uint64_t weight = 1; - uint64_t total = 0; - while (level < num_levels_) { - const auto from_index(levels_[level]); - const auto to_index(levels_[level + 1]); // exclusive - for (uint32_t i = from_index; i < to_index; i++) { - if (C()(items_[i], value)) { - total += weight; - } else if ((level > 0) || is_level_zero_sorted_) { - break; // levels above 0 are sorted, no point comparing further - } - } - level++; - weight *= 2; - } - return (double) total / n_; -} - -template -vector_d kll_sketch::get_PMF(const T* split_points, uint32_t size) const { - return get_PMF_or_CDF(split_points, size, false); + return get_quantiles(ranks.data(), num, inclusive); } -template -vector_d kll_sketch::get_CDF(const T* split_points, uint32_t size) const { - return get_PMF_or_CDF(split_points, size, true); -} - -template -double kll_sketch::get_normalized_rank_error(bool pmf) const { +template +double kll_sketch::get_normalized_rank_error(bool pmf) const { return get_normalized_rank_error(min_k_, pmf); } // implementation for fixed-size arithmetic types (integral and floating point) -template -template::value, int>::type> -size_t kll_sketch::get_serialized_size_bytes() const { +template +template::value, int>::type> +size_t kll_sketch::get_serialized_size_bytes(const SerDe&) const { if (is_empty()) { return EMPTY_SIZE_BYTES; } if (num_levels_ == 1 && get_num_retained() == 1) { return DATA_START_SINGLE_ITEM + sizeof(TT); @@ -369,25 +376,25 @@ size_t kll_sketch::get_serialized_size_bytes() const { } // implementation for all other types -template -template::value, int>::type> -size_t kll_sketch::get_serialized_size_bytes() const { +template +template::value, int>::type> +size_t kll_sketch::get_serialized_size_bytes(const SerDe& sd) const { if (is_empty()) { return EMPTY_SIZE_BYTES; } if (num_levels_ == 1 && get_num_retained() == 1) { - return DATA_START_SINGLE_ITEM + S().size_of_item(items_[levels_[0]]); + return DATA_START_SINGLE_ITEM + sd.size_of_item(items_[levels_[0]]); } // the last integer in the levels_ array is not serialized because it can be derived size_t size = DATA_START + num_levels_ * sizeof(uint32_t); - size += S().size_of_item(*min_value_); - size += S().size_of_item(*max_value_); - for (auto it: *this) size += S().size_of_item(it.first); + size += sd.size_of_item(*min_item_); + size += sd.size_of_item(*max_item_); + for (auto it: *this) size += sd.size_of_item(it.first); return size; } // implementation for fixed-size arithmetic types (integral and floating point) -template +template template::value, int>::type> -size_t kll_sketch::get_max_serialized_size_bytes(uint16_t k, uint64_t n) { +size_t kll_sketch::get_max_serialized_size_bytes(uint16_t k, uint64_t n) { const uint8_t num_levels = kll_helper::ub_on_num_levels(n); const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, DEFAULT_M, num_levels); // the last integer in the levels_ array is not serialized because it can be derived @@ -395,17 +402,18 @@ size_t kll_sketch::get_max_serialized_size_bytes(uint16_t k, uint64_ } // implementation for all other types -template +template template::value, int>::type> -size_t kll_sketch::get_max_serialized_size_bytes(uint16_t k, uint64_t n, size_t max_item_size_bytes) { +size_t kll_sketch::get_max_serialized_size_bytes(uint16_t k, uint64_t n, size_t max_item_size_bytes) { const uint8_t num_levels = kll_helper::ub_on_num_levels(n); const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, DEFAULT_M, num_levels); // the last integer in the levels_ array is not serialized because it can be derived return DATA_START + num_levels * sizeof(uint32_t) + (max_num_retained + 2) * max_item_size_bytes; } -template -void kll_sketch::serialize(std::ostream& os) const { +template +template +void kll_sketch::serialize(std::ostream& os, const SerDe& sd) const { const bool is_single_item = n_ == 1; const uint8_t preamble_ints(is_empty() || is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL); write(os, preamble_ints); @@ -430,17 +438,18 @@ void kll_sketch::serialize(std::ostream& os) const { write(os, num_levels_); write(os, unused); write(os, levels_.data(), sizeof(levels_[0]) * num_levels_); - S().serialize(os, min_value_, 1); - S().serialize(os, max_value_, 1); + sd.serialize(os, min_item_, 1); + sd.serialize(os, max_item_, 1); } - S().serialize(os, &items_[levels_[0]], get_num_retained()); + sd.serialize(os, &items_[levels_[0]], get_num_retained()); } -template -vector_u8 kll_sketch::serialize(unsigned header_size_bytes) const { +template +template +auto kll_sketch::serialize(unsigned header_size_bytes, const SerDe& sd) const -> vector_bytes { const bool is_single_item = n_ == 1; - const size_t size = header_size_bytes + get_serialized_size_bytes(); - vector_u8 bytes(size, 0, allocator_); + const size_t size = header_size_bytes + get_serialized_size_bytes(sd); + vector_bytes bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; const uint8_t* end_ptr = ptr + size; const uint8_t preamble_ints(is_empty() || is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL); @@ -465,19 +474,22 @@ vector_u8 kll_sketch::serialize(unsigned header_size_bytes) const ptr += copy_to_mem(num_levels_, ptr); ptr += sizeof(uint8_t); // unused ptr += copy_to_mem(levels_.data(), ptr, sizeof(levels_[0]) * num_levels_); - ptr += S().serialize(ptr, end_ptr - ptr, min_value_, 1); - ptr += S().serialize(ptr, end_ptr - ptr, max_value_, 1); + ptr += sd.serialize(ptr, end_ptr - ptr, min_item_, 1); + ptr += sd.serialize(ptr, end_ptr - ptr, max_item_, 1); } const size_t bytes_remaining = end_ptr - ptr; - ptr += S().serialize(ptr, bytes_remaining, &items_[levels_[0]], get_num_retained()); + ptr += sd.serialize(ptr, bytes_remaining, &items_[levels_[0]], get_num_retained()); } const size_t delta = ptr - bytes.data(); - if (delta != size) throw std::logic_error("serialized size mismatch: " + std::to_string(delta) + " != " + std::to_string(size)); + if (delta != size) throw std::logic_error("serialized size mismatch: " + std::to_string(delta) + + " != " + std::to_string(size)); return bytes; } -template -kll_sketch kll_sketch::deserialize(std::istream& is, const A& allocator) { +template +template +kll_sketch kll_sketch::deserialize(std::istream& is, const SerDe& sd, + const C& comparator, const A& allocator) { const auto preamble_ints = read(is); const auto serial_version = read(is); const auto family_id = read(is); @@ -493,7 +505,7 @@ kll_sketch kll_sketch::deserialize(std::istream& is, con if (!is.good()) throw std::runtime_error("error reading from std::istream"); const bool is_empty(flags_byte & (1 << flags::IS_EMPTY)); - if (is_empty) return kll_sketch(k, allocator); + if (is_empty) return kll_sketch(k, comparator, allocator); uint64_t n; uint16_t min_k; @@ -509,7 +521,7 @@ kll_sketch kll_sketch::deserialize(std::istream& is, con num_levels = read(is); read(is); // skip unused byte } - vector_u32 levels(num_levels + 1, 0, allocator); + vector_u32 levels(num_levels + 1, 0, allocator); const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels)); if (is_single_item) { levels[0] = capacity - 1; @@ -520,41 +532,43 @@ kll_sketch kll_sketch::deserialize(std::istream& is, con levels[num_levels] = capacity; A alloc(allocator); auto item_buffer_deleter = [&alloc](T* ptr) { alloc.deallocate(ptr, 1); }; - std::unique_ptr min_value_buffer(alloc.allocate(1), item_buffer_deleter); - std::unique_ptr max_value_buffer(alloc.allocate(1), item_buffer_deleter); - std::unique_ptr min_value(nullptr, item_deleter(allocator)); - std::unique_ptr max_value(nullptr, item_deleter(allocator)); + std::unique_ptr min_item_buffer(alloc.allocate(1), item_buffer_deleter); + std::unique_ptr max_item_buffer(alloc.allocate(1), item_buffer_deleter); + std::unique_ptr min_item(nullptr, item_deleter(allocator)); + std::unique_ptr max_item(nullptr, item_deleter(allocator)); if (!is_single_item) { - S().deserialize(is, min_value_buffer.get(), 1); + sd.deserialize(is, min_item_buffer.get(), 1); // serde call did not throw, repackage with destrtuctor - min_value = std::unique_ptr(min_value_buffer.release(), item_deleter(allocator)); - S().deserialize(is, max_value_buffer.get(), 1); + min_item = std::unique_ptr(min_item_buffer.release(), item_deleter(allocator)); + sd.deserialize(is, max_item_buffer.get(), 1); // serde call did not throw, repackage with destrtuctor - max_value = std::unique_ptr(max_value_buffer.release(), item_deleter(allocator)); + max_item = std::unique_ptr(max_item_buffer.release(), item_deleter(allocator)); } auto items_buffer_deleter = [capacity, &alloc](T* ptr) { alloc.deallocate(ptr, capacity); }; std::unique_ptr items_buffer(alloc.allocate(capacity), items_buffer_deleter); const auto num_items = levels[num_levels] - levels[0]; - S().deserialize(is, &items_buffer.get()[levels[0]], num_items); + sd.deserialize(is, &items_buffer.get()[levels[0]], num_items); // serde call did not throw, repackage with destrtuctors std::unique_ptr items(items_buffer.release(), items_deleter(levels[0], capacity, allocator)); const bool is_level_zero_sorted = (flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED)) > 0; if (is_single_item) { - new (min_value_buffer.get()) T(items.get()[levels[0]]); + new (min_item_buffer.get()) T(items.get()[levels[0]]); // copy did not throw, repackage with destrtuctor - min_value = std::unique_ptr(min_value_buffer.release(), item_deleter(allocator)); - new (max_value_buffer.get()) T(items.get()[levels[0]]); + min_item = std::unique_ptr(min_item_buffer.release(), item_deleter(allocator)); + new (max_item_buffer.get()) T(items.get()[levels[0]]); // copy did not throw, repackage with destrtuctor - max_value = std::unique_ptr(max_value_buffer.release(), item_deleter(allocator)); + max_item = std::unique_ptr(max_item_buffer.release(), item_deleter(allocator)); } if (!is.good()) throw std::runtime_error("error reading from std::istream"); return kll_sketch(k, min_k, n, num_levels, std::move(levels), std::move(items), capacity, - std::move(min_value), std::move(max_value), is_level_zero_sorted); + std::move(min_item), std::move(max_item), is_level_zero_sorted, comparator); } -template -kll_sketch kll_sketch::deserialize(const void* bytes, size_t size, const A& allocator) { +template +template +kll_sketch kll_sketch::deserialize(const void* bytes, size_t size, const SerDe& sd, + const C& comparator, const A& allocator) { ensure_minimum_memory(size, 8); const char* ptr = static_cast(bytes); uint8_t preamble_ints; @@ -578,7 +592,7 @@ kll_sketch kll_sketch::deserialize(const void* bytes, si ensure_minimum_memory(size, preamble_ints * sizeof(uint32_t)); const bool is_empty(flags_byte & (1 << flags::IS_EMPTY)); - if (is_empty) return kll_sketch(k, allocator); + if (is_empty) return kll_sketch(k, comparator, allocator); uint64_t n; uint16_t min_k; @@ -595,7 +609,7 @@ kll_sketch kll_sketch::deserialize(const void* bytes, si ptr += copy_from_mem(ptr, num_levels); ptr += sizeof(uint8_t); // skip unused byte } - vector_u32 levels(num_levels + 1, 0, allocator); + vector_u32 levels(num_levels + 1, 0, allocator); const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels)); if (is_single_item) { levels[0] = capacity - 1; @@ -606,37 +620,37 @@ kll_sketch kll_sketch::deserialize(const void* bytes, si levels[num_levels] = capacity; A alloc(allocator); auto item_buffer_deleter = [&alloc](T* ptr) { alloc.deallocate(ptr, 1); }; - std::unique_ptr min_value_buffer(alloc.allocate(1), item_buffer_deleter); - std::unique_ptr max_value_buffer(alloc.allocate(1), item_buffer_deleter); - std::unique_ptr min_value(nullptr, item_deleter(allocator)); - std::unique_ptr max_value(nullptr, item_deleter(allocator)); + std::unique_ptr min_item_buffer(alloc.allocate(1), item_buffer_deleter); + std::unique_ptr max_item_buffer(alloc.allocate(1), item_buffer_deleter); + std::unique_ptr min_item(nullptr, item_deleter(allocator)); + std::unique_ptr max_item(nullptr, item_deleter(allocator)); if (!is_single_item) { - ptr += S().deserialize(ptr, end_ptr - ptr, min_value_buffer.get(), 1); + ptr += sd.deserialize(ptr, end_ptr - ptr, min_item_buffer.get(), 1); // serde call did not throw, repackage with destrtuctor - min_value = std::unique_ptr(min_value_buffer.release(), item_deleter(allocator)); - ptr += S().deserialize(ptr, end_ptr - ptr, max_value_buffer.get(), 1); + min_item = std::unique_ptr(min_item_buffer.release(), item_deleter(allocator)); + ptr += sd.deserialize(ptr, end_ptr - ptr, max_item_buffer.get(), 1); // serde call did not throw, repackage with destrtuctor - max_value = std::unique_ptr(max_value_buffer.release(), item_deleter(allocator)); + max_item = std::unique_ptr(max_item_buffer.release(), item_deleter(allocator)); } auto items_buffer_deleter = [capacity, &alloc](T* ptr) { alloc.deallocate(ptr, capacity); }; std::unique_ptr items_buffer(alloc.allocate(capacity), items_buffer_deleter); const auto num_items = levels[num_levels] - levels[0]; - ptr += S().deserialize(ptr, end_ptr - ptr, &items_buffer.get()[levels[0]], num_items); + ptr += sd.deserialize(ptr, end_ptr - ptr, &items_buffer.get()[levels[0]], num_items); // serde call did not throw, repackage with destrtuctors std::unique_ptr items(items_buffer.release(), items_deleter(levels[0], capacity, allocator)); const size_t delta = ptr - static_cast(bytes); if (delta != size) throw std::logic_error("deserialized size mismatch: " + std::to_string(delta) + " != " + std::to_string(size)); const bool is_level_zero_sorted = (flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED)) > 0; if (is_single_item) { - new (min_value_buffer.get()) T(items.get()[levels[0]]); + new (min_item_buffer.get()) T(items.get()[levels[0]]); // copy did not throw, repackage with destrtuctor - min_value = std::unique_ptr(min_value_buffer.release(), item_deleter(allocator)); - new (max_value_buffer.get()) T(items.get()[levels[0]]); + min_item = std::unique_ptr(min_item_buffer.release(), item_deleter(allocator)); + new (max_item_buffer.get()) T(items.get()[levels[0]]); // copy did not throw, repackage with destrtuctor - max_value = std::unique_ptr(max_value_buffer.release(), item_deleter(allocator)); + max_item = std::unique_ptr(max_item_buffer.release(), item_deleter(allocator)); } return kll_sketch(k, min_k, n, num_levels, std::move(levels), std::move(items), capacity, - std::move(min_value), std::move(max_value), is_level_zero_sorted); + std::move(min_item), std::move(max_item), is_level_zero_sorted, comparator); } /* @@ -646,36 +660,38 @@ kll_sketch kll_sketch::deserialize(const void* bytes, si * Otherwise, it is the "single-sided" normalized rank error for all the other queries. * Constants were derived as the best fit to 99 percentile empirically measured max error in thousands of trials */ -template -double kll_sketch::get_normalized_rank_error(uint16_t k, bool pmf) { +template +double kll_sketch::get_normalized_rank_error(uint16_t k, bool pmf) { return pmf ? 2.446 / pow(k, 0.9433) : 2.296 / pow(k, 0.9723); } // for deserialization -template -kll_sketch::kll_sketch(uint16_t k, uint16_t min_k, uint64_t n, uint8_t num_levels, vector_u32&& levels, - std::unique_ptr items, uint32_t items_size, std::unique_ptr min_value, - std::unique_ptr max_value, bool is_level_zero_sorted): +template +kll_sketch::kll_sketch(uint16_t k, uint16_t min_k, uint64_t n, uint8_t num_levels, vector_u32&& levels, + std::unique_ptr items, uint32_t items_size, std::unique_ptr min_item, + std::unique_ptr max_item, bool is_level_zero_sorted, const C& comparator): +comparator_(comparator), allocator_(levels.get_allocator()), k_(k), m_(DEFAULT_M), min_k_(min_k), -n_(n), num_levels_(num_levels), +is_level_zero_sorted_(is_level_zero_sorted), +n_(n), levels_(std::move(levels)), items_(items.release()), items_size_(items_size), -min_value_(min_value.release()), -max_value_(max_value.release()), -is_level_zero_sorted_(is_level_zero_sorted) +min_item_(min_item.release()), +max_item_(max_item.release()), +sorted_view_(nullptr) {} // The following code is only valid in the special case of exactly reaching capacity while updating. // It cannot be used while merging, while reducing k, or anything else. -template -void kll_sketch::compress_while_updating(void) { +template +void kll_sketch::compress_while_updating(void) { const uint8_t level = find_level_to_compact(); // It is important to add the new top level right here. Be aware that this operation @@ -699,7 +715,7 @@ void kll_sketch::compress_while_updating(void) { // level zero might not be sorted, so we must sort it if we wish to compact it // sort_level_zero() is not used here because of the adjustment for odd number of items if ((level == 0) && !is_level_zero_sorted_) { - std::sort(&items_[adj_beg], &items_[adj_beg + adj_pop], C()); + std::sort(items_ + adj_beg, items_ + adj_beg + adj_pop, comparator_); } if (pop_above == 0) { kll_helper::randomly_halve_up(items_, adj_beg, adj_pop); @@ -722,14 +738,14 @@ void kll_sketch::compress_while_updating(void) { // so that the freed-up space can be used by level zero if (level > 0) { const uint32_t amount = raw_beg - levels_[0]; - std::move_backward(&items_[levels_[0]], &items_[levels_[0] + amount], &items_[levels_[0] + half_adj_pop + amount]); + std::move_backward(items_ + levels_[0], items_ + levels_[0] + amount, items_ + levels_[0] + half_adj_pop + amount); for (uint8_t lvl = 0; lvl < level; lvl++) levels_[lvl] += half_adj_pop; } for (uint32_t i = 0; i < half_adj_pop; i++) items_[i + destroy_beg].~T(); } -template -uint8_t kll_sketch::find_level_to_compact() const { +template +uint8_t kll_sketch::find_level_to_compact() const { uint8_t level = 0; while (true) { if (level >= num_levels_) throw std::logic_error("capacity calculation error"); @@ -742,8 +758,8 @@ uint8_t kll_sketch::find_level_to_compact() const { } } -template -void kll_sketch::add_empty_top_level_to_completely_full_sketch() { +template +void kll_sketch::add_empty_top_level_to_completely_full_sketch() { const uint32_t cur_total_cap = levels_[num_levels_]; // make sure that we are following a certain growth scheme @@ -777,107 +793,50 @@ void kll_sketch::add_empty_top_level_to_completely_full_sketch() { levels_[num_levels_] = new_total_cap; // initialize the new "extra" index at the top } -template -void kll_sketch::sort_level_zero() { +template +void kll_sketch::sort_level_zero() { if (!is_level_zero_sorted_) { - std::sort(&items_[levels_[0]], &items_[levels_[1]], C()); + std::sort(items_ + levels_[0], items_ + levels_[1], comparator_); is_level_zero_sorted_ = true; } } -template -std::unique_ptr, std::function*)>> kll_sketch::get_quantile_calculator() { - sort_level_zero(); - using AllocCalc = typename std::allocator_traits::template rebind_alloc>; - AllocCalc alloc(allocator_); - std::unique_ptr, std::function*)>> quantile_calculator( - new (alloc.allocate(1)) kll_quantile_calculator(*this), - [&alloc](kll_quantile_calculator* ptr){ ptr->~kll_quantile_calculator(); alloc.deallocate(ptr, 1); } - ); - return quantile_calculator; -} - -template -vector_d kll_sketch::get_PMF_or_CDF(const T* split_points, uint32_t size, bool is_CDF) const { - if (is_empty()) return vector_d(allocator_); - kll_helper::validate_values(split_points, size); - vector_d buckets(size + 1, 0, allocator_); - uint8_t level = 0; - uint64_t weight = 1; - while (level < num_levels_) { - const auto from_index = levels_[level]; - const auto to_index = levels_[level + 1]; // exclusive - if ((level == 0) && !is_level_zero_sorted_) { - increment_buckets_unsorted_level(from_index, to_index, weight, split_points, size, buckets.data()); - } else { - increment_buckets_sorted_level(from_index, to_index, weight, split_points, size, buckets.data()); +template +void kll_sketch::check_sorting() const { + // not checking level 0 + for (uint8_t level = 1; level < num_levels_; ++level) { + const auto from = items_ + levels_[level]; + const auto to = items_ + levels_[level + 1]; + if (!std::is_sorted(from, to, comparator_)) { + throw std::logic_error("levels must be sorted"); } - level++; - weight *= 2; } - // normalize and, if CDF, convert to cumulative - if (is_CDF) { - double subtotal = 0; - for (uint32_t i = 0; i <= size; i++) { - subtotal += buckets[i]; - buckets[i] = subtotal / n_; - } - } else { - for (uint32_t i = 0; i <= size; i++) { - buckets[i] /= n_; - } - } - return buckets; } -template -void kll_sketch::increment_buckets_unsorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, - const T* split_points, uint32_t size, double* buckets) const -{ - for (uint32_t i = from_index; i < to_index; i++) { - uint32_t j; - for (j = 0; j < size; j++) { - if (C()(items_[i], split_points[j])) { - break; - } - } - buckets[j] += weight; +template +quantiles_sorted_view kll_sketch::get_sorted_view() const { + const_cast(this)->sort_level_zero(); // allow this side effect + quantiles_sorted_view view(get_num_retained(), comparator_, allocator_); + for (uint8_t level = 0; level < num_levels_; ++level) { + const auto from = items_ + levels_[level]; + const auto to = items_ + levels_[level + 1]; // exclusive + view.add(from, to, 1 << level); } + view.convert_to_cummulative(); + return view; } -template -void kll_sketch::increment_buckets_sorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, - const T* split_points, uint32_t size, double* buckets) const -{ - uint32_t i = from_index; - uint32_t j = 0; - while ((i < to_index) && (j < size)) { - if (C()(items_[i], split_points[j])) { - buckets[j] += weight; // this sample goes into this bucket - i++; // move on to next sample and see whether it also goes into this bucket - } else { - j++; // no more samples for this bucket - } - } - // now either i == to_index (we are out of samples), or - // j == size (we are out of buckets, but there are more samples remaining) - // we only need to do something in the latter case - if (j == size) { - buckets[j] += weight * (to_index - i); - } -} - -template +template template -void kll_sketch::merge_higher_levels(O&& other, uint64_t final_n) { +void kll_sketch::merge_higher_levels(O&& other, uint64_t final_n) { const uint32_t tmp_num_items = get_num_retained() + other.get_num_retained_above_level_zero(); A alloc(allocator_); auto tmp_items_deleter = [tmp_num_items, &alloc](T* ptr) { alloc.deallocate(ptr, tmp_num_items); }; // no destructor needed const std::unique_ptr workbuf(allocator_.allocate(tmp_num_items), tmp_items_deleter); const uint8_t ub = kll_helper::ub_on_num_levels(final_n); const size_t work_levels_size = ub + 2; // ub+1 does not work - vector_u32 worklevels(work_levels_size, 0, allocator_); - vector_u32 outlevels(work_levels_size, 0, allocator_); + vector_u32 worklevels(work_levels_size, 0, allocator_); + vector_u32 outlevels(work_levels_size, 0, allocator_); const uint8_t provisional_num_levels = std::max(num_levels_, other.num_levels_); @@ -903,16 +862,16 @@ void kll_sketch::merge_higher_levels(O&& other, uint64_t final_n) { levels_.resize(new_levels_size); } const uint32_t offset = free_space_at_bottom - outlevels[0]; - for (uint8_t lvl = 0; lvl < static_cast(levels_.size()); lvl++) { // includes the "extra" index + for (uint8_t lvl = 0; lvl < levels_.size(); lvl++) { // includes the "extra" index levels_[lvl] = outlevels[lvl] + offset; } num_levels_ = result.final_num_levels; } // this leaves items_ uninitialized (all objects moved out and destroyed) -// this version copies objects from the incoming sketch -template -void kll_sketch::populate_work_arrays(const kll_sketch& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels) { +template +template +void kll_sketch::populate_work_arrays(FwdSk&& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels) { worklevels[0] = 0; // the level zero data from "other" was already inserted into "this" @@ -927,68 +886,45 @@ void kll_sketch::populate_work_arrays(const kll_sketch& other, T* wo if ((self_pop > 0) && (other_pop == 0)) { kll_helper::move_construct(items_, levels_[lvl], levels_[lvl] + self_pop, workbuf, worklevels[lvl], true); } else if ((self_pop == 0) && (other_pop > 0)) { - kll_helper::copy_construct(other.items_, other.levels_[lvl], other.levels_[lvl] + other_pop, workbuf, worklevels[lvl]); - } else if ((self_pop > 0) && (other_pop > 0)) { - kll_helper::merge_sorted_arrays(items_, levels_[lvl], self_pop, other.items_, other.levels_[lvl], other_pop, workbuf, worklevels[lvl]); - } - } -} - -// this leaves items_ uninitialized (all objects moved out and destroyed) -// this version moves objects from the incoming sketch -template -void kll_sketch::populate_work_arrays(kll_sketch&& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels) { - worklevels[0] = 0; - - // the level zero data from "other" was already inserted into "this" - kll_helper::move_construct(items_, levels_[0], levels_[1], workbuf, 0, true); - worklevels[1] = safe_level_size(0); - - for (uint8_t lvl = 1; lvl < provisional_num_levels; lvl++) { - const uint32_t self_pop = safe_level_size(lvl); - const uint32_t other_pop = other.safe_level_size(lvl); - worklevels[lvl + 1] = worklevels[lvl] + self_pop + other_pop; - - if ((self_pop > 0) && (other_pop == 0)) { - kll_helper::move_construct(items_, levels_[lvl], levels_[lvl] + self_pop, workbuf, worklevels[lvl], true); - } else if ((self_pop == 0) && (other_pop > 0)) { - kll_helper::move_construct(other.items_, other.levels_[lvl], other.levels_[lvl] + other_pop, workbuf, worklevels[lvl], false); + for (auto i = other.levels_[lvl], j = worklevels[lvl]; i < other.levels_[lvl] + other_pop; ++i, ++j) { + new (&workbuf[j]) T(conditional_forward(other.items_[i])); + } } else if ((self_pop > 0) && (other_pop > 0)) { kll_helper::merge_sorted_arrays(items_, levels_[lvl], self_pop, other.items_, other.levels_[lvl], other_pop, workbuf, worklevels[lvl]); } } } -template -void kll_sketch::assert_correct_total_weight() const { +template +void kll_sketch::assert_correct_total_weight() const { const uint64_t total(kll_helper::sum_the_sample_weights(num_levels_, levels_.data())); if (total != n_) { throw std::logic_error("Total weight does not match N"); } } -template -uint32_t kll_sketch::safe_level_size(uint8_t level) const { +template +uint32_t kll_sketch::safe_level_size(uint8_t level) const { if (level >= num_levels_) return 0; return levels_[level + 1] - levels_[level]; } -template -uint32_t kll_sketch::get_num_retained_above_level_zero() const { +template +uint32_t kll_sketch::get_num_retained_above_level_zero() const { if (num_levels_ == 1) return 0; return levels_[num_levels_] - levels_[1]; } -template -void kll_sketch::check_m(uint8_t m) { +template +void kll_sketch::check_m(uint8_t m) { if (m != DEFAULT_M) { throw std::invalid_argument("Possible corruption: M must be " + std::to_string(DEFAULT_M) + ": " + std::to_string(m)); } } -template -void kll_sketch::check_preamble_ints(uint8_t preamble_ints, uint8_t flags_byte) { +template +void kll_sketch::check_preamble_ints(uint8_t preamble_ints, uint8_t flags_byte) { const bool is_empty(flags_byte & (1 << flags::IS_EMPTY)); const bool is_single_item(flags_byte & (1 << flags::IS_SINGLE_ITEM)); if (is_empty || is_single_item) { @@ -1004,8 +940,8 @@ void kll_sketch::check_preamble_ints(uint8_t preamble_ints, uint8_t } } -template -void kll_sketch::check_serial_version(uint8_t serial_version) { +template +void kll_sketch::check_serial_version(uint8_t serial_version) { if (serial_version != SERIAL_VERSION_1 && serial_version != SERIAL_VERSION_2) { throw std::invalid_argument("Possible corruption: serial version mismatch: expected " + std::to_string(SERIAL_VERSION_1) + " or " + std::to_string(SERIAL_VERSION_2) @@ -1013,16 +949,16 @@ void kll_sketch::check_serial_version(uint8_t serial_version) { } } -template -void kll_sketch::check_family_id(uint8_t family_id) { +template +void kll_sketch::check_family_id(uint8_t family_id) { if (family_id != FAMILY) { throw std::invalid_argument("Possible corruption: family mismatch: expected " + std::to_string(FAMILY) + ", got " + std::to_string(family_id)); } } -template -string kll_sketch::to_string(bool print_levels, bool print_items) const { +template +string kll_sketch::to_string(bool print_levels, bool print_items) const { // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements. // The stream does not support passing an allocator instance, and alternatives are complicated. std::ostringstream os; @@ -1039,10 +975,9 @@ string kll_sketch::to_string(bool print_levels, bool print_items) os << " Sorted : " << (is_level_zero_sorted_ ? "true" : "false") << std::endl; os << " Capacity items : " << items_size_ << std::endl; os << " Retained items : " << get_num_retained() << std::endl; - os << " Storage bytes : " << get_serialized_size_bytes() << std::endl; if (!is_empty()) { - os << " Min value : " << *min_value_ << std::endl; - os << " Max value : " << *max_value_ << std::endl; + os << " Min item : " << *min_item_ << std::endl; + os << " Max item : " << *max_item_ << std::endl; } os << "### End sketch summary" << std::endl; @@ -1074,25 +1009,74 @@ string kll_sketch::to_string(bool print_levels, bool print_items) return string(os.str().c_str(), allocator_); } -template -typename kll_sketch::const_iterator kll_sketch::begin() const { - return kll_sketch::const_iterator(items_, levels_.data(), num_levels_); +template +typename kll_sketch::const_iterator kll_sketch::begin() const { + return kll_sketch::const_iterator(items_, levels_.data(), num_levels_); } -template -typename kll_sketch::const_iterator kll_sketch::end() const { - return kll_sketch::const_iterator(nullptr, levels_.data(), num_levels_); +template +typename kll_sketch::const_iterator kll_sketch::end() const { + return kll_sketch::const_iterator(nullptr, levels_.data(), num_levels_); +} + +template +class kll_sketch::item_deleter { + public: + item_deleter(const A& allocator): allocator_(allocator) {} + void operator() (T* ptr) { + if (ptr != nullptr) { + ptr->~T(); + allocator_.deallocate(ptr, 1); + } + } + private: + A allocator_; +}; + +template +class kll_sketch::items_deleter { + public: + items_deleter(uint32_t start, uint32_t num, const A& allocator): + allocator_(allocator), start_(start), num_(num) {} + void operator() (T* ptr) { + if (ptr != nullptr) { + for (uint32_t i = start_; i < num_; ++i) ptr[i].~T(); + allocator_.deallocate(ptr, num_); + } + } + private: + A allocator_; + uint32_t start_; + uint32_t num_; +}; + +template +void kll_sketch::setup_sorted_view() const { + if (sorted_view_ == nullptr) { + using AllocSortedView = typename std::allocator_traits::template rebind_alloc>; + sorted_view_ = new (AllocSortedView(allocator_).allocate(1)) quantiles_sorted_view(get_sorted_view()); + } +} + +template +void kll_sketch::reset_sorted_view() { + if (sorted_view_ != nullptr) { + sorted_view_->~quantiles_sorted_view(); + using AllocSortedView = typename std::allocator_traits::template rebind_alloc>; + AllocSortedView(allocator_).deallocate(sorted_view_, 1); + sorted_view_ = nullptr; + } } // kll_sketch::const_iterator implementation -template -kll_sketch::const_iterator::const_iterator(const T* items, const uint32_t* levels, const uint8_t num_levels): +template +kll_sketch::const_iterator::const_iterator(const T* items, const uint32_t* levels, const uint8_t num_levels): items(items), levels(levels), num_levels(num_levels), index(items == nullptr ? levels[num_levels] : levels[0]), level(items == nullptr ? num_levels : 0), weight(1) {} -template -typename kll_sketch::const_iterator& kll_sketch::const_iterator::operator++() { +template +typename kll_sketch::const_iterator& kll_sketch::const_iterator::operator++() { ++index; if (index == levels[level + 1]) { // go to the next non-empty level do { @@ -1103,58 +1087,32 @@ typename kll_sketch::const_iterator& kll_sketch::const_i return *this; } -template -typename kll_sketch::const_iterator& kll_sketch::const_iterator::operator++(int) { +template +typename kll_sketch::const_iterator& kll_sketch::const_iterator::operator++(int) { const_iterator tmp(*this); operator++(); return tmp; } -template -bool kll_sketch::const_iterator::operator==(const const_iterator& other) const { +template +bool kll_sketch::const_iterator::operator==(const const_iterator& other) const { return index == other.index; } -template -bool kll_sketch::const_iterator::operator!=(const const_iterator& other) const { +template +bool kll_sketch::const_iterator::operator!=(const const_iterator& other) const { return !operator==(other); } -template -const std::pair kll_sketch::const_iterator::operator*() const { - return std::pair(items[index], weight); +template +auto kll_sketch::const_iterator::operator*() const -> reference { + return value_type(items[index], weight); } -template -class kll_sketch::item_deleter { - public: - item_deleter(const A& allocator): allocator_(allocator) {} - void operator() (T* ptr) { - if (ptr != nullptr) { - ptr->~T(); - allocator_.deallocate(ptr, 1); - } - } - private: - A allocator_; -}; - -template -class kll_sketch::items_deleter { - public: - items_deleter(uint32_t start, uint32_t num, const A& allocator): - allocator_(allocator), start_(start), num_(num) {} - void operator() (T* ptr) { - if (ptr != nullptr) { - for (uint32_t i = start_; i < num_; ++i) ptr[i].~T(); - allocator_.deallocate(ptr, num_); - } - } - private: - A allocator_; - uint32_t start_; - uint32_t num_; -}; +template +auto kll_sketch::const_iterator::operator->() const -> pointer { + return **this; +} } /* namespace datasketches */ diff --git a/3rd/fstrm/CMakeLists.txt b/3rd/fstrm/CMakeLists.txt index 982fd713a..ed7f03032 100644 --- a/3rd/fstrm/CMakeLists.txt +++ b/3rd/fstrm/CMakeLists.txt @@ -16,7 +16,7 @@ add_library(fstrm libmy/my_queue_mutex.c ) add_library(fstrm::fstrm ALIAS fstrm) -target_compile_options(fstrm PRIVATE -Wno-pedantic -Wno-unused-variable) +target_compile_options(fstrm PRIVATE -Wno-pedantic -Wno-unused-variable -Wno-unused-but-set-variable) target_include_directories(fstrm PUBLIC diff --git a/CMakeLists.txt b/CMakeLists.txt index ffda16259..ba067218d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.13) ####################################################### # this is the source of truth for semver version -project(visor VERSION 4.4.0) +project(visor VERSION 4.5.0) # for main line release, this is empty # for development release, this is "-develop" @@ -47,7 +47,8 @@ configure_file(golang/pkg/client/version.go.in ${PROJECT_SOURCE_DIR}/golang/pkg/ if(NOT DEFINED ENV{VERSION_ONLY}) -list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_BINARY_DIR}) +list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/automated_tests/requirements.txt b/automated_tests/requirements.txt index a7322c07f..ba4327fe9 100644 --- a/automated_tests/requirements.txt +++ b/automated_tests/requirements.txt @@ -1,7 +1,7 @@ attrs==21.4.0 behave==1.2.6 behavex==1.5.11 -certifi==2022.6.15 +certifi==2023.7.22 charset-normalizer==2.1.0 configobj==5.0.6 csscompressor==0.9.5 @@ -19,8 +19,8 @@ parse-type==0.6.0 PyHamcrest==2.0.3 pyrsistent==0.18.1 PyYAML==6.0 -requests==2.28.1 +requests==2.31.0 six==1.16.0 -urllib3==1.26.11 +urllib3==1.26.18 websocket-client==1.3.3 zipp==3.8.1 diff --git a/cmake/FindCorrade.cmake b/cmake/FindCorrade.cmake deleted file mode 100644 index 6851f0a5a..000000000 --- a/cmake/FindCorrade.cmake +++ /dev/null @@ -1,633 +0,0 @@ -#.rst: -# Find Corrade -# ------------ -# -# Finds the Corrade library. Basic usage:: -# -# find_package(Corrade REQUIRED) -# -# This module tries to find the base Corrade library and then defines the -# following: -# -# Corrade_FOUND - Whether the base library was found -# CORRADE_LIB_SUFFIX_MODULE - Path to CorradeLibSuffix.cmake module -# -# This command will try to find only the base library, not the optional -# components, which are: -# -# Containers - Containers library -# Interconnect - Interconnect library -# Main - Main library -# PluginManager - PluginManager library -# TestSuite - TestSuite library -# Utility - Utility library -# rc - corrade-rc executable -# -# Example usage with specifying additional components is:: -# -# find_package(Corrade REQUIRED Utility TestSuite) -# -# For each component is then defined: -# -# Corrade_*_FOUND - Whether the component was found -# Corrade::* - Component imported target -# -# The package is found if either debug or release version of each library is -# found. If both debug and release libraries are found, proper version is -# chosen based on actual build configuration of the project (i.e. Debug build -# is linked to debug libraries, Release build to release libraries). -# -# Corrade conditionally defines ``CORRADE_IS_DEBUG_BUILD`` preprocessor -# variable in case build configuration is ``Debug`` (not Corrade itself, but -# build configuration of the project using it). Useful e.g. for selecting -# proper plugin directory. -# -# Corrade defines the following custom target properties: -# -# CORRADE_CXX_STANDARD - C++ standard to require when compiling given -# target. Does nothing if :variable:`CMAKE_CXX_FLAGS` already contains -# particular standard setting flag or if given target contains -# :prop_tgt:`CMAKE_CXX_STANDARD` property. Allowed value is 11, 14 or 17. -# INTERFACE_CORRADE_CXX_STANDARD - C++ standard to require when using given -# target. Does nothing if :variable:`CMAKE_CXX_FLAGS` already contains -# particular standard setting flag or if given target contains -# :prop_tgt:`CMAKE_CXX_STANDARD` property. Allowed value is 11, 14 or 17. -# CORRADE_USE_PEDANTIC_FLAGS - Enable additional compiler/linker flags. -# Boolean. -# -# These properties are inherited from directory properties, meaning that if you -# set them on directories, they get implicitly set on all targets in given -# directory (with a possibility to do target-specific overrides). All Corrade -# libraries have the :prop_tgt:`INTERFACE_CORRADE_CXX_STANDARD` property set to -# 11, meaning that you will always have at least C++11 enabled once you link to -# any Corrade library. -# -# Features of found Corrade library are exposed in these variables: -# -# CORRADE_MSVC_COMPATIBILITY - Defined if compiled with compatibility -# mode for MSVC 2019+ without the /permissive- flag set -# CORRADE_MSVC2017_COMPATIBILITY - Defined if compiled with compatibility -# mode for MSVC 2017 -# CORRADE_MSVC2015_COMPATIBILITY - Defined if compiled with compatibility -# mode for MSVC 2015 -# CORRADE_BUILD_DEPRECATED - Defined if compiled with deprecated APIs -# included -# CORRADE_BUILD_STATIC - Defined if compiled as static libraries. -# Default are shared libraries. -# CORRADE_BUILD_STATIC_UNIQUE_GLOBALS - Defined if static libraries keep their -# globals unique even across different shared libraries. Enabled by default -# for static builds. -# CORRADE_BUILD_MULTITHREADED - Defined if compiled in a way that makes it -# possible to safely use certain Corrade features simultaneously in multiple -# threads -# CORRADE_BUILD_CPU_RUNTIME_DISPATCH - Defined if built with code paths -# optimized for multiple architectres with the best matching variant selected -# at runtime based on detected CPU features -# CORRADE_TARGET_UNIX - Defined if compiled for some Unix flavor -# (Linux, BSD, macOS) -# CORRADE_TARGET_APPLE - Defined if compiled for Apple platforms -# CORRADE_TARGET_IOS - Defined if compiled for iOS (device or -# simulator) -# CORRADE_TARGET_IOS_SIMULATOR - Defined if compiled for iOS Simulator -# CORRADE_TARGET_WINDOWS - Defined if compiled for Windows -# CORRADE_TARGET_WINDOWS_RT - Defined if compiled for Windows RT -# CORRADE_TARGET_EMSCRIPTEN - Defined if compiled for Emscripten -# CORRADE_TARGET_ANDROID - Defined if compiled for Android -# CORRADE_TARGET_GCC - Defined if compiling with GCC or GCC- -# compatible Clang -# CORRADE_TARGET_CLANG - Defined if compiling with Clang or any of its -# variants -# CORRADE_TARGET_APPLE_CLANG - Defined if compiling with Apple's Clang -# CORRADE_TARGET_CLANG_CL - Defined if compiling with Clang-CL (Clang -# with a MSVC frontend) -# CORRADE_TARGET_MSVC - Defined if compiling with MSVC or Clang with -# a MSVC frontend -# CORRADE_TARGET_MINGW - Defined if compiling under MinGW -# CORRADE_CPU_USE_IFUNC - Defined if GNU IFUNC is allowed to be used -# for runtime dispatch in the Cpu library -# CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT - Defined if PluginManager -# doesn't support dynamic plugin loading due to platform limitations -# CORRADE_TESTSUITE_TARGET_XCTEST - Defined if TestSuite is targeting Xcode -# XCTest -# CORRADE_UTILITY_USE_ANSI_COLORS - Defined if ANSI escape sequences are used -# for colored output with Utility::Debug on Windows -# -# Additionally these variables are defined for internal usage: -# -# CORRADE_INCLUDE_DIR - Root include dir -# CORRADE_*_LIBRARY_DEBUG - Debug version of given library, if found -# CORRADE_*_LIBRARY_RELEASE - Release version of given library, if found -# CORRADE_*_EXECUTABLE - Location of given executable, if found -# CORRADE_USE_MODULE - Path to UseCorrade.cmake module (included -# automatically) -# CORRADE_TESTSUITE_XCTEST_RUNNER - Path to XCTestRunner.mm.in file -# CORRADE_TESTSUITE_ADB_RUNNER - Path to AdbRunner.sh file -# CORRADE_PEDANTIC_COMPILER_OPTIONS - List of pedantic compiler options used -# for targets with :prop_tgt:`CORRADE_USE_PEDANTIC_FLAGS` enabled -# CORRADE_PEDANTIC_COMPILER_DEFINITIONS - List of pedantic compiler -# definitions used for targets with :prop_tgt:`CORRADE_USE_PEDANTIC_FLAGS` -# enabled -# CORRADE_CXX{11,14,17,20}_STANDARD_FLAG - Compiler flag to use for targeting -# C++11, 14, 17 or 20 in cases where it's not possible to use -# :prop_tgt:`CORRADE_CXX_STANDARD`. Not defined if a standard switch is -# already present in :variable:`CMAKE_CXX_FLAGS`. -# -# Corrade provides these macros and functions: -# -# .. command:: corrade_add_test -# -# Add unit test using Corrade's TestSuite:: -# -# corrade_add_test( -# ... -# [LIBRARIES ...] -# [FILES ...] -# [ARGUMENTS ...]) -# -# Test name is also executable name. You can use ``LIBRARIES`` to specify -# libraries to link with instead of using :command:`target_link_libraries()`. -# The ``Corrade::TestSuite`` target is linked automatically to each test. Note -# that the :command:`enable_testing()` function must be called explicitly. -# Arguments passed after ``ARGUMENTS`` will be appended to the test -# command line. ``ARGUMENTS`` are supported everywhere except when -# ``CORRADE_TESTSUITE_TARGET_XCTEST`` is enabled. -# -# You can list files needed by the test in the ``FILES`` section. If given -# filename is relative, it is treated relatively to `CMAKE_CURRENT_SOURCE_DIR`. -# The files are added to the :prop_test:`REQUIRED_FILES` target property. On -# Emscripten they are bundled to the executable and available in the virtual -# filesystem root. On Android they are copied along the executable to the -# target. In case of Emscripten and Android, if the file is absolute or -# contains ``..``, only the leaf name is used. Alternatively you can have a -# filename formatted as ``@``, in which case the ```` is -# treated as local filesystem location and ```` as remote/virtual -# filesystem location. The remote location can't be absolute or contain ``..`` -# / ``@`` characters. -# -# Unless :variable:`CORRADE_TESTSUITE_TARGET_XCTEST` is set, test cases on iOS -# targets are created as bundles with bundle identifier set to CMake project -# name by default. Use the cache variable :variable:`CORRADE_TESTSUITE_BUNDLE_IDENTIFIER_PREFIX` -# to change it to something else. -# -# .. command:: corrade_add_resource -# -# Compile data resources into application binary:: -# -# corrade_add_resource( ) -# -# Depends on ``Corrade::rc``, which is part of Corrade utilities. This command -# generates resource data using given configuration file in current build -# directory. Argument name is name under which the resources can be explicitly -# loaded. Variable ```` contains compiled resource filename, which is -# then used for compiling library / executable. On CMake >= 3.1 the -# `resources.conf` file can contain UTF-8-encoded filenames. Example usage:: -# -# corrade_add_resource(app_resources resources.conf) -# add_executable(app source1 source2 ... ${app_resources}) -# -# .. command:: corrade_add_plugin -# -# Add dynamic plugin:: -# -# corrade_add_plugin( -# ";" -# ";" -# -# ...) -# -# The macro adds a preprocessor directive ``CORRADE_DYNAMIC_PLUGIN`` when -# compiling ````. Additional libraries can be linked in via -# :command:`target_link_libraries(plugin_name ...) `. -# On DLL platforms, the plugin DLLs and metadata files are put into -# ```` / ```` and the -# ``*.lib`` files into ```` / -# ````. On non-DLL platforms everything is put -# into ```` / ````. -# -# If the plugin interface disables plugin metadata files, the -# ```` can be set to ``""``, in which case no metadata file is -# copied anywhere. Otherwise the metadata file is copied and renamed to -# ````, retaining its original extension. -# -# corrade_add_plugin( -# -# -# -# ...) -# -# Unline the above version this puts everything into ```` on -# both DLL and non-DLL platforms. If ```` is set to -# :variable:`CMAKE_CURRENT_BINARY_DIR` (e.g. for testing purposes), the files -# are copied directly, without the need to perform install step. Note that the -# files are actually put into configuration-based subdirectory, i.e. -# ``${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}``. See documentation of -# :variable:`CMAKE_CFG_INTDIR` variable for more information. -# -# .. command:: corrade_add_static_plugin -# -# Add static plugin:: -# -# corrade_add_static_plugin( -# ";" -# -# ...) -# -# The macro adds a preprocessor directive ``CORRADE_STATIC_PLUGIN`` when -# compiling ````. Additional libraries can be linked in via -# :command:`target_link_libraries(plugin_name ...) `. -# The ```` is ignored and included just for compatibility -# with the :command:`corrade_add_plugin` command, everything is installed into -# ````. Note that plugins built in debug configuration -# (e.g. with :variable:`CMAKE_BUILD_TYPE` set to ``Debug``) have ``"-d"`` -# suffix to make it possible to have both debug and release plugins installed -# alongside each other. -# -# If the plugin interface disables plugin metadata files, the -# ```` can be set to ``""``, in which case no metadata file is -# used. Otherwise the metadata file is bundled and renamed to -# ````, retaining its original extension. -# -# corrade_add_static_plugin( -# -# -# ...) -# -# Equivalent to the above with ```` set to ````. -# If ```` is set to :variable:`CMAKE_CURRENT_BINARY_DIR` (e.g. for -# testing purposes), no installation rules are added. -# -# .. command:: corrade_find_dlls_for_libs -# -# Find corresponding DLLs for library files:: -# -# corrade_find_dlls_for_libs( ...) -# -# Available only on Windows, for all ``*.lib`` files tries to find -# corresponding DLL file. Useful for bundling dependencies for e.g. WinRT -# packages. -# - -# -# This file is part of Corrade. -# -# Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, -# 2017, 2018, 2019, 2020, 2021, 2022 -# Vladimír Vondruš -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. -# - -# Root include dir -find_path(CORRADE_INCLUDE_DIR - NAMES Corrade/Corrade.h) -mark_as_advanced(CORRADE_INCLUDE_DIR) - -# Configuration file -find_file(_CORRADE_CONFIGURE_FILE configure.h - HINTS ${CORRADE_INCLUDE_DIR}/Corrade/) -mark_as_advanced(_CORRADE_CONFIGURE_FILE) - -# We need to open configure.h file from CORRADE_INCLUDE_DIR before we check for -# the components. Bail out with proper error message if it wasn't found. The -# complete check with all components is further below. -if(NOT CORRADE_INCLUDE_DIR) - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(Corrade - REQUIRED_VARS CORRADE_INCLUDE_DIR _CORRADE_CONFIGURE_FILE) -endif() - -# Read flags from configuration -file(READ ${_CORRADE_CONFIGURE_FILE} _corradeConfigure) -string(REGEX REPLACE ";" "\\\\;" _corradeConfigure "${_corradeConfigure}") -string(REGEX REPLACE "\n" ";" _corradeConfigure "${_corradeConfigure}") -set(_corradeFlags - MSVC2015_COMPATIBILITY - MSVC2017_COMPATIBILITY - MSVC_COMPATIBILITY - BUILD_DEPRECATED - BUILD_STATIC - BUILD_STATIC_UNIQUE_GLOBALS - BUILD_MULTITHREADED - BUILD_CPU_RUNTIME_DISPATCH - TARGET_UNIX - TARGET_APPLE - TARGET_IOS - TARGET_IOS_SIMULATOR - TARGET_WINDOWS - TARGET_WINDOWS_RT - TARGET_EMSCRIPTEN - TARGET_ANDROID - # TARGET_X86 etc, TARGET_32BIT, TARGET_BIG_ENDIAN and TARGET_LIBCXX etc. - # are not exposed to CMake as the meaning is unclear on platforms with - # multi-arch binaries or when mixing different STL implementations. - # TARGET_GCC etc are figured out via UseCorrade.cmake, as the compiler can - # be different when compiling the lib & when using it. - CPU_USE_IFUNC - PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT - TESTSUITE_TARGET_XCTEST - UTILITY_USE_ANSI_COLORS) -foreach(_corradeFlag ${_corradeFlags}) - list(FIND _corradeConfigure "#define CORRADE_${_corradeFlag}" _corrade_${_corradeFlag}) - if(NOT _corrade_${_corradeFlag} EQUAL -1) - set(CORRADE_${_corradeFlag} 1) - endif() -endforeach() - -# CMake module dir -find_path(_CORRADE_MODULE_DIR - NAMES UseCorrade.cmake CorradeLibSuffix.cmake - PATH_SUFFIXES share/cmake/Corrade lib/cmake) -mark_as_advanced(_CORRADE_MODULE_DIR) - -set(CORRADE_USE_MODULE ${_CORRADE_MODULE_DIR}/UseCorrade.cmake) -set(CORRADE_LIB_SUFFIX_MODULE ${_CORRADE_MODULE_DIR}/CorradeLibSuffix.cmake) - -# Component distinction (listing them explicitly to avoid mistakes with finding -# unknown components) -set(_CORRADE_LIBRARY_COMPONENTS - Containers Interconnect Main PluginManager TestSuite Utility) -set(_CORRADE_HEADER_ONLY_COMPONENTS Containers) -if(NOT CORRADE_TARGET_WINDOWS) - # CorradeMain is a real library only on windows, a dummy target elsewhere - list(APPEND _CORRADE_HEADER_ONLY_COMPONENTS Main) -endif() -set(_CORRADE_EXECUTABLE_COMPONENTS rc) -# Currently everything is enabled implicitly. Keep in sync with Corrade's root -# CMakeLists.txt. -set(_CORRADE_IMPLICITLY_ENABLED_COMPONENTS - Containers Interconnect Main PluginManager TestSuite Utility rc) - -# Inter-component dependencies -set(_CORRADE_Containers_DEPENDENCIES Utility) -set(_CORRADE_Interconnect_DEPENDENCIES Containers Utility) -set(_CORRADE_PluginManager_DEPENDENCIES Containers Utility rc) -set(_CORRADE_TestSuite_DEPENDENCIES Containers Utility Main) # see below -set(_CORRADE_Utility_DEPENDENCIES Containers rc) - -# Ensure that all inter-component dependencies are specified as well -foreach(_component ${Corrade_FIND_COMPONENTS}) - # Mark the dependencies as required if the component is also required - if(Corrade_FIND_REQUIRED_${_component}) - foreach(_dependency ${_CORRADE_${_component}_DEPENDENCIES}) - set(Corrade_FIND_REQUIRED_${_dependency} TRUE) - endforeach() - endif() - - list(APPEND _CORRADE_ADDITIONAL_COMPONENTS ${_CORRADE_${_component}_DEPENDENCIES}) -endforeach() - -# Main is linked only in corrade_add_test(), not to everything that depends on -# TestSuite, so remove it from the list again once we filled the above -# variables -set(_CORRADE_TestSuite_DEPENDENCIES Containers Utility) - -# Join the lists, remove duplicate components -set(_CORRADE_ORIGINAL_FIND_COMPONENTS ${Corrade_FIND_COMPONENTS}) -if(_CORRADE_ADDITIONAL_COMPONENTS) - list(INSERT Corrade_FIND_COMPONENTS 0 ${_CORRADE_ADDITIONAL_COMPONENTS}) -endif() -if(Corrade_FIND_COMPONENTS) - list(REMOVE_DUPLICATES Corrade_FIND_COMPONENTS) -endif() - -# Find all components -foreach(_component ${Corrade_FIND_COMPONENTS}) - string(TOUPPER ${_component} _COMPONENT) - - # Create imported target in case the library is found. If the project is - # added as subproject to CMake, the target already exists and all the - # required setup is already done from the build tree. - if(TARGET Corrade::${_component}) - set(Corrade_${_component}_FOUND TRUE) - else() - # Library (and not header-only) components - if(_component IN_LIST _CORRADE_LIBRARY_COMPONENTS AND NOT _component IN_LIST _CORRADE_HEADER_ONLY_COMPONENTS) - add_library(Corrade::${_component} UNKNOWN IMPORTED) - - # Try to find both debug and release version - find_library(CORRADE_${_COMPONENT}_LIBRARY_DEBUG Corrade${_component}-d) - find_library(CORRADE_${_COMPONENT}_LIBRARY_RELEASE Corrade${_component}) - mark_as_advanced(CORRADE_${_COMPONENT}_LIBRARY_DEBUG - CORRADE_${_COMPONENT}_LIBRARY_RELEASE) - - if(CORRADE_${_COMPONENT}_LIBRARY_RELEASE) - set_property(TARGET Corrade::${_component} APPEND PROPERTY - IMPORTED_CONFIGURATIONS RELEASE) - set_property(TARGET Corrade::${_component} PROPERTY - IMPORTED_LOCATION_RELEASE ${CORRADE_${_COMPONENT}_LIBRARY_RELEASE}) - endif() - - if(CORRADE_${_COMPONENT}_LIBRARY_DEBUG) - set_property(TARGET Corrade::${_component} APPEND PROPERTY - IMPORTED_CONFIGURATIONS DEBUG) - set_property(TARGET Corrade::${_component} PROPERTY - IMPORTED_LOCATION_DEBUG ${CORRADE_${_COMPONENT}_LIBRARY_DEBUG}) - endif() - endif() - - # Header-only library components - if(_component IN_LIST _CORRADE_HEADER_ONLY_COMPONENTS) - add_library(Corrade::${_component} INTERFACE IMPORTED) - endif() - - # Default include path names to look for for library / header-only - # components - if(_component IN_LIST _CORRADE_LIBRARY_COMPONENTS) - set(_CORRADE_${_COMPONENT}_INCLUDE_PATH_SUFFIX Corrade/${_component}) - set(_CORRADE_${_COMPONENT}_INCLUDE_PATH_NAMES ${_component}.h) - endif() - - # Executable components - if(_component IN_LIST _CORRADE_EXECUTABLE_COMPONENTS) - add_executable(Corrade::${_component} IMPORTED) - - find_program(CORRADE_${_COMPONENT}_EXECUTABLE corrade-${_component}) - mark_as_advanced(CORRADE_${_COMPONENT}_EXECUTABLE) - - if(CORRADE_${_COMPONENT}_EXECUTABLE) - set_property(TARGET Corrade::${_component} PROPERTY - IMPORTED_LOCATION ${CORRADE_${_COMPONENT}_EXECUTABLE}) - endif() - endif() - - # No special setup for Containers library - - # Interconnect library - if(_component STREQUAL Interconnect) - # Disable /OPT:ICF on MSVC, which merges functions with identical - # contents and thus breaks signal comparison - if(CORRADE_TARGET_WINDOWS AND CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - if(CMAKE_VERSION VERSION_LESS 3.13) - set_property(TARGET Corrade::${_component} PROPERTY - INTERFACE_LINK_LIBRARIES "-OPT:NOICF,REF") - else() - set_property(TARGET Corrade::${_component} PROPERTY - INTERFACE_LINK_OPTIONS "/OPT:NOICF,REF") - endif() - endif() - - # Main library - elseif(_component STREQUAL Main) - set(_CORRADE_${_COMPONENT}_INCLUDE_PATH_SUFFIX Corrade) - set(_CORRADE_${_COMPONENT}_INCLUDE_PATH_NAMES Corrade.h) - - if(CORRADE_TARGET_WINDOWS) - if(NOT MINGW) - # Abusing INTERFACE_LINK_LIBRARIES because - # INTERFACE_LINK_OPTIONS is only since 3.13. They treat - # things with `-` in front as linker flags and fortunately - # I can use `-ENTRY` instead of `/ENTRY`. - # https://gitlab.kitware.com/cmake/cmake/issues/16543 - set_property(TARGET Corrade::${_component} APPEND PROPERTY - INTERFACE_LINK_LIBRARIES "-ENTRY:$<$>>:wmainCRTStartup>$<$>:wWinMainCRTStartup>") - else() - set_property(TARGET Corrade::${_component} APPEND PROPERTY - INTERFACE_LINK_LIBRARIES "-municode") - endif() - endif() - - # PluginManager library - elseif(_component STREQUAL PluginManager) - # -ldl is handled by Utility now - - # TestSuite library has some additional files - elseif(_component STREQUAL TestSuite) - # XCTest runner file - if(CORRADE_TESTSUITE_TARGET_XCTEST) - find_file(CORRADE_TESTSUITE_XCTEST_RUNNER XCTestRunner.mm.in - PATH_SUFFIXES share/corrade/TestSuite) - set(CORRADE_TESTSUITE_XCTEST_RUNNER_NEEDED CORRADE_TESTSUITE_XCTEST_RUNNER) - - # ADB runner file - elseif(CORRADE_TARGET_ANDROID) - find_file(CORRADE_TESTSUITE_ADB_RUNNER AdbRunner.sh - PATH_SUFFIXES share/corrade/TestSuite) - set(CORRADE_TESTSUITE_ADB_RUNNER_NEEDED CORRADE_TESTSUITE_ADB_RUNNER) - - # Emscripten runner file - elseif(CORRADE_TARGET_EMSCRIPTEN) - find_file(CORRADE_TESTSUITE_EMSCRIPTEN_RUNNER EmscriptenRunner.html.in - PATH_SUFFIXES share/corrade/TestSuite) - set(CORRADE_TESTSUITE_EMSCRIPTEN_RUNNER_NEEDED CORRADE_TESTSUITE_EMSCRIPTEN_RUNNER) - endif() - - # Utility library (contains all setup that is used by others) - elseif(_component STREQUAL Utility) - # Top-level include directory - set_property(TARGET Corrade::${_component} APPEND PROPERTY - INTERFACE_INCLUDE_DIRECTORIES ${CORRADE_INCLUDE_DIR}) - - # Require (at least) C++11 for users - set_property(TARGET Corrade::${_component} PROPERTY - INTERFACE_CORRADE_CXX_STANDARD 11) - set_property(TARGET Corrade::${_component} APPEND PROPERTY - COMPATIBLE_INTERFACE_NUMBER_MAX CORRADE_CXX_STANDARD) - - # Path::libraryLocation() needs this - if(CORRADE_TARGET_UNIX) - set_property(TARGET Corrade::${_component} APPEND PROPERTY - INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}) - endif() - # AndroidLogStreamBuffer class needs to be linked to log library - if(CORRADE_TARGET_ANDROID) - set_property(TARGET Corrade::${_component} APPEND PROPERTY - INTERFACE_LINK_LIBRARIES "log") - endif() - endif() - - # Find library includes - if(_component IN_LIST _CORRADE_LIBRARY_COMPONENTS) - find_path(_CORRADE_${_COMPONENT}_INCLUDE_DIR - NAMES ${_CORRADE_${_COMPONENT}_INCLUDE_PATH_NAMES} - HINTS ${CORRADE_INCLUDE_DIR}/${_CORRADE_${_COMPONENT}_INCLUDE_PATH_SUFFIX}) - mark_as_advanced(_CORRADE_${_COMPONENT}_INCLUDE_DIR) - endif() - - # Add inter-library dependencies - if(_component IN_LIST _CORRADE_LIBRARY_COMPONENTS OR _component IN_LIST _CORRADE_HEADER_ONLY_COMPONENTS) - foreach(_dependency ${_CORRADE_${_component}_DEPENDENCIES}) - if(_dependency IN_LIST _CORRADE_LIBRARY_COMPONENTS OR _dependency IN_LIST _CORRADE_HEADER_ONLY_COMPONENTS) - set_property(TARGET Corrade::${_component} APPEND PROPERTY - INTERFACE_LINK_LIBRARIES Corrade::${_dependency}) - endif() - endforeach() - endif() - - # Decide if the component was found - if((_component IN_LIST _CORRADE_LIBRARY_COMPONENTS AND _CORRADE_${_COMPONENT}_INCLUDE_DIR AND (_component IN_LIST _CORRADE_HEADER_ONLY_COMPONENTS OR CORRADE_${_COMPONENT}_LIBRARY_RELEASE OR CORRADE_${_COMPONENT}_LIBRARY_DEBUG)) OR (_component IN_LIST _CORRADE_EXECUTABLE_COMPONENTS AND CORRADE_${_COMPONENT}_EXECUTABLE)) - set(Corrade_${_component}_FOUND TRUE) - else() - set(Corrade_${_component}_FOUND FALSE) - endif() - endif() -endforeach() - -# For CMake 3.16+ with REASON_FAILURE_MESSAGE, provide additional potentially -# useful info about the failed components. -if(NOT CMAKE_VERSION VERSION_LESS 3.16) - set(_CORRADE_REASON_FAILURE_MESSAGE ) - # Go only through the originally specified find_package() components, not - # the dependencies added by us afterwards - foreach(_component ${_CORRADE_ORIGINAL_FIND_COMPONENTS}) - if(Corrade_${_component}_FOUND) - continue() - endif() - - # If it's not known at all, tell the user -- it might be a new library - # and an old Find module, or something platform-specific. - if(NOT _component IN_LIST _CORRADE_LIBRARY_COMPONENTS AND NOT _component IN_LIST _CORRADE_EXECUTABLE_COMPONENTS) - list(APPEND _CORRADE_REASON_FAILURE_MESSAGE "${_component} is not a known component on this platform.") - # Otherwise, if it's not among implicitly built components, hint that - # the user may need to enable it. - # TODO: currently, the _FOUND variable doesn't reflect if dependencies - # were found. When it will, this needs to be updated to avoid - # misleading messages. - elseif(NOT _component IN_LIST _CORRADE_IMPLICITLY_ENABLED_COMPONENTS) - string(TOUPPER ${_component} _COMPONENT) - list(APPEND _CORRADE_REASON_FAILURE_MESSAGE "${_component} is not built by default. Make sure you enabled CORRADE_WITH_${_COMPONENT} when building Corrade.") - # Otherwise we have no idea. Better be silent than to print something - # misleading. - else() - endif() - endforeach() - - string(REPLACE ";" " " _CORRADE_REASON_FAILURE_MESSAGE "${_CORRADE_REASON_FAILURE_MESSAGE}") - set(_CORRADE_REASON_FAILURE_MESSAGE REASON_FAILURE_MESSAGE "${_CORRADE_REASON_FAILURE_MESSAGE}") -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Corrade REQUIRED_VARS - CORRADE_INCLUDE_DIR - _CORRADE_MODULE_DIR - _CORRADE_CONFIGURE_FILE - ${CORRADE_TESTSUITE_XCTEST_RUNNER_NEEDED} - ${CORRADE_TESTSUITE_ADB_RUNNER_NEEDED} - ${CORRADE_TESTSUITE_EMSCRIPTEN_RUNNER_NEEDED} - HANDLE_COMPONENTS - ${_CORRADE_REASON_FAILURE_MESSAGE}) - -# Finalize the finding process -include(${CORRADE_USE_MODULE}) - -set(CORRADE_INCLUDE_INSTALL_DIR include/Corrade) - -if(CORRADE_BUILD_DEPRECATED AND CORRADE_INCLUDE_INSTALL_PREFIX AND NOT CORRADE_INCLUDE_INSTALL_PREFIX STREQUAL ".") - message(DEPRECATION "CORRADE_INCLUDE_INSTALL_PREFIX is obsolete as its primary use was for old Android NDK versions. Please switch to the NDK r19+ layout instead of using this variable and recreate your build directory to get rid of this warning.") - set(CORRADE_INCLUDE_INSTALL_DIR ${CORRADE_INCLUDE_INSTALL_PREFIX}/${CORRADE_INCLUDE_INSTALL_DIR}) -endif() \ No newline at end of file diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 0edeab933..6a52cb3af 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -2,32 +2,33 @@ # SPDX-License-Identifier: Apache-2.0 # -# The dependency on opentelemetry-proto can be provided different ways. -# By order of decreasing priority, options are: +# The dependency on opentelemetry-proto can be provided different ways. By order +# of decreasing priority, options are: # # 1 - Use a provided package # # This is useful to build opentelemetry-cpp as part of a super project. # -# The super project provides the path to the opentelemetry-proto -# source code using variable ${OTELCPP_PROTO_PATH} +# The super project provides the path to the opentelemetry-proto source code +# using variable ${OTELCPP_PROTO_PATH} # # 2 - Search for a opentelemetry-proto git submodule # -# When git submodule is used, -# the opentelemetry-proto code is located in: +# When git submodule is used, the opentelemetry-proto code is located in: # third_party/opentelemetry-proto # # 3 - Download opentelemetry-proto from github # -# Code from the required version is used, -# unless a specific release tag is provided -# in variable ${opentelemetry-proto} +# Code from the required version is used, unless a specific release tag is +# provided in variable ${opentelemetry-proto} # if(OTELCPP_PROTO_PATH) - if(NOT EXISTS "${OTELCPP_PROTO_PATH}/opentelemetry/proto/common/v1/common.proto") - message(FATAL_ERROR "OTELCPP_PROTO_PATH does not point to a opentelemetry-proto repository") + if(NOT EXISTS + "${OTELCPP_PROTO_PATH}/opentelemetry/proto/common/v1/common.proto") + message( + FATAL_ERROR + "OTELCPP_PROTO_PATH does not point to a opentelemetry-proto repository") endif() message(STATUS "opentelemetry-proto dependency satisfied by: external path") set(PROTO_PATH ${OTELCPP_PROTO_PATH}) @@ -35,12 +36,14 @@ if(OTELCPP_PROTO_PATH) else() if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opentelemetry-proto/.git) message(STATUS "opentelemetry-proto dependency satisfied by: git submodule") - set(PROTO_PATH "${CMAKE_CURRENT_SOURCE_DIR}/third_party/opentelemetry-proto") + set(PROTO_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/third_party/opentelemetry-proto") set(needs_proto_download FALSE) else() - message(STATUS "opentelemetry-proto dependency satisfied by: github download") + message( + STATUS "opentelemetry-proto dependency satisfied by: github download") if("${opentelemetry-proto}" STREQUAL "") - set(opentelemetry-proto "v0.19.0") + set(opentelemetry-proto "v1.0.0") endif() include(ExternalProject) ExternalProject_Add( @@ -62,8 +65,6 @@ else() endif() endif() -include(${PROJECT_SOURCE_DIR}/cmake/proto-options-patch.cmake) - set(COMMON_PROTO "${PROTO_PATH}/opentelemetry/proto/common/v1/common.proto") set(RESOURCE_PROTO "${PROTO_PATH}/opentelemetry/proto/resource/v1/resource.proto") @@ -166,91 +167,129 @@ if(WITH_OTLP_GRPC) message(STATUS "gRPC_CPP_PLUGIN_EXECUTABLE=${gRPC_CPP_PLUGIN_EXECUTABLE}") endif() -if(WITH_OTLP_GRPC) - add_custom_command( - OUTPUT ${COMMON_PB_H_FILE} - ${COMMON_PB_CPP_FILE} - ${RESOURCE_PB_H_FILE} - ${RESOURCE_PB_CPP_FILE} - ${TRACE_PB_H_FILE} - ${TRACE_PB_CPP_FILE} - ${LOGS_PB_H_FILE} - ${LOGS_PB_CPP_FILE} - ${METRICS_PB_H_FILE} - ${METRICS_PB_CPP_FILE} - ${TRACE_SERVICE_PB_H_FILE} - ${TRACE_SERVICE_PB_CPP_FILE} - ${TRACE_SERVICE_GRPC_PB_H_FILE} - ${TRACE_SERVICE_GRPC_PB_CPP_FILE} - ${LOGS_SERVICE_PB_H_FILE} - ${LOGS_SERVICE_PB_CPP_FILE} - ${LOGS_SERVICE_GRPC_PB_H_FILE} - ${LOGS_SERVICE_GRPC_PB_CPP_FILE} - ${METRICS_SERVICE_PB_H_FILE} - ${METRICS_SERVICE_PB_CPP_FILE} - ${METRICS_SERVICE_GRPC_PB_H_FILE} - ${METRICS_SERVICE_GRPC_PB_CPP_FILE} - COMMAND - ${PROTOBUF_PROTOC_EXECUTABLE} ARGS "--experimental_allow_proto3_optional" - "--proto_path=${PROTO_PATH}" ${PROTOBUF_INCLUDE_FLAGS} - "--cpp_out=${GENERATED_PROTOBUF_PATH}" - "--grpc_out=generate_mock_code=true:${GENERATED_PROTOBUF_PATH}" - --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}" ${COMMON_PROTO} - ${RESOURCE_PROTO} ${TRACE_PROTO} ${LOGS_PROTO} ${METRICS_PROTO} - ${TRACE_SERVICE_PROTO} ${LOGS_SERVICE_PROTO} ${METRICS_SERVICE_PROTO}) -else() - add_custom_command( - OUTPUT ${COMMON_PB_H_FILE} - ${COMMON_PB_CPP_FILE} - ${RESOURCE_PB_H_FILE} - ${RESOURCE_PB_CPP_FILE} - ${TRACE_PB_H_FILE} - ${TRACE_PB_CPP_FILE} - ${LOGS_PB_H_FILE} - ${LOGS_PB_CPP_FILE} - ${METRICS_PB_H_FILE} - ${METRICS_PB_CPP_FILE} - ${TRACE_SERVICE_PB_H_FILE} - ${TRACE_SERVICE_PB_CPP_FILE} - ${LOGS_SERVICE_PB_H_FILE} - ${LOGS_SERVICE_PB_CPP_FILE} - ${METRICS_SERVICE_PB_H_FILE} - ${METRICS_SERVICE_PB_CPP_FILE} - COMMAND - ${PROTOBUF_PROTOC_EXECUTABLE} ARGS "--experimental_allow_proto3_optional" - "--proto_path=${PROTO_PATH}" ${PROTOBUF_INCLUDE_FLAGS} - "--cpp_out=${GENERATED_PROTOBUF_PATH}" ${COMMON_PROTO} ${RESOURCE_PROTO} - ${TRACE_PROTO} ${LOGS_PROTO} ${METRICS_PROTO} ${TRACE_SERVICE_PROTO} - ${LOGS_SERVICE_PROTO} ${METRICS_SERVICE_PROTO}) +set(PROTOBUF_COMMON_FLAGS "--proto_path=${PROTO_PATH}" + "--cpp_out=${GENERATED_PROTOBUF_PATH}") +# --experimental_allow_proto3_optional is available from 3.13 and be stable and +# enabled by default from 3.16 +if(Protobuf_VERSION AND Protobuf_VERSION VERSION_LESS "3.16") + list(APPEND PROTOBUF_COMMON_FLAGS "--experimental_allow_proto3_optional") +elseif(PROTOBUF_VERSION AND PROTOBUF_VERSION VERSION_LESS "3.16") + list(APPEND PROTOBUF_COMMON_FLAGS "--experimental_allow_proto3_optional") endif() -include_directories("${GENERATED_PROTOBUF_PATH}") - -if(WITH_OTLP_GRPC) - add_library( - opentelemetry_proto STATIC +set(PROTOBUF_GENERATED_FILES + ${COMMON_PB_H_FILE} ${COMMON_PB_CPP_FILE} + ${RESOURCE_PB_H_FILE} ${RESOURCE_PB_CPP_FILE} + ${TRACE_PB_H_FILE} ${TRACE_PB_CPP_FILE} + ${LOGS_PB_H_FILE} ${LOGS_PB_CPP_FILE} + ${METRICS_PB_H_FILE} ${METRICS_PB_CPP_FILE} + ${TRACE_SERVICE_PB_H_FILE} ${TRACE_SERVICE_PB_CPP_FILE} - ${TRACE_SERVICE_GRPC_PB_CPP_FILE} + ${LOGS_SERVICE_PB_H_FILE} ${LOGS_SERVICE_PB_CPP_FILE} + ${METRICS_SERVICE_PB_H_FILE} + ${METRICS_SERVICE_PB_CPP_FILE}) + +if(WITH_OTLP_GRPC) + list(APPEND PROTOBUF_COMMON_FLAGS + "--grpc_out=generate_mock_code=true:${GENERATED_PROTOBUF_PATH}" + --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}") + + list( + APPEND + PROTOBUF_GENERATED_FILES + ${TRACE_SERVICE_GRPC_PB_H_FILE} + ${TRACE_SERVICE_GRPC_PB_CPP_FILE} + ${LOGS_SERVICE_GRPC_PB_H_FILE} ${LOGS_SERVICE_GRPC_PB_CPP_FILE} - ${METRICS_SERVICE_PB_CPP_FILE} + ${METRICS_SERVICE_GRPC_PB_H_FILE} ${METRICS_SERVICE_GRPC_PB_CPP_FILE}) +endif() + +set(PROTOBUF_RUN_PROTOC_COMMAND "\"${PROTOBUF_PROTOC_EXECUTABLE}\"") +foreach( + PROTOBUF_RUN_PROTOC_ARG + ${PROTOBUF_COMMON_FLAGS} + ${PROTOBUF_INCLUDE_FLAGS} + ${COMMON_PROTO} + ${RESOURCE_PROTO} + ${TRACE_PROTO} + ${LOGS_PROTO} + ${METRICS_PROTO} + ${TRACE_SERVICE_PROTO} + ${LOGS_SERVICE_PROTO} + ${METRICS_SERVICE_PROTO}) + set(PROTOBUF_RUN_PROTOC_COMMAND + "${PROTOBUF_RUN_PROTOC_COMMAND} \"${PROTOBUF_RUN_PROTOC_ARG}\"") +endforeach() + +add_custom_command( + OUTPUT ${PROTOBUF_GENERATED_FILES} + COMMAND + ${PROTOBUF_PROTOC_EXECUTABLE} ${PROTOBUF_COMMON_FLAGS} + ${PROTOBUF_INCLUDE_FLAGS} ${COMMON_PROTO} ${RESOURCE_PROTO} ${TRACE_PROTO} + ${LOGS_PROTO} ${METRICS_PROTO} ${TRACE_SERVICE_PROTO} ${LOGS_SERVICE_PROTO} + ${METRICS_SERVICE_PROTO} + COMMENT "[Run]: ${PROTOBUF_RUN_PROTOC_COMMAND}") + +include_directories("${GENERATED_PROTOBUF_PATH}") + +unset(OTELCPP_PROTO_TARGET_OPTIONS) +if(CMAKE_SYSTEM_NAME MATCHES "Windows|MinGW|WindowsStore") + list(APPEND OTELCPP_PROTO_TARGET_OPTIONS STATIC) +elseif(NOT DEFINED BUILD_SHARED_LIBS OR BUILD_SHARED_LIBS) + list(APPEND OTELCPP_PROTO_TARGET_OPTIONS SHARED) else() + list(APPEND OTELCPP_PROTO_TARGET_OPTIONS STATIC) +endif() + +set(OPENTELEMETRY_PROTO_TARGETS opentelemetry_proto) +add_library( + opentelemetry_proto + ${OTELCPP_PROTO_TARGET_OPTIONS} + ${COMMON_PB_CPP_FILE} + ${RESOURCE_PB_CPP_FILE} + ${TRACE_PB_CPP_FILE} + ${LOGS_PB_CPP_FILE} + ${METRICS_PB_CPP_FILE} + ${TRACE_SERVICE_PB_CPP_FILE} + ${LOGS_SERVICE_PB_CPP_FILE} + ${METRICS_SERVICE_PB_CPP_FILE}) + +if(WITH_ABSEIL) + target_link_libraries(opentelemetry_proto PUBLIC absl::bad_variant_access) +endif() + +if(WITH_OTLP_GRPC) add_library( - opentelemetry_proto STATIC - ${COMMON_PB_CPP_FILE} - ${RESOURCE_PB_CPP_FILE} - ${TRACE_PB_CPP_FILE} - ${LOGS_PB_CPP_FILE} - ${METRICS_PB_CPP_FILE} - ${TRACE_SERVICE_PB_CPP_FILE} - ${LOGS_SERVICE_PB_CPP_FILE} - ${METRICS_SERVICE_PB_CPP_FILE}) + opentelemetry_proto_grpc + ${OTELCPP_PROTO_TARGET_OPTIONS} ${TRACE_SERVICE_GRPC_PB_CPP_FILE} + ${LOGS_SERVICE_GRPC_PB_CPP_FILE} ${METRICS_SERVICE_GRPC_PB_CPP_FILE}) + + list(APPEND OPENTELEMETRY_PROTO_TARGETS opentelemetry_proto_grpc) + target_link_libraries(opentelemetry_proto_grpc + PUBLIC opentelemetry_proto) + + get_target_property(grpc_lib_type gRPC::grpc++ TYPE) + if (grpc_lib_type STREQUAL "SHARED_LIBRARY") + target_link_libraries(opentelemetry_proto_grpc + PUBLIC gRPC::grpc++) + endif() + set_target_properties(opentelemetry_proto_grpc PROPERTIES EXPORT_NAME + proto_grpc) + patch_protobuf_targets(opentelemetry_proto_grpc) + get_target_property(GRPC_INCLUDE_DIRECTORY gRPC::grpc++ + INTERFACE_INCLUDE_DIRECTORIES) + if(GRPC_INCLUDE_DIRECTORY) + target_include_directories( + opentelemetry_proto_grpc + PUBLIC "$") + endif() endif() if(needs_proto_download) @@ -259,27 +298,41 @@ endif() set_target_properties(opentelemetry_proto PROPERTIES EXPORT_NAME proto) patch_protobuf_targets(opentelemetry_proto) -install( - TARGETS opentelemetry_proto - EXPORT "${PROJECT_NAME}-target" - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +if(OPENTELEMETRY_INSTALL) + install( + TARGETS ${OPENTELEMETRY_PROTO_TARGETS} + EXPORT "${PROJECT_NAME}-target" + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -install( - DIRECTORY ${GENERATED_PROTOBUF_PATH}/opentelemetry - DESTINATION include - FILES_MATCHING - PATTERN "*.h") + install( + DIRECTORY ${GENERATED_PROTOBUF_PATH}/opentelemetry + DESTINATION include + FILES_MATCHING + PATTERN "*.h") +endif() if(TARGET protobuf::libprotobuf) target_link_libraries(opentelemetry_proto PUBLIC ${CONAN_LIBS_PROTOBUF}) else() # cmake 3.8 or lower target_include_directories(opentelemetry_proto PUBLIC ${Protobuf_INCLUDE_DIRS}) - target_link_libraries(opentelemetry_proto INTERFACE ${Protobuf_LIBRARIES}) + target_link_libraries(opentelemetry_proto PUBLIC ${Protobuf_LIBRARIES}) endif() -if(BUILD_SHARED_LIBS) - set_property(TARGET opentelemetry_proto PROPERTY POSITION_INDEPENDENT_CODE ON) +if(WITH_OTLP_GRPC) + if(WITH_ABSEIL) + find_package(absl CONFIG) + if(TARGET absl::synchronization) + target_link_libraries(opentelemetry_proto_grpc + PRIVATE absl::synchronization) + endif() + endif() endif() + +if(BUILD_SHARED_LIBS) + foreach(proto_target ${OPENTELEMETRY_PROTO_TARGETS}) + set_property(TARGET ${proto_target} PROPERTY POSITION_INDEPENDENT_CODE ON) + endforeach() +endif() \ No newline at end of file diff --git a/cmake/proto-options-patch.cmake b/cmake/opentelemetry-tools.cmake similarity index 96% rename from cmake/proto-options-patch.cmake rename to cmake/opentelemetry-tools.cmake index ed294ea79..84ce8be91 100644 --- a/cmake/proto-options-patch.cmake +++ b/cmake/opentelemetry-tools.cmake @@ -11,7 +11,7 @@ endmacro() if(NOT PATCH_PROTOBUF_SOURCES_OPTIONS_SET) if(MSVC) unset(PATCH_PROTOBUF_SOURCES_OPTIONS CACHE) - set(PATCH_PROTOBUF_SOURCES_OPTIONS /wd4244 /wd4251 /wd4267 /wd4309) + set(PATCH_PROTOBUF_SOURCES_OPTIONS /wd4244 /wd4251 /wd4267 /wd4309 /wd4668 /wd4946 /wd6001 /wd6244 /wd6246) if(MSVC_VERSION GREATER_EQUAL 1922) # see @@ -31,7 +31,7 @@ if(NOT PATCH_PROTOBUF_SOURCES_OPTIONS_SET) unset(PATCH_PROTOBUF_SOURCES_OPTIONS CACHE) include(CheckCXXCompilerFlag) check_append_cxx_compiler_flag( - PATCH_PROTOBUF_SOURCES_OPTIONS -Wno-type-limits + PATCH_PROTOBUF_SOURCES_OPTIONS -Wno-type-limits -Wno-missing-declarations -Wno-deprecated-declarations -Wno-unused-parameter) endif() set(PATCH_PROTOBUF_SOURCES_OPTIONS_SET TRUE) diff --git a/cmd/pktvisord/main.cpp b/cmd/pktvisord/main.cpp index f01563b3b..60c3bbed9 100644 --- a/cmd/pktvisord/main.cpp +++ b/cmd/pktvisord/main.cpp @@ -593,7 +593,7 @@ int main(int argc, char *argv[]) // if we are demonized, change to root directory now that (potentially) logs are open if (options.daemon) { #if __has_include() - chdir("/"); + [[maybe_unused]] int result = chdir("/"); #endif } diff --git a/conanfile.txt b/conanfile.txt index 76e6331dc..cb8d7c1f6 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -1,30 +1,32 @@ [requires] -benchmark/1.7.0 -catch2/2.13.9 +catch2/3.5.0 corrade/2020.06 -cpp-httplib/0.11.2 +cpp-httplib/0.14.3 docopt.cpp/0.6.3 -fmt/8.1.1 fast-cpp-csv-parser/cci.20211104 json-schema-validator/2.2.0 -libmaxminddb/1.7.1 +libmaxminddb/1.8.0 nlohmann_json/3.11.2 -openssl/1.1.1k -opentelemetry-proto/0.19.0 -pcapplusplus/22.05 -protobuf/3.19.2 -sigslot/1.2.1 -spdlog/1.11.0 -uvw/2.12.1 -libpcap/1.10.1 -yaml-cpp/0.7.0 +openssl/1.1.1w +opentelemetry-proto/1.0.0 +pcapplusplus/23.09 +protobuf/3.21.12 +sigslot/1.2.2 +spdlog/1.12.0 +uvw/3.2.0 +libpcap/1.10.4 +yaml-cpp/0.8.0 robin-hood-hashing/3.11.5 -zlib/1.2.12 +libcurl/8.5.0 crashpad/cci.20220219 +zlib/[>=1.2.10 <1.3] -[build_requires] +[tool_requires] corrade/2020.06 -protobuf/3.19.2 +protobuf/3.21.12 [options] pcapplusplus:immediate_mode=True + +[generators] +cmake_find_package \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 44b82b7ad..fa53ffbd5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ RUN \ apt-get update && \ apt-get upgrade --yes --force-yes && \ apt-get install --yes --force-yes --no-install-recommends ${BUILD_DEPS} && \ - pip3 install "conan==1.59.0" + pip3 install "conan==1.61.0" # need git for current hash for VERSION COPY ./.git/ /pktvisor-src/.git/ diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 046236340..e6480f56e 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -1,5 +1,6 @@ message(STATUS "Custom Libraries") +add_subdirectory(visor_test) add_subdirectory(visor_transaction) add_subdirectory(visor_tcp) add_subdirectory(visor_dns) diff --git a/libs/visor_dns/CMakeLists.txt b/libs/visor_dns/CMakeLists.txt index 06191035e..a7c4e8aec 100644 --- a/libs/visor_dns/CMakeLists.txt +++ b/libs/visor_dns/CMakeLists.txt @@ -24,4 +24,16 @@ target_link_libraries(VisorLibDns ${CONAN_LIBS_FMT} ) -add_subdirectory(tests) \ No newline at end of file +## TEST SUITE +add_executable(unit-tests-visor-dns + tests/test_dns.cpp + tests/benchmark_dns.cpp) + +target_link_libraries(unit-tests-visor-dns + PRIVATE + Visor::Lib::Dns + ${CONAN_LIBS_CATCH2}) + +add_test(NAME unit-tests-visor-dns + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/libs/visor_dns + COMMAND unit-tests-visor-dns) \ No newline at end of file diff --git a/libs/visor_dns/DnsLayer.cpp b/libs/visor_dns/DnsLayer.cpp index 006615f15..a38354b0b 100644 --- a/libs/visor_dns/DnsLayer.cpp +++ b/libs/visor_dns/DnsLayer.cpp @@ -2,12 +2,12 @@ #include "DnsLayer.h" #include "EndianPortable.h" -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic push #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/libs/visor_dns/DnsLayer.h b/libs/visor_dns/DnsLayer.h index ba4567eb8..c28c4e742 100644 --- a/libs/visor_dns/DnsLayer.h +++ b/libs/visor_dns/DnsLayer.h @@ -10,7 +10,7 @@ #include "DnsLayerEnums.h" #include "DnsResource.h" #include "DnsResourceData.h" -#include +#include #include #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/libs/visor_dns/DnsResource.cpp b/libs/visor_dns/DnsResource.cpp index 8b329d6e5..97ad533c9 100644 --- a/libs/visor_dns/DnsResource.cpp +++ b/libs/visor_dns/DnsResource.cpp @@ -7,7 +7,7 @@ #pragma GCC diagnostic push #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/libs/visor_dns/DnsResourceData.cpp b/libs/visor_dns/DnsResourceData.cpp index 61ef5eb3e..e13821d5e 100644 --- a/libs/visor_dns/DnsResourceData.cpp +++ b/libs/visor_dns/DnsResourceData.cpp @@ -2,14 +2,14 @@ #include "DnsResourceData.h" #include "EndianPortable.h" -#include +#include #include #include #ifdef __GNUC__ #pragma GCC diagnostic push #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #pragma GCC diagnostic ignored "-Wunused-parameter" diff --git a/libs/visor_dns/DnsResourceData.h b/libs/visor_dns/DnsResourceData.h index b7c683920..4c80d3172 100644 --- a/libs/visor_dns/DnsResourceData.h +++ b/libs/visor_dns/DnsResourceData.h @@ -2,7 +2,7 @@ #define PV_PACKETPP_DNS_RESOURCE_DATA #include "DnsResource.h" -#include +#include #include #include #include diff --git a/libs/visor_dns/dns.h b/libs/visor_dns/dns.h index e457c5fa6..d3daea8a7 100644 --- a/libs/visor_dns/dns.h +++ b/libs/visor_dns/dns.h @@ -8,6 +8,7 @@ #include "DnsResource.h" #include "DnsResourceData.h" #include +#include #include namespace visor::lib::dns { diff --git a/libs/visor_dns/tests/CMakeLists.txt b/libs/visor_dns/tests/CMakeLists.txt deleted file mode 100644 index 396462e66..000000000 --- a/libs/visor_dns/tests/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# Unit -add_executable(unit-tests-visor-dns - main.cpp - test_dns.cpp - ) - -target_link_libraries(unit-tests-visor-dns - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Lib::Dns) - -add_test(NAME unit-tests-visor-dns - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND unit-tests-visor-dns - ) - -# Benchmark -add_executable(benchmark-visor-dns - benchmark_dns.cpp - ) - -target_link_libraries(benchmark-visor-dns PRIVATE - Visor::Lib::Dns - ${CONAN_LIBS_BENCHMARK}) \ No newline at end of file diff --git a/libs/visor_dns/tests/benchmark_dns.cpp b/libs/visor_dns/tests/benchmark_dns.cpp index 46d604342..09ee28b34 100644 --- a/libs/visor_dns/tests/benchmark_dns.cpp +++ b/libs/visor_dns/tests/benchmark_dns.cpp @@ -1,8 +1,9 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +#include +#include -#include "benchmark/benchmark.h" #include "dns.h" #ifdef __GNUC__ #pragma GCC diagnostic push @@ -12,75 +13,74 @@ #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #pragma clang diagnostic ignored "-Wc99-extensions" #endif -#include "IPv4Layer.h" -#include "PacketUtils.h" -#include "PcapFileDevice.h" +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif using namespace visor::lib::dns; -static void BM_aggregateDomain(benchmark::State &state) +void BM_aggregateDomain(const std::string &domain) { AggDomainResult result; - std::string domain{"biz.foo.bar.com"}; - for (auto _ : state) { - result = aggregateDomain(domain); - } -} -BENCHMARK(BM_aggregateDomain); - -static void BM_aggregateDomainLong(benchmark::State &state) -{ - AggDomainResult result; - std::string domain{"long1.long2.long3.long4.long5.long6.long7.long8.biz.foo.bar.com"}; - for (auto _ : state) { - result = aggregateDomain(domain); - } + result = aggregateDomain(domain); } -BENCHMARK(BM_aggregateDomainLong); - -static void BM_pcapReadNoParse(benchmark::State &state) +void BM_pcapReadNoParse() { + auto reader = pcpp::IFileReaderDevice::getReader("tests/dns_udp_tcp_random.pcap"); - for (auto _ : state) { - auto reader = pcpp::IFileReaderDevice::getReader("fixtures/dns_udp_tcp_random.pcap"); - - if (!reader->open()) { - throw std::runtime_error("Cannot open pcap/pcapng file"); - } - - pcpp::RawPacket rawPacket; - while (reader->getNextPacket(rawPacket)) { - } + if (!reader->open()) { + throw std::runtime_error("Cannot open pcap/pcapng file"); + } - reader->close(); - delete reader; + pcpp::RawPacket rawPacket; + while (reader->getNextPacket(rawPacket)) { } + + reader->close(); + delete reader; } -BENCHMARK(BM_pcapReadNoParse); -static void BM_pcapReadParse1(benchmark::State &state) +void BM_pcapReadParse() { - for (auto _ : state) { - auto reader = pcpp::IFileReaderDevice::getReader("fixtures/dns_udp_tcp_random.pcap"); - - if (!reader->open()) { - throw std::runtime_error("Cannot open pcap/pcapng file"); - } + auto reader = pcpp::IFileReaderDevice::getReader("tests/dns_udp_tcp_random.pcap"); - pcpp::RawPacket rawPacket; - while (reader->getNextPacket(rawPacket)) { - pcpp::Packet packet(&rawPacket, pcpp::OsiModelTransportLayer); - } + if (!reader->open()) { + throw std::runtime_error("Cannot open pcap/pcapng file"); + } - reader->close(); - delete reader; + pcpp::RawPacket rawPacket; + while (reader->getNextPacket(rawPacket)) { + pcpp::Packet packet(&rawPacket, pcpp::OsiModelTransportLayer); } + + reader->close(); + delete reader; } -BENCHMARK(BM_pcapReadParse1); -BENCHMARK_MAIN(); \ No newline at end of file +TEST_CASE("DNS benchmark") +{ + BENCHMARK("Aggregate Domain") + { + return BM_aggregateDomain("biz.foo.bar.com"); + }; + + BENCHMARK("Aggregate Domain Long") + { + return BM_aggregateDomain("long1.long2.long3.long4.long5.long6.long7.long8.biz.foo.bar.com"); + }; + + BENCHMARK("Pcap Read No Parse") + { + return BM_pcapReadNoParse(); + }; + + BENCHMARK("Pcap Read No Parse") + { + return BM_pcapReadParse(); + }; +} diff --git a/libs/visor_dns/tests/dns_udp_tcp_random.pcap b/libs/visor_dns/tests/dns_udp_tcp_random.pcap new file mode 100644 index 000000000..c831d0546 Binary files /dev/null and b/libs/visor_dns/tests/dns_udp_tcp_random.pcap differ diff --git a/libs/visor_dns/tests/main.cpp b/libs/visor_dns/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/libs/visor_dns/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/libs/visor_dns/tests/test_dns.cpp b/libs/visor_dns/tests/test_dns.cpp index 4234e1bef..732f30827 100644 --- a/libs/visor_dns/tests/test_dns.cpp +++ b/libs/visor_dns/tests/test_dns.cpp @@ -1,117 +1,119 @@ -#include "catch2/catch.hpp" +#include #include "dns.h" using namespace visor::lib::dns; +static std::pair convert(const AggDomainResult &result) +{ + return {std::string(result.first), std::string(result.second)}; +} + TEST_CASE("DNS Utilities", "[dns]") { + std::pair result; + std::string domain; SECTION("aggregateDomain") { - AggDomainResult result; - std::string domain; - domain = "biz.foo.bar.com"; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == ".bar.com"); CHECK(result.second == ".foo.bar.com"); domain = "a.com"; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == "a.com"); CHECK(result.second == ""); domain = "abcdefg.com."; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == "abcdefg.com."); CHECK(result.second == ""); domain = "foo.bar.com"; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == ".bar.com"); CHECK(result.second == "foo.bar.com"); domain = "."; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == "."); CHECK(result.second == ""); domain = ".."; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == ".."); CHECK(result.second == ""); domain = "a"; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == "a"); CHECK(result.second == ""); domain = "a."; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == "a."); CHECK(result.second == ""); domain = "foo.bar.com."; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == ".bar.com."); CHECK(result.second == "foo.bar.com."); domain = ".foo.bar.com"; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == ".bar.com"); CHECK(result.second == ".foo.bar.com"); domain = "a.b.c"; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == ".b.c"); CHECK(result.second == "a.b.c"); domain = ".b.c"; - result = aggregateDomain(domain); + result = convert(aggregateDomain(domain)); CHECK(result.first == ".b.c"); CHECK(result.second == ""); } SECTION("aggregateDomain with static suffix") { - AggDomainResult result; - std::string domain; std::string static_suffix; domain = "biz.foo.bar.com"; static_suffix = ".bar.com"; - result = aggregateDomain(domain, static_suffix.size()); + result = convert(aggregateDomain(domain, static_suffix.size())); CHECK(result.first == ".foo.bar.com"); CHECK(result.second == "biz.foo.bar.com"); domain = "biz.foo.bar.com"; static_suffix = "bar.com"; - result = aggregateDomain(domain, static_suffix.size()); + result = convert(aggregateDomain(domain, static_suffix.size())); CHECK(result.first == ".foo.bar.com"); CHECK(result.second == "biz.foo.bar.com"); domain = "biz.foo.bar.com"; static_suffix = "foo.bar.com"; - result = aggregateDomain(domain, static_suffix.size()); + result = convert(aggregateDomain(domain, static_suffix.size())); CHECK(result.first == "biz.foo.bar.com"); CHECK(result.second == ""); domain = "foo.bar.com."; static_suffix = "biz.foo.bar.com"; - result = aggregateDomain(domain, static_suffix.size()); + result = convert(aggregateDomain(domain, static_suffix.size())); CHECK(result.first == ".bar.com."); CHECK(result.second == "foo.bar.com."); domain = "www.google.co.uk"; static_suffix = ".co.uk"; - result = aggregateDomain(domain, static_suffix.size()); + result = convert(aggregateDomain(domain, static_suffix.size())); CHECK(result.first == ".google.co.uk"); CHECK(result.second == "www.google.co.uk"); domain = "www.google.co.uk"; static_suffix = "google.co.uk"; - result = aggregateDomain(domain, static_suffix.size()); + result = convert(aggregateDomain(domain, static_suffix.size())); CHECK(result.first == "www.google.co.uk"); CHECK(result.second == ""); } diff --git a/libs/visor_tcp/VisorTcpLayer.h b/libs/visor_tcp/VisorTcpLayer.h index 414357fb8..b1f4b5a27 100644 --- a/libs/visor_tcp/VisorTcpLayer.h +++ b/libs/visor_tcp/VisorTcpLayer.h @@ -18,7 +18,7 @@ #ifdef TCPOPT_CCECHO #undef TCPOPT_CCECHO #endif // TCPOPT_CCECHO -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/libs/visor_test/CMakeLists.txt b/libs/visor_test/CMakeLists.txt new file mode 100644 index 000000000..d664f26ce --- /dev/null +++ b/libs/visor_test/CMakeLists.txt @@ -0,0 +1,17 @@ +message(STATUS "Visor Lib Test Helper") + +add_library(VisorLibTest INTERFACE) +add_library(Visor::Lib::Test ALIAS VisorLibTest) + +target_include_directories(VisorLibTest + INTERFACE + $) + +target_link_libraries(VisorLibTest + INTERFACE + ${CONAN_LIBS_SPDLOG} + ${CONAN_LIBS_CATCH2}) + +target_compile_features(VisorLibTest INTERFACE cxx_std_17) + +target_sources(VisorLibTest INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/catch2/catch_test_visor.hpp) \ No newline at end of file diff --git a/libs/visor_test/catch2/catch_test_visor.hpp b/libs/visor_test/catch2/catch_test_visor.hpp new file mode 100644 index 000000000..99a40d5ab --- /dev/null +++ b/libs/visor_test/catch2/catch_test_visor.hpp @@ -0,0 +1,24 @@ +#include +#include +#include +#include + +class SpdlogInitializer : public Catch::EventListenerBase +{ +public: + using Catch::EventListenerBase::EventListenerBase; + + void testCaseStarting([[maybe_unused]] Catch::TestCaseInfo const &testInfo) override + { + static bool initialized = false; + if (!initialized) { + auto logger = spdlog::get("visor"); + if (!logger) { + spdlog::stderr_color_mt("visor"); + } + initialized = true; + } + } +}; + +CATCH_REGISTER_LISTENER(SpdlogInitializer) \ No newline at end of file diff --git a/libs/visor_utils/CMakeLists.txt b/libs/visor_utils/CMakeLists.txt index 9eb2b25a0..4e2af3d4e 100644 --- a/libs/visor_utils/CMakeLists.txt +++ b/libs/visor_utils/CMakeLists.txt @@ -20,4 +20,14 @@ target_link_libraries(VisorLibUtils ${CONAN_LIBS_FMT} ) -add_subdirectory(tests) \ No newline at end of file +## TEST SUITE +add_executable(unit-tests-visor-utils test_utils.cpp) + +target_link_libraries(unit-tests-visor-utils + PRIVATE + Visor::Lib::Utils + ${CONAN_LIBS_CATCH2}) + +add_test(NAME unit-tests-visor-utils + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/libs + COMMAND unit-tests-visor-utils) \ No newline at end of file diff --git a/libs/visor_utils/tests/test_utils.cpp b/libs/visor_utils/test_utils.cpp similarity index 98% rename from libs/visor_utils/tests/test_utils.cpp rename to libs/visor_utils/test_utils.cpp index 8440994ed..7b5dce792 100644 --- a/libs/visor_utils/tests/test_utils.cpp +++ b/libs/visor_utils/test_utils.cpp @@ -1,4 +1,4 @@ -#include "catch2/catch.hpp" +#include #include "utils.h" #ifdef _WIN32 diff --git a/libs/visor_utils/tests/CMakeLists.txt b/libs/visor_utils/tests/CMakeLists.txt deleted file mode 100644 index f4e4e65d7..000000000 --- a/libs/visor_utils/tests/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Unit -add_executable(unit-tests-visor-utils - main.cpp - test_utils.cpp - ) - -target_link_libraries(unit-tests-visor-utils - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Lib::Utils) - -add_test(NAME unit-tests-visor-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND unit-tests-visor-utils - ) \ No newline at end of file diff --git a/libs/visor_utils/tests/main.cpp b/libs/visor_utils/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/libs/visor_utils/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/libs/visor_utils/utils.cpp b/libs/visor_utils/utils.cpp index 7702801ea..e9ca8d431 100644 --- a/libs/visor_utils/utils.cpp +++ b/libs/visor_utils/utils.cpp @@ -1,6 +1,6 @@ #include "utils.h" #include "EndianPortable.h" -#include +#include #include #include diff --git a/libs/visor_utils/utils.h b/libs/visor_utils/utils.h index 1488bc1d5..28c6053b9 100644 --- a/libs/visor_utils/utils.h +++ b/libs/visor_utils/utils.h @@ -15,7 +15,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6ce187312..1d3d8de5e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,12 +1,13 @@ message(STATUS "visor-core") -find_package(Corrade REQUIRED PluginManager) - set_directory_properties(PROPERTIES CORRADE_USE_PEDANTIC_FLAGS ON) +#generate cpp code from opentelemetry-proto find_package(Protobuf REQUIRED) set(OTELCPP_PROTO_PATH ${CONAN_OPENTELEMETRY-PROTO_ROOT}/res) +set(BUILD_SHARED_LIBS FALSE) +include(opentelemetry-tools) include(opentelemetry-proto) add_library(visor-core @@ -56,23 +57,24 @@ add_subdirectory(handlers) set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} PARENT_SCOPE) ## TEST SUITE -add_executable(unit-tests-vizor-core - tests/main.cpp +add_executable(unit-tests-visor-core tests/test_sketches.cpp tests/test_metrics.cpp tests/test_geoip.cpp tests/test_ipport.cpp tests/test_taps.cpp tests/test_policies.cpp + tests/test_handlers.cpp ) -target_include_directories(unit-tests-vizor-core - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - ) +target_include_directories(unit-tests-visor-core PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(unit-tests-vizor-core PRIVATE Visor::Core ${VISOR_STATIC_PLUGINS}) +target_link_libraries(unit-tests-visor-core + PRIVATE + Visor::Core + ${VISOR_STATIC_PLUGINS} + ${CONAN_LIBS_CATCH2}) -add_test(NAME unit-tests-vizor-core +add_test(NAME unit-tests-visor-core WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-vizor-core - ) \ No newline at end of file + COMMAND unit-tests-visor-core) \ No newline at end of file diff --git a/src/CoreServer.cpp b/src/CoreServer.cpp index bcdf252f3..0d31a1a9c 100644 --- a/src/CoreServer.cpp +++ b/src/CoreServer.cpp @@ -448,7 +448,10 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config) try { auto [policy, lock] = _registry->policy_manager()->module_get_locked(p_mname); auto scope = resource.add_scope_metrics(); - scope->mutable_scope()->set_name(p_mname); + scope->mutable_scope()->set_name("pktvisor/" + p_mname); + auto attr = scope->mutable_scope()->add_attributes(); + attr->set_key("policy_name"); + attr->mutable_value()->set_string_value(p_mname); policy->opentelemetry_metrics(*scope); } catch (const std::exception &) { return false; diff --git a/src/GeoDB.cpp b/src/GeoDB.cpp index 9b3f0b2fc..a6e70a49f 100644 --- a/src/GeoDB.cpp +++ b/src/GeoDB.cpp @@ -4,6 +4,7 @@ #include "GeoDB.h" #include +#include #include namespace visor::geo { @@ -112,7 +113,7 @@ City MaxmindDB::getGeoLoc(const struct sockaddr_in6 *sa6) const std::string ip_address; if (_lru_geo_cache) { - ip_address = fmt::format("{}", sa6->sin6_addr.s6_addr); + ip_address = fmt::format("{}", fmt::join(sa6->sin6_addr.s6_addr, "")); std::shared_lock lock(_cache_mutex); if (auto geoloc = _lru_geo_cache->getValue(ip_address); geoloc.has_value()) { return geoloc.value(); @@ -304,7 +305,7 @@ std::string MaxmindDB::getASNString(const struct sockaddr_in6 *sa6) const std::string ip_address; if (_lru_asn_cache) { - ip_address = fmt::format("{}", sa6->sin6_addr.s6_addr); + ip_address = fmt::format("{}", fmt::join(sa6->sin6_addr.s6_addr, "")); std::shared_lock lock(_cache_mutex); if (auto asn = _lru_asn_cache->getValue(ip_address); asn.has_value()) { return asn.value(); diff --git a/src/GeoDB.h b/src/GeoDB.h index 672f3dc67..f8b1ccbdf 100644 --- a/src/GeoDB.h +++ b/src/GeoDB.h @@ -8,7 +8,6 @@ #pragma GCC diagnostic ignored "-Wpedantic" #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif -#include #include #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/src/IpPort.h b/src/IpPort.h index bedd177e1..4f3236b81 100644 --- a/src/IpPort.h +++ b/src/IpPort.h @@ -6,6 +6,7 @@ #ifdef __GNUC__ #pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-truncation" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif #include diff --git a/src/Metrics.h b/src/Metrics.h index f5fdd38de..d3c716f51 100644 --- a/src/Metrics.h +++ b/src/Metrics.h @@ -4,7 +4,6 @@ #pragma once #include -#include #include #include #ifdef __GNUC__ @@ -17,6 +16,7 @@ #include #include #include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif @@ -180,26 +180,14 @@ class Counter final : public Metric template class Histogram final : public Metric { - static_assert(std::is_integral::value || std::is_floating_point::value); - - static constexpr T _get_pace() - { - if constexpr (std::is_integral::value) { - return 1; - } else { - return 0.0000001; - } - } - // calculated at compile time static constexpr std::pair, size_t> _get_boundaries() { - auto pace = _get_pace(); std::array boundaries{}; size_t index = 0; for (auto exponent = HIST_MIN_EXP; exponent < HIST_MAX_EXP; exponent++) { for (auto buckets = 0; buckets < HIST_LOG_BUCK; buckets++) { - boundaries[index++] = static_cast((std::pow(10.0, static_cast(buckets) / HIST_LOG_BUCK) * std::pow(10.0, exponent)) + pace); + boundaries[index++] = static_cast((std::pow(10.0, static_cast(buckets) / HIST_LOG_BUCK) * std::pow(10.0, exponent))); } } auto itr = std::unique(boundaries.begin(), boundaries.end()); @@ -235,12 +223,12 @@ class Histogram final : public Metric auto get_min() const { - return _sketch.get_min_value(); + return _sketch.get_min_item(); } auto get_max() const { - return _sketch.get_max_value(); + return _sketch.get_max_item(); } // Metric @@ -253,14 +241,13 @@ class Histogram final : public Metric auto histogram_pmf = _sketch.get_PMF(bins_pmf.first.data(), bins_pmf.second); std::vector bins; for (size_t i = 0; i < bins_pmf.second; ++i) { - if (histogram_pmf[i]) { + if (histogram_pmf[i] != 0.0) { bins.push_back(bins_pmf.first[i]); } } auto histogram = _sketch.get_CDF(bins.data(), bins.size()); - auto pace = _get_pace(); for (std::size_t i = 0; i < bins.size(); ++i) { - name_json_assign(j, {"buckets", std::to_string(bins[i] - pace)}, histogram[i] * _sketch.get_n()); + name_json_assign(j, {"buckets", std::to_string(bins[i])}, histogram[i] * _sketch.get_n()); } name_json_assign(j, {"buckets", "+Inf"}, histogram[bins.size()] * _sketch.get_n()); } @@ -274,17 +261,16 @@ class Histogram final : public Metric auto histogram_pmf = _sketch.get_PMF(bins_pmf.first.data(), bins_pmf.second); std::vector bins; for (size_t i = 0; i < bins_pmf.second; ++i) { - if (histogram_pmf[i]) { + if (histogram_pmf[i] != 0.0) { bins.push_back(bins_pmf.first[i]); } } auto histogram = _sketch.get_CDF(bins.data(), bins.size()); - auto pace = _get_pace(); out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl; out << "# TYPE " << base_name_snake() << " histogram" << std::endl; for (std::size_t i = 0; i < bins.size(); ++i) { LabelMap le(add_labels); - le["le"] = std::to_string(bins[i] - pace); + le["le"] = std::to_string(bins[i]); out << name_snake({"bucket"}, le) << ' ' << histogram[i] * _sketch.get_n() << std::endl; } LabelMap le(add_labels); @@ -293,7 +279,7 @@ class Histogram final : public Metric out << name_snake({"count"}, add_labels) << ' ' << _sketch.get_n() << std::endl; } - void to_opentelemetry(metrics::v1::ScopeMetrics &scope, timespec &start, timespec &end, LabelMap add_labels = {}) const + void to_opentelemetry(metrics::v1::ScopeMetrics &scope, timespec &start, timespec &end, LabelMap add_labels = {}) const override { if (_sketch.is_empty()) { return; @@ -302,12 +288,11 @@ class Histogram final : public Metric auto histogram_pmf = _sketch.get_PMF(bins_pmf.first.data(), bins_pmf.second); std::vector bins; for (size_t i = 0; i < bins_pmf.second; ++i) { - if (histogram_pmf[i]) { + if (histogram_pmf[i] != 0.0) { bins.push_back(bins_pmf.first[i]); } } auto histogram = _sketch.get_CDF(bins.data(), bins.size()); - auto pace = _get_pace(); auto metric = scope.add_metrics(); metric->set_name(base_name_snake()); @@ -319,8 +304,8 @@ class Histogram final : public Metric hist_data_point->set_time_unix_nano(timespec_to_uint64(end)); for (std::size_t i = 0; i < bins.size(); ++i) { - hist_data_point->add_explicit_bounds(bins[i] - pace); - hist_data_point->add_bucket_counts(histogram[i] * _sketch.get_n()); + hist_data_point->add_explicit_bounds(bins[i]); + hist_data_point->add_bucket_counts(static_cast(histogram[i]) * _sketch.get_n()); } hist_data_point->set_count(_sketch.get_n()); @@ -383,24 +368,28 @@ class Quantile final : public Metric return _quantile.get_n(); } - auto get_quantile(float p) const + auto get_quantile(double p) const { return _quantile.get_quantile(p); } auto get_min() const { - return _quantile.get_min_value(); + return _quantile.get_min_item(); } auto get_max() const { - return _quantile.get_max_value(); + return _quantile.get_max_item(); } // Metric void to_json(json &j) const override { + if (_quantile.is_empty()) { + return; + } + std::vector quantiles; if (_quantiles_sum.empty()) { const double fractions[4]{0.50, 0.90, 0.95, 0.99}; @@ -447,7 +436,7 @@ class Quantile final : public Metric out << name_snake({}, l9) << ' ' << quantiles[1] << std::endl; out << name_snake({}, l95) << ' ' << quantiles[2] << std::endl; out << name_snake({}, l99) << ' ' << quantiles[3] << std::endl; - out << name_snake({"sum"}, add_labels) << ' ' << _quantile.get_max_value() << std::endl; + out << name_snake({"sum"}, add_labels) << ' ' << _quantile.get_max_item() << std::endl; out << name_snake({"count"}, add_labels) << ' ' << _quantile.get_n() << std::endl; } } diff --git a/src/Taps.cpp b/src/Taps.cpp index 2d61bee38..55153128d 100644 --- a/src/Taps.cpp +++ b/src/Taps.cpp @@ -183,15 +183,15 @@ bool Tap::tags_match_selector_yaml(const YAML::Node &tag_yaml, bool all) } const auto it_module = it->begin(); + const auto &key = it_module->first.as(); if (!it_module->first.IsScalar()) { - throw TapException(fmt::format("tag key '{}' have be scalar", it_module->first)); + throw TapException(fmt::format("tag key '{}' have be scalar", key)); } if (!it_module->second.IsScalar()) { - throw TapException(fmt::format("tag key '{}' must have scalar value", it_module->first)); + throw TapException(fmt::format("tag key '{}' must have scalar value", key)); } - auto key = it_module->first.as(); if (!_tags->config_exists(key)) { if (all) { return false; diff --git a/src/handlers/bgp/BgpStreamHandler.cpp b/src/handlers/bgp/BgpStreamHandler.cpp index 270c513e7..3dd09b471 100644 --- a/src/handlers/bgp/BgpStreamHandler.cpp +++ b/src/handlers/bgp/BgpStreamHandler.cpp @@ -3,7 +3,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ #include "BgpStreamHandler.h" -#include +#include namespace visor::handler::bgp { diff --git a/src/handlers/bgp/BgpStreamHandler.h b/src/handlers/bgp/BgpStreamHandler.h index 5e74788dc..2974c367e 100644 --- a/src/handlers/bgp/BgpStreamHandler.h +++ b/src/handlers/bgp/BgpStreamHandler.h @@ -13,7 +13,7 @@ #pragma GCC diagnostic ignored "-Wold-style-cast" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/src/handlers/bgp/CMakeLists.txt b/src/handlers/bgp/CMakeLists.txt index 6794a2bbd..4ed0e0d65 100644 --- a/src/handlers/bgp/CMakeLists.txt +++ b/src/handlers/bgp/CMakeLists.txt @@ -11,14 +11,25 @@ add_library(Visor::Handler::Bgp ALIAS VisorHandlerBgp) target_include_directories(VisorHandlerBgp INTERFACE - $ - ) + $) target_link_libraries(VisorHandlerBgp PUBLIC - Visor::Input::Pcap - ) + Visor::Input::Pcap) set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Bgp PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file +## TEST SUITE +add_executable(unit-tests-handler-bgp + tests/test_bgp_layer.cpp + tests/test_json_schema.cpp) + +target_link_libraries(unit-tests-handler-bgp + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Bgp + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-bgp + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-bgp) \ No newline at end of file diff --git a/src/handlers/bgp/tests/CMakeLists.txt b/src/handlers/bgp/tests/CMakeLists.txt deleted file mode 100644 index ab824f241..000000000 --- a/src/handlers/bgp/tests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -## TEST SUITE -add_executable(unit-tests-handler-bgp - main.cpp - test_bgp_layer.cpp - test_json_schema.cpp - ) - -target_link_libraries(unit-tests-handler-bgp - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Bgp) - -add_test(NAME unit-tests-handler-bgp - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-bgp - ) diff --git a/src/handlers/bgp/tests/main.cpp b/src/handlers/bgp/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/handlers/bgp/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/bgp/tests/test_bgp_layer.cpp b/src/handlers/bgp/tests/test_bgp_layer.cpp index 1893040c7..cebba7668 100644 --- a/src/handlers/bgp/tests/test_bgp_layer.cpp +++ b/src/handlers/bgp/tests/test_bgp_layer.cpp @@ -1,4 +1,5 @@ -#include +#include +#include #include "PcapInputStream.h" #include "BgpStreamHandler.h" diff --git a/src/handlers/bgp/tests/test_json_schema.cpp b/src/handlers/bgp/tests/test_json_schema.cpp index c242f9a72..d591ae128 100644 --- a/src/handlers/bgp/tests/test_json_schema.cpp +++ b/src/handlers/bgp/tests/test_json_schema.cpp @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#include +#include #include #include #include diff --git a/src/handlers/dhcp/CMakeLists.txt b/src/handlers/dhcp/CMakeLists.txt index 208b41c8c..1fb89114b 100644 --- a/src/handlers/dhcp/CMakeLists.txt +++ b/src/handlers/dhcp/CMakeLists.txt @@ -6,21 +6,32 @@ corrade_add_static_plugin(VisorHandlerDhcp ${CMAKE_CURRENT_BINARY_DIR} DhcpHandler.conf DhcpHandlerModulePlugin.cpp - DhcpStreamHandler.cpp - ) + DhcpStreamHandler.cpp) add_library(Visor::Handler::Dhcp ALIAS VisorHandlerDhcp) target_include_directories(VisorHandlerDhcp INTERFACE - $ - ) + $) target_link_libraries(VisorHandlerDhcp PUBLIC Visor::Lib::Transaction - Visor::Input::Pcap - ) + Visor::Input::Pcap) set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Dhcp PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file + +## TEST SUITE +add_executable(unit-tests-handler-dhcp + tests/test_dhcp_layer.cpp + tests/test_json_schema.cpp) + +target_link_libraries(unit-tests-handler-dhcp + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Dhcp + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-dhcp + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-dhcp) diff --git a/src/handlers/dhcp/DhcpStreamHandler.h b/src/handlers/dhcp/DhcpStreamHandler.h index 8c4fb4faf..fe53e4d69 100644 --- a/src/handlers/dhcp/DhcpStreamHandler.h +++ b/src/handlers/dhcp/DhcpStreamHandler.h @@ -12,10 +12,10 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include "DhcpLayer.h" -#include "DhcpV6Layer.h" -#include "EthLayer.h" -#include "IPv6Layer.h" +#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/src/handlers/dhcp/tests/CMakeLists.txt b/src/handlers/dhcp/tests/CMakeLists.txt deleted file mode 100644 index 5e5890380..000000000 --- a/src/handlers/dhcp/tests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -## TEST SUITE -add_executable(unit-tests-handler-dhcp - main.cpp - test_dhcp_layer.cpp - test_json_schema.cpp - ) - -target_link_libraries(unit-tests-handler-dhcp - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Dhcp) - -add_test(NAME unit-tests-handler-dhcp - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-dhcp - ) diff --git a/src/handlers/dhcp/tests/main.cpp b/src/handlers/dhcp/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/handlers/dhcp/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/dhcp/tests/test_dhcp_layer.cpp b/src/handlers/dhcp/tests/test_dhcp_layer.cpp index ef86494a1..d99888b56 100644 --- a/src/handlers/dhcp/tests/test_dhcp_layer.cpp +++ b/src/handlers/dhcp/tests/test_dhcp_layer.cpp @@ -1,4 +1,5 @@ -#include +#include +#include #include "DhcpStreamHandler.h" #include "PcapInputStream.h" diff --git a/src/handlers/dhcp/tests/test_json_schema.cpp b/src/handlers/dhcp/tests/test_json_schema.cpp index 655998dee..2d90de62f 100644 --- a/src/handlers/dhcp/tests/test_json_schema.cpp +++ b/src/handlers/dhcp/tests/test_json_schema.cpp @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#include +#include #include #include #include diff --git a/src/handlers/dns/v1/CMakeLists.txt b/src/handlers/dns/v1/CMakeLists.txt index bee5d1049..669eebe2c 100644 --- a/src/handlers/dns/v1/CMakeLists.txt +++ b/src/handlers/dns/v1/CMakeLists.txt @@ -26,4 +26,18 @@ target_link_libraries(VisorHandlerDns set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Dns PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file +## TEST SUITE +add_executable(unit-tests-handler-dns + tests/test_dns_layer.cpp + tests/test_dnstap.cpp + tests/test_json_schema.cpp) + +target_link_libraries(unit-tests-handler-dns + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Dns + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-dns + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-dns) \ No newline at end of file diff --git a/src/handlers/dns/v1/DnsStreamHandler.cpp b/src/handlers/dns/v1/DnsStreamHandler.cpp index b4d693528..5cc0c0d30 100644 --- a/src/handlers/dns/v1/DnsStreamHandler.cpp +++ b/src/handlers/dns/v1/DnsStreamHandler.cpp @@ -15,9 +15,9 @@ #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #pragma clang diagnostic ignored "-Wc99-extensions" #endif -#include -#include -#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif @@ -64,7 +64,7 @@ void DnsStreamHandler::start() _f_rcodes.push_back(NoError); } else if (config_exists("only_rcode")) { std::vector rcodes; - uint64_t want_code; + uint64_t want_code{0}; try { want_code = config_get("only_rcode"); } catch (const std::exception &e) { diff --git a/src/handlers/dns/v1/DnsStreamHandler.h b/src/handlers/dns/v1/DnsStreamHandler.h index 2d10cc64a..da5d7ed56 100644 --- a/src/handlers/dns/v1/DnsStreamHandler.h +++ b/src/handlers/dns/v1/DnsStreamHandler.h @@ -11,7 +11,7 @@ #include "StreamHandler.h" #include "TransactionManager.h" #include "dns.h" -#include "dnstap.pb.h" +#include "pb/dnstap.pb.h" #include #include #include @@ -239,8 +239,8 @@ class DnsMetricsManager final : public visor::AbstractMetricsManager; typedef lib::transaction::TransactionManager DnsTransactionManager; std::unique_ptr _qr_pair_manager; - float _to90th{0.0}; - float _from90th{0.0}; + float _to90th{0.0f}; + float _from90th{0.0f}; public: DnsMetricsManager(const Configurable *window_config) diff --git a/src/handlers/dns/v1/tests/CMakeLists.txt b/src/handlers/dns/v1/tests/CMakeLists.txt deleted file mode 100644 index 151ce1886..000000000 --- a/src/handlers/dns/v1/tests/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ - -# Unit -add_executable(unit-tests-handler-dns - main.cpp - test_dns_layer.cpp - test_dnstap.cpp - test_json_schema.cpp - ) - -target_link_libraries(unit-tests-handler-dns - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Dns) - -add_test(NAME unit-tests-handler-dns - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-dns - ) \ No newline at end of file diff --git a/src/handlers/dns/v1/tests/main.cpp b/src/handlers/dns/v1/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/handlers/dns/v1/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/dns/v1/tests/test_dns_layer.cpp b/src/handlers/dns/v1/tests/test_dns_layer.cpp index d85a8aec2..4c7c52898 100644 --- a/src/handlers/dns/v1/tests/test_dns_layer.cpp +++ b/src/handlers/dns/v1/tests/test_dns_layer.cpp @@ -1,4 +1,6 @@ -#include +#include +#include +#include #include "DnsStreamHandler.h" #include "GeoDB.h" @@ -12,11 +14,11 @@ #pragma clang diagnostic ignored "-Wc99-extensions" #pragma clang diagnostic ignored "-Wrange-loop-analysis" #endif -#include "DnsLayer.h" -#include -#include -#include -#include +#include +#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #pragma GCC diagnostic ignored "-Wold-style-cast" diff --git a/src/handlers/dns/v1/tests/test_dnstap.cpp b/src/handlers/dns/v1/tests/test_dnstap.cpp index 5d4447040..a35e80211 100644 --- a/src/handlers/dns/v1/tests/test_dnstap.cpp +++ b/src/handlers/dns/v1/tests/test_dnstap.cpp @@ -1,4 +1,6 @@ -#include +#include +#include +#include #include "DnsStreamHandler.h" #include "DnstapInputStream.h" diff --git a/src/handlers/dns/v1/tests/test_json_schema.cpp b/src/handlers/dns/v1/tests/test_json_schema.cpp index 9adbb49a7..eac352bc6 100644 --- a/src/handlers/dns/v1/tests/test_json_schema.cpp +++ b/src/handlers/dns/v1/tests/test_json_schema.cpp @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#include +#include #include #include #include diff --git a/src/handlers/dns/v2/CMakeLists.txt b/src/handlers/dns/v2/CMakeLists.txt index 42c9b274f..1814e775d 100644 --- a/src/handlers/dns/v2/CMakeLists.txt +++ b/src/handlers/dns/v2/CMakeLists.txt @@ -26,4 +26,17 @@ target_link_libraries(VisorHandlerDnsV2 set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Dns::V2 PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file +## TEST SUITE +add_executable(unit-tests-handler-dns-v2 + tests/test_dns_layer.cpp + tests/test_dnstap.cpp) + +target_link_libraries(unit-tests-handler-dns-v2 + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Dns::V2 + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-dns-v2 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-dns-v2) \ No newline at end of file diff --git a/src/handlers/dns/v2/DnsStreamHandler.cpp b/src/handlers/dns/v2/DnsStreamHandler.cpp index a2454e8df..2ad2b7325 100644 --- a/src/handlers/dns/v2/DnsStreamHandler.cpp +++ b/src/handlers/dns/v2/DnsStreamHandler.cpp @@ -15,9 +15,9 @@ #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #pragma clang diagnostic ignored "-Wc99-extensions" #endif -#include -#include -#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif @@ -64,7 +64,7 @@ void DnsStreamHandler::start() _f_rcodes.push_back(NoError); } else if (config_exists("only_rcode")) { std::vector rcodes; - uint64_t want_code; + uint64_t want_code{0}; try { want_code = config_get("only_rcode"); } catch (const std::exception &e) { diff --git a/src/handlers/dns/v2/DnsStreamHandler.h b/src/handlers/dns/v2/DnsStreamHandler.h index 1bd18e5c7..45a0cb3cd 100644 --- a/src/handlers/dns/v2/DnsStreamHandler.h +++ b/src/handlers/dns/v2/DnsStreamHandler.h @@ -11,7 +11,7 @@ #include "StreamHandler.h" #include "TransactionManager.h" #include "dns.h" -#include "dnstap.pb.h" +#include "pb/dnstap.pb.h" #include #include #include @@ -387,7 +387,7 @@ class DnsMetricsManager final : public visor::AbstractMetricsManager DnsTransactionManager; struct DirTransaction { std::unique_ptr xact_map; - float per_90th{0.0}; + float per_90th{0.0f}; DirTransaction() : xact_map(std::make_unique()) diff --git a/src/handlers/dns/v2/tests/CMakeLists.txt b/src/handlers/dns/v2/tests/CMakeLists.txt deleted file mode 100644 index 5972a423a..000000000 --- a/src/handlers/dns/v2/tests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -# Unit -add_executable(unit-tests-handler-dns-v2 - main.cpp - test_dns_layer.cpp - test_dnstap.cpp - ) - -target_link_libraries(unit-tests-handler-dns-v2 - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Dns::V2) - -add_test(NAME unit-tests-handler-dns-v2 - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-dns-v2 - ) \ No newline at end of file diff --git a/src/handlers/dns/v2/tests/main.cpp b/src/handlers/dns/v2/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/handlers/dns/v2/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/dns/v2/tests/test_dns_layer.cpp b/src/handlers/dns/v2/tests/test_dns_layer.cpp index be8ffe423..d5f15a2b9 100644 --- a/src/handlers/dns/v2/tests/test_dns_layer.cpp +++ b/src/handlers/dns/v2/tests/test_dns_layer.cpp @@ -1,4 +1,6 @@ -#include +#include +#include +#include #include "DnsStreamHandler.h" #include "GeoDB.h" @@ -12,11 +14,11 @@ #pragma clang diagnostic ignored "-Wc99-extensions" #pragma clang diagnostic ignored "-Wrange-loop-analysis" #endif -#include "DnsLayer.h" -#include -#include -#include -#include +#include +#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #pragma GCC diagnostic ignored "-Wold-style-cast" diff --git a/src/handlers/dns/v2/tests/test_dnstap.cpp b/src/handlers/dns/v2/tests/test_dnstap.cpp index 9513a8a6e..9da42dd8a 100644 --- a/src/handlers/dns/v2/tests/test_dnstap.cpp +++ b/src/handlers/dns/v2/tests/test_dnstap.cpp @@ -1,4 +1,6 @@ -#include +#include +#include +#include #include "DnsStreamHandler.h" #include "DnstapInputStream.h" diff --git a/src/handlers/dns/v2/tests/test_json_schema.cpp b/src/handlers/dns/v2/tests/test_json_schema.cpp index 394174cc2..ca043f84d 100644 --- a/src/handlers/dns/v2/tests/test_json_schema.cpp +++ b/src/handlers/dns/v2/tests/test_json_schema.cpp @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#include +#include #include #include #include diff --git a/src/handlers/flow/CMakeLists.txt b/src/handlers/flow/CMakeLists.txt index b938cf004..cf7234587 100644 --- a/src/handlers/flow/CMakeLists.txt +++ b/src/handlers/flow/CMakeLists.txt @@ -23,4 +23,14 @@ target_link_libraries(VisorHandlerFlow set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Flow PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file +## TEST SUITE +add_executable(unit-tests-handler-flow test_flows.cpp) + +target_link_libraries(unit-tests-handler-flow + PRIVATE + Visor::Handler::Flow + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-flow + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-flow) \ No newline at end of file diff --git a/src/handlers/flow/tests/test_flows.cpp b/src/handlers/flow/test_flows.cpp similarity index 99% rename from src/handlers/flow/tests/test_flows.cpp rename to src/handlers/flow/test_flows.cpp index 2b689945e..a0aeded68 100644 --- a/src/handlers/flow/tests/test_flows.cpp +++ b/src/handlers/flow/test_flows.cpp @@ -1,4 +1,6 @@ -#include +#include +#include +#include #include "FlowInputStream.h" #include "FlowStreamHandler.h" diff --git a/src/handlers/flow/tests/CMakeLists.txt b/src/handlers/flow/tests/CMakeLists.txt deleted file mode 100644 index 91fc406cc..000000000 --- a/src/handlers/flow/tests/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -## TEST SUITE -add_executable(unit-tests-handler-flow - main.cpp - test_flows.cpp - ) - -target_link_libraries(unit-tests-handler-flow - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Flow) - -add_test(NAME unit-tests-handler-flow - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-flow - ) diff --git a/src/handlers/flow/tests/main.cpp b/src/handlers/flow/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/handlers/flow/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/input_resources/CMakeLists.txt b/src/handlers/input_resources/CMakeLists.txt index fc0076272..bee66ddcd 100644 --- a/src/handlers/input_resources/CMakeLists.txt +++ b/src/handlers/input_resources/CMakeLists.txt @@ -26,4 +26,14 @@ target_link_libraries(VisorHandlerInputResources set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::InputResources PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file +## TEST SUITE +add_executable(unit-tests-handler-input-resources test_resources_layer.cpp) + +target_link_libraries(unit-tests-handler-input-resources + PRIVATE + Visor::Handler::InputResources + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-input-resources + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-input-resources) \ No newline at end of file diff --git a/src/handlers/input_resources/tests/test_resources_layer.cpp b/src/handlers/input_resources/test_resources_layer.cpp similarity index 97% rename from src/handlers/input_resources/tests/test_resources_layer.cpp rename to src/handlers/input_resources/test_resources_layer.cpp index 7c0c34101..f73e9c664 100644 --- a/src/handlers/input_resources/tests/test_resources_layer.cpp +++ b/src/handlers/input_resources/test_resources_layer.cpp @@ -1,4 +1,5 @@ -#include +#include +#include #include "DnstapInputStream.h" #include "FlowInputStream.h" diff --git a/src/handlers/input_resources/tests/CMakeLists.txt b/src/handlers/input_resources/tests/CMakeLists.txt deleted file mode 100644 index 6e3075cb2..000000000 --- a/src/handlers/input_resources/tests/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ - -## TEST SUITE -add_executable(unit-tests-handler-input-resources - main.cpp - test_resources_layer.cpp - ) - -target_link_libraries(unit-tests-handler-input-resources - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::InputResources) - -add_test(NAME unit-tests-handler-input-resources - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-input-resources - ) diff --git a/src/handlers/input_resources/tests/main.cpp b/src/handlers/input_resources/tests/main.cpp deleted file mode 100644 index 3b6e7e5b3..000000000 --- a/src/handlers/input_resources/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/net/v1/CMakeLists.txt b/src/handlers/net/v1/CMakeLists.txt index e5e6ed287..e6e542480 100644 --- a/src/handlers/net/v1/CMakeLists.txt +++ b/src/handlers/net/v1/CMakeLists.txt @@ -24,4 +24,24 @@ target_link_libraries(VisorHandlerNet set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Net PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file + +## TEST SUITE +if(WIN32) + #dnstap not supported + add_executable(unit-tests-handler-net tests/test_json_schema.cpp) +else() + add_executable(unit-tests-handler-net + tests/test_net_layer.cpp + tests/test_json_schema.cpp) +endif() + +target_link_libraries(unit-tests-handler-net + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Net + Visor::Handler::Dns + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-net + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-net) diff --git a/src/handlers/net/v1/NetStreamHandler.cpp b/src/handlers/net/v1/NetStreamHandler.cpp index 48bf4d900..5ade9ca2a 100644 --- a/src/handlers/net/v1/NetStreamHandler.cpp +++ b/src/handlers/net/v1/NetStreamHandler.cpp @@ -15,10 +15,10 @@ #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #pragma clang diagnostic ignored "-Wc99-extensions" #endif -#include "IPv4Layer.h" -#include "IPv6Layer.h" -#include "PacketUtils.h" -#include "TimespecTimeval.h" +#include +#include +#include +#include #include "VisorTcpLayer.h" #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/src/handlers/net/v1/tests/CMakeLists.txt b/src/handlers/net/v1/tests/CMakeLists.txt deleted file mode 100644 index cf0f0efb7..000000000 --- a/src/handlers/net/v1/tests/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ - -## TEST SUITE -if(WIN32) - #dnstap not supported - add_executable(unit-tests-handler-net - main.cpp - test_json_schema.cpp - ) -else() - add_executable(unit-tests-handler-net - main.cpp - test_net_layer.cpp - test_json_schema.cpp - ) -endif() - -target_link_libraries(unit-tests-handler-net - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Net - Visor::Handler::Dns) - -add_test(NAME unit-tests-handler-net - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-net - ) diff --git a/src/handlers/net/v1/tests/main.cpp b/src/handlers/net/v1/tests/main.cpp deleted file mode 100644 index cd298f86e..000000000 --- a/src/handlers/net/v1/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include "catch2/catch.hpp" -#include "spdlog/sinks/stdout_color_sinks.h" -#include "spdlog/spdlog.h" -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/net/v1/tests/test_json_schema.cpp b/src/handlers/net/v1/tests/test_json_schema.cpp index 9ae5d3c9f..608256829 100644 --- a/src/handlers/net/v1/tests/test_json_schema.cpp +++ b/src/handlers/net/v1/tests/test_json_schema.cpp @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#include "catch2/catch.hpp" +#include #include "nlohmann/json-schema.hpp" #include #include diff --git a/src/handlers/net/v1/tests/test_net_layer.cpp b/src/handlers/net/v1/tests/test_net_layer.cpp index df36c2360..5dfb37dbd 100644 --- a/src/handlers/net/v1/tests/test_net_layer.cpp +++ b/src/handlers/net/v1/tests/test_net_layer.cpp @@ -1,4 +1,6 @@ -#include "catch2/catch.hpp" +#include +#include +#include #include "DnsStreamHandler.h" #include "DnstapInputStream.h" diff --git a/src/handlers/net/v2/CMakeLists.txt b/src/handlers/net/v2/CMakeLists.txt index a26bfa33f..38e017234 100644 --- a/src/handlers/net/v2/CMakeLists.txt +++ b/src/handlers/net/v2/CMakeLists.txt @@ -24,4 +24,20 @@ target_link_libraries(VisorHandlerNetV2 set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Net::V2 PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file + +## TEST SUITE +if (NOT WIN32) + add_executable(unit-tests-handler-net-v2 tests/test_net_layer.cpp) + + target_link_libraries(unit-tests-handler-net-v2 + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Net::V2 + Visor::Handler::Dns::V2 + Visor::Lib::Test) + + add_test(NAME unit-tests-handler-net-v2 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-net-v2) +endif() + \ No newline at end of file diff --git a/src/handlers/net/v2/NetStreamHandler.cpp b/src/handlers/net/v2/NetStreamHandler.cpp index 64517dfef..225f60ed4 100644 --- a/src/handlers/net/v2/NetStreamHandler.cpp +++ b/src/handlers/net/v2/NetStreamHandler.cpp @@ -15,10 +15,10 @@ #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #pragma clang diagnostic ignored "-Wc99-extensions" #endif -#include "IPv4Layer.h" -#include "IPv6Layer.h" -#include "PacketUtils.h" -#include "TimespecTimeval.h" +#include +#include +#include +#include #include "VisorTcpLayer.h" #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/src/handlers/net/v2/tests/CMakeLists.txt b/src/handlers/net/v2/tests/CMakeLists.txt deleted file mode 100644 index 35204889b..000000000 --- a/src/handlers/net/v2/tests/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ - -## TEST SUITE -if(WIN32) - #dnstap not supported - add_executable(unit-tests-handler-net-v2 - main.cpp - ) -else() - add_executable(unit-tests-handler-net-v2 - main.cpp - test_net_layer.cpp - ) -endif() - -target_link_libraries(unit-tests-handler-net-v2 - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Net::V2 - Visor::Handler::Dns::V2) - -add_test(NAME unit-tests-handler-net-v2 - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-net-v2 - ) diff --git a/src/handlers/net/v2/tests/main.cpp b/src/handlers/net/v2/tests/main.cpp deleted file mode 100644 index cd298f86e..000000000 --- a/src/handlers/net/v2/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include "catch2/catch.hpp" -#include "spdlog/sinks/stdout_color_sinks.h" -#include "spdlog/spdlog.h" -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/net/v2/tests/test_json_schema.cpp b/src/handlers/net/v2/tests/test_json_schema.cpp deleted file mode 100644 index aa3c14d05..000000000 --- a/src/handlers/net/v2/tests/test_json_schema.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -#include "catch2/catch.hpp" -#include "nlohmann/json-schema.hpp" -#include -#include -#include - -#include "PcapInputStream.h" -#include "NetStreamHandler.h" - -using namespace visor::handler::net::v2; -using namespace visor::input::pcap; -using namespace nlohmann; -using nlohmann::json_schema::json_validator; - -TEST_CASE("Net JSON Schema", "[net][iface][json]") -{ - - SECTION("json iface") - { - - PcapInputStream stream{"pcap-test"}; - stream.config_set("pcap_file", "tests/fixtures/dns_udp_tcp_random.pcap"); - stream.config_set("bpf", ""); - stream.config_set("host_spec", "192.168.0.0/24"); - stream.parse_host_spec(); - - visor::Config c; - auto stream_proxy = stream.add_event_proxy(c); - NetStreamHandler net_handler{"net-test", stream_proxy, &c}; - net_handler.config_set("recorded_stream", true); - - net_handler.start(); - stream.start(); - stream.stop(); - net_handler.stop(); - - json net_json; - net_handler.metrics()->window_merged_json(net_json, net_handler.schema_key(), 5); - std::ifstream sfile("handlers/net/v2/tests/window-schema.json"); - CHECK(sfile.is_open()); - std::string schema; - - sfile.seekg(0, std::ios::end); - schema.reserve(sfile.tellg()); - sfile.seekg(0, std::ios::beg); - - schema.assign((std::istreambuf_iterator(sfile)), std::istreambuf_iterator()); - json_validator validator; - - auto schema_json = json::parse(schema); - - try { - validator.set_root_schema(schema_json); - validator.validate(net_json); - } catch (const std::exception &e) { - FAIL(e.what()); - } - } -} diff --git a/src/handlers/net/v2/tests/test_net_layer.cpp b/src/handlers/net/v2/tests/test_net_layer.cpp index de015f048..27714c2c8 100644 --- a/src/handlers/net/v2/tests/test_net_layer.cpp +++ b/src/handlers/net/v2/tests/test_net_layer.cpp @@ -1,4 +1,6 @@ -#include "catch2/catch.hpp" +#include +#include +#include #include "DnsStreamHandler.h" #include "DnstapInputStream.h" diff --git a/src/handlers/netprobe/CMakeLists.txt b/src/handlers/netprobe/CMakeLists.txt index c3ba79537..8c8bb145e 100644 --- a/src/handlers/netprobe/CMakeLists.txt +++ b/src/handlers/netprobe/CMakeLists.txt @@ -23,4 +23,15 @@ target_link_libraries(VisorHandlerNetProbe set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::NetProbe PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file + +## TEST SUITE +add_executable(unit-tests-handler-netprobe test_net_probe.cpp) + +target_link_libraries(unit-tests-handler-netprobe + PRIVATE + Visor::Handler::NetProbe + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-netprobe + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-netprobe) diff --git a/src/handlers/netprobe/NetProbeStreamHandler.cpp b/src/handlers/netprobe/NetProbeStreamHandler.cpp index f41c21a50..8fa5ff6b7 100644 --- a/src/handlers/netprobe/NetProbeStreamHandler.cpp +++ b/src/handlers/netprobe/NetProbeStreamHandler.cpp @@ -79,7 +79,7 @@ void NetProbeStreamHandler::probe_signal_send(pcpp::Packet &payload, TestType ty } } else if (type == TestType::TCP) { if (auto tcp = payload.getLayerOfType(); tcp != nullptr) { - _metrics->process_netprobe_tcp(static_cast(tcp->getSrcPort()), true, name, stamp); + _metrics->process_netprobe_tcp(true, name, stamp); } } } @@ -92,7 +92,7 @@ void NetProbeStreamHandler::probe_signal_recv(pcpp::Packet &payload, TestType ty } } else if (type == TestType::TCP) { if (auto tcp = payload.getLayerOfType(); tcp != nullptr) { - _metrics->process_netprobe_tcp(static_cast(tcp->getDstPort()), false, name, stamp); + _metrics->process_netprobe_tcp(false, name, stamp); } } } @@ -119,6 +119,7 @@ void NetProbeMetricsBucket::specialized_merge(const AbstractMetricsBucket &o, Me if (group_enabled(group::NetProbeMetrics::Counters)) { _targets_metrics[targetId]->attempts += target.second->attempts; _targets_metrics[targetId]->successes += target.second->successes; + _targets_metrics[targetId]->connect_failures += target.second->connect_failures; _targets_metrics[targetId]->dns_failures += target.second->dns_failures; _targets_metrics[targetId]->timed_out += target.second->timed_out; } @@ -143,6 +144,7 @@ void NetProbeMetricsBucket::to_prometheus(std::stringstream &out, Metric::LabelM if (group_enabled(group::NetProbeMetrics::Counters)) { target.second->attempts.to_prometheus(out, target_labels); target.second->successes.to_prometheus(out, target_labels); + target.second->connect_failures.to_prometheus(out, target_labels); target.second->dns_failures.to_prometheus(out, target_labels); target.second->timed_out.to_prometheus(out, target_labels); } @@ -198,6 +200,7 @@ void NetProbeMetricsBucket::to_opentelemetry(metrics::v1::ScopeMetrics &scope, t if (group_enabled(group::NetProbeMetrics::Counters)) { target.second->attempts.to_opentelemetry(scope, start_ts, end_ts, target_labels); target.second->successes.to_opentelemetry(scope, start_ts, end_ts, target_labels); + target.second->connect_failures.to_opentelemetry(scope, start_ts, end_ts, target_labels); target.second->dns_failures.to_opentelemetry(scope, start_ts, end_ts, target_labels); target.second->timed_out.to_opentelemetry(scope, start_ts, end_ts, target_labels); } @@ -252,6 +255,7 @@ void NetProbeMetricsBucket::to_json(json &j) const if (group_enabled(group::NetProbeMetrics::Counters)) { target.second->attempts.to_json(j["targets"][targetId]); target.second->successes.to_json(j["targets"][targetId]); + target.second->connect_failures.to_json(j["targets"][targetId]); target.second->dns_failures.to_json(j["targets"][targetId]); target.second->timed_out.to_json(j["targets"][targetId]); } @@ -311,9 +315,12 @@ void NetProbeMetricsBucket::process_failure(ErrorType error, const std::string & break; case ErrorType::Timeout: ++_targets_metrics[target]->timed_out; + break; case ErrorType::SocketError: case ErrorType::InvalidIp: - case ErrorType::ConnectionFailure: + case ErrorType::ConnectFailure: + ++_targets_metrics[target]->connect_failures; + break; default: break; } @@ -374,12 +381,14 @@ void NetProbeMetricsManager::process_netprobe_icmp(pcpp::IcmpLayer *layer, const if (layer->getMessageType() == pcpp::ICMP_ECHO_REQUEST) { if (auto request = layer->getEchoRequestData(); request != nullptr) { - _request_reply_manager->start_transaction((static_cast(request->header->id) << 16) | request->header->sequence, {{stamp, {0, 0}}, target}); + auto ping_id = (static_cast(request->header->id) << 16) | request->header->sequence; + _request_reply_manager->start_transaction(std::to_string(ping_id), {{stamp, {0, 0}}, target}); } live_bucket()->process_attempts(_deep_sampling_now, target); } else if (layer->getMessageType() == pcpp::ICMP_ECHO_REPLY) { if (auto reply = layer->getEchoReplyData(); reply != nullptr) { - auto xact = _request_reply_manager->maybe_end_transaction((static_cast(reply->header->id) << 16) | reply->header->sequence, stamp); + auto ping_id = (static_cast(reply->header->id) << 16) | reply->header->sequence; + auto xact = _request_reply_manager->maybe_end_transaction(std::to_string(ping_id), stamp); if (xact.first == Result::Valid) { live_bucket()->new_transaction(_deep_sampling_now, xact.second); } else if (xact.first == Result::TimedOut) { @@ -389,16 +398,16 @@ void NetProbeMetricsManager::process_netprobe_icmp(pcpp::IcmpLayer *layer, const } } -void NetProbeMetricsManager::process_netprobe_tcp(uint32_t port, bool send, const std::string &target, timespec stamp) +void NetProbeMetricsManager::process_netprobe_tcp(bool send, const std::string &target, timespec stamp) { // base event new_event(stamp); if (send) { - _request_reply_manager->start_transaction(port, {{stamp, {0, 0}}, target}); + _request_reply_manager->start_transaction(target, {{stamp, {0, 0}}, target}); live_bucket()->process_attempts(_deep_sampling_now, target); } else { - auto xact = _request_reply_manager->maybe_end_transaction(port, stamp); + auto xact = _request_reply_manager->maybe_end_transaction(target, stamp); if (xact.first == Result::Valid) { live_bucket()->new_transaction(_deep_sampling_now, xact.second); } else if (xact.first == Result::TimedOut) { diff --git a/src/handlers/netprobe/NetProbeStreamHandler.h b/src/handlers/netprobe/NetProbeStreamHandler.h index 0841fd48f..ba351cc37 100644 --- a/src/handlers/netprobe/NetProbeStreamHandler.h +++ b/src/handlers/netprobe/NetProbeStreamHandler.h @@ -15,7 +15,7 @@ #pragma GCC diagnostic ignored "-Wold-style-cast" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include +#include #include #ifdef __GNUC__ #pragma GCC diagnostic pop @@ -49,6 +49,7 @@ struct Target { Counter successes; Counter minimum; Counter maximum; + Counter connect_failures; Counter dns_failures; Counter timed_out; @@ -59,7 +60,8 @@ struct Target { , successes(NET_PROBE_SCHEMA, {"successes"}, "Total Net Probe successes") , minimum(NET_PROBE_SCHEMA, {"response_min_us"}, "Minimum response time measured in the reporting interval") , maximum(NET_PROBE_SCHEMA, {"response_max_us"}, "Maximum response time measured in the reporting interval") - , dns_failures(NET_PROBE_SCHEMA, {"dns_lookup_failures"}, "Total Net Probe failures when performed DNS lookup") + , connect_failures(NET_PROBE_SCHEMA, {"connect_failures"}, "Total Net Probe failures when performing a TCP socket connection") + , dns_failures(NET_PROBE_SCHEMA, {"dns_lookup_failures"}, "Total Net Probe failures when performing a DNS lookup") , timed_out(NET_PROBE_SCHEMA, {"packets_timeout"}, "Total Net Probe timeout transactions") { } @@ -98,7 +100,7 @@ class NetProbeMetricsBucket final : public visor::AbstractMetricsBucket class NetProbeMetricsManager final : public visor::AbstractMetricsManager { - typedef TransactionManager> NetProbeTransactionManager; + typedef TransactionManager> NetProbeTransactionManager; std::unique_ptr _request_reply_manager; public: @@ -122,7 +124,7 @@ class NetProbeMetricsManager final : public visor::AbstractMetricsManager diff --git a/src/handlers/netprobe/tests/test_net_probe.cpp b/src/handlers/netprobe/test_net_probe.cpp similarity index 97% rename from src/handlers/netprobe/tests/test_net_probe.cpp rename to src/handlers/netprobe/test_net_probe.cpp index e1fd0119f..8aeae7fb5 100644 --- a/src/handlers/netprobe/tests/test_net_probe.cpp +++ b/src/handlers/netprobe/test_net_probe.cpp @@ -1,4 +1,5 @@ -#include +#include +#include #include "NetProbeInputStream.h" #include "NetProbeStreamHandler.h" diff --git a/src/handlers/netprobe/tests/CMakeLists.txt b/src/handlers/netprobe/tests/CMakeLists.txt deleted file mode 100644 index 593c542fe..000000000 --- a/src/handlers/netprobe/tests/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ - -## TEST SUITE -add_executable(unit-tests-handler-netprobe - main.cpp - test_net_probe.cpp - ) - -target_link_libraries(unit-tests-handler-netprobe - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::NetProbe) - -add_test(NAME unit-tests-handler-netprobe - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-netprobe - ) diff --git a/src/handlers/netprobe/tests/main.cpp b/src/handlers/netprobe/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/handlers/netprobe/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/pcap/CMakeLists.txt b/src/handlers/pcap/CMakeLists.txt index 38c91ca7d..ef8ce2e92 100644 --- a/src/handlers/pcap/CMakeLists.txt +++ b/src/handlers/pcap/CMakeLists.txt @@ -21,4 +21,18 @@ target_link_libraries(VisorHandlerPcap set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Pcap PARENT_SCOPE) -add_subdirectory(tests) \ No newline at end of file + +## TEST SUITE +add_executable(unit-tests-handler-pcap + tests/test_pcap_layer.cpp + tests/test_json_schema.cpp) + +target_link_libraries(unit-tests-handler-pcap + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Pcap + Visor::Lib::Test) + +add_test(NAME unit-tests-handler-pcap + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src + COMMAND unit-tests-handler-pcap) diff --git a/src/handlers/pcap/tests/CMakeLists.txt b/src/handlers/pcap/tests/CMakeLists.txt deleted file mode 100644 index 943453032..000000000 --- a/src/handlers/pcap/tests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -## TEST SUITE -add_executable(unit-tests-handler-pcap - main.cpp - test_pcap_layer.cpp - test_json_schema.cpp - ) - -target_link_libraries(unit-tests-handler-pcap - PRIVATE - ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} - Visor::Handler::Pcap) - -add_test(NAME unit-tests-handler-pcap - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-handler-pcap - ) diff --git a/src/handlers/pcap/tests/main.cpp b/src/handlers/pcap/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/handlers/pcap/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/handlers/pcap/tests/test_json_schema.cpp b/src/handlers/pcap/tests/test_json_schema.cpp index ee0eb9bc8..be045da1d 100644 --- a/src/handlers/pcap/tests/test_json_schema.cpp +++ b/src/handlers/pcap/tests/test_json_schema.cpp @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#include +#include #include #include #include diff --git a/src/handlers/pcap/tests/test_pcap_layer.cpp b/src/handlers/pcap/tests/test_pcap_layer.cpp index ccc643a26..102ab325d 100644 --- a/src/handlers/pcap/tests/test_pcap_layer.cpp +++ b/src/handlers/pcap/tests/test_pcap_layer.cpp @@ -1,4 +1,5 @@ -#include +#include +#include #include "GeoDB.h" #include "PcapInputStream.h" diff --git a/src/handlers/static_plugins.h b/src/handlers/static_plugins.h index 3f0652f43..2a968cca2 100644 --- a/src/handlers/static_plugins.h +++ b/src/handlers/static_plugins.h @@ -1,6 +1,6 @@ #pragma once -int import_handler_plugins() +static int import_handler_plugins() { CORRADE_PLUGIN_IMPORT(VisorHandlerNet); CORRADE_PLUGIN_IMPORT(VisorHandlerNetV2); diff --git a/src/inputs/dnstap/CMakeLists.txt b/src/inputs/dnstap/CMakeLists.txt index 717fda680..33f3f518f 100644 --- a/src/inputs/dnstap/CMakeLists.txt +++ b/src/inputs/dnstap/CMakeLists.txt @@ -2,9 +2,15 @@ message(STATUS "Input Module: Dnstap") set_directory_properties(PROPERTIES CORRADE_USE_PEDANTIC_FLAGS ON) -find_package(Protobuf REQUIRED) protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS pb/dnstap.proto) +if(NOT WIN32) + # Suppress the warning for the generated files + set_source_files_properties(${PROTO_SRCS} ${PROTO_HDRS} + PROPERTIES COMPILE_FLAGS "-Wno-missing-declarations -Wno-unused-parameter" + ) +endif() + corrade_add_static_plugin(VisorInputDnstap ${CMAKE_CURRENT_BINARY_DIR} Dnstap.conf DnstapInputModulePlugin.cpp @@ -38,16 +44,13 @@ target_link_libraries(VisorInputDnstap set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Input::Dnstap PARENT_SCOPE) ## TEST SUITE -add_executable(unit-tests-input-dnstap - tests/main.cpp - tests/test_dnstap.cpp - ) +add_executable(unit-tests-input-dnstap tests/test_dnstap.cpp) target_link_libraries(unit-tests-input-dnstap - PRIVATE Visor::Input::Dnstap - ) + PRIVATE + Visor::Input::Dnstap + Visor::Lib::Test) add_test(NAME unit-tests-input-dnstap WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-input-dnstap - ) \ No newline at end of file + COMMAND unit-tests-input-dnstap) \ No newline at end of file diff --git a/src/inputs/dnstap/DnstapInputStream.cpp b/src/inputs/dnstap/DnstapInputStream.cpp index 2a10a0d5a..741c5f356 100644 --- a/src/inputs/dnstap/DnstapInputStream.cpp +++ b/src/inputs/dnstap/DnstapInputStream.cpp @@ -6,12 +6,19 @@ #include "DnstapException.h" #include "ThreadName.h" #include +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif #include #include #include #include #include #include +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif namespace visor::input::dnstap { @@ -75,7 +82,7 @@ void DnstapInputStream::_read_frame_stream_file() break; } else { // Abnormal end - _logger->warn("fstrm_reader_read() data stream ended abnormally: {}", result); + _logger->warn("fstrm_reader_read() data stream ended abnormally: {}", static_cast(result)); break; } } @@ -129,16 +136,16 @@ void DnstapInputStream::_create_frame_stream_tcp_socket() } // main io loop, run in its own thread - _io_loop = uvw::Loop::create(); + _io_loop = uvw::loop::create(); if (!_io_loop) { throw DnstapException("unable to create io loop"); } // AsyncHandle lets us stop the loop from its own thread - _async_h = _io_loop->resource(); + _async_h = _io_loop->resource(); if (!_async_h) { throw DnstapException("unable to initialize AsyncHandle"); } - _async_h->once([this](const auto &, auto &handle) { + _async_h->on([this](const auto &, auto &handle) { _timer->stop(); _timer->close(); _tcp_server_h->stop(); @@ -147,16 +154,16 @@ void DnstapInputStream::_create_frame_stream_tcp_socket() _io_loop->close(); handle.close(); }); - _async_h->on([this](const auto &err, auto &handle) { + _async_h->on([this](const auto &err, auto &handle) { _logger->error("[{}] AsyncEvent error: {}", _name, err.what()); handle.close(); }); - _timer = _io_loop->resource(); + _timer = _io_loop->resource(); if (!_timer) { throw DnstapException("unable to initialize TimerHandle"); } - _timer->on([this](const auto &, auto &) { + _timer->on([this](const auto &, auto &) { timespec stamp; // use now() std::timespec_get(&stamp, TIME_UTC); @@ -165,25 +172,25 @@ void DnstapInputStream::_create_frame_stream_tcp_socket() static_cast(proxy.get())->heartbeat_cb(stamp); } }); - _timer->on([this](const auto &err, auto &handle) { + _timer->on([this](const auto &err, auto &handle) { _logger->error("[{}] TimerEvent error: {}", _name, err.what()); handle.close(); }); // setup server socket - _tcp_server_h = _io_loop->resource(); + _tcp_server_h = _io_loop->resource(); if (!_tcp_server_h) { - throw DnstapException("unable to initialize server PipeHandle"); + throw DnstapException("unable to initialize server pipe_handle"); } - _tcp_server_h->on([this](const auto &err, auto &) { + _tcp_server_h->on([this](const auto &err, auto &) { _logger->error("[{}] socket error: {}", _name, err.what()); throw DnstapException(err.what()); }); - // ListenEvent happens on client connection - _tcp_server_h->on([this](const uvw::ListenEvent &, uvw::TCPHandle &) { - auto client = _io_loop->resource(); + // listen_event happens on client connection + _tcp_server_h->on([this](const uvw::listen_event &, uvw::tcp_handle &) { + auto client = _io_loop->resource(); if (!client) { throw DnstapException("unable to initialize connected client TCPHandle"); } @@ -209,14 +216,14 @@ void DnstapInputStream::_create_frame_stream_tcp_socket() } }; - client->on([this](const uvw::ErrorEvent &err, uvw::TCPHandle &c_sock) { + client->on([this](const uvw::error_event &err, uvw::tcp_handle &c_sock) { _logger->error("[{}]: dnstap client socket error: {}", _name, err.what()); c_sock.stop(); c_sock.close(); }); // client sent data - client->on([this](const uvw::DataEvent &data, uvw::TCPHandle &c_sock) { + client->on([this](const uvw::data_event &data, uvw::tcp_handle &c_sock) { assert(_tcp_sessions[c_sock.fd()]); try { _tcp_sessions[c_sock.fd()]->receive_socket_data(reinterpret_cast(data.data.get()), data.length); @@ -227,12 +234,12 @@ void DnstapInputStream::_create_frame_stream_tcp_socket() } }); // client was closed - client->on([this](const uvw::CloseEvent &, uvw::TCPHandle &c_sock) { + client->on([this](const uvw::close_event &, uvw::tcp_handle &c_sock) { _logger->info("[{}]: dnstap client disconnected", _name); _tcp_sessions.erase(c_sock.fd()); }); // client read EOF - client->on([this](const uvw::EndEvent &, uvw::TCPHandle &c_sock) { + client->on([this](const uvw::end_event &, uvw::tcp_handle &c_sock) { _logger->info("[{}]: dnstap client EOF {}", _name, c_sock.peer().ip); c_sock.stop(); c_sock.close(); @@ -240,7 +247,7 @@ void DnstapInputStream::_create_frame_stream_tcp_socket() _tcp_server_h->accept(*client); _logger->info("[{}]: dnstap client connected {}", _name, client->peer().ip); - _tcp_sessions[client->fd()] = std::make_unique>(client, CONTENT_TYPE, on_data_frame); + _tcp_sessions[client->fd()] = std::make_unique>(client, CONTENT_TYPE, on_data_frame); client->read(); }); @@ -250,7 +257,7 @@ void DnstapInputStream::_create_frame_stream_tcp_socket() // spawn the loop _io_thread = std::make_unique([this] { - _timer->start(uvw::TimerHandle::Time{1000}, uvw::TimerHandle::Time{HEARTBEAT_INTERVAL * 1000}); + _timer->start(uvw::timer_handle::time{1000}, uvw::timer_handle::time{HEARTBEAT_INTERVAL * 1000}); thread::change_self_name(schema_key(), name()); _io_loop->run(); }); @@ -260,16 +267,16 @@ void DnstapInputStream::_create_frame_stream_unix_socket() assert(config_exists("socket")); // main io loop, run in its own thread - _io_loop = uvw::Loop::create(); + _io_loop = uvw::loop::create(); if (!_io_loop) { throw DnstapException("unable to create io loop"); } // AsyncHandle lets us stop the loop from its own thread - _async_h = _io_loop->resource(); + _async_h = _io_loop->resource(); if (!_async_h) { throw DnstapException("unable to initialize AsyncHandle"); } - _async_h->once([this](const auto &, auto &handle) { + _async_h->on([this](const auto &, auto &handle) { _timer->stop(); _timer->close(); _unix_server_h->stop(); @@ -278,16 +285,16 @@ void DnstapInputStream::_create_frame_stream_unix_socket() _io_loop->close(); handle.close(); }); - _async_h->on([this](const auto &err, auto &handle) { + _async_h->on([this](const auto &err, auto &handle) { _logger->error("[{}] AsyncEvent error: {}", _name, err.what()); handle.close(); }); - _timer = _io_loop->resource(); + _timer = _io_loop->resource(); if (!_timer) { throw DnstapException("unable to initialize TimerHandle"); } - _timer->on([this](const auto &, auto &) { + _timer->on([this](const auto &, auto &) { timespec stamp; // use now() std::timespec_get(&stamp, TIME_UTC); @@ -296,27 +303,27 @@ void DnstapInputStream::_create_frame_stream_unix_socket() static_cast(proxy.get())->heartbeat_cb(stamp); } }); - _timer->on([this](const auto &err, auto &handle) { + _timer->on([this](const auto &err, auto &handle) { _logger->error("[{}] TimerEvent error: {}", _name, err.what()); handle.close(); }); // setup server socket - _unix_server_h = _io_loop->resource(); + _unix_server_h = _io_loop->resource(); if (!_unix_server_h) { - throw DnstapException("unable to initialize server PipeHandle"); + throw DnstapException("unable to initialize server pipe_handle"); } - _unix_server_h->on([this](const auto &err, auto &) { + _unix_server_h->on([this](const auto &err, auto &) { _logger->error("[{}] socket error: {}", _name, err.what()); throw DnstapException(err.what()); }); - // ListenEvent happens on client connection - _unix_server_h->on([this](const uvw::ListenEvent &, uvw::PipeHandle &) { - auto client = _io_loop->resource(); + // listen_event happens on client connection + _unix_server_h->on([this](const uvw::listen_event &, uvw::pipe_handle &) { + auto client = _io_loop->resource(); if (!client) { - throw DnstapException("unable to initialize connected client PipeHandle"); + throw DnstapException("unable to initialize connected client pipe_handle"); } auto on_data_frame = [this](const void *data, std::size_t len_data) { @@ -339,14 +346,14 @@ void DnstapInputStream::_create_frame_stream_unix_socket() } }; - client->on([this](const uvw::ErrorEvent &err, uvw::PipeHandle &c_sock) { + client->on([this](const uvw::error_event &err, uvw::pipe_handle &c_sock) { _logger->error("[{}]: dnstap client socket error: {}", _name, err.what()); c_sock.stop(); c_sock.close(); }); // client sent data - client->on([this](const uvw::DataEvent &data, uvw::PipeHandle &c_sock) { + client->on([this](const uvw::data_event &data, uvw::pipe_handle &c_sock) { assert(_unix_sessions[c_sock.fd()]); try { _unix_sessions[c_sock.fd()]->receive_socket_data(reinterpret_cast(data.data.get()), data.length); @@ -357,20 +364,20 @@ void DnstapInputStream::_create_frame_stream_unix_socket() } }); // client was closed - client->on([this](const uvw::CloseEvent &, uvw::PipeHandle &c_sock) { + client->on([this](const uvw::close_event &, uvw::pipe_handle &c_sock) { _logger->info("[{}]: dnstap client disconnected", _name); _unix_sessions.erase(c_sock.fd()); }); // client read EOF - client->on([this](const uvw::EndEvent &, uvw::PipeHandle &c_sock) { - _logger->info("[{}]: dnstap client EOF {}", _name, c_sock.fd()); + client->on([this](const uvw::end_event &, uvw::pipe_handle &c_sock) { + _logger->info("[{}]: dnstap client EOF {}", _name, c_sock.sock()); c_sock.stop(); c_sock.close(); }); _unix_server_h->accept(*client); - _logger->info("[{}]: dnstap client connected {}", _name, client->fd()); - _unix_sessions[client->fd()] = std::make_unique>(client, CONTENT_TYPE, on_data_frame); + _logger->info("[{}]: dnstap client connected {}", _name, client->sock()); + _unix_sessions[client->fd()] = std::make_unique>(client, CONTENT_TYPE, on_data_frame); client->read(); }); @@ -383,7 +390,7 @@ void DnstapInputStream::_create_frame_stream_unix_socket() // spawn the loop _io_thread = std::make_unique([this] { - _timer->start(uvw::TimerHandle::Time{1000}, uvw::TimerHandle::Time{HEARTBEAT_INTERVAL * 1000}); + _timer->start(uvw::timer_handle::time{1000}, uvw::timer_handle::time{HEARTBEAT_INTERVAL * 1000}); thread::change_self_name(schema_key(), name()); _io_loop->run(); }); diff --git a/src/inputs/dnstap/DnstapInputStream.h b/src/inputs/dnstap/DnstapInputStream.h index 857a6b906..eec4b674d 100644 --- a/src/inputs/dnstap/DnstapInputStream.h +++ b/src/inputs/dnstap/DnstapInputStream.h @@ -11,7 +11,7 @@ #include "UnixFrameSession.h" #endif #include "InputStream.h" -#include "dnstap.pb.h" +#include "pb/dnstap.pb.h" #include "utils.h" #ifdef __GNUC__ #pragma GCC diagnostic push @@ -19,7 +19,7 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif @@ -30,11 +30,11 @@ #include namespace uvw { -class Loop; -class AsyncHandle; -class PipeHandle; -class TCPHandle; -class TimerHandle; +class loop; +class async_handle; +class pipe_handle; +class tcp_handle; +class timer_handle; } struct fstrm_reader; @@ -48,15 +48,15 @@ class DnstapInputStream : public visor::InputStream std::shared_ptr _logger; std::unique_ptr _io_thread; - std::shared_ptr _io_loop; - std::shared_ptr _async_h; - std::shared_ptr _timer; + std::shared_ptr _io_loop; + std::shared_ptr _async_h; + std::shared_ptr _timer; - std::shared_ptr _unix_server_h; - std::unordered_map>> _unix_sessions; + std::shared_ptr _unix_server_h; + std::unordered_map>> _unix_sessions; - std::shared_ptr _tcp_server_h; - std::unordered_map>> _tcp_sessions; + std::shared_ptr _tcp_server_h; + std::unordered_map>> _tcp_sessions; static const inline ConfigsDefType _config_defs = { "tcp", diff --git a/src/inputs/dnstap/tests/main.cpp b/src/inputs/dnstap/tests/main.cpp deleted file mode 100644 index 3b6e7e5b3..000000000 --- a/src/inputs/dnstap/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/inputs/dnstap/tests/test_dnstap.cpp b/src/inputs/dnstap/tests/test_dnstap.cpp index 68c7f83da..42a1426dd 100644 --- a/src/inputs/dnstap/tests/test_dnstap.cpp +++ b/src/inputs/dnstap/tests/test_dnstap.cpp @@ -1,5 +1,8 @@ #include "DnstapInputStream.h" -#include + +#include +#include +#include using namespace visor::input::dnstap; using namespace std::chrono; diff --git a/src/inputs/flow/CMakeLists.txt b/src/inputs/flow/CMakeLists.txt index 77309a8b5..f822d0be3 100644 --- a/src/inputs/flow/CMakeLists.txt +++ b/src/inputs/flow/CMakeLists.txt @@ -30,16 +30,13 @@ target_link_libraries(VisorInputFlow set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Input::Flow PARENT_SCOPE) ## TEST SUITE -add_executable(unit-tests-input-flow - tests/main.cpp - tests/test_flow.cpp - ) +add_executable(unit-tests-input-flow test_flow.cpp) target_link_libraries(unit-tests-input-flow - PRIVATE Visor::Input::Flow - ) + PRIVATE + Visor::Input::Flow + Visor::Lib::Test) add_test(NAME unit-tests-input-flow WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-input-flow - ) \ No newline at end of file + COMMAND unit-tests-input-flow) \ No newline at end of file diff --git a/src/inputs/flow/FlowInputStream.cpp b/src/inputs/flow/FlowInputStream.cpp index 18bf8287f..9d9f16ef5 100644 --- a/src/inputs/flow/FlowInputStream.cpp +++ b/src/inputs/flow/FlowInputStream.cpp @@ -12,19 +12,18 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include -#include -#include -#include -#include -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - +#include +#include +#include +#include +#include #include #include #include #include +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif namespace visor::input::flow { @@ -134,16 +133,16 @@ void FlowInputStream::_create_frame_stream_udp_socket() auto bind = config_get("bind"); auto port = config_get("port"); // main io loop, run in its own thread - _io_loop = uvw::Loop::create(); + _io_loop = uvw::loop::create(); if (!_io_loop) { throw FlowException("unable to create io loop"); } // AsyncHandle lets us stop the loop from its own thread - _async_h = _io_loop->resource(); + _async_h = _io_loop->resource(); if (!_async_h) { throw FlowException("unable to initialize AsyncHandle"); } - _async_h->once([this](const auto &, auto &handle) { + _async_h->on([this](const auto &, auto &handle) { _timer->stop(); _timer->close(); _udp_server_h->stop(); @@ -152,16 +151,16 @@ void FlowInputStream::_create_frame_stream_udp_socket() _io_loop->close(); handle.close(); }); - _async_h->on([this](const auto &err, auto &handle) { + _async_h->on([this](const auto &err, auto &handle) { _logger->error("[{}] AsyncEvent error: {}", _name, err.what()); handle.close(); }); - _timer = _io_loop->resource(); + _timer = _io_loop->resource(); if (!_timer) { throw FlowException("unable to initialize TimerHandle"); } - _timer->on([this](const auto &, auto &) { + _timer->on([this](const auto &, auto &) { timespec stamp; // use now() std::timespec_get(&stamp, TIME_UTC); @@ -170,25 +169,25 @@ void FlowInputStream::_create_frame_stream_udp_socket() static_cast(proxy.get())->heartbeat_cb(stamp); } }); - _timer->on([this](const auto &err, auto &handle) { + _timer->on([this](const auto &err, auto &handle) { _logger->error("[{}] TimerEvent error: {}", _name, err.what()); handle.close(); }); // setup server socket - _udp_server_h = _io_loop->resource(); + _udp_server_h = _io_loop->resource(); if (!_udp_server_h) { - throw FlowException("unable to initialize server PipeHandle"); + throw FlowException("unable to initialize server pipe_handle"); } - _udp_server_h->on([this](const auto &err, auto &) { + _udp_server_h->on([this](const auto &err, auto &) { _logger->error("[{}] socket error: {}", _name, err.what()); throw FlowException(err.what()); }); - // ListenEvent happens on client connection + // listen_event happens on client connection if (_flow_type == Type::SFLOW) { - _udp_server_h->on([this](const uvw::UDPDataEvent &event, uvw::UDPHandle &) { + _udp_server_h->on([this](const uvw::udp_data_event &event, uvw::udp_handle &) { SFSample sample; sample.rawSample = reinterpret_cast(event.data.get()); sample.rawSampleLen = event.length; @@ -207,7 +206,7 @@ void FlowInputStream::_create_frame_stream_udp_socket() } }); } else if (_flow_type == Type::NETFLOW) { - _udp_server_h->on([this](const uvw::UDPDataEvent &event, uvw::UDPHandle &) { + _udp_server_h->on([this](const uvw::udp_data_event &event, uvw::udp_handle &) { NFSample sample; sample.raw_sample = reinterpret_cast(event.data.get()); sample.raw_sample_len = event.length; @@ -228,7 +227,7 @@ void FlowInputStream::_create_frame_stream_udp_socket() // spawn the loop _io_thread = std::make_unique([this] { - _timer->start(uvw::TimerHandle::Time{1000}, uvw::TimerHandle::Time{HEARTBEAT_INTERVAL * 1000}); + _timer->start(uvw::timer_handle::time{1000}, uvw::timer_handle::time{HEARTBEAT_INTERVAL * 1000}); thread::change_self_name(schema_key(), name()); _io_loop->run(); }); diff --git a/src/inputs/flow/FlowInputStream.h b/src/inputs/flow/FlowInputStream.h index 772a5d175..870a61997 100644 --- a/src/inputs/flow/FlowInputStream.h +++ b/src/inputs/flow/FlowInputStream.h @@ -21,10 +21,10 @@ #include namespace uvw { -class Loop; -class AsyncHandle; -class UDPHandle; -class TimerHandle; +class loop; +class async_handle; +class udp_handle; +class timer_handle; } namespace visor::input::flow { @@ -41,11 +41,11 @@ class FlowInputStream : public visor::InputStream std::shared_ptr _logger; std::unique_ptr _io_thread; - std::shared_ptr _io_loop; - std::shared_ptr _async_h; - std::shared_ptr _timer; + std::shared_ptr _io_loop; + std::shared_ptr _async_h; + std::shared_ptr _timer; - std::shared_ptr _udp_server_h; + std::shared_ptr _udp_server_h; static const inline ConfigsDefType _config_defs = { "flow_type", diff --git a/src/inputs/flow/tests/test_flow.cpp b/src/inputs/flow/test_flow.cpp similarity index 75% rename from src/inputs/flow/tests/test_flow.cpp rename to src/inputs/flow/test_flow.cpp index 9774becaa..bef5e9ba1 100644 --- a/src/inputs/flow/tests/test_flow.cpp +++ b/src/inputs/flow/test_flow.cpp @@ -1,8 +1,18 @@ #include "FlowInputStream.h" -#include +#include +#include +#include + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif #include #include +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif using namespace visor::input::flow; @@ -56,13 +66,13 @@ TEST_CASE("sflow udp socket", "[sflow][udp]") CHECK_NOTHROW(stream.start()); - auto loop = uvw::Loop::getDefault(); - auto client = loop->resource(); - client->once([](const uvw::SendEvent &, uvw::UDPHandle &handle) { + auto loop = uvw::loop::get_default(); + auto client = loop->resource(); + client->on([](const uvw::send_event &, uvw::udp_handle &handle) { handle.close(); }); auto dataSend = std::unique_ptr(new char[2]{'b', 'c'}); - client->send(uvw::Addr{bind, static_cast(port)}, dataSend.get(), 2); + client->send(uvw::socket_address{bind, static_cast(port)}, dataSend.get(), 2); client->send(bind, port, nullptr, 0); uv_sleep(100); @@ -87,13 +97,13 @@ TEST_CASE("netflow udp socket", "[netflow][udp]") CHECK_NOTHROW(stream.start()); - auto loop = uvw::Loop::getDefault(); - auto client = loop->resource(); - client->once([](const uvw::SendEvent &, uvw::UDPHandle &handle) { + auto loop = uvw::loop::get_default(); + auto client = loop->resource(); + client->on([](const uvw::send_event &, uvw::udp_handle &handle) { handle.close(); }); auto dataSend = std::unique_ptr(new char[2]{'b', 'c'}); - client->send(uvw::Addr{bind, static_cast(port)}, dataSend.get(), 2); + client->send(uvw::socket_address{bind, static_cast(port)}, dataSend.get(), 2); client->send(bind, port, nullptr, 0); uv_sleep(100); diff --git a/src/inputs/flow/tests/main.cpp b/src/inputs/flow/tests/main.cpp deleted file mode 100644 index 3b6e7e5b3..000000000 --- a/src/inputs/flow/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/inputs/mock/CMakeLists.txt b/src/inputs/mock/CMakeLists.txt index 7f9b279d5..7069b7683 100644 --- a/src/inputs/mock/CMakeLists.txt +++ b/src/inputs/mock/CMakeLists.txt @@ -22,16 +22,13 @@ target_link_libraries(VisorInputMock set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Input::Mock PARENT_SCOPE) ## TEST SUITE -add_executable(unit-tests-input-mock - tests/main.cpp - tests/test_mock.cpp - ) +add_executable(unit-tests-input-mock test_mock.cpp) target_link_libraries(unit-tests-input-mock - PRIVATE Visor::Input::Mock - ) + PRIVATE + Visor::Input::Mock + ${CONAN_LIBS_CATCH2}) add_test(NAME unit-tests-input-mock WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-input-mock - ) + COMMAND unit-tests-input-mock) diff --git a/src/inputs/mock/tests/test_mock.cpp b/src/inputs/mock/test_mock.cpp similarity index 71% rename from src/inputs/mock/tests/test_mock.cpp rename to src/inputs/mock/test_mock.cpp index 1c73796a3..90ae121fb 100644 --- a/src/inputs/mock/tests/test_mock.cpp +++ b/src/inputs/mock/test_mock.cpp @@ -1,5 +1,5 @@ -#include +#include //using namespace visor; //using namespace visor::input::mock; diff --git a/src/inputs/mock/tests/main.cpp b/src/inputs/mock/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/inputs/mock/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/inputs/netprobe/CMakeLists.txt b/src/inputs/netprobe/CMakeLists.txt index 63a39a1ac..84246e4ee 100644 --- a/src/inputs/netprobe/CMakeLists.txt +++ b/src/inputs/netprobe/CMakeLists.txt @@ -32,16 +32,13 @@ target_link_libraries(VisorInputNetProbe set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Input::NetProbe PARENT_SCOPE) ## TEST SUITE -add_executable(unit-tests-input-netprobe - tests/main.cpp - tests/test_netprobe.cpp - ) +add_executable(unit-tests-input-netprobe test_netprobe.cpp) target_link_libraries(unit-tests-input-netprobe - PRIVATE Visor::Input::NetProbe - ) + PRIVATE + Visor::Input::NetProbe + Visor::Lib::Test) add_test(NAME unit-tests-input-netprobe WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-input-netprobe - ) \ No newline at end of file + COMMAND unit-tests-input-netprobe) \ No newline at end of file diff --git a/src/inputs/netprobe/NetProbe.h b/src/inputs/netprobe/NetProbe.h index 18c24732c..e8597fb24 100644 --- a/src/inputs/netprobe/NetProbe.h +++ b/src/inputs/netprobe/NetProbe.h @@ -7,15 +7,16 @@ #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include -#include +#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif -#include -#include namespace visor::input::netprobe { @@ -24,7 +25,7 @@ enum class ErrorType { SocketError, DnsLookupFailure, InvalidIp, - ConnectionFailure + ConnectFailure }; enum class TestType { @@ -55,15 +56,15 @@ class NetProbe std::string _name; pcpp::IPAddress _ip; std::string _dns; - std::shared_ptr _io_loop; + std::shared_ptr _io_loop; RecvCallback _recv; SendCallback _send; FailCallback _fail; std::pair _resolve_dns(bool first_match = true, bool ipv4 = false) { - auto request = _io_loop->resource(); - auto response = request->nodeAddrInfoSync(_dns); + auto request = _io_loop->resource(); + auto response = request->node_addr_info_sync(_dns); if (!response.first) { return {std::string(), false}; } @@ -109,7 +110,7 @@ class NetProbe _fail = fail; } - virtual bool start(std::shared_ptr io_loop) = 0; + virtual bool start(std::shared_ptr io_loop) = 0; virtual bool stop() = 0; }; } \ No newline at end of file diff --git a/src/inputs/netprobe/NetProbeInputStream.cpp b/src/inputs/netprobe/NetProbeInputStream.cpp index f7f77cf41..e8bc702b8 100644 --- a/src/inputs/netprobe/NetProbeInputStream.cpp +++ b/src/inputs/netprobe/NetProbeInputStream.cpp @@ -14,11 +14,11 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif @@ -154,32 +154,32 @@ void NetProbeInputStream::_fail_cb(ErrorType error, TestType type, const std::st void NetProbeInputStream::_create_netprobe_loop() { // main io loop, run in its own thread - _io_loop = uvw::Loop::create(); + _io_loop = uvw::loop::create(); if (!_io_loop) { throw NetProbeException("unable to create io loop"); } // AsyncHandle lets us stop the loop from its own thread - _async_h = _io_loop->resource(); + _async_h = _io_loop->resource(); if (!_async_h) { throw NetProbeException("unable to initialize AsyncHandle"); } - _async_h->once([this](const auto &, auto &handle) { + _async_h->on([this](const auto &, auto &handle) { _timer->stop(); _timer->close(); _io_loop->stop(); _io_loop->close(); handle.close(); }); - _async_h->on([this](const auto &err, auto &handle) { + _async_h->on([this](const auto &err, auto &handle) { _logger->error("[{}] AsyncEvent error: {}", _name, err.what()); handle.close(); }); - _timer = _io_loop->resource(); + _timer = _io_loop->resource(); if (!_timer) { throw NetProbeException("unable to initialize TimerHandle"); } - _timer->on([this](const auto &, auto &) { + _timer->on([this](const auto &, auto &) { timespec stamp; // use now() std::timespec_get(&stamp, TIME_UTC); @@ -188,7 +188,7 @@ void NetProbeInputStream::_create_netprobe_loop() proxy->heartbeat_cb(stamp); } }); - _timer->on([this](const auto &err, auto &handle) { + _timer->on([this](const auto &err, auto &handle) { _logger->error("[{}] TimerEvent error: {}", _name, err.what()); handle.close(); }); @@ -231,7 +231,7 @@ void NetProbeInputStream::_create_netprobe_loop() // spawn the loop _io_thread = std::make_unique([this] { - _timer->start(uvw::TimerHandle::Time{1000}, uvw::TimerHandle::Time{HEARTBEAT_INTERVAL * 1000}); + _timer->start(uvw::timer_handle::time{1000}, uvw::timer_handle::time{HEARTBEAT_INTERVAL * 1000}); thread::change_self_name(schema_key(), name()); _io_loop->run(); }); diff --git a/src/inputs/netprobe/NetProbeInputStream.h b/src/inputs/netprobe/NetProbeInputStream.h index fe9b0e129..7225d6f55 100644 --- a/src/inputs/netprobe/NetProbeInputStream.h +++ b/src/inputs/netprobe/NetProbeInputStream.h @@ -6,7 +6,7 @@ #include "InputStream.h" #include "NetProbe.h" -#include +#include #include namespace uvw { @@ -32,9 +32,9 @@ class NetProbeInputStream : public visor::InputStream std::shared_ptr _logger; std::unique_ptr _io_thread; - std::shared_ptr _io_loop; - std::shared_ptr _async_h; - std::shared_ptr _timer; + std::shared_ptr _io_loop; + std::shared_ptr _async_h; + std::shared_ptr _timer; std::vector> _probes; diff --git a/src/inputs/netprobe/PingProbe.cpp b/src/inputs/netprobe/PingProbe.cpp index 8404515f1..974549b01 100644 --- a/src/inputs/netprobe/PingProbe.cpp +++ b/src/inputs/netprobe/PingProbe.cpp @@ -2,8 +2,8 @@ #include "NetProbeException.h" #include "ThreadName.h" -#include -#include +#include +#include #include namespace visor::input::netprobe { @@ -38,16 +38,16 @@ PingReceiver::~PingReceiver() void PingReceiver::_setup_receiver() { - _io_loop = uvw::Loop::create(); + _io_loop = uvw::loop::create(); if (!_io_loop) { throw NetProbeException("unable to create io loop"); } // AsyncHandle lets us stop the loop from its own thread - _async_h = _io_loop->resource(); + _async_h = _io_loop->resource(); if (!_async_h) { throw NetProbeException("unable to initialize AsyncHandle"); } - _async_h->once([this](const auto &, auto &handle) { + _async_h->on([this](const auto &, auto &handle) { _io_loop->stop(); _io_loop->close(); handle.close(); @@ -75,15 +75,15 @@ void PingReceiver::_setup_receiver() } #endif - _poll = _io_loop->resource(static_cast(_sock)); + _poll = _io_loop->resource(static_cast(_sock)); if (!_poll) { throw NetProbeException("PingProbe - unable to initialize PollHandle"); } - _poll->on([](const auto &, auto &handler) { + _poll->on([](const auto &, auto &handler) { handler.close(); }); - _poll->on([this](const uvw::PollEvent &, uvw::PollHandle &) { + _poll->on([this](const uvw::poll_event &, uvw::poll_handle &) { int rc{0}; while (rc != SOCKET_ERROR) { rc = recv(_sock, _array.data(), _array.size(), 0); @@ -98,8 +98,8 @@ void PingReceiver::_setup_receiver() } }); - _timer = _io_loop->resource(); - _timer->on([this](const auto &, auto &) { + _timer = _io_loop->resource(); + _timer->on([this](const auto &, auto &) { if (!_recv_packets.empty()) { recv_packets = _recv_packets; _recv_packets.clear(); @@ -108,10 +108,10 @@ void PingReceiver::_setup_receiver() } } }); - _timer->start(uvw::TimerHandle::Time{100}, uvw::TimerHandle::Time{100}); + _timer->start(uvw::timer_handle::time{100}, uvw::timer_handle::time{100}); _poll->init(); - _poll->start(uvw::PollHandle::Event::READABLE); + _poll->start(uvw::poll_handle::poll_event_flags::READABLE); // spawn the loop _io_thread = std::make_unique([this] { @@ -120,7 +120,7 @@ void PingReceiver::_setup_receiver() }); } -bool PingProbe::start(std::shared_ptr io_loop) +bool PingProbe::start(std::shared_ptr io_loop) { if (_init || (!_ip.isValid() && _dns.empty())) { return false; @@ -145,11 +145,11 @@ bool PingProbe::start(std::shared_ptr io_loop) _receiver = std::make_unique(); } - _interval_timer = _io_loop->resource(); + _interval_timer = _io_loop->resource(); if (!_interval_timer) { throw NetProbeException("PingProbe - unable to initialize interval TimerHandle"); } - _interval_timer->on([this](const auto &, auto &) { + _interval_timer->on([this](const auto &, auto &) { _internal_sequence = 0; if (auto error = _create_socket(); error.has_value()) { @@ -162,8 +162,8 @@ bool PingProbe::start(std::shared_ptr io_loop) return; } - _internal_timer = _io_loop->resource(); - _internal_timer->on([this](const auto &, auto &handle) { + _internal_timer = _io_loop->resource(); + _internal_timer->on([this](const auto &, auto &handle) { if (_internal_sequence < static_cast(_config.packets_per_test)) { _internal_sequence++; _send_icmp_v4(_internal_sequence); @@ -176,14 +176,17 @@ bool PingProbe::start(std::shared_ptr io_loop) (_sequence == UCHAR_MAX) ? _sequence = 0 : _sequence++; _send_icmp_v4(_internal_sequence); _internal_sequence++; - _internal_timer->start(uvw::TimerHandle::Time{_config.packets_interval_msec}, uvw::TimerHandle::Time{_config.packets_interval_msec}); + _internal_timer->start(uvw::timer_handle::time{_config.packets_interval_msec}, uvw::timer_handle::time{_config.packets_interval_msec}); }); - _recv_handler = _io_loop->resource(); + _recv_handler = _io_loop->resource(); if (!_recv_handler) { throw NetProbeException("PingProbe - unable to initialize AsyncHandle receiver"); } - _recv_handler->on([this](const auto &, auto &) { + _recv_handler->on([this](const auto &, auto &) { + // note this processes received packets across ALL active ping probes (because of the single receiver thread) + // the expectation is that packets which did not originate from this probe will be ignored by the handler attached to this probe, + // since it did not originate from it for (auto &[packet, stamp] : PingReceiver::recv_packets) { _recv(packet, TestType::Ping, _name, stamp); } @@ -192,7 +195,7 @@ bool PingProbe::start(std::shared_ptr io_loop) _recv_handler->init(); ++sock_count; - _interval_timer->start(uvw::TimerHandle::Time{0}, uvw::TimerHandle::Time{_config.interval_msec}); + _interval_timer->start(uvw::timer_handle::time{0}, uvw::timer_handle::time{_config.interval_msec}); _init = true; return true; } @@ -240,8 +243,8 @@ std::optional PingProbe::_get_addr() } // do Dns lookup for interval loop - auto request = _io_loop->resource(); - auto response = request->nodeAddrInfoSync(_dns); + auto request = _io_loop->resource(); + auto response = request->node_addr_info_sync(_dns); if (!response.first) { return ErrorType::DnsLookupFailure; } diff --git a/src/inputs/netprobe/PingProbe.h b/src/inputs/netprobe/PingProbe.h index 01288ad46..e3407c728 100644 --- a/src/inputs/netprobe/PingProbe.h +++ b/src/inputs/netprobe/PingProbe.h @@ -26,8 +26,8 @@ typedef int SOCKET; #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #pragma clang diagnostic ignored "-Wc99-extensions" #endif -#include -#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif @@ -54,12 +54,12 @@ class PingReceiver { std::array _array; SOCKET _sock{INVALID_SOCKET}; - std::shared_ptr _poll; + std::shared_ptr _poll; std::unique_ptr _io_thread; - std::shared_ptr _io_loop; - std::shared_ptr _async_h; - std::vector> _callbacks; - std::shared_ptr _timer; + std::shared_ptr _io_loop; + std::shared_ptr _async_h; + std::vector> _callbacks; + std::shared_ptr _timer; std::vector> _recv_packets; void _setup_receiver(); @@ -69,12 +69,12 @@ class PingReceiver PingReceiver(); ~PingReceiver(); - void register_async_callback(std::shared_ptr callback) + void register_async_callback(std::shared_ptr callback) { _callbacks.push_back(callback); } - void remove_async_callback(std::shared_ptr callback) + void remove_async_callback(std::shared_ptr callback) { _callbacks.erase(std::remove(_callbacks.begin(), _callbacks.end(), callback), _callbacks.end()); } @@ -97,9 +97,9 @@ class PingProbe final : public NetProbe bool _ip_set{false}; uint8_t _sequence{0}; uint8_t _internal_sequence{0}; - std::shared_ptr _interval_timer; - std::shared_ptr _internal_timer; - std::shared_ptr _recv_handler; + std::shared_ptr _interval_timer; + std::shared_ptr _internal_timer; + std::shared_ptr _recv_handler; SOCKETLEN _sin_length{0}; std::vector _payload_array; sockaddr_in _sa; @@ -116,7 +116,7 @@ class PingProbe final : public NetProbe PingProbe(uint16_t id, const std::string &name, const pcpp::IPAddress &ip, const std::string &dns) : NetProbe(id, name, ip, dns){}; ~PingProbe() = default; - bool start(std::shared_ptr io_loop) override; + bool start(std::shared_ptr io_loop) override; bool stop() override; }; } diff --git a/src/inputs/netprobe/TcpProbe.cpp b/src/inputs/netprobe/TcpProbe.cpp index 625cdd0bb..2cb61f978 100644 --- a/src/inputs/netprobe/TcpProbe.cpp +++ b/src/inputs/netprobe/TcpProbe.cpp @@ -13,7 +13,7 @@ #endif namespace visor::input::netprobe { -bool TcpProbe::start(std::shared_ptr io_loop) +bool TcpProbe::start(std::shared_ptr io_loop) { if (_init || (!_ip.isValid() && _dns.empty())) { return false; @@ -25,89 +25,56 @@ bool TcpProbe::start(std::shared_ptr io_loop) } _io_loop = io_loop; - _interval_timer = _io_loop->resource(); + _interval_timer = _io_loop->resource(); if (!_interval_timer) { throw NetProbeException("Netprobe - unable to initialize interval TimerHandle"); } - _interval_timer->on([this](const auto &, auto &) { - _internal_sequence = 0; - _timeout_timer = _io_loop->resource(); - if (!_timeout_timer) { - throw NetProbeException("Netprobe - unable to initialize timeout TimerHandle"); - } - - _timeout_timer->on([this](const auto &, auto &) { - _internal_sequence = _config.packets_per_test; - _fail(ErrorType::Timeout, TestType::Ping, _name); - if (_internal_timer) { - _internal_timer->stop(); - } - _interval_timer->again(); - }); - + _interval_timer->on([this](const auto &, auto &) { if (!_dns.empty()) { auto [ip, ipv4] = _resolve_dns(); _ip_str = ip; _is_ipv4 = ipv4; if (_ip_str.empty()) { - _fail(ErrorType::DnsLookupFailure, TestType::Ping, _name); + _fail(ErrorType::DnsLookupFailure, TestType::TCP, _name); return; } } - - _internal_timer = _io_loop->resource(); - _internal_timer->on([this](const auto &, auto &) { - if (_internal_sequence < _config.packets_per_test) { - _internal_sequence++; - _timeout_timer->stop(); - _timeout_timer->start(uvw::TimerHandle::Time{_config.timeout_msec}, uvw::TimerHandle::Time{0}); - _perform_tcp_process(); - } - }); - _timeout_timer->start(uvw::TimerHandle::Time{_config.timeout_msec}, uvw::TimerHandle::Time{0}); _perform_tcp_process(); - _internal_sequence++; - _internal_timer->start(uvw::TimerHandle::Time{_config.packets_interval_msec}, uvw::TimerHandle::Time{_config.packets_interval_msec}); }); - _interval_timer->start(uvw::TimerHandle::Time{0}, uvw::TimerHandle::Time{_config.interval_msec}); + _interval_timer->start(uvw::timer_handle::time{0}, uvw::timer_handle::time{_config.interval_msec}); _init = true; return true; } void TcpProbe::_perform_tcp_process() { - _client = _io_loop->resource(); - _client->on([this](const auto &, auto &) { - _fail(ErrorType::ConnectionFailure, TestType::Ping, _name); + _client = _io_loop->resource(); + _client->on([this](const auto &, auto &) { + _fail(ErrorType::ConnectFailure, TestType::TCP, _name); }); - _client->once([this](const uvw::CloseEvent &, uvw::TCPHandle &) { - timespec stamp; - std::timespec_get(&stamp, TIME_UTC); - pcpp::Packet packet; - auto layer = pcpp::TcpLayer(static_cast(_port), _client_port); - packet.addLayer(&layer); - _recv(packet, TestType::TCP, _name, stamp); + _client->on([this](const uvw::close_event &, uvw::tcp_handle &) { }); - _client->once([](const uvw::ShutdownEvent &, uvw::TCPHandle &handle) { + _client->on([this](const uvw::shutdown_event &, uvw::tcp_handle &handle) { handle.close(); }); - _client->once([this](const uvw::ConnectEvent &, uvw::TCPHandle &handle) { + _client->on([this](const uvw::connect_event &, uvw::tcp_handle &handle) { timespec stamp; std::timespec_get(&stamp, TIME_UTC); - _client_port = static_cast(handle.sock().port); pcpp::Packet packet; - auto layer = pcpp::TcpLayer(_client_port, static_cast(_port)); + auto layer = pcpp::TcpLayer(0, static_cast(_dst_port)); packet.addLayer(&layer); - _send(packet, TestType::TCP, _name, stamp); + _recv(packet, TestType::TCP, _name, stamp); handle.shutdown(); }); - if (_is_ipv4) { - _client->connect(_ip_str, _port); - } else { - _client->connect(_ip_str, _port); - } + timespec stamp; + std::timespec_get(&stamp, TIME_UTC); + pcpp::Packet packet; + auto layer = pcpp::TcpLayer(0, static_cast(_dst_port)); + packet.addLayer(&layer); + _send(packet, TestType::TCP, _name, stamp); + _client->connect(_ip_str, _dst_port); } bool TcpProbe::stop() diff --git a/src/inputs/netprobe/TcpProbe.h b/src/inputs/netprobe/TcpProbe.h index fe598db2a..db14d11af 100644 --- a/src/inputs/netprobe/TcpProbe.h +++ b/src/inputs/netprobe/TcpProbe.h @@ -11,36 +11,24 @@ namespace visor::input::netprobe { -/** - * @class PingProbe - * @brief PingProbe class used for sending ICMP Echo Requests. - * - * This class is created for each specified target. However, it reuses a shared socket per thread (per UV_LOOP). - * I.e. each unique NetProbeInputStream with Ping Type will have a socket to send ICMP Echo Request. - */ class TcpProbe final : public NetProbe { - uint32_t _port; - uint16_t _client_port; + uint32_t _dst_port; bool _init{false}; bool _is_ipv4{false}; - uint16_t _internal_sequence{0}; std::string _ip_str; - std::shared_ptr _interval_timer; - std::shared_ptr _internal_timer; - std::shared_ptr _timeout_timer; + std::shared_ptr _interval_timer; - std::shared_ptr _client; + std::shared_ptr _client; void _perform_tcp_process(); public: TcpProbe(uint16_t id, const std::string &name, const pcpp::IPAddress &ip, const std::string &dns, uint32_t port) : NetProbe(id, name, ip, dns) - , _port(port) - , _client_port(0){}; + , _dst_port(port) {}; ~TcpProbe() = default; - bool start(std::shared_ptr io_loop) override; + bool start(std::shared_ptr io_loop) override; bool stop() override; }; } diff --git a/src/inputs/netprobe/tests/test_netprobe.cpp b/src/inputs/netprobe/test_netprobe.cpp similarity index 96% rename from src/inputs/netprobe/tests/test_netprobe.cpp rename to src/inputs/netprobe/test_netprobe.cpp index 100950faf..12f64bcc9 100644 --- a/src/inputs/netprobe/tests/test_netprobe.cpp +++ b/src/inputs/netprobe/test_netprobe.cpp @@ -1,5 +1,8 @@ #include "NetProbeInputStream.h" -#include + +#include +#include +#include using namespace visor::input::netprobe; using namespace std::chrono; diff --git a/src/inputs/netprobe/tests/main.cpp b/src/inputs/netprobe/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/inputs/netprobe/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/inputs/pcap/CMakeLists.txt b/src/inputs/pcap/CMakeLists.txt index 59eeb4264..a32635af7 100644 --- a/src/inputs/pcap/CMakeLists.txt +++ b/src/inputs/pcap/CMakeLists.txt @@ -29,16 +29,13 @@ set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Input::Pcap PARENT_SCOPE ## TEST SUITE add_executable(unit-tests-input-pcap - tests/main.cpp tests/test_mock_traffic.cpp - tests/test_parse_pcap.cpp - ) + tests/test_parse_pcap.cpp) target_link_libraries(unit-tests-input-pcap PRIVATE Visor::Input::Pcap - ) + ${CONAN_LIBS_CATCH2}) add_test(NAME unit-tests-input-pcap WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src - COMMAND unit-tests-input-pcap - ) \ No newline at end of file + COMMAND unit-tests-input-pcap) \ No newline at end of file diff --git a/src/inputs/pcap/PcapInputStream.cpp b/src/inputs/pcap/PcapInputStream.cpp index fd04cc7c3..c1cbf1875 100644 --- a/src/inputs/pcap/PcapInputStream.cpp +++ b/src/inputs/pcap/PcapInputStream.cpp @@ -15,20 +15,20 @@ #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #pragma clang diagnostic ignored "-Wc99-extensions" #endif -#include // used only for mock generator -#include -#include -#include -#include -#include -#include -#include +#include // used only for mock generator +#include +#include +#include +#include +#include +#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif -#include #include #include +#include #include using namespace std::chrono; @@ -76,7 +76,7 @@ PcapInputStream::PcapInputStream(const std::string &name) this, _tcp_connection_start_cb, _tcp_connection_end_cb, - {true, 1, 1000, 50}) + pcpp::TcpReassemblyConfiguration(true, 1, 1000, 50)) { pcpp::Logger::getInstance().suppressLogs(); } @@ -111,7 +111,7 @@ void PcapInputStream::start() } if (config_exists("debug") && config_get("debug")) { - pcpp::Logger::getInstance().setAllModlesToLogLevel(pcpp::Logger::LogLevel::Debug); + pcpp::Logger::getInstance().setAllModulesToLogLevel(pcpp::Logger::LogLevel::Debug); } _cur_pcap_source = PcapInputStream::DefaultPcapSource; @@ -257,7 +257,7 @@ void PcapInputStream::tcp_message_ready(int8_t side, const pcpp::TcpStreamData & for (auto &proxy : _event_proxies) { dynamic_cast(proxy.get())->tcp_message_ready_cb(side, tcpData, _packet_dir_cache); } - if (_lru_list->put(tcpData.getConnectionData().flowKey, tcpData.getConnectionData().endTime, &_deleted_data)){ + if (_lru_list->put(tcpData.getConnectionData().flowKey, tcpData.getConnectionData().endTime, &_deleted_data)) { _lru_overflow.push_back(_deleted_data.first); } } diff --git a/src/inputs/pcap/PcapInputStream.h b/src/inputs/pcap/PcapInputStream.h index 647c2fdd0..c327dd53e 100644 --- a/src/inputs/pcap/PcapInputStream.h +++ b/src/inputs/pcap/PcapInputStream.h @@ -16,10 +16,10 @@ #pragma GCC diagnostic ignored "-Wold-style-cast" #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif -#include -#include -#include -#include +#include +#include +#include +#include #include #include #ifdef __GNUC__ diff --git a/src/inputs/pcap/afpacket.cpp b/src/inputs/pcap/afpacket.cpp index 7d6952b08..688a4a9b4 100644 --- a/src/inputs/pcap/afpacket.cpp +++ b/src/inputs/pcap/afpacket.cpp @@ -6,7 +6,7 @@ #include "afpacket.h" #include "utils.h" -#include +#include #include #include #include @@ -60,7 +60,7 @@ AFPacket::~AFPacket() fd = -1; } if (map != nullptr) { - munmap(map, static_cast(block_size) * num_blocks); + munmap(map, static_cast(block_size) * num_blocks); } } @@ -274,17 +274,26 @@ void filter_try_compile(const std::string &filter, struct sock_fprog *bpf, int l link_type = 1; } - ret = pcap_compile_nopcap(65535, link_type, &prog, filter.c_str(), 1, 0xffffffff); + pcap_t *handle = pcap_open_dead(link_type, 65535); + if (handle == nullptr) { + throw PcapException("Failed to open pcap handle"); + } + + ret = pcap_compile(handle, &prog, filter.c_str(), 1, 0xffffffff); if (ret < 0) { + pcap_close(handle); throw PcapException("Failed to parse bpf filter: " + filter); } bpf->len = prog.bf_len; bpf->filter = reinterpret_cast(malloc(bpf->len * sizeof(struct sock_filter))); if (bpf->filter == nullptr) { + pcap_close(handle); throw PcapException("Failed to generating bpf filter: Out of memory"); } + pcap_close(handle); + for (i = 0, ins = prog.bf_insns, out = bpf->filter; i < bpf->len; ++i, ++ins, ++out) { out->code = ins->code; diff --git a/src/inputs/pcap/afpacket.h b/src/inputs/pcap/afpacket.h index 1c2b33b40..87c5a8d93 100644 --- a/src/inputs/pcap/afpacket.h +++ b/src/inputs/pcap/afpacket.h @@ -10,7 +10,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/src/inputs/pcap/tests/main.cpp b/src/inputs/pcap/tests/main.cpp deleted file mode 100644 index 3ab9e7842..000000000 --- a/src/inputs/pcap/tests/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - auto logger = spdlog::get("visor"); - if (!logger) { - spdlog::stderr_color_mt("visor"); - } - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/inputs/pcap/tests/test_mock_traffic.cpp b/src/inputs/pcap/tests/test_mock_traffic.cpp index 2a00ded0a..afb609994 100644 --- a/src/inputs/pcap/tests/test_mock_traffic.cpp +++ b/src/inputs/pcap/tests/test_mock_traffic.cpp @@ -1,5 +1,5 @@ #include "PcapInputStream.h" -#include +#include using namespace visor::input::pcap; using namespace std::chrono; diff --git a/src/inputs/pcap/tests/test_parse_pcap.cpp b/src/inputs/pcap/tests/test_parse_pcap.cpp index 83e38ef59..9710c22b0 100644 --- a/src/inputs/pcap/tests/test_parse_pcap.cpp +++ b/src/inputs/pcap/tests/test_parse_pcap.cpp @@ -5,11 +5,11 @@ #pragma clang diagnostic ignored "-Wc99-extensions" #pragma clang diagnostic ignored "-Wrange-loop-analysis" #endif -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/src/inputs/static_plugins.h b/src/inputs/static_plugins.h index 6cb7047b9..9d0e68d39 100644 --- a/src/inputs/static_plugins.h +++ b/src/inputs/static_plugins.h @@ -1,6 +1,6 @@ #pragma once -int import_input_plugins() +static int import_input_plugins() { CORRADE_PLUGIN_IMPORT(VisorInputMock); CORRADE_PLUGIN_IMPORT(VisorInputPcap); diff --git a/src/tests/main.cpp b/src/tests/main.cpp deleted file mode 100644 index 6cfe48b5e..000000000 --- a/src/tests/main.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#define CATCH_CONFIG_RUNNER -#include -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmissing-declarations" -#endif -#include "inputs/static_plugins.h" -#include "handlers/static_plugins.h" -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif -#include -#include - -int main(int argc, char *argv[]) -{ - Catch::Session session; - - int result = session.applyCommandLine(argc, argv); - if (result != 0) { - return result; - } - - result = session.run(); - - return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/tests/test_geoip.cpp b/src/tests/test_geoip.cpp index cf275d474..7288a75fb 100644 --- a/src/tests/test_geoip.cpp +++ b/src/tests/test_geoip.cpp @@ -4,7 +4,7 @@ #else #include #endif -#include +#include #ifdef __GNUC__ #pragma GCC diagnostic ignored "-Wold-style-cast" #endif diff --git a/src/tests/test_handlers.cpp b/src/tests/test_handlers.cpp new file mode 100644 index 000000000..6e96d3392 --- /dev/null +++ b/src/tests/test_handlers.cpp @@ -0,0 +1,225 @@ +#include "StreamHandler.h" + +#include + +using namespace visor; + +namespace group { +enum TestHandler : MetricGroupIntType { + Test, +}; +} + +class HandlerBucket : public AbstractMetricsBucket +{ +public: + std::string type; + void specialized_merge(const AbstractMetricsBucket &, Metric::Aggregate) override{}; + void to_json(json &) const override{}; + void to_prometheus(std::stringstream &, + Metric::LabelMap) const override{}; + void to_opentelemetry(metrics::v1::ScopeMetrics &, timespec &, timespec &, Metric::LabelMap) const override{}; + void update_topn_metrics(size_t, uint64_t) override{}; +}; + +class TestHandlerMetricsManager : public AbstractMetricsManager +{ + const Configurable _config; + HandlerBucket _bucket; + +public: + TestHandlerMetricsManager(const Configurable *window_config) + : AbstractMetricsManager(window_config) + , _config(*window_config){}; + auto current_periods() const + { + return _config.config_get("period"); + } + const HandlerBucket *bucket(uint64_t) const + { + return &_bucket; + } + void window_single_json(json &j, const std::string &, uint64_t) const + { + j["window"] = "single"; + } + void window_single_prometheus(std::stringstream &out, uint64_t period, Metric::LabelMap) const + { + if (period) { + out << "first_window"; + } else { + out << "live_window"; + } + } + void window_single_opentelemetry(metrics::v1::ScopeMetrics &scope, uint64_t period, Metric::LabelMap) const + { + if (period) { + scope.add_metrics()->set_name("first_window"); + } else { + scope.add_metrics()->set_name("live_window"); + } + } + void window_external_opentelemetry(metrics::v1::ScopeMetrics &scope, AbstractMetricsBucket *, Metric::LabelMap) const + { + scope.add_metrics()->set_name("external_window"); + } + void window_external_prometheus(std::stringstream &out, AbstractMetricsBucket *, Metric::LabelMap) const + { + out << "external_window"; + } + void window_external_json(json &j, const std::string &, AbstractMetricsBucket *) const + { + j["window"] = "external"; + } + void window_merged_json(json &j, const std::string &, uint64_t) const + { + j["window"] = "merged"; + } + std::unique_ptr simple_merge(AbstractMetricsBucket *, uint64_t) + { + auto result = std::make_unique(); + result->type = "simple"; + return result; + } + std::unique_ptr multiple_merge(AbstractMetricsBucket *, uint64_t) + { + auto result = std::make_unique(); + result->type = "multiple"; + return result; + } +}; + +class TestStreamMetricsHandler : public StreamMetricsHandler +{ +public: + TestStreamMetricsHandler(const std::string &name, const Configurable *config) + : StreamMetricsHandler(name, config){}; + void start() override{}; + void stop() override{}; + std::string schema_key() const override + { + return "test"; + } + void test_common_info(json &j) const + { + common_info_json(j); + } + void test_process_groups() + { + const StreamMetricsHandler::GroupDefType group_defs = { + {"test", group::TestHandler::Test}, + }; + process_groups(group_defs); + } +}; + +TEST_CASE("StreamMetricsHandler tests", "[metrics][handler]") +{ + Configurable config; + + SECTION("Common info") + { + config.config_set("period", 1); + TestStreamMetricsHandler handler("my_handler", &config); + nlohmann::json j; + handler.test_common_info(j); + CHECK(j["module"]["name"] == "my_handler"); + CHECK(handler.metrics()->current_periods() == 1); + } + + SECTION("Process groups") + { + config.config_set("period", 1); + TestStreamMetricsHandler handler("my_handler", &config); + handler.config_set("disable", {"all"}); + handler.config_set("enable", {"all"}); + CHECK_NOTHROW(handler.test_process_groups()); + + handler.config_set("disable", {"test"}); + handler.config_set("enable", {"test"}); + CHECK_NOTHROW(handler.test_process_groups()); + + handler.config_set("disable", {"invalid"}); + CHECK_THROWS(handler.test_process_groups()); + + handler.config_set("disable", {"test"}); + handler.config_set("enable", {"invalid"}); + CHECK_THROWS(handler.test_process_groups()); + } + + SECTION("Period shift") + { + config.config_set("period", 0); + TestStreamMetricsHandler handler("my_handler", &config); + timespec now; + timespec_get(&now, TIME_UTC); + CHECK_NOTHROW(handler.check_period_shift(now)); + } + + SECTION("JSON window") + { + TestStreamMetricsHandler handler("my_handler", &config); + json j; + handler.window_json(j, 0, false); + CHECK(j["window"] == "single"); + handler.window_json(j, 0, true); + CHECK(j["window"] == "merged"); + handler.window_json(j, nullptr); + CHECK(j["window"] == "external"); + } + + SECTION("Prometheus window") + { + std::string line; + std::stringstream out; + config.config_set("period", 0); + auto handler = std::make_unique("my_handler", &config); + handler->window_prometheus(out); + std::getline(out, line); + CHECK(line == "live_window"); + out.clear(); + + config.config_set("period", 3); + handler = std::make_unique("my_handler", &config); + handler->window_prometheus(out); + std::getline(out, line); + CHECK(line == "first_window"); + out.clear(); + + handler->window_prometheus(out, nullptr); + std::getline(out, line); + CHECK(line == "external_window"); + out.clear(); + } + + SECTION("Opentelemetry window") + { + metrics::v1::ScopeMetrics scope; + config.config_set("period", 0); + auto handler = std::make_unique("my_handler", &config); + handler->window_opentelemetry(scope); + CHECK(scope.metrics().at(0).name() == "live_window"); + scope.Clear(); + + config.config_set("period", 3); + handler = std::make_unique("my_handler", &config); + handler->window_opentelemetry(scope); + CHECK(scope.metrics().at(0).name() == "first_window"); + scope.Clear(); + + handler->window_opentelemetry(scope, nullptr); + CHECK(scope.metrics().at(0).name() == "external_window"); + scope.Clear(); + } + + SECTION("Merge") + { + config.config_set("period", 0); + auto handler = std::make_unique("my_handler", &config); + auto result = handler->merge(nullptr, 0, false, true); + CHECK(dynamic_cast(result.get())->type == "multiple"); + + result = handler->merge(nullptr, 0, true, false); + CHECK(dynamic_cast(result.get())->type == "simple"); + } +} \ No newline at end of file diff --git a/src/tests/test_ipport.cpp b/src/tests/test_ipport.cpp index 54c42464e..b7b1e7312 100644 --- a/src/tests/test_ipport.cpp +++ b/src/tests/test_ipport.cpp @@ -1,6 +1,6 @@ #include "IpPort.h" -#include +#include using namespace visor::network; diff --git a/src/tests/test_metrics.cpp b/src/tests/test_metrics.cpp index 710b8cdcb..1c263ec2b 100644 --- a/src/tests/test_metrics.cpp +++ b/src/tests/test_metrics.cpp @@ -1,5 +1,7 @@ #include "AbstractMetricsManager.h" -#include + +#include +#include using namespace visor; diff --git a/src/tests/test_policies.cpp b/src/tests/test_policies.cpp index df3275198..f0e483815 100644 --- a/src/tests/test_policies.cpp +++ b/src/tests/test_policies.cpp @@ -1,9 +1,23 @@ +#include +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmissing-declarations" +#endif +#include "inputs/static_plugins.h" +#include "handlers/static_plugins.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + #include "CoreRegistry.h" #include "HandlerManager.h" +#include "HandlerModulePlugin.h" #include "InputStream.h" #include "InputStreamManager.h" #include "Policies.h" -#include + +#include +#include #include #include #include @@ -62,6 +76,38 @@ version: "1.0" - "slack.com" )"; +auto policies_config_merge = R"( +visor: + taps: + anycast_merge: + input_type: mock + config: + iface: eth0 + policies: + default_merge: + config: + merge_like_handlers: true + kind: collection + input: + tap: anycast_merge + input_type: mock + config: + sample: value + handlers: + window_config: + num_periods: 5 + deep_sample_rate: 100 + modules: + default_dns_1: + type: dns + default_dns_2: + type: dns + default_net_1: + type: net + default_net_2: + type: net +)"; + auto policies_config_hseq = R"( visor: taps: @@ -200,6 +246,13 @@ version: "1.0" type: net )"; +auto policies_config_bad0 = R"( +version: "1.0" + +visor: + no_policies: +)"; + auto policies_config_bad1 = R"( version: "2.0" @@ -250,8 +303,6 @@ version: "1.0" )"; auto policies_config_bad4 = R"( -version: "1.0" - visor: taps: anycast: @@ -491,6 +542,28 @@ version: "1.0" type: net )"; +auto policies_config_bad14 = R"( +version: "1.0" + +visor: + taps: + anycast: + input_type: mock + config: + iface: eth0 + policies: + default_view: + config: not_a_map + kind: collection + input: + tap: anycast + input_type: mock + handlers: + modules: + default_net: + type: net +)"; + auto policies_config_hseq_bad2 = R"( version: "1.0" @@ -753,9 +826,12 @@ TEST_CASE("Policies", "[policies]") CHECK(policy->input_stream().back()->config_get("bpf") == "tcp or udp"); // TODO this will move to filter member variable CHECK(policy->input_stream().back()->config_get("sample") == "value"); CHECK(policy->modules()[0]->name() == "default_view-anycast-default_net"); + CHECK(dynamic_cast(policy->modules()[0])->version() == "1.0"); CHECK(policy->modules()[1]->name() == "default_view-anycast-default_dns"); + CHECK(dynamic_cast(policy->modules()[1])->version() == "1.0"); CHECK(policy->modules()[1]->config_get("only_rcode") == 2); CHECK(policy->modules()[2]->name() == "default_view-anycast-special_domain"); + CHECK(dynamic_cast(policy->modules()[2])->version() == "1.0"); CHECK(policy->modules()[2]->config_get("only_qname_suffix")[0] == ".google.com"); CHECK(policy->modules()[2]->config_get("only_rcode") == 2); // TODO check window config settings made it through @@ -763,6 +839,35 @@ TEST_CASE("Policies", "[policies]") CHECK(policy->modules()[0]->running()); CHECK(policy->modules()[1]->running()); CHECK(policy->modules()[2]->running()); + + json j; + REQUIRE_NOTHROW(policy->info_json(j)); + CHECK(j["input"] != nullptr); + CHECK(j["modules"]["default_view-anycast-default_net"] != nullptr); + CHECK(j["taps"]["anycast"]["config"]["iface"] == "eth0"); + } + + SECTION("Good Config merge like handlers") + { + CoreRegistry registry; + registry.start(nullptr); + YAML::Node config_file = YAML::Load(policies_config_merge); + CHECK(config_file["visor"]["policies"]); + CHECK(config_file["visor"]["policies"].IsMap()); + REQUIRE_NOTHROW(registry.tap_manager()->load(config_file["visor"]["taps"])); + REQUIRE_NOTHROW(registry.policy_manager()->load(config_file["visor"]["policies"])); + + REQUIRE(registry.policy_manager()->module_exists("default_merge")); + auto [policy, lock] = registry.policy_manager()->module_get_locked("default_merge"); + CHECK(policy->name() == "default_merge"); + CHECK(policy->input_stream().back()->name().find("anycast_merge-") != std::string::npos); + CHECK(policy->modules()[0]->name() == "default_merge-anycast_merge-default_dns_1"); + CHECK(policy->modules()[1]->name() == "default_merge-anycast_merge-default_dns_2"); + CHECK(policy->modules()[2]->name() == "default_merge-anycast_merge-default_net_1"); + CHECK(policy->modules()[3]->name() == "default_merge-anycast_merge-default_net_2"); + CHECK(policy->input_stream().back()->running()); + CHECK(policy->modules()[0]->running()); + CHECK(policy->modules()[1]->running()); } SECTION("Good Config sequence modules") @@ -830,7 +935,13 @@ TEST_CASE("Policies", "[policies]") REQUIRE(registry.policy_manager()->module_exists("default_view")); } - SECTION("Bad Config") + SECTION("Bad Config: no policies schema") + { + CoreRegistry registry; + REQUIRE_THROWS_WITH(registry.policy_manager()->load_from_str(policies_config_bad0), "no policies found in schema"); + } + + SECTION("Bad Config: no policies map config") { CoreRegistry registry; YAML::Node config_file = YAML::Load(policies_config_bad1); @@ -973,6 +1084,16 @@ TEST_CASE("Policies", "[policies]") REQUIRE_THROWS_WITH(registry.handler_manager()->set_default_handler_config(config_file["visor"]["global_handler_config"]), "expecting global_handler_config configuration map"); } + SECTION("Bad Config: policy config not a map") + { + CoreRegistry registry; + registry.start(nullptr); + YAML::Node config_file = YAML::Load(policies_config_bad14); + + REQUIRE_NOTHROW(registry.tap_manager()->load(config_file["visor"]["taps"], true)); + REQUIRE_THROWS_WITH(registry.policy_manager()->load(config_file["visor"]["policies"]), "policy configuration is not a map"); + } + SECTION("Bad Config: invalid handler modules YAML type") { CoreRegistry registry; @@ -1196,3 +1317,69 @@ TEST_CASE("Policies", "[policies]") lock2.unlock(); } } + +TEST_CASE("Policies and Metrics", "[policies][metrics]") +{ + CoreRegistry registry; + registry.start(nullptr); + YAML::Node config_file = YAML::Load(policies_config); + CHECK(config_file["visor"]["policies"]); + CHECK(config_file["visor"]["policies"].IsMap()); + REQUIRE_NOTHROW(registry.tap_manager()->load(config_file["visor"]["taps"])); + REQUIRE_NOTHROW(registry.policy_manager()->load(config_file["visor"]["policies"])); + REQUIRE(registry.policy_manager()->module_exists("default_view")); + + SECTION("JSON") + { + auto [policy, lock] = registry.policy_manager()->module_get_locked("default_view"); + json j; + policy->json_metrics(j, 0, false); + CHECK(j["default_view"]["default_view-anycast-default_dns"] != nullptr); + CHECK(j["default_view"]["default_view-anycast-default_net"] != nullptr); + } + + SECTION("Prometheus") + { + auto [policy, lock] = registry.policy_manager()->module_get_locked("default_view"); + std::stringstream out; + REQUIRE_NOTHROW(policy->prometheus_metrics(out)); + } + + SECTION("Opentelemetry") + { + auto [policy, lock] = registry.policy_manager()->module_get_locked("default_view"); + metrics::v1::ScopeMetrics scope; + REQUIRE_NOTHROW(policy->opentelemetry_metrics(scope)); + } + + config_file = YAML::Load(policies_config_merge); + CHECK(config_file["visor"]["policies"]); + CHECK(config_file["visor"]["policies"].IsMap()); + REQUIRE_NOTHROW(registry.tap_manager()->load(config_file["visor"]["taps"])); + REQUIRE_NOTHROW(registry.policy_manager()->load(config_file["visor"]["policies"])); + REQUIRE(registry.policy_manager()->module_exists("default_merge")); + + SECTION("JSON merge_like_handlers") + { + auto [policy, lock] = registry.policy_manager()->module_get_locked("default_merge"); + json j; + policy->json_metrics(j, 0, false); + std::cout << j.dump(); + CHECK(j["default_merge"]["dns_merged"] != nullptr); + CHECK(j["default_merge"]["packets_merged"] != nullptr); + } + + SECTION("Prometheus merge_like_handlers") + { + auto [policy, lock] = registry.policy_manager()->module_get_locked("default_merge"); + std::stringstream out; + REQUIRE_NOTHROW(policy->prometheus_metrics(out)); + } + + SECTION("Opentelemetry merge_like_handlers") + { + auto [policy, lock] = registry.policy_manager()->module_get_locked("default_merge"); + metrics::v1::ScopeMetrics scope; + REQUIRE_NOTHROW(policy->opentelemetry_metrics(scope)); + } +} \ No newline at end of file diff --git a/src/tests/test_sketches.cpp b/src/tests/test_sketches.cpp index 04f236216..4f89f7714 100644 --- a/src/tests/test_sketches.cpp +++ b/src/tests/test_sketches.cpp @@ -4,13 +4,12 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma clang diagnostic ignored "-Wrange-loop-analysis" #endif -#include +#include #include #include #include #ifdef __GNUC__ #pragma GCC diagnostic pop -#pragma GCC diagnostic ignored "-Wold-style-cast" #endif TEST_CASE("Top-K", "[topk]") @@ -41,22 +40,19 @@ TEST_CASE("Top-K", "[topk]") CHECK(sketch.get_maximum_error() > 0); // estimation mode CHECK(!sketch.is_empty()); - CHECK(35 == (int) sketch.get_total_weight()); + CHECK(35ul == sketch.get_total_weight()); auto items = sketch.get_frequent_items(datasketches::frequent_items_error_type::NO_FALSE_POSITIVES); - CHECK(2 == (int) items.size()); // only 2 items (1 and 7) should have counts more than 1 + CHECK(2ul == items.size()); // only 2 items (1 and 7) should have counts more than 1 CHECK(7 == items[0].get_item()); - CHECK(15 == (int) items[0].get_estimate()); + CHECK(15ul == items[0].get_estimate()); CHECK(1 == items[1].get_item()); - CHECK(10 == (int) items[1].get_estimate()); + CHECK(10ul == items[1].get_estimate()); items = sketch.get_frequent_items(datasketches::frequent_items_error_type::NO_FALSE_NEGATIVES); - CHECK(2 <= (int) items.size()); // at least 2 items - CHECK(12 >= (int) items.size()); // but not more than 12 items - + CHECK(2ul <= items.size()); // at least 2 items + CHECK(12ul >= items.size()); // but not more than 12 items } - - } TEST_CASE("Distinct Count", "[cpc]") @@ -66,13 +62,13 @@ TEST_CASE("Distinct Count", "[cpc]") { datasketches::cpc_sketch sketch(11); const int n(10000); - for (int i = 0; i < n; i++) sketch.update(i); + for (int i = 0; i < n; i++) + sketch.update(i); CHECK(!sketch.is_empty()); CHECK(sketch.get_estimate() >= sketch.get_lower_bound(1)); CHECK(sketch.get_estimate() <= sketch.get_upper_bound(1)); CHECK(sketch.validate()); } - } TEST_CASE("Quantiles", "[kll]") { @@ -83,28 +79,38 @@ TEST_CASE("Quantiles", "[kll]") const uint32_t n(200); for (uint32_t i = 0; i < n; i++) { sketch.update(i); - CHECK((uint64_t) i + 1 == sketch.get_n()); + CHECK(static_cast(i + 1) == sketch.get_n()); } CHECK(!sketch.is_empty()); CHECK(!sketch.is_estimation_mode()); CHECK(n == sketch.get_num_retained()); - CHECK(0.0f == sketch.get_min_value()); + CHECK(0.0f == sketch.get_min_item()); CHECK(0.0f == sketch.get_quantile(0)); - CHECK((float) n - 1 == sketch.get_max_value()); - CHECK((float) n - 1 == sketch.get_quantile(1)); + CHECK(static_cast(n - 1) == sketch.get_max_item()); + CHECK(static_cast(n - 1) == sketch.get_quantile(1)); - const double fractions[3] {0, 0.5, 1}; + const double fractions[3]{0.0, 0.5, 1.0}; + // inclusive (default) auto quantiles = sketch.get_quantiles(fractions, 3); - CHECK(3 == (int) quantiles.size()); + CHECK(3ul == quantiles.size()); + CHECK(0.0f == quantiles[0]); + CHECK(static_cast((n - 1) / 2) == quantiles[1]); + CHECK(static_cast(n - 1) == quantiles[2]); + // exclusive + quantiles = sketch.get_quantiles(fractions, 3, false); + CHECK(3ul == quantiles.size()); CHECK(0.0f == quantiles[0]); - CHECK((float) n / 2 == quantiles[1]); - CHECK((float) n - 1 == quantiles[2]); + CHECK(static_cast(n) / 2 == quantiles[1]); + CHECK(static_cast(n - 1) == quantiles[2]); for (uint32_t i = 0; i < n; i++) { - const double trueRank = (double) i / n; - CHECK(trueRank == sketch.get_rank(i)); + const double trueRank = static_cast(i) / n; + // exclusive + CHECK(trueRank == sketch.get_rank(i, false)); + // inclusive + if (i != 0) { + CHECK(trueRank == sketch.get_rank(i - 1)); + } } } - } - diff --git a/src/tests/test_taps.cpp b/src/tests/test_taps.cpp index fd2c8f5fb..fa8867542 100644 --- a/src/tests/test_taps.cpp +++ b/src/tests/test_taps.cpp @@ -1,6 +1,19 @@ +#include +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmissing-declarations" +#endif +#include "inputs/static_plugins.h" +#include "handlers/static_plugins.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + #include "CoreRegistry.h" #include "Taps.h" -#include + +#include +#include #include #include #include