From 0b0261a8ede4e2ed1fef9c8f68714b26a2bbffc0 Mon Sep 17 00:00:00 2001 From: Alex Harker Date: Wed, 16 Aug 2023 23:24:35 +0100 Subject: [PATCH 1/4] Adapt YINFFT to store its FFT, rather than allocating per frame (using the default allocator) --- examples/describe.cpp | 2 +- include/algorithms/public/YINFFT.hpp | 10 ++++++++-- include/clients/rt/NoveltyFeatureClient.hpp | 1 + include/clients/rt/NoveltySliceClient.hpp | 4 ++-- include/clients/rt/PitchClient.hpp | 19 ++++++++++--------- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/examples/describe.cpp b/examples/describe.cpp index 199e68862..33440b9e9 100644 --- a/examples/describe.cpp +++ b/examples/describe.cpp @@ -91,7 +91,7 @@ int main(int argc, char* argv[]) STFT stft{windowSize, fftSize, hopSize}; MelBands bands{nBands, fftSize}; DCT dct{nBands, nCoefs}; - YINFFT yin; + YINFFT yin{nBins, FluidDefaultAllocator()}; SpectralShape shape(FluidDefaultAllocator()); Loudness loudness{windowSize}; MultiStats stats; diff --git a/include/algorithms/public/YINFFT.hpp b/include/algorithms/public/YINFFT.hpp index 2b27c6651..3194fbda9 100644 --- a/include/algorithms/public/YINFFT.hpp +++ b/include/algorithms/public/YINFFT.hpp @@ -25,6 +25,11 @@ class YINFFT { public: + + YINFFT(index maxInputSize, Allocator& alloc) + : mFFT(2 * maxInputSize - 1, alloc) + {} + void processFrame(const RealVectorView& input, RealVectorView output, double minFreq, double maxFreq, double sampleRate, Allocator& alloc = FluidDefaultAllocator()) @@ -36,14 +41,13 @@ class YINFFT squareMag = _impl::asEigen(input).square(); index nBins = input.size(); - FFT fft(2 * (input.size() - 1)); double squareMagSum = 2 * squareMag.sum(); ScopedEigenMap squareMagSym(2 * (nBins - 1), alloc); squareMagSym << squareMag[0], squareMag.segment(1, nBins - 1), squareMag.segment(1, nBins - 2).reverse(); - Eigen::Map squareMagFFT = fft.process(squareMagSym); + Eigen::Map squareMagFFT = mFFT.process(squareMagSym); ScopedEigenMap yin(squareMagFFT.size(), alloc); yin = squareMagSum - squareMagFFT.real(); @@ -83,6 +87,8 @@ class YINFFT output(0) = pitch; output(1) = pitchConfidence; } + + FFT mFFT; }; } // namespace algorithm } // namespace fluid diff --git a/include/clients/rt/NoveltyFeatureClient.hpp b/include/clients/rt/NoveltyFeatureClient.hpp index e986352cf..a6330d0fb 100644 --- a/include/clients/rt/NoveltyFeatureClient.hpp +++ b/include/clients/rt/NoveltyFeatureClient.hpp @@ -79,6 +79,7 @@ class NoveltyFeatureClient : public FluidBaseClient, mMelBands{40, get().max(), c.allocator()}, mDCT{40, 13, c.allocator()}, mChroma{12, get().max(), c.allocator()}, + mYinFFT(get().maxFrameSize(), c.allocator()), mLoudness{get().max(), c.allocator()} { audioChannelsIn(1); diff --git a/include/clients/rt/NoveltySliceClient.hpp b/include/clients/rt/NoveltySliceClient.hpp index cb6652a91..c0484f205 100644 --- a/include/clients/rt/NoveltySliceClient.hpp +++ b/include/clients/rt/NoveltySliceClient.hpp @@ -89,8 +89,8 @@ class NoveltySliceClient : public FluidBaseClient, c.allocator()}, mChroma{12, get().max(), c.allocator()}, mLoudness{ get().max(), - c.allocator()} - + c.allocator()}, + mYinFFT(get().maxFrameSize(), c.allocator()) { audioChannelsIn(1); audioChannelsOut(1); diff --git a/include/clients/rt/PitchClient.hpp b/include/clients/rt/PitchClient.hpp index d0c1491be..dadc1bb3d 100644 --- a/include/clients/rt/PitchClient.hpp +++ b/include/clients/rt/PitchClient.hpp @@ -78,7 +78,8 @@ class PitchClient : public FluidBaseClient, public AudioIn, public ControlOut PitchClient(ParamSetViewType& p, FluidContext& c) : mParams(p), mSTFTBufferedProcess(get(), 1, 0, c.hostVectorSize(), c.allocator()), - cepstrumF0(get().maxFrameSize(), c.allocator()), + mCepstrumF0(get().maxFrameSize(), c.allocator()), + mYinFFT(get().maxFrameSize(), c.allocator()), mMagnitude(get().maxFrameSize(), c.allocator()), mDescriptors(2, c.allocator()) { @@ -99,7 +100,7 @@ class PitchClient : public FluidBaseClient, public AudioIn, public ControlOut if (mParamTracker.changed(get().frameSize(), sampleRate(), c.hostVectorSize())) { - cepstrumF0.init(get().frameSize(), c.allocator()); + mCepstrumF0.init(get().frameSize(), c.allocator()); mSTFTBufferedProcess = STFTBufferedProcess(get(), 1, 0, c.hostVectorSize(), c.allocator()); // mMagnitude.resize(get().frameSize()); } @@ -112,15 +113,15 @@ class PitchClient : public FluidBaseClient, public AudioIn, public ControlOut switch (get()) { case 0: - cepstrumF0.processFrame(mags, mDescriptors, get(), + mCepstrumF0.processFrame(mags, mDescriptors, get(), get(), sampleRate(),c.allocator()); break; case 1: - hps.processFrame(mags, mDescriptors, 4, get(), + mHPS.processFrame(mags, mDescriptors, 4, get(), get(), sampleRate(), c.allocator()); break; case 2: - yinFFT.processFrame(mags, mDescriptors, get(), + mYinFFT.processFrame(mags, mDescriptors, get(), get(), sampleRate(), c.allocator()); break; } @@ -157,7 +158,7 @@ class PitchClient : public FluidBaseClient, public AudioIn, public ControlOut void reset(FluidContext& c) { mSTFTBufferedProcess.reset(); - cepstrumF0.init(get().frameSize(), c.allocator()); + mCepstrumF0.init(get().frameSize(), c.allocator()); // mMagnitude.resize(get().frameSize()); } @@ -166,9 +167,9 @@ class PitchClient : public FluidBaseClient, public AudioIn, public ControlOut STFTBufferedProcess<> mSTFTBufferedProcess; - CepstrumF0 cepstrumF0; - HPS hps; - YINFFT yinFFT; + CepstrumF0 mCepstrumF0; + HPS mHPS; + YINFFT mYinFFT; FluidTensor mMagnitude; FluidTensor mDescriptors; }; From 965cf27ce6205f87650b35d5989bb362be49fd87 Mon Sep 17 00:00:00 2001 From: Alex Harker Date: Wed, 16 Aug 2023 23:34:56 +0100 Subject: [PATCH 2/4] Update test code --- tests/algorithms/public/TestNoveltySegmentation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/algorithms/public/TestNoveltySegmentation.cpp b/tests/algorithms/public/TestNoveltySegmentation.cpp index df8d6e0a9..9b7338ca0 100644 --- a/tests/algorithms/public/TestNoveltySegmentation.cpp +++ b/tests/algorithms/public/TestNoveltySegmentation.cpp @@ -144,7 +144,7 @@ NoveltyPitchTest(fluid::FluidTensorView testSignal, Params p) FluidTensor pitchFrame(2); auto stft = STFT{p.window, p.fft, p.hop}; - auto pitch = fluid::algorithm::YINFFT(); + auto pitch = fluid::algorithm::YINFFT((p.fft / 2) + 1); auto makeInput = [&stft, &pitch, &stftFrame, &magnitudes, &pitchFrame](auto source) { From 35f06276dff150251532df7b58ac2847102c8ba9 Mon Sep 17 00:00:00 2001 From: Alex Harker Date: Wed, 16 Aug 2023 23:48:29 +0100 Subject: [PATCH 3/4] Add default allocator to yinFFT constructor --- include/algorithms/public/YINFFT.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/algorithms/public/YINFFT.hpp b/include/algorithms/public/YINFFT.hpp index 3194fbda9..332f61017 100644 --- a/include/algorithms/public/YINFFT.hpp +++ b/include/algorithms/public/YINFFT.hpp @@ -26,7 +26,7 @@ class YINFFT public: - YINFFT(index maxInputSize, Allocator& alloc) + YINFFT(index maxInputSize, Allocator& alloc = FluidDefaultAllocator()) : mFFT(2 * maxInputSize - 1, alloc) {} From 31d03d318b43fbcd015731bd94042e41e7e09cde Mon Sep 17 00:00:00 2001 From: Alex Harker Date: Thu, 17 Aug 2023 08:04:33 +0100 Subject: [PATCH 4/4] Resize mFFT to the correct size --- include/algorithms/public/YINFFT.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/algorithms/public/YINFFT.hpp b/include/algorithms/public/YINFFT.hpp index 332f61017..0408ba7b9 100644 --- a/include/algorithms/public/YINFFT.hpp +++ b/include/algorithms/public/YINFFT.hpp @@ -43,6 +43,8 @@ class YINFFT index nBins = input.size(); double squareMagSum = 2 * squareMag.sum(); + mFFT.resize(2 * (nBins - 1)); + ScopedEigenMap squareMagSym(2 * (nBins - 1), alloc); squareMagSym << squareMag[0], squareMag.segment(1, nBins - 1), squareMag.segment(1, nBins - 2).reverse();