Skip to content

Commit 7b73128

Browse files
nyoungbqJDuffeyBQ
andauthored
BUG/API: Histogram Sync (#1073)
* Add HistogramUtilities File * Integrated Histogram API into Compute Array Statistics and Compute Array Histogram * complete extensive documentation for histogram utilities * Update test case to reflect new histogram that produces results inline with plot.ly histogram * repair broken pipelines after key updates * Added a size() function to IDataStore for std::vector compatibility * Update CalculateBin for special bool case --------- Co-authored-by: Jared Duffey <[email protected]>
1 parent 99a8fe1 commit 7b73128

21 files changed

+747
-366
lines changed

CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ set(SIMPLNX_HDRS
499499
${SIMPLNX_SOURCE_DIR}/Utilities/FilterUtilities.hpp
500500
${SIMPLNX_SOURCE_DIR}/Utilities/GeometryUtilities.hpp
501501
${SIMPLNX_SOURCE_DIR}/Utilities/GeometryHelpers.hpp
502+
${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
502503
${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
503504
${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
504505
${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.hpp

src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,8 @@
117117
{
118118
"args": {
119119
"new_data_group_path": "DataContainer/Statistics",
120-
"histogram_suffix": " Histogram",
120+
"histogram_bin_count_suffix": " Histogram Counts",
121+
"histogram_bin_range_suffix": " Histogram Bin Ranges",
121122
"max_range": 1.0,
122123
"min_range": 0.0,
123124
"create_new_data_group": true,
@@ -141,11 +142,11 @@
141142
"file_extension": ".csv",
142143
"header_option_index": 1,
143144
"max_val_per_line": 0,
144-
"output_dir": "Data/Output/OrientationAnalysis/",
145-
"output_path": "Data/Output/OrientationAnalysis/Test/CI_Histogram.csv",
146-
"output_style_index": 1,
145+
"output_dir": "Data/Output/OrientationAnalysis/Test",
146+
"output_style_index": 0,
147147
"input_data_array_paths": [
148-
"DataContainer/Statistics/Confidence Index Histogram"
148+
"DataContainer/Statistics/Confidence Index Histogram Counts",
149+
"DataContainer/Statistics/Confidence Index Histogram Bin Ranges"
149150
]
150151
},
151152
"comments": "",

src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,8 @@
448448
{
449449
"args": {
450450
"create_new_data_group": true,
451-
"histogram_suffix": " Histogram",
451+
"histogram_bin_count_suffix": " Histogram Counts",
452+
"histogram_bin_range_suffix": " Histogram Bin Ranges",
452453
"max_range": 1.0,
453454
"min_range": 0.0,
454455
"new_data_group_path": "fw-ar-IF1-aptr12-corr/Histograms",
@@ -472,12 +473,12 @@
472473
"file_extension": ".csv",
473474
"header_option_index": 1,
474475
"input_data_array_paths": [
475-
"fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram"
476+
"fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Counts",
477+
"fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges"
476478
],
477479
"max_val_per_line": 0,
478-
"output_dir": "",
479-
"output_path": "Data/Output/fw-ar-IF1-aptr12-corr/EqDiamHistogram.csv",
480-
"output_style_index": 1
480+
"output_dir": "Data/Output/fw-ar-IF1-aptr12-corr",
481+
"output_style_index": 0
481482
},
482483
"comments": "",
483484
"filter": {

src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,8 @@
448448
{
449449
"args": {
450450
"create_new_data_group": true,
451-
"histogram_suffix": "Histogram",
451+
"histogram_bin_count_suffix": " Histogram Counts",
452+
"histogram_bin_range_suffix": " Histogram Bin Ranges",
452453
"max_range": 1.0,
453454
"min_range": 0.0,
454455
"new_data_group_path": "fw-ar-IF1-avtr12-corr/Histograms",
@@ -472,12 +473,12 @@
472473
"file_extension": ".csv",
473474
"header_option_index": 1,
474475
"input_data_array_paths": [
475-
"fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiametersHistogram"
476+
"fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Counts",
477+
"fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges"
476478
],
477479
"max_val_per_line": 0,
478-
"output_dir": "",
479-
"output_path": "Data/Output/fw-ar-IF1-avtr12-corr/EqDiamHistogram.csv",
480-
"output_style_index": 1
480+
"output_dir": "Data/Output/fw-ar-IF1-avtr12-corr",
481+
"output_style_index": 0
481482
},
482483
"comments": "",
483484
"filter": {

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp

Lines changed: 23 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -1,106 +1,16 @@
11
#include "ComputeArrayHistogram.hpp"
22

33
#include "SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp"
4-
54
#include "simplnx/DataStructure/DataArray.hpp"
65
#include "simplnx/DataStructure/DataGroup.hpp"
6+
#include "simplnx/Utilities/HistogramUtilities.hpp"
77
#include "simplnx/Utilities/ParallelAlgorithmUtilities.hpp"
88
#include "simplnx/Utilities/ParallelTaskAlgorithm.hpp"
99

10-
#include <chrono>
1110
#include <tuple>
1211

1312
using namespace nx::core;
1413

15-
namespace
16-
{
17-
template <typename Type>
18-
class GenerateHistogramFromData
19-
{
20-
public:
21-
GenerateHistogramFromData(ComputeArrayHistogram& filter, const int32 numBins, const IDataArray& inputArray, AbstractDataStore<float64>& histogram, std::atomic<usize>& overflow,
22-
std::tuple<bool, float64, float64>& range, size_t progressIncrement)
23-
: m_Filter(filter)
24-
, m_NumBins(numBins)
25-
, m_InputArray(inputArray)
26-
, m_Histogram(histogram)
27-
, m_Overflow(overflow)
28-
, m_Range(range)
29-
, m_ProgressIncrement(progressIncrement)
30-
{
31-
}
32-
~GenerateHistogramFromData() = default;
33-
34-
void operator()() const
35-
{
36-
const auto& inputStore = m_InputArray.template getIDataStoreRefAs<AbstractDataStore<Type>>();
37-
auto end = inputStore.getSize();
38-
39-
// tuple visualization: Histogram = {(bin maximum, count), (bin maximum, count), ... }
40-
float64 min = 0.0;
41-
float64 max = 0.0;
42-
if(std::get<0>(m_Range))
43-
{
44-
min = std::get<1>(m_Range);
45-
max = std::get<2>(m_Range);
46-
}
47-
else
48-
{
49-
auto minMax = std::minmax_element(inputStore.begin(), inputStore.end());
50-
min = (static_cast<float64>(*minMax.first) - 1); // ensure upper limit encapsulates max value
51-
max = (static_cast<float64>(*minMax.second) + 1); // ensure lower limit encapsulates min value
52-
}
53-
54-
const float64 increment = (max - min) / static_cast<float64>(m_NumBins);
55-
if(m_NumBins == 1) // if one bin, just set the first element to total number of points
56-
{
57-
m_Histogram[0] = max;
58-
m_Histogram[1] = end;
59-
}
60-
else
61-
{
62-
size_t progressCounter = 0;
63-
for(usize i = 0; i < end; i++)
64-
{
65-
if(progressCounter > m_ProgressIncrement)
66-
{
67-
m_Filter.updateThreadSafeProgress(progressCounter);
68-
progressCounter = 0;
69-
}
70-
if(m_Filter.getCancel())
71-
{
72-
return;
73-
}
74-
const auto bin = std::floor((inputStore[i] - min) / increment);
75-
if((bin >= 0) && (bin < m_NumBins))
76-
{
77-
m_Histogram[bin * 2 + 1]++;
78-
}
79-
else
80-
{
81-
m_Overflow++;
82-
}
83-
progressCounter++;
84-
}
85-
}
86-
87-
for(int64 i = 0; i < m_NumBins; i++)
88-
{
89-
m_Histogram[(i * 2)] = static_cast<float64>(min + (increment * (static_cast<float64>(i) + 1.0))); // load bin maximum into respective position {(x, ), (x , ), ...}
90-
}
91-
}
92-
93-
private:
94-
ComputeArrayHistogram& m_Filter;
95-
const int32 m_NumBins = 1;
96-
std::tuple<bool, float64, float64>& m_Range;
97-
const IDataArray& m_InputArray;
98-
AbstractDataStore<float64>& m_Histogram;
99-
std::atomic<usize>& m_Overflow;
100-
size_t m_ProgressIncrement = 100;
101-
};
102-
} // namespace
103-
10414
// -----------------------------------------------------------------------------
10515
ComputeArrayHistogram::ComputeArrayHistogram(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel,
10616
ComputeArrayHistogramInputValues* inputValues)
@@ -119,22 +29,6 @@ void ComputeArrayHistogram::updateProgress(const std::string& progressMessage)
11929
{
12030
m_MessageHandler({IFilter::Message::Type::Info, progressMessage});
12131
}
122-
// -----------------------------------------------------------------------------
123-
void ComputeArrayHistogram::updateThreadSafeProgress(size_t counter)
124-
{
125-
std::lock_guard<std::mutex> guard(m_ProgressMessage_Mutex);
126-
127-
m_ProgressCounter += counter;
128-
129-
auto now = std::chrono::steady_clock::now();
130-
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - m_InitialTime).count() > 1000) // every second update
131-
{
132-
auto progressInt = static_cast<size_t>((static_cast<double>(m_ProgressCounter) / static_cast<double>(m_TotalElements)) * 100.0);
133-
std::string progressMessage = "Calculating... ";
134-
m_MessageHandler(IFilter::ProgressMessage{IFilter::Message::Type::Progress, progressMessage, static_cast<int32_t>(progressInt)});
135-
m_InitialTime = std::chrono::steady_clock::now();
136-
}
137-
}
13832

13933
// -----------------------------------------------------------------------------
14034
const std::atomic_bool& ComputeArrayHistogram::getCancel()
@@ -145,16 +39,9 @@ const std::atomic_bool& ComputeArrayHistogram::getCancel()
14539
// -----------------------------------------------------------------------------
14640
Result<> ComputeArrayHistogram::operator()()
14741
{
148-
const auto numBins = m_InputValues->NumberOfBins;
149-
const auto selectedArrayPaths = m_InputValues->SelectedArrayPaths;
150-
151-
for(const auto& arrayPath : selectedArrayPaths)
152-
{
153-
m_TotalElements += m_DataStructure.getDataAs<IDataArray>(arrayPath)->getSize();
154-
}
155-
auto progressIncrement = m_TotalElements / 100;
42+
const int32 numBins = m_InputValues->NumberOfBins;
43+
const std::vector<DataPath> selectedArrayPaths = m_InputValues->SelectedArrayPaths;
15644

157-
std::tuple<bool, float64, float64> range = std::make_tuple(m_InputValues->UserDefinedRange, m_InputValues->MinRange, m_InputValues->MaxRange); // Custom bool, min, max
15845
ParallelTaskAlgorithm taskRunner;
15946

16047
std::atomic<usize> overflow = 0;
@@ -165,13 +52,29 @@ Result<> ComputeArrayHistogram::operator()()
16552
{
16653
return {};
16754
}
168-
const auto& inputData = m_DataStructure.getDataRefAs<IDataArray>(selectedArrayPaths[i]);
169-
auto& histogram = m_DataStructure.getDataAs<DataArray<float64>>(m_InputValues->CreatedHistogramDataPaths.at(i))->getDataStoreRef();
170-
ExecuteParallelFunction<GenerateHistogramFromData>(inputData.getDataType(), taskRunner, *this, numBins, inputData, histogram, overflow, range, progressIncrement);
55+
56+
const auto* inputData = m_DataStructure.getDataAs<IDataArray>(selectedArrayPaths[i]);
57+
auto* binRanges = m_DataStructure.getDataAs<IDataArray>(m_InputValues->CreatedBinRangeDataPaths.at(i));
58+
auto& counts = m_DataStructure.getDataAs<DataArray<uint64>>(m_InputValues->CreatedHistogramCountsDataPaths.at(i))->getDataStoreRef();
59+
Result<> result = {};
60+
if(m_InputValues->UserDefinedRange)
61+
{
62+
ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges,
63+
std::make_pair(m_InputValues->MinRange, m_InputValues->MaxRange), m_ShouldCancel, numBins, counts, overflow);
64+
}
65+
else
66+
{
67+
ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges, m_ShouldCancel, numBins, counts, overflow);
68+
}
69+
70+
if(result.invalid())
71+
{
72+
return result;
73+
}
17174

17275
if(overflow > 0)
17376
{
174-
const std::string arrayName = inputData.getName();
77+
const std::string arrayName = inputData->getName();
17578
ComputeArrayHistogram::updateProgress(fmt::format("{} values not categorized into bin for array {}", overflow.load(), arrayName));
17679
}
17780
}

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,23 @@
77
#include "simplnx/Filter/IFilter.hpp"
88
#include "simplnx/Parameters/MultiArraySelectionParameter.hpp"
99

10-
#include <chrono>
11-
#include <mutex>
12-
1310
namespace nx::core
1411
{
15-
1612
struct SIMPLNXCORE_EXPORT ComputeArrayHistogramInputValues
1713
{
18-
int32 NumberOfBins = 0;
1914
bool UserDefinedRange = false;
15+
int32 NumberOfBins = 0;
2016
float64 MinRange = 0.0;
2117
float64 MaxRange = 0.0;
2218
MultiArraySelectionParameter::ValueType SelectedArrayPaths = {};
23-
MultiArraySelectionParameter::ValueType CreatedHistogramDataPaths = {};
19+
MultiArraySelectionParameter::ValueType CreatedBinRangeDataPaths = {};
20+
MultiArraySelectionParameter::ValueType CreatedHistogramCountsDataPaths = {};
2421
};
2522

2623
/**
2724
* @class ComputeArrayHistogram
2825
* @brief This filter calculates a Histogram according to user specification and stores it accordingly
2926
*/
30-
3127
class SIMPLNXCORE_EXPORT ComputeArrayHistogram
3228
{
3329
public:
@@ -42,20 +38,12 @@ class SIMPLNXCORE_EXPORT ComputeArrayHistogram
4238
Result<> operator()();
4339

4440
void updateProgress(const std::string& progMessage);
45-
void updateThreadSafeProgress(size_t counter);
4641
const std::atomic_bool& getCancel();
4742

4843
private:
4944
DataStructure& m_DataStructure;
5045
const ComputeArrayHistogramInputValues* m_InputValues = nullptr;
5146
const std::atomic_bool& m_ShouldCancel;
5247
const IFilter::MessageHandler& m_MessageHandler;
53-
54-
// Threadsafe Progress Message
55-
mutable std::mutex m_ProgressMessage_Mutex;
56-
size_t m_TotalElements = 0;
57-
size_t m_ProgressCounter = 0;
58-
std::chrono::steady_clock::time_point m_InitialTime = std::chrono::steady_clock::now();
5948
};
60-
6149
} // namespace nx::core

0 commit comments

Comments
 (0)