diff --git a/src/neuralnet/superblock.cpp b/src/neuralnet/superblock.cpp index b8ad205bc8..fe154db4c3 100644 --- a/src/neuralnet/superblock.cpp +++ b/src/neuralnet/superblock.cpp @@ -73,9 +73,9 @@ class ScraperStatsSuperblockBuilder continue; case statsobjecttype::byCPID: - m_superblock.m_cpids.Add( + m_superblock.m_cpids.RoundAndAdd( Cpid::Parse(object_id), - std::round(entry.second.statsvalue.dMag)); + entry.second.statsvalue.dMag); break; @@ -83,9 +83,9 @@ class ScraperStatsSuperblockBuilder m_superblock.m_projects.Add( object_id, Superblock::ProjectStats( - std::round(entry.second.statsvalue.dTC), - std::round(entry.second.statsvalue.dAvgRAC), - std::round(entry.second.statsvalue.dRAC)) + std::nearbyint(entry.second.statsvalue.dTC), + std::nearbyint(entry.second.statsvalue.dAvgRAC), + std::nearbyint(entry.second.statsvalue.dRAC)) ); break; @@ -114,6 +114,10 @@ class ScraperStatsSuperblockBuilder //! class generates quorum hashes from scraper statistics that will match the //! hashes of corresponding superblock objects. //! +//! CONSENSUS: This class will only produce a SHA256 quorum hash for versions +//! 2+ superblocks. Do not use it to produce hashes of scraper statistics for +//! legacy superblocks. +//! class ScraperStatsQuorumHasher { public: @@ -198,6 +202,18 @@ class ScraperStatsQuorumHasher } } + //! + //! \brief Round and hash a CPID/magnitude pair as it would exist + //! in the Superblock::CpidIndex container. + //! + //! \param cpid The CPID value to hash. + //! \param magnitude The magnitude value to hash. + //! + void RoundAndAdd(Cpid cpid, double magnitude) + { + Add(cpid, std::nearbyint(magnitude)); + } + //! //! \brief Hash a project statistics entry as it would exist in the //! Superblock::ProjectIndex container. @@ -302,7 +318,7 @@ class SuperblockValidator return Result::UNKNOWN; } - if (m_superblock.Age() < SCRAPER_CMANIFEST_RETENTION_TIME) { + if (m_superblock.Age() > SCRAPER_CMANIFEST_RETENTION_TIME) { return Result::HISTORICAL; } @@ -1402,9 +1418,11 @@ Superblock::Superblock(uint32_t version) { } -Superblock Superblock::FromConvergence(const ConvergedScraperStats& stats) +Superblock Superblock::FromConvergence( + const ConvergedScraperStats& stats, + const uint32_t version) { - Superblock superblock = Superblock::FromStats(stats.mScraperConvergedStats); + Superblock superblock = Superblock::FromStats(stats.mScraperConvergedStats, version); superblock.m_convergence_hint = stats.Convergence.nContentHash.GetUint64() >> 32; @@ -1430,11 +1448,20 @@ Superblock Superblock::FromConvergence(const ConvergedScraperStats& stats) return superblock; } -Superblock Superblock::FromStats(const ScraperStats& stats) +Superblock Superblock::FromStats(const ScraperStats& stats, const uint32_t version) { - Superblock superblock; + Superblock superblock(version); ScraperStatsSuperblockBuilder builder(superblock); + if (version == 1) { + // Force the CPID index into legacy mode to capture zero-magnitude + // CPIDs that result from rounding. + // + // TODO: encapsulate this + // + superblock.m_cpids = Superblock::CpidIndex(0); + } + builder.BuildFromStats(stats); return superblock; @@ -1625,6 +1652,25 @@ void Superblock::CpidIndex::Add(const MiningId id, const uint16_t magnitude) } } +void Superblock::CpidIndex::RoundAndAdd(const MiningId id, const double magnitude) +{ + // The ScraperGetNeuralContract() function that these classes replace + // rounded magnitude values using a half-away-from-zero rounding mode + // to determine whether floating-point magnitudes round-down to zero, + // but it added the magnitude values to the superblock with half-even + // rounding. This caused legacy superblock contracts to contain CPIDs + // with zero magnitude when the rounding results differed. + // + // To create legacy superblocks from scraper statistics with matching + // hashes, we filter magnitudes using the same rounding rules: + // + if (!m_legacy || std::round(magnitude) > 0) { + Add(id, std::nearbyint(magnitude)); + } else { + m_zero_magnitude_count++; + } +} + // ----------------------------------------------------------------------------- // Class: Superblock::ProjectStats // ----------------------------------------------------------------------------- diff --git a/src/neuralnet/superblock.h b/src/neuralnet/superblock.h index 4adb4603a4..2f211485ab 100644 --- a/src/neuralnet/superblock.h +++ b/src/neuralnet/superblock.h @@ -86,6 +86,18 @@ class QuorumHash //! static QuorumHash Hash(const Superblock& superblock); + //! + //! \brief Hash the provided scraper statistics to produce the same quorum + //! hash that would be generated for a superblock created from the stats. + //! + //! CONSENSUS: This method will only produce a SHA256 quorum hash matching + //! version 2+ superblocks. Do not use it to produce hashes of the scraper + //! statistics for legacy superblocks. + //! + //! \param stats Scraper statistics from a convergence to hash. + //! + //! \return A SHA256 quorum hash of the scraper statistics. + //! static QuorumHash Hash(const ScraperStats& stats); //! @@ -383,6 +395,18 @@ class Superblock //! void Add(const MiningId id, const uint16_t magnitude); + //! + //! \brief Add the supplied mining ID to the index if it represents a + //! valid CPID after rounding the magnitude to an integer. + //! + //! This method ignores an attempt to add a duplicate entry if a CPID + //! already exists. + //! + //! \param id May contain a CPID. + //! \param magnitude Total magnitude to associate with the CPID. + //! + void RoundAndAdd(const MiningId id, const double magnitude); + //! //! \brief Serialize the object to the provided stream. //! @@ -772,7 +796,9 @@ class Superblock //! \return A new superblock instance that contains the imported scraper //! statistics. //! - static Superblock FromConvergence(const ConvergedScraperStats& stats); + static Superblock FromConvergence( + const ConvergedScraperStats& stats, + const uint32_t version = Superblock::CURRENT_VERSION); //! //! \brief Initialize a superblock from the provided scraper statistics. @@ -783,7 +809,9 @@ class Superblock //! \return A new superblock instance that contains the imported scraper //! statistics. //! - static Superblock FromStats(const ScraperStats& stats); + static Superblock FromStats( + const ScraperStats& stats, + const uint32_t version = Superblock::CURRENT_VERSION); //! //! \brief Initialize a superblock from a legacy superblock contract. diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index ab817ebd4a..d977067047 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -1528,6 +1528,24 @@ UniValue network(const UniValue& params, bool fHelp) return res; } +UniValue parselegacysb(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() < 1) + throw runtime_error( + "parselegacysb\n" + "\n" + "Convert a legacy superblock contract to JSON.\n"); + + UniValue json(UniValue::VOBJ); + + NN::Superblock superblock = NN::Superblock::UnpackLegacy(params[0].get_str()); + + json.pushKV("contract", SuperblockToJson(superblock)); + json.pushKV("legacy_hash", superblock.GetHash().ToString()); + + return json; +} + UniValue projects(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 0) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 9a748bce8a..fd6c822efb 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -373,6 +373,7 @@ static const CRPCCommand vRPCCommands[] = { "listprojects", &listprojects, cat_developer }, { "memorizekeys", &memorizekeys, cat_developer }, { "network", &network, cat_developer }, + { "parselegacysb", &parselegacysb, cat_developer }, { "projects", &projects, cat_developer }, { "readconfig", &readconfig, cat_developer }, { "readdata", &readdata, cat_developer }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 39b5f64d6b..3aeaf578a7 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -191,6 +191,7 @@ extern UniValue listdata(const UniValue& params, bool fHelp); extern UniValue listprojects(const UniValue& params, bool fHelp); extern UniValue memorizekeys(const UniValue& params, bool fHelp); extern UniValue network(const UniValue& params, bool fHelp); +extern UniValue parselegacysb(const UniValue& params, bool fHelp); extern UniValue projects(const UniValue& params, bool fHelp); extern UniValue readconfig(const UniValue& params, bool fHelp); extern UniValue readdata(const UniValue& params, bool fHelp); diff --git a/src/scraper/scraper.cpp b/src/scraper/scraper.cpp index 76e88918d4..dad63045cd 100755 --- a/src/scraper/scraper.cpp +++ b/src/scraper/scraper.cpp @@ -4533,10 +4533,10 @@ NN::Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats, bool bCon ConvergedScraperStatsCache.nTime = GetAdjustedTime(); ConvergedScraperStatsCache.Convergence = StructConvergedManifest; - superblock = NN::Superblock::FromConvergence(ConvergedScraperStatsCache); - - if (!IsV11Enabled(nBestHeight)) { - superblock.m_version = 1; + if (IsV11Enabled(nBestHeight)) { + superblock = NN::Superblock::FromConvergence(ConvergedScraperStatsCache); + } else { + superblock = NN::Superblock::FromConvergence(ConvergedScraperStatsCache, 1); } ConvergedScraperStatsCache.NewFormatSuperblock = superblock; diff --git a/src/test/neuralnet/superblock_tests.cpp b/src/test/neuralnet/superblock_tests.cpp index f28563c676..d95c1a5aa9 100644 --- a/src/test/neuralnet/superblock_tests.cpp +++ b/src/test/neuralnet/superblock_tests.cpp @@ -991,6 +991,58 @@ BOOST_AUTO_TEST_CASE(it_adds_a_cpid_magnitude_to_the_index) BOOST_CHECK(cpids.size() == 1); } +BOOST_AUTO_TEST_CASE(it_adds_a_rounded_cpid_magnitude_to_the_index) +{ + NN::Superblock::CpidIndex cpids; + + NN::Cpid cpid1 = NN::Cpid::Parse("00010203040506070809101112131415"); + NN::Cpid cpid2 = NN::Cpid::Parse("15141312111009080706050403020100"); + + cpids.RoundAndAdd(cpid1, 123.5); + cpids.RoundAndAdd(cpid2, 123.4); + + BOOST_CHECK(cpids.MagnitudeOf(cpid1) == 124); + BOOST_CHECK(cpids.MagnitudeOf(cpid2) == 123); +} + +BOOST_AUTO_TEST_CASE(it_adds_a_legacy_rounded_cpid_magnitude_to_the_index) +{ + // Initializing the CPID index with a count of zero-magnitude CPIDs + // switches it to legacy mode: + // + NN::Superblock::CpidIndex cpids(0); + + NN::Cpid cpid1 = NN::Cpid::Parse("11111111111111111111111111111111"); + NN::Cpid cpid2 = NN::Cpid::Parse("22222222222222222222222222222222"); + NN::Cpid cpid3 = NN::Cpid::Parse("33333333333333333333333333333333"); + NN::Cpid cpid4 = NN::Cpid::Parse("44444444444444444444444444444444"); + + // The ScraperGetNeuralContract() function that these classes replace + // rounded magnitude values using a half-away-from-zero rounding mode + // to determine whether floating-point magnitudes round-down to zero, + // but it added the magnitude values to the superblock with half-even + // rounding. This caused legacy superblock contracts to contain CPIDs + // with zero magnitude when the rounding results differed. + // + // To create legacy superblocks from scraper statistics with matching + // hashes, we filter magnitudes using the same rounding rules: + // + // - A magnitude of 0.5 is rounded-down to zero. + // - A magnitude less than 0.5 is omitted from the superblock. + // + cpids.RoundAndAdd(cpid1, 1.1); + cpids.RoundAndAdd(cpid2, 1.5); + cpids.RoundAndAdd(cpid3, 0.5); + cpids.RoundAndAdd(cpid4, 0.4); + + BOOST_CHECK(cpids.size() == 3); // cpid4 is omitted + + BOOST_CHECK(cpids.MagnitudeOf(cpid1) == 1); + BOOST_CHECK(cpids.MagnitudeOf(cpid2) == 2); + BOOST_CHECK(cpids.MagnitudeOf(cpid3) == 0); + BOOST_CHECK(cpids.MagnitudeOf(cpid4) == 0); +} + BOOST_AUTO_TEST_CASE(it_ignores_insertion_of_a_duplicate_cpid) { NN::Superblock::CpidIndex cpids;