Skip to content

Add support for generating legacy superblocks from scraper stats #1603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 23, 2019
Merged

Add support for generating legacy superblocks from scraper stats #1603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f6b13b1
Add support for generating legacy superblocks from scraper stats
cyrossignol Nov 22, 2019
729ab65
Add RPC method to parse a legacy superblock contract to JSON
cyrossignol Nov 22, 2019
7aeb2be
Fix comparison operator for historical superblock validation
cyrossignol Nov 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 56 additions & 10 deletions src/neuralnet/superblock.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,19 +73,19 @@ class ScraperStatsSuperblockBuilder
continue;

case statsobjecttype::byCPID:
m_superblock.m_cpids.Add(
m_superblock.m_cpids.RoundAndAdd(
Cpid::Parse(object_id),
std::round(entry.second.statsvalue.dMag));
entry.second.statsvalue.dMag);

break;

case statsobjecttype::byProject:
m_superblock.m_projects.Add(
object_id,
Superblock::ProjectStats(
std::round(entry.second.statsvalue.dTC),
std::round(entry.second.statsvalue.dAvgRAC),
std::round(entry.second.statsvalue.dRAC))
std::nearbyint(entry.second.statsvalue.dTC),
std::nearbyint(entry.second.statsvalue.dAvgRAC),
std::nearbyint(entry.second.statsvalue.dRAC))
);

break;
Expand Down Expand Up @@ -114,6 +114,10 @@ class ScraperStatsSuperblockBuilder
//! class generates quorum hashes from scraper statistics that will match the
//! hashes of corresponding superblock objects.
//!
//! CONSENSUS: This class will only produce a SHA256 quorum hash for versions
//! 2+ superblocks. Do not use it to produce hashes of scraper statistics for
//! legacy superblocks.
//!
class ScraperStatsQuorumHasher
{
public:
Expand Down Expand Up @@ -198,6 +202,18 @@ class ScraperStatsQuorumHasher
}
}

//!
//! \brief Round and hash a CPID/magnitude pair as it would exist
//! in the Superblock::CpidIndex container.
//!
//! \param cpid The CPID value to hash.
//! \param magnitude The magnitude value to hash.
//!
void RoundAndAdd(Cpid cpid, double magnitude)
{
Add(cpid, std::nearbyint(magnitude));
}

//!
//! \brief Hash a project statistics entry as it would exist in the
//! Superblock::ProjectIndex container.
Expand Down Expand Up @@ -302,7 +318,7 @@ class SuperblockValidator
return Result::UNKNOWN;
}

if (m_superblock.Age() < SCRAPER_CMANIFEST_RETENTION_TIME) {
if (m_superblock.Age() > SCRAPER_CMANIFEST_RETENTION_TIME) {
return Result::HISTORICAL;
}

Expand Down Expand Up @@ -1402,9 +1418,11 @@ Superblock::Superblock(uint32_t version)
{
}

Superblock Superblock::FromConvergence(const ConvergedScraperStats& stats)
Superblock Superblock::FromConvergence(
const ConvergedScraperStats& stats,
const uint32_t version)
{
Superblock superblock = Superblock::FromStats(stats.mScraperConvergedStats);
Superblock superblock = Superblock::FromStats(stats.mScraperConvergedStats, version);

superblock.m_convergence_hint = stats.Convergence.nContentHash.GetUint64() >> 32;

Expand All @@ -1430,11 +1448,20 @@ Superblock Superblock::FromConvergence(const ConvergedScraperStats& stats)
return superblock;
}

Superblock Superblock::FromStats(const ScraperStats& stats)
Superblock Superblock::FromStats(const ScraperStats& stats, const uint32_t version)
{
Superblock superblock;
Superblock superblock(version);
ScraperStatsSuperblockBuilder<Superblock> builder(superblock);

if (version == 1) {
// Force the CPID index into legacy mode to capture zero-magnitude
// CPIDs that result from rounding.
//
// TODO: encapsulate this
//
superblock.m_cpids = Superblock::CpidIndex(0);
}

builder.BuildFromStats(stats);

return superblock;
Expand Down Expand Up @@ -1625,6 +1652,25 @@ void Superblock::CpidIndex::Add(const MiningId id, const uint16_t magnitude)
}
}

void Superblock::CpidIndex::RoundAndAdd(const MiningId id, const double magnitude)
{
// The ScraperGetNeuralContract() function that these classes replace
// rounded magnitude values using a half-away-from-zero rounding mode
// to determine whether floating-point magnitudes round-down to zero,
// but it added the magnitude values to the superblock with half-even
// rounding. This caused legacy superblock contracts to contain CPIDs
// with zero magnitude when the rounding results differed.
//
// To create legacy superblocks from scraper statistics with matching
// hashes, we filter magnitudes using the same rounding rules:
//
if (!m_legacy || std::round(magnitude) > 0) {
Add(id, std::nearbyint(magnitude));
} else {
m_zero_magnitude_count++;
}
}

// -----------------------------------------------------------------------------
// Class: Superblock::ProjectStats
// -----------------------------------------------------------------------------
Expand Down
32 changes: 30 additions & 2 deletions src/neuralnet/superblock.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,18 @@ class QuorumHash
//!
static QuorumHash Hash(const Superblock& superblock);

//!
//! \brief Hash the provided scraper statistics to produce the same quorum
//! hash that would be generated for a superblock created from the stats.
//!
//! CONSENSUS: This method will only produce a SHA256 quorum hash matching
//! version 2+ superblocks. Do not use it to produce hashes of the scraper
//! statistics for legacy superblocks.
//!
//! \param stats Scraper statistics from a convergence to hash.
//!
//! \return A SHA256 quorum hash of the scraper statistics.
//!
static QuorumHash Hash(const ScraperStats& stats);

//!
Expand Down Expand Up @@ -383,6 +395,18 @@ class Superblock
//!
void Add(const MiningId id, const uint16_t magnitude);

//!
//! \brief Add the supplied mining ID to the index if it represents a
//! valid CPID after rounding the magnitude to an integer.
//!
//! This method ignores an attempt to add a duplicate entry if a CPID
//! already exists.
//!
//! \param id May contain a CPID.
//! \param magnitude Total magnitude to associate with the CPID.
//!
void RoundAndAdd(const MiningId id, const double magnitude);

//!
//! \brief Serialize the object to the provided stream.
//!
Expand Down Expand Up @@ -772,7 +796,9 @@ class Superblock
//! \return A new superblock instance that contains the imported scraper
//! statistics.
//!
static Superblock FromConvergence(const ConvergedScraperStats& stats);
static Superblock FromConvergence(
const ConvergedScraperStats& stats,
const uint32_t version = Superblock::CURRENT_VERSION);

//!
//! \brief Initialize a superblock from the provided scraper statistics.
Expand All @@ -783,7 +809,9 @@ class Superblock
//! \return A new superblock instance that contains the imported scraper
//! statistics.
//!
static Superblock FromStats(const ScraperStats& stats);
static Superblock FromStats(
const ScraperStats& stats,
const uint32_t version = Superblock::CURRENT_VERSION);

//!
//! \brief Initialize a superblock from a legacy superblock contract.
Expand Down
18 changes: 18 additions & 0 deletions src/rpcblockchain.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1528,6 +1528,24 @@ UniValue network(const UniValue& params, bool fHelp)
return res;
}

UniValue parselegacysb(const UniValue& params, bool fHelp)
{
if (fHelp || params.size() < 1)
throw runtime_error(
"parselegacysb\n"
"\n"
"Convert a legacy superblock contract to JSON.\n");

UniValue json(UniValue::VOBJ);

NN::Superblock superblock = NN::Superblock::UnpackLegacy(params[0].get_str());

json.pushKV("contract", SuperblockToJson(superblock));
json.pushKV("legacy_hash", superblock.GetHash().ToString());

return json;
}

UniValue projects(const UniValue& params, bool fHelp)
{
if (fHelp || params.size() != 0)
Expand Down
1 change: 1 addition & 0 deletions src/rpcserver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,7 @@ static const CRPCCommand vRPCCommands[] =
{ "listprojects", &listprojects, cat_developer },
{ "memorizekeys", &memorizekeys, cat_developer },
{ "network", &network, cat_developer },
{ "parselegacysb", &parselegacysb, cat_developer },
{ "projects", &projects, cat_developer },
{ "readconfig", &readconfig, cat_developer },
{ "readdata", &readdata, cat_developer },
Expand Down
1 change: 1 addition & 0 deletions src/rpcserver.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ extern UniValue listdata(const UniValue& params, bool fHelp);
extern UniValue listprojects(const UniValue& params, bool fHelp);
extern UniValue memorizekeys(const UniValue& params, bool fHelp);
extern UniValue network(const UniValue& params, bool fHelp);
extern UniValue parselegacysb(const UniValue& params, bool fHelp);
extern UniValue projects(const UniValue& params, bool fHelp);
extern UniValue readconfig(const UniValue& params, bool fHelp);
extern UniValue readdata(const UniValue& params, bool fHelp);
Expand Down
8 changes: 4 additions & 4 deletions src/scraper/scraper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4533,10 +4533,10 @@ NN::Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats, bool bCon
ConvergedScraperStatsCache.nTime = GetAdjustedTime();
ConvergedScraperStatsCache.Convergence = StructConvergedManifest;

superblock = NN::Superblock::FromConvergence(ConvergedScraperStatsCache);

if (!IsV11Enabled(nBestHeight)) {
superblock.m_version = 1;
if (IsV11Enabled(nBestHeight)) {
superblock = NN::Superblock::FromConvergence(ConvergedScraperStatsCache);
} else {
superblock = NN::Superblock::FromConvergence(ConvergedScraperStatsCache, 1);
}

ConvergedScraperStatsCache.NewFormatSuperblock = superblock;
Expand Down
52 changes: 52 additions & 0 deletions src/test/neuralnet/superblock_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -991,6 +991,58 @@ BOOST_AUTO_TEST_CASE(it_adds_a_cpid_magnitude_to_the_index)
BOOST_CHECK(cpids.size() == 1);
}

BOOST_AUTO_TEST_CASE(it_adds_a_rounded_cpid_magnitude_to_the_index)
{
NN::Superblock::CpidIndex cpids;

NN::Cpid cpid1 = NN::Cpid::Parse("00010203040506070809101112131415");
NN::Cpid cpid2 = NN::Cpid::Parse("15141312111009080706050403020100");

cpids.RoundAndAdd(cpid1, 123.5);
cpids.RoundAndAdd(cpid2, 123.4);

BOOST_CHECK(cpids.MagnitudeOf(cpid1) == 124);
BOOST_CHECK(cpids.MagnitudeOf(cpid2) == 123);
}

BOOST_AUTO_TEST_CASE(it_adds_a_legacy_rounded_cpid_magnitude_to_the_index)
{
// Initializing the CPID index with a count of zero-magnitude CPIDs
// switches it to legacy mode:
//
NN::Superblock::CpidIndex cpids(0);

NN::Cpid cpid1 = NN::Cpid::Parse("11111111111111111111111111111111");
NN::Cpid cpid2 = NN::Cpid::Parse("22222222222222222222222222222222");
NN::Cpid cpid3 = NN::Cpid::Parse("33333333333333333333333333333333");
NN::Cpid cpid4 = NN::Cpid::Parse("44444444444444444444444444444444");

// The ScraperGetNeuralContract() function that these classes replace
// rounded magnitude values using a half-away-from-zero rounding mode
// to determine whether floating-point magnitudes round-down to zero,
// but it added the magnitude values to the superblock with half-even
// rounding. This caused legacy superblock contracts to contain CPIDs
// with zero magnitude when the rounding results differed.
//
// To create legacy superblocks from scraper statistics with matching
// hashes, we filter magnitudes using the same rounding rules:
//
// - A magnitude of 0.5 is rounded-down to zero.
// - A magnitude less than 0.5 is omitted from the superblock.
//
cpids.RoundAndAdd(cpid1, 1.1);
cpids.RoundAndAdd(cpid2, 1.5);
cpids.RoundAndAdd(cpid3, 0.5);
cpids.RoundAndAdd(cpid4, 0.4);

BOOST_CHECK(cpids.size() == 3); // cpid4 is omitted

BOOST_CHECK(cpids.MagnitudeOf(cpid1) == 1);
BOOST_CHECK(cpids.MagnitudeOf(cpid2) == 2);
BOOST_CHECK(cpids.MagnitudeOf(cpid3) == 0);
BOOST_CHECK(cpids.MagnitudeOf(cpid4) == 0);
}

BOOST_AUTO_TEST_CASE(it_ignores_insertion_of_a_duplicate_cpid)
{
NN::Superblock::CpidIndex cpids;
Expand Down