Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Merge pull request #4334 from pwojcikdev/hinted-fixes

Browse source 
Fixes for hinted scheduler & vote cache
This commit is contained in:
Piotr Wójcik 2023-11-08 15:11:10 +01:00 • committed by GitHub
commit 58a8a6bd19
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 184 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

3
nano/core_test/toml.cpp
View file

@ -275,6 +275,7 @@ TEST (toml, daemon_config_deserialize_defaults)
ASSERT_EQ (conf.node.hinted_scheduler.hinting_threshold_percent, defaults.node.hinted_scheduler.hinting_threshold_percent);
ASSERT_EQ (conf.node.hinted_scheduler.check_interval.count (), defaults.node.hinted_scheduler.check_interval.count ());
ASSERT_EQ (conf.node.hinted_scheduler.block_cooldown.count (), defaults.node.hinted_scheduler.block_cooldown.count ());
ASSERT_EQ (conf.node.hinted_scheduler.vacancy_threshold_percent, defaults.node.hinted_scheduler.vacancy_threshold_percent);
ASSERT_EQ (conf.node.vote_cache.max_size, defaults.node.vote_cache.max_size);
ASSERT_EQ (conf.node.vote_cache.max_voters, defaults.node.vote_cache.max_voters);
@ -543,6 +544,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
hinting_threshold = 99
check_interval = 999
block_cooldown = 999
vacancy_threshold = 99
[node.rocksdb]
enable = true
@ -720,6 +722,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
ASSERT_NE (conf.node.hinted_scheduler.hinting_threshold_percent, defaults.node.hinted_scheduler.hinting_threshold_percent);
ASSERT_NE (conf.node.hinted_scheduler.check_interval.count (), defaults.node.hinted_scheduler.check_interval.count ());
ASSERT_NE (conf.node.hinted_scheduler.block_cooldown.count (), defaults.node.hinted_scheduler.block_cooldown.count ());
ASSERT_NE (conf.node.hinted_scheduler.vacancy_threshold_percent, defaults.node.hinted_scheduler.vacancy_threshold_percent);
ASSERT_NE (conf.node.vote_cache.max_size, defaults.node.vote_cache.max_size);
ASSERT_NE (conf.node.vote_cache.max_voters, defaults.node.vote_cache.max_voters);

66
nano/core_test/vote_cache.cpp
View file

@ -36,8 +36,9 @@ nano::keypair create_rep (nano::uint128_t weight)
TEST (vote_cache, construction)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
ASSERT_EQ (0, vote_cache.size ());
ASSERT_TRUE (vote_cache.empty ());
auto hash1 = nano::test::random_hash ();
@ -49,8 +50,9 @@ TEST (vote_cache, construction)
*/
TEST (vote_cache, insert_one_hash)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
auto rep1 = create_rep (7);
auto hash1 = nano::test::random_hash ();
@ -79,8 +81,9 @@ TEST (vote_cache, insert_one_hash)
*/
TEST (vote_cache, insert_one_hash_many_votes)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
auto hash1 = nano::test::random_hash ();
auto rep1 = create_rep (7);
@ -114,8 +117,9 @@ TEST (vote_cache, insert_one_hash_many_votes)
*/
TEST (vote_cache, insert_many_hashes_many_votes)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
// There will be 3 random hashes to vote for
auto hash1 = nano::test::random_hash ();
@ -194,8 +198,9 @@ TEST (vote_cache, insert_many_hashes_many_votes)
*/
TEST (vote_cache, insert_duplicate)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
auto hash1 = nano::test::random_hash ();
auto rep1 = create_rep (9);
@ -211,8 +216,9 @@ TEST (vote_cache, insert_duplicate)
*/
TEST (vote_cache, insert_newer)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
auto hash1 = nano::test::random_hash ();
auto rep1 = create_rep (9);
@ -236,8 +242,9 @@ TEST (vote_cache, insert_newer)
*/
TEST (vote_cache, insert_older)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
auto hash1 = nano::test::random_hash ();
auto rep1 = create_rep (9);
@ -259,8 +266,9 @@ TEST (vote_cache, insert_older)
*/
TEST (vote_cache, erase)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
auto hash1 = nano::test::random_hash ();
auto hash2 = nano::test::random_hash ();
@ -298,10 +306,11 @@ TEST (vote_cache, erase)
*/
TEST (vote_cache, overfill)
{
nano::test::system system;
// Create a vote cache with max size set to 1024
nano::vote_cache_config cfg;
cfg.max_size = 1024;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
const int count = 16 * 1024;
for (int n = 0; n < count; ++n)
@ -324,8 +333,9 @@ TEST (vote_cache, overfill)
*/
TEST (vote_cache, overfill_entry)
{
nano::test::system system;
nano::vote_cache_config cfg;
nano::vote_cache vote_cache{ cfg };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
const int count = 1024;
auto hash1 = nano::test::random_hash ();
@ -336,4 +346,40 @@ TEST (vote_cache, overfill_entry)
vote_cache.vote (vote1->hashes.front (), vote1);
}
ASSERT_EQ (1, vote_cache.size ());
}
TEST (vote_cache, age_cutoff)
{
nano::test::system system;
nano::vote_cache_config cfg;
cfg.age_cutoff = std::chrono::seconds{ 3 };
nano::vote_cache vote_cache{ cfg, system.stats };
vote_cache.rep_weight_query = rep_weight_query ();
auto hash1 = nano::test::random_hash ();
auto rep1 = create_rep (9);
auto vote1 = nano::test::make_vote (rep1, { hash1 }, 3);
vote_cache.vote (vote1->hashes.front (), vote1);
ASSERT_EQ (1, vote_cache.size ());
ASSERT_TRUE (vote_cache.find (hash1));
auto tops1 = vote_cache.top (0);
ASSERT_EQ (tops1.size (), 1);
ASSERT_EQ (tops1[0].hash, hash1);
ASSERT_EQ (system.stats.count (nano::stat::type::vote_cache, nano::stat::detail::cleanup), 0);
// Wait for first cleanup
auto check = [&] () {
// Cleanup is performed periodically when calling `top ()`
vote_cache.top (0);
return system.stats.count (nano::stat::type::vote_cache, nano::stat::detail::cleanup);
};
ASSERT_TIMELY_EQ (5s, 1, check ());
// After first cleanup the entry should still be there
auto tops2 = vote_cache.top (0);
ASSERT_EQ (tops2.size (), 1);
// After 3 seconds the entry should be removed
ASSERT_TIMELY (5s, vote_cache.top (0).empty ());
}

1
nano/lib/CMakeLists.txt
View file

@ -38,6 +38,7 @@ add_library(
errors.hpp
errors.cpp
id_dispenser.hpp
interval.hpp
ipc.hpp
ipc.cpp
ipc_client.hpp

30
nano/lib/interval.hpp Normal file
View file

@ -0,0 +1,30 @@
#pragma once
#include <chrono>
namespace nano
{
class interval
{
public:
explicit interval (std::chrono::milliseconds target) :
target{ target }
{
}
bool elapsed ()
{
auto const now = std::chrono::steady_clock::now ();
if (now - last >= target)
{
last = now;
return true;
}
return false;
}
private:
std::chrono::milliseconds const target;
std::chrono::steady_clock::time_point last{ std::chrono::steady_clock::now () };
};
}

2
nano/lib/stats_enums.hpp
View file

@ -63,6 +63,8 @@ enum class detail : uint8_t
update,
request,
broadcast,
cleanup,
top,
// processing queue
queue,

2
nano/node/node.cpp
View file

@ -182,7 +182,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, std::filesystem::path cons
history{ config.network_params.voting },
vote_uniquer (block_uniquer),
confirmation_height_processor (ledger, write_database_queue, config.conf_height_processor_batch_min_time, config.logging, logger, node_initialized_latch, flags.confirmation_height_processor_mode),
vote_cache{ config.vote_cache },
vote_cache{ config.vote_cache, stats },
generator{ config, ledger, wallets, vote_processor, history, network, stats, /* non-final */ false },
final_generator{ config, ledger, wallets, vote_processor, history, network, stats, /* final */ true },
active (*this, confirmation_height_processor),

12
nano/node/scheduler/hinted.cpp
View file

@ -47,7 +47,7 @@ void nano::scheduler::hinted::notify ()
{
// Avoid notifying when there is very little space inside AEC
auto const limit = active.limit (nano::election_behavior::hinted);
if (active.vacancy (nano::election_behavior::hinted) >= (limit / 5))
if (active.vacancy (nano::election_behavior::hinted) >= (limit * config.vacancy_threshold_percent / 100))
{
condition.notify_all ();
}
@ -107,7 +107,8 @@ void nano::scheduler::hinted::activate (const nano::store::read_transaction & tr
else
{
stats.inc (nano::stat::type::hinting, nano::stat::detail::missing_block);
node.bootstrap_block (current_hash);
// TODO: Block is missing, bootstrap it
}
}
}
@ -244,6 +245,7 @@ nano::error nano::scheduler::hinted_config::serialize (nano::tomlconfig & toml)
toml.put ("hinting_threshold", hinting_threshold_percent, "Percentage of online weight needed to start a hinted election. \ntype:uint32,[0,100]");
toml.put ("check_interval", check_interval.count (), "Interval between scans of the vote cache for possible hinted elections. \ntype:milliseconds");
toml.put ("block_cooldown", block_cooldown.count (), "Cooldown period for blocks that failed to start an election. \ntype:milliseconds");
toml.put ("vacancy_threshold", vacancy_threshold_percent, "Percentage of available space in the active elections container needed to trigger a scan for hinted elections (before the check interval elapses). \ntype:uint32,[0,100]");
return toml.get_error ();
}
@ -260,10 +262,16 @@ nano::error nano::scheduler::hinted_config::deserialize (nano::tomlconfig & toml
toml.get ("block_cooldown", block_cooldown_l);
block_cooldown = std::chrono::milliseconds{ block_cooldown_l };
toml.get ("vacancy_threshold", vacancy_threshold_percent);
if (hinting_threshold_percent > 100)
{
toml.get_error ().set ("hinting_threshold must be a number between 0 and 100");
}
if (vacancy_threshold_percent > 100)
{
toml.get_error ().set ("vacancy_threshold must be a number between 0 and 100");
}
return toml.get_error ();
}

3
nano/node/scheduler/hinted.hpp
View file

@ -37,8 +37,9 @@ public:
public:
std::chrono::milliseconds check_interval{ 1000 };
std::chrono::milliseconds block_cooldown{ 5000 };
std::chrono::milliseconds block_cooldown{ 10000 };
unsigned hinting_threshold_percent{ 10 };
unsigned vacancy_threshold_percent{ 20 };
};
/*

66
nano/node/vote_cache.cpp
View file

@ -12,6 +12,16 @@ nano::vote_cache::entry::entry (const nano::block_hash & hash) :
}
bool nano::vote_cache::entry::vote (const nano::account & representative, const uint64_t & timestamp, const nano::uint128_t & rep_weight, std::size_t max_voters)
{
bool updated = vote_impl (representative, timestamp, rep_weight, max_voters);
if (updated)
{
last_vote_m = std::chrono::steady_clock::now ();
}
return updated;
}
bool nano::vote_cache::entry::vote_impl (const nano::account & representative, const uint64_t & timestamp, const nano::uint128_t & rep_weight, std::size_t max_voters)
{
auto existing = std::find_if (voters_m.begin (), voters_m.end (), [&representative] (auto const & item) { return item.representative == representative; });
if (existing != voters_m.end ())
@ -92,17 +102,27 @@ std::vector<nano::vote_cache::entry::voter_entry> nano::vote_cache::entry::voter
return voters_m;
}
std::chrono::steady_clock::time_point nano::vote_cache::entry::last_vote () const
{
return last_vote_m;
}
/*
* vote_cache
*/
nano::vote_cache::vote_cache (vote_cache_config const & config_a) :
config{ config_a }
nano::vote_cache::vote_cache (vote_cache_config const & config_a, nano::stats & stats_a) :
config{ config_a },
stats{ stats_a },
cleanup_interval{ config_a.age_cutoff / 2 }
{
}
void nano::vote_cache::vote (const nano::block_hash & hash, const std::shared_ptr<nano::vote> vote)
{
// Assert that supplied hash corresponds to a one of the hashes stored in vote
debug_assert (std::find (vote->hashes.begin (), vote->hashes.end (), hash) != vote->hashes.end ());
auto const representative = vote->account;
auto const timestamp = vote->timestamp ();
auto const rep_weight = rep_weight_query (representative);
@ -112,12 +132,16 @@ void nano::vote_cache::vote (const nano::block_hash & hash, const std::shared_pt
auto & cache_by_hash = cache.get<tag_hash> ();
if (auto existing = cache_by_hash.find (hash); existing != cache_by_hash.end ())
{
stats.inc (nano::stat::type::vote_cache, nano::stat::detail::update);
cache_by_hash.modify (existing, [this, &representative, &timestamp, &rep_weight] (entry & ent) {
ent.vote (representative, timestamp, rep_weight, config.max_voters);
});
}
else
{
stats.inc (nano::stat::type::vote_cache, nano::stat::detail::insert);
entry cache_entry{ hash };
cache_entry.vote (representative, timestamp, rep_weight, config.max_voters);
@ -169,12 +193,19 @@ bool nano::vote_cache::erase (const nano::block_hash & hash)
return result;
}
std::vector<nano::vote_cache::top_entry> nano::vote_cache::top (const nano::uint128_t & min_tally) const
std::vector<nano::vote_cache::top_entry> nano::vote_cache::top (const nano::uint128_t & min_tally)
{
stats.inc (nano::stat::type::vote_cache, nano::stat::detail::top);
std::vector<top_entry> results;
{
nano::lock_guard<nano::mutex> lock{ mutex };
if (cleanup_interval.elapsed ())
{
cleanup ();
}
for (auto & entry : cache.get<tag_tally> ())
{
if (entry.tally () < min_tally)
@ -200,7 +231,29 @@ std::vector<nano::vote_cache::top_entry> nano::vote_cache::top (const nano::uint
return results;
}
std::unique_ptr<nano::container_info_component> nano::vote_cache::collect_container_info (const std::string & name)
void nano::vote_cache::cleanup ()
{
debug_assert (!mutex.try_lock ());
stats.inc (nano::stat::type::vote_cache, nano::stat::detail::cleanup);
auto const cutoff = std::chrono::steady_clock::now () - config.age_cutoff;
auto it = cache.begin ();
while (it != cache.end ())
{
if (it->last_vote () < cutoff)
{
it = cache.erase (it);
}
else
{
++it;
}
}
}
std::unique_ptr<nano::container_info_component> nano::vote_cache::collect_container_info (const std::string & name) const
{
auto composite = std::make_unique<container_info_composite> (name);
composite->add_component (std::make_unique<container_info_leaf> (container_info{ "cache", size (), sizeof (ordered_cache::value_type) }));
@ -215,6 +268,7 @@ nano::error nano::vote_cache_config::serialize (nano::tomlconfig & toml) const
{
toml.put ("max_size", max_size, "Maximum number of blocks to cache votes for. \ntype:uint64");
toml.put ("max_voters", max_voters, "Maximum number of voters to cache per block. \ntype:uint64");
toml.put ("age_cutoff", age_cutoff.count (), "Maximum age of votes to keep in cache. \ntype:seconds");
return toml.get_error ();
}
@ -224,5 +278,9 @@ nano::error nano::vote_cache_config::deserialize (nano::tomlconfig & toml)
toml.get ("max_size", max_size);
toml.get ("max_voters", max_voters);
auto age_cutoff_l = age_cutoff.count ();
toml.get ("age_cutoff", age_cutoff_l);
age_cutoff = std::chrono::seconds{ age_cutoff_l };
return toml.get_error ();
}

21
nano/node/vote_cache.hpp
View file

@ -1,5 +1,6 @@
#pragma once
#include <nano/lib/interval.hpp>
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/utility.hpp>
@ -38,6 +39,7 @@ public:
public:
std::size_t max_size{ 1024 * 128 };
std::size_t max_voters{ 128 };
std::chrono::seconds age_cutoff{ 5 * 60 };
};
class vote_cache final
@ -63,6 +65,7 @@ public:
* @return true if current tally changed, false otherwise
*/
bool vote (nano::account const & representative, uint64_t const & timestamp, nano::uint128_t const & rep_weight, std::size_t max_voters);
/**
* Inserts votes stored in this entry into an election
*/
@ -73,17 +76,22 @@ public:
nano::uint128_t tally () const;
nano::uint128_t final_tally () const;
std::vector<voter_entry> voters () const;
std::chrono::steady_clock::time_point last_vote () const;
private:
bool vote_impl (nano::account const & representative, uint64_t const & timestamp, nano::uint128_t const & rep_weight, std::size_t max_voters);
nano::block_hash const hash_m;
std::vector<voter_entry> voters_m;
nano::uint128_t tally_m{ 0 };
nano::uint128_t final_tally_m{ 0 };
std::chrono::steady_clock::time_point last_vote_m{};
};
public:
explicit vote_cache (vote_cache_config const &);
explicit vote_cache (vote_cache_config const &, nano::stats &);
/**
* Adds a new vote to cache
@ -115,10 +123,10 @@ public:
* The blocks are sorted in descending order by final tally, then by tally
* @param min_tally minimum tally threshold, entries below with their voting weight below this will be ignored
*/
std::vector<top_entry> top (nano::uint128_t const & min_tally) const;
std::vector<top_entry> top (nano::uint128_t const & min_tally);
public: // Container info
std::unique_ptr<nano::container_info_component> collect_container_info (std::string const & name);
std::unique_ptr<nano::container_info_component> collect_container_info (std::string const & name) const;
public:
/**
@ -126,8 +134,12 @@ public:
*/
std::function<nano::uint128_t (nano::account const &)> rep_weight_query{ [] (nano::account const & rep) { debug_assert (false); return 0; } };
private:
private: // Dependencies
vote_cache_config const & config;
nano::stats & stats;
private:
void cleanup ();
// clang-format off
class tag_sequenced {};
@ -148,5 +160,6 @@ private:
ordered_cache cache;
mutable nano::mutex mutex;
nano::interval cleanup_interval;
};
}