Use frontiers age option for frontier_req client (#3098)

* Use frontiers age option for frontier_req client

to reduce bandwidth usage and failure rate for millions of frontiers.

- 66.7% of bootstrap attempts after initial bootstrap and after node warm up after startup to request only frontier modified since 1 day ago (1 hour for beta)
- for long inactive node use last stored online weight timestamp minus 1 hour

Additionally disable bulk push for requests containing age option.

nano/core_test/node.cpp

@@ -325,6 +325,30 @@ TEST (node, auto_bootstrap_reverse)
 	ASSERT_TIMELY (10s, node1->balance (key2.pub) == node0->config.receive_minimum.number ());
 }
 
+TEST (node, auto_bootstrap_age)
+{
+	nano::system system;
+	nano::node_config config (nano::get_available_port (), system.logging);
+	config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	nano::node_flags node_flags;
+	node_flags.disable_bootstrap_bulk_push_client = true;
+	node_flags.disable_lazy_bootstrap = true;
+	node_flags.bootstrap_interval = 1;
+	auto node0 = system.add_node (config, node_flags);
+	auto node1 (std::make_shared<nano::node> (system.io_ctx, nano::get_available_port (), nano::unique_path (), system.logging, system.work, node_flags));
+	ASSERT_FALSE (node1->init_error ());
+	node1->start ();
+	system.nodes.push_back (node1);
+	ASSERT_NE (nullptr, nano::establish_tcp (system, *node1, node0->network.endpoint ()));
+	ASSERT_TIMELY (10s, node1->bootstrap_initiator.in_progress ());
+	// 4 bootstraps with frontiers age
+	ASSERT_TIMELY (10s, node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out) >= 3);
+	// More attempts with frontiers age
+	ASSERT_GE (node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out), node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out));
+
+	node1->stop ();
+}
+
 TEST (node, receive_gap)
 {
 	nano::system system (1);

nano/lib/stats.cpp

@@ -658,6 +658,9 @@ std::string nano::stat::detail_to_string (uint32_t key)
 		case nano::stat::detail::initiate:
 			res = "initiate";
 			break;
+		case nano::stat::detail::initiate_legacy_age:
+			res = "initiate_legacy_age";
+			break;
 		case nano::stat::detail::initiate_lazy:
 			res = "initiate_lazy";
 			break;

nano/lib/stats.hpp

@@ -285,6 +285,7 @@ public:
 
 		// bootstrap, callback
 		initiate,
+		initiate_legacy_age,
 		initiate_lazy,
 		initiate_wallet_lazy,
 

nano/node/bootstrap/bootstrap.cpp

@@ -31,7 +31,7 @@ nano::bootstrap_initiator::~bootstrap_initiator ()
 	stop ();
 }
 
-void nano::bootstrap_initiator::bootstrap (bool force, std::string id_a)
+void nano::bootstrap_initiator::bootstrap (bool force, std::string id_a, uint32_t const frontiers_age_a)
 {
 	if (force)
 	{
@@ -40,8 +40,8 @@ void nano::bootstrap_initiator::bootstrap (bool force, std::string id_a)
 	nano::unique_lock<nano::mutex> lock (mutex);
 	if (!stopped && find_attempt (nano::bootstrap_mode::legacy) == nullptr)
 	{
-		node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out);
-		auto legacy_attempt (std::make_shared<nano::bootstrap_attempt_legacy> (node.shared (), attempts.incremental++, id_a));
+		node.stats.inc (nano::stat::type::bootstrap, frontiers_age_a == std::numeric_limits<uint32_t>::max () ? nano::stat::detail::initiate : nano::stat::detail::initiate_legacy_age, nano::stat::dir::out);
+		auto legacy_attempt (std::make_shared<nano::bootstrap_attempt_legacy> (node.shared (), attempts.incremental++, id_a, frontiers_age_a));
 		attempts_list.push_back (legacy_attempt);
 		attempts.add (legacy_attempt);
 		lock.unlock ();
@@ -67,7 +67,7 @@ void nano::bootstrap_initiator::bootstrap (nano::endpoint const & endpoint_a, bo
 		stop_attempts ();
 		node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out);
 		nano::lock_guard<nano::mutex> lock (mutex);
-		auto legacy_attempt (std::make_shared<nano::bootstrap_attempt_legacy> (node.shared (), attempts.incremental++, id_a));
+		auto legacy_attempt (std::make_shared<nano::bootstrap_attempt_legacy> (node.shared (), attempts.incremental++, id_a, std::numeric_limits<uint32_t>::max ()));
 		attempts_list.push_back (legacy_attempt);
 		attempts.add (legacy_attempt);
 		if (frontiers_confirmed)

nano/node/bootstrap/bootstrap.hpp

@@ -82,7 +82,7 @@ public:
 	explicit bootstrap_initiator (nano::node &);
 	~bootstrap_initiator ();
 	void bootstrap (nano::endpoint const &, bool add_to_peers = true, bool frontiers_confirmed = false, std::string id_a = "");
-	void bootstrap (bool force = false, std::string id_a = "");
+	void bootstrap (bool force = false, std::string id_a = "", uint32_t const frontiers_age_a = std::numeric_limits<uint32_t>::max ());
 	void bootstrap_lazy (nano::hash_or_account const &, bool force = false, bool confirmed = true, std::string id_a = "");
 	void bootstrap_wallet (std::deque<nano::account> &);
 	void run_bootstrap ();

nano/node/bootstrap/bootstrap_frontier.cpp

@@ -12,12 +12,13 @@ constexpr unsigned nano::bootstrap_limits::bulk_push_cost_limit;
 
 constexpr size_t nano::frontier_req_client::size_frontier;
 
-void nano::frontier_req_client::run ()
+void nano::frontier_req_client::run (uint32_t const frontiers_age_a)
 {
 	nano::frontier_req request;
 	request.start.clear ();
-	request.age = std::numeric_limits<decltype (request.age)>::max ();
+	request.age = frontiers_age_a;
 	request.count = std::numeric_limits<decltype (request.count)>::max ();
+	frontiers_age = frontiers_age_a;
 	auto this_l (shared_from_this ());
 	connection->channel->send (
 	request, [this_l](boost::system::error_code const & ec, size_t size_a) {
@@ -68,7 +69,7 @@ void nano::frontier_req_client::receive_frontier ()
 
 void nano::frontier_req_client::unsynced (nano::block_hash const & head, nano::block_hash const & end)
 {
-	if (bulk_push_cost < nano::bootstrap_limits::bulk_push_cost_limit)
+	if (bulk_push_cost < nano::bootstrap_limits::bulk_push_cost_limit && frontiers_age == std::numeric_limits<decltype (frontiers_age)>::max ())
 	{
 		attempt->add_bulk_push_target (head, end);
 		if (end.is_zero ())

nano/node/bootstrap/bootstrap_frontier.hpp

@@ -13,7 +13,7 @@ class frontier_req_client final : public std::enable_shared_from_this<nano::fron
 {
 public:
 	explicit frontier_req_client (std::shared_ptr<nano::bootstrap_client> const &, std::shared_ptr<nano::bootstrap_attempt> const &);
-	void run ();
+	void run (uint32_t const frontiers_age_a);
 	void receive_frontier ();
 	void received_frontier (boost::system::error_code const &, size_t);
 	void unsynced (nano::block_hash const &, nano::block_hash const &);
@@ -30,6 +30,7 @@ public:
 	/** A very rough estimate of the cost of `bulk_push`ing missing blocks */
 	uint64_t bulk_push_cost;
 	std::deque<std::pair<nano::account, nano::block_hash>> accounts;
+	uint32_t frontiers_age{ std::numeric_limits<uint32_t>::max () };
 	static size_t constexpr size_frontier = sizeof (nano::account) + sizeof (nano::block_hash);
 };
 class bootstrap_server;

nano/node/bootstrap/bootstrap_legacy.cpp

@@ -5,8 +5,9 @@
 
 #include <boost/format.hpp>
 
-nano::bootstrap_attempt_legacy::bootstrap_attempt_legacy (std::shared_ptr<nano::node> const & node_a, uint64_t incremental_id_a, std::string const & id_a) :
-nano::bootstrap_attempt (node_a, nano::bootstrap_mode::legacy, incremental_id_a, id_a)
+nano::bootstrap_attempt_legacy::bootstrap_attempt_legacy (std::shared_ptr<nano::node> const & node_a, uint64_t const incremental_id_a, std::string const & id_a, uint32_t const frontiers_age_a) :
+nano::bootstrap_attempt (node_a, nano::bootstrap_mode::legacy, incremental_id_a, id_a),
+frontiers_age (frontiers_age_a)
 {
 	node->bootstrap_initiator.notify_listeners (true);
 }
@@ -152,11 +153,14 @@ void nano::bootstrap_attempt_legacy::attempt_restart_check (nano::unique_lock<na
 			// Start new bootstrap connection
 			auto node_l (node->shared ());
 			auto this_l (shared_from_this ());
-			node->background ([node_l, this_l]() {
+			auto duration (std::chrono::duration_cast<std::chrono::seconds> (std::chrono::steady_clock::now () - attempt_start).count ());
+			auto frontiers_age_l (frontiers_age != std::numeric_limits<uint32_t>::max () ? frontiers_age + duration : frontiers_age);
+			node->background ([node_l, this_l, frontiers_age_l]() {
 				node_l->bootstrap_initiator.remove_attempt (this_l);
+				auto id_l (this_l->id);
 				// Delay after removing current attempt
-				node_l->workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::milliseconds (50), [node_l]() {
-					node_l->bootstrap_initiator.bootstrap (true);
+				node_l->workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::milliseconds (50), [node_l, id_l, frontiers_age_l]() {
+					node_l->bootstrap_initiator.bootstrap (true, id_l, frontiers_age_l);
 				});
 			});
 		}
@@ -314,7 +318,7 @@ bool nano::bootstrap_attempt_legacy::request_frontier (nano::unique_lock<nano::m
 		{
 			auto this_l (shared_from_this ());
 			auto client (std::make_shared<nano::frontier_req_client> (connection_l, this_l));
-			client->run ();
+			client->run (frontiers_age);
 			frontiers = client;
 			future = client->promise.get_future ();
 		}

nano/node/bootstrap/bootstrap_legacy.hpp

@@ -16,7 +16,7 @@ class node;
 class bootstrap_attempt_legacy : public bootstrap_attempt
 {
 public:
-	explicit bootstrap_attempt_legacy (std::shared_ptr<nano::node> const & node_a, uint64_t incremental_id_a, std::string const & id_a = "");
+	explicit bootstrap_attempt_legacy (std::shared_ptr<nano::node> const & node_a, uint64_t const incremental_id_a, std::string const & id_a = "", uint32_t const frontiers_age_a = std::numeric_limits<uint32_t>::max ());
 	void run () override;
 	bool consume_future (std::future<bool> &);
 	void stop () override;
@@ -39,5 +39,6 @@ public:
 	std::vector<std::pair<nano::block_hash, nano::block_hash>> bulk_push_targets;
 	std::atomic<unsigned> account_count{ 0 };
 	std::atomic<bool> frontiers_confirmation_pending{ false };
+	uint32_t frontiers_age;
 };
 }

nano/node/node.cpp

@@ -851,7 +851,45 @@ void nano::node::ongoing_bootstrap ()
 			++warmed_up;
 		}
 	}
-	bootstrap_initiator.bootstrap ();
+	if (network_params.network.is_dev_network () && flags.bootstrap_interval != 0)
+	{
+		// For test purposes allow faster automatic bootstraps
+		next_wakeup = std::chrono::seconds (flags.bootstrap_interval);
+		++warmed_up;
+	}
+	// Differential bootstrap with max age (75% of all legacy attempts)
+	uint32_t frontiers_age (std::numeric_limits<uint32_t>::max ());
+	auto bootstrap_weight_reached (ledger.cache.block_count >= ledger.bootstrap_weight_max_blocks);
+	auto previous_bootstrap_count (stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out) + stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out));
+	/* 
+	- Maximum value for 25% of attempts or if block count is below preconfigured value (initial bootstrap not finished)
+	- Node shutdown time minus 1 hour for start attempts (warm up)
+	- Default age value otherwise (1 day for live network, 1 hour for beta)
+	*/
+	if (bootstrap_weight_reached)
+	{
+		if (warmed_up < 3)
+		{
+			// Find last online weight sample (last active time for node)
+			uint64_t last_sample_time (0);
+			auto transaction (store.tx_begin_read ());
+			for (auto i (store.online_weight_begin (transaction)), n (store.online_weight_end ()); i != n; ++i)
+			{
+				last_sample_time = i->first;
+			}
+			uint64_t time_since_last_sample = std::chrono::duration_cast<std::chrono::seconds> (std::chrono::system_clock::now ().time_since_epoch ()).count () - last_sample_time / std::pow (10, 9); // Nanoseconds to seconds
+			if (time_since_last_sample + 60 * 60 < std::numeric_limits<uint32_t>::max ())
+			{
+				frontiers_age = std::max<uint32_t> (time_since_last_sample + 60 * 60, network_params.bootstrap.default_frontiers_age_seconds);
+			}
+		}
+		else if (previous_bootstrap_count % 4 != 0)
+		{
+			frontiers_age = network_params.bootstrap.default_frontiers_age_seconds;
+		}
+	}
+	// Bootstrap and schedule for next attempt
+	bootstrap_initiator.bootstrap (false, boost::str (boost::format ("auto_bootstrap_%1%") % previous_bootstrap_count), frontiers_age);
 	std::weak_ptr<nano::node> node_w (shared_from_this ());
 	workers.add_timed_task (std::chrono::steady_clock::now () + next_wakeup, [node_w]() {
 		if (auto node_l = node_w.lock ())

nano/node/nodeconfig.hpp

@@ -154,5 +154,6 @@ public:
 	size_t block_processor_verification_size{ 0 };
 	size_t inactive_votes_cache_size{ 16 * 1024 };
 	size_t vote_processor_capacity{ 144 * 1024 };
+	size_t bootstrap_interval{ 0 }; // For testing only
 };
 }

nano/secure/common.cpp

@@ -175,6 +175,7 @@ nano::bootstrap_constants::bootstrap_constants (nano::network_constants & networ
 	lazy_retry_limit = network_constants.is_dev_network () ? 2 : frontier_retry_limit * 4;
 	lazy_destinations_retry_limit = network_constants.is_dev_network () ? 1 : frontier_retry_limit / 4;
 	gap_cache_bootstrap_start_interval = network_constants.is_dev_network () ? std::chrono::milliseconds (5) : std::chrono::milliseconds (30 * 1000);
+	default_frontiers_age_seconds = network_constants.is_dev_network () ? 1 : network_constants.is_beta_network () ? 60 * 60 : 24 * 60 * 60; // 1 second for dev network, 1 hour for beta, 24 hours for live
 }
 
 // Create a new random keypair

nano/secure/common.hpp

@@ -448,6 +448,7 @@ public:
 	unsigned lazy_retry_limit;
 	unsigned lazy_destinations_retry_limit;
 	std::chrono::milliseconds gap_cache_bootstrap_start_interval;
+	uint32_t default_frontiers_age_seconds;
 };
 
 /** Constants whose value depends on the active network */