Merge pull request #4889 from pwojcikdev/missing-configs

Implement missing configuration components

nano/core_test/node.cpp

@@ -3615,7 +3615,7 @@ TEST (node, local_block_broadcast)
 
 	// Wait until a broadcast is attempted
 	ASSERT_TIMELY_EQ (5s, node1.local_block_broadcaster.size (), 1);
-	ASSERT_TIMELY (5s, node1.stats.count (nano::stat::type::local_block_broadcaster, nano::stat::detail::broadcast, nano::stat::dir::out) >= 1);
+	ASSERT_TIMELY (5s, node1.stats.count (nano::stat::type::local_block_broadcaster, nano::stat::detail::broadcast) >= 1);
 
 	// The other node should not have received the block
 	ASSERT_NEVER (500ms, node2.block (send1->hash ()));

nano/core_test/toml.cpp

@@ -288,7 +288,6 @@ TEST (toml_config, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads);
 	ASSERT_EQ (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count);
 	ASSERT_EQ (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator);
-	ASSERT_EQ (conf.node.cementing_set_batch_time, defaults.node.cementing_set_batch_time);
 	ASSERT_EQ (conf.node.enable_voting, defaults.node.enable_voting);
 	ASSERT_EQ (conf.node.external_address, defaults.node.external_address);
 	ASSERT_EQ (conf.node.external_port, defaults.node.external_port);
@@ -456,7 +455,6 @@ TEST (toml_config, daemon_config_deserialize_no_defaults)
 	bootstrap_serving_threads = 999
 	bootstrap_frontier_request_count = 9999
 	bootstrap_fraction_numerator = 999
-	cementing_set_batch_time = 999
 	enable_voting = false
 	external_address = "0:0:0:0:0:ffff:7f01:101"
 	external_port = 999
@@ -720,7 +718,6 @@ TEST (toml_config, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads);
 	ASSERT_NE (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count);
 	ASSERT_NE (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator);
-	ASSERT_NE (conf.node.cementing_set_batch_time, defaults.node.cementing_set_batch_time);
 	ASSERT_NE (conf.node.enable_voting, defaults.node.enable_voting);
 	ASSERT_NE (conf.node.external_address, defaults.node.external_address);
 	ASSERT_NE (conf.node.external_port, defaults.node.external_port);

nano/node/cementing_set.cpp

@@ -340,3 +340,31 @@ nano::container_info nano::cementing_set::container_info () const
 	info.add ("workers", workers.container_info ());
 	return info;
 }
+
+/*
+ * cementing_set_config
+ */
+
+nano::error nano::cementing_set_config::serialize (nano::tomlconfig & toml) const
+{
+	toml.put ("enable", enable, "Enable or disable cementing set.\ntype:bool");
+	toml.put ("batch_size", batch_size, "Number of blocks to cement in a single batch.\ntype:uint64");
+	toml.put ("max_blocks", max_blocks, "Maximum number of dependent blocks to be stored in memory during processing.\ntype:uint64");
+	toml.put ("max_queued_notifications", max_queued_notifications, "Maximum number of notification batches to queue.\ntype:uint64");
+	toml.put ("max_deferred", max_deferred, "Maximum number of failed blocks to keep for requeuing.\ntype:uint64");
+	toml.put ("deferred_age_cutoff", deferred_age_cutoff.count (), "Max age of deferred blocks before they are dropped.\ntype:seconds");
+
+	return toml.get_error ();
+}
+
+nano::error nano::cementing_set_config::deserialize (nano::tomlconfig & toml)
+{
+	toml.get ("enable", enable);
+	toml.get ("batch_size", batch_size);
+	toml.get ("max_blocks", max_blocks);
+	toml.get ("max_queued_notifications", max_queued_notifications);
+	toml.get ("max_deferred", max_deferred);
+	toml.get_duration ("deferred_age_cutoff", deferred_age_cutoff);
+
+	return toml.get_error ();
+}

nano/node/cementing_set.hpp

@@ -27,7 +27,8 @@ namespace nano
 class cementing_set_config final
 {
 public:
-	// TODO: Serialization & deserialization
+	nano::error deserialize (nano::tomlconfig &);
+	nano::error serialize (nano::tomlconfig &) const;
 
 public:
 	bool enable{ true };

nano/node/local_block_broadcaster.cpp

@@ -195,8 +195,11 @@ void nano::local_block_broadcaster::run_broadcasts (nano::unique_lock<nano::mute
 		entry.block->hash (),
 		entry.rebroadcasts);
 
-		stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::broadcast, nano::stat::dir::out);
+		stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::broadcast);
-		network.flood_block_initial (entry.block);
+
+		auto sent = network.flood_block_initial (entry.block);
+
+		stats.add (nano::stat::type::local_block_broadcaster, nano::stat::detail::sent, sent);
 	}
 }
 
@@ -243,3 +246,31 @@ nano::container_info nano::local_block_broadcaster::container_info () const
 	info.put ("local", local_blocks);
 	return info;
 }
+
+/*
+ * local_block_broadcaster_config
+ */
+
+nano::error nano::local_block_broadcaster_config::serialize (nano::tomlconfig & toml) const
+{
+	toml.put ("max_size", max_size, "Maximum number of blocks to keep in the local block broadcaster set. \ntype:uint64");
+	toml.put ("rebroadcast_interval", rebroadcast_interval.count (), "Interval between rebroadcasts of the same block. Interval increases with each rebroadcast. \ntype:seconds");
+	toml.put ("max_rebroadcast_interval", max_rebroadcast_interval.count (), "Maximum interval between rebroadcasts of the same block. \ntype:seconds");
+	toml.put ("broadcast_rate_limit", broadcast_rate_limit, "Rate limit for broadcasting blocks. \ntype:uint64");
+	toml.put ("broadcast_rate_burst_ratio", broadcast_rate_burst_ratio, "Burst ratio for broadcasting blocks. \ntype:double");
+	toml.put ("cleanup_interval", cleanup_interval.count (), "Cleanup interval of the local blocks broadcaster set. \ntype:seconds");
+
+	return toml.get_error ();
+}
+
+nano::error nano::local_block_broadcaster_config::deserialize (nano::tomlconfig & toml)
+{
+	toml.get ("max_size", max_size);
+	toml.get_duration ("rebroadcast_interval", rebroadcast_interval);
+	toml.get_duration ("max_rebroadcast_interval", max_rebroadcast_interval);
+	toml.get ("broadcast_rate_limit", broadcast_rate_limit);
+	toml.get ("broadcast_rate_burst_ratio", broadcast_rate_burst_ratio);
+	toml.get_duration ("cleanup_interval", cleanup_interval);
+
+	return toml.get_error ();
+}

nano/node/local_block_broadcaster.hpp

@@ -35,7 +35,8 @@ public:
 		}
 	}
 
-	// TODO: Serialization & deserialization
+	nano::error deserialize (nano::tomlconfig &);
+	nano::error serialize (nano::tomlconfig &) const;
 
 public:
 	std::size_t max_size{ 1024 * 8 };

nano/node/nodeconfig.cpp

@@ -133,7 +133,6 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	toml.put ("bootstrap_bandwidth_limit", bootstrap_bandwidth_limit, "Outbound bootstrap traffic limit in bytes/sec after which messages will be dropped.\nNote: changing to unlimited bandwidth (0) is not recommended for limited connections.\ntype:uint64");
 	toml.put ("bootstrap_bandwidth_burst_ratio", bootstrap_bandwidth_burst_ratio, "Burst ratio for outbound bootstrap traffic.\ntype:double");
 
-	toml.put ("cementing_set_batch_time", cementing_set_batch_time.count (), "Maximum time the confirming set will hold the database write transaction.\ntype:milliseconds");
 	toml.put ("backup_before_upgrade", backup_before_upgrade, "Backup the ledger database before performing upgrades.\nWarning: uses more disk storage and increases startup time when upgrading.\ntype:bool");
 	toml.put ("max_work_generate_multiplier", max_work_generate_multiplier, "Maximum allowed difficulty multiplier for work generation.\ntype:double,[1..]");
 	toml.put ("max_queued_requests", max_queued_requests, "Limit for number of queued confirmation requests for one channel, after which new requests are dropped until the queue drops below this value.\ntype:uint32");
@@ -211,6 +210,14 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	optimistic_scheduler.serialize (optimistic_l);
 	toml.put_child ("optimistic_scheduler", optimistic_l);
 
+	nano::tomlconfig hinted_l;
+	hinted_scheduler.serialize (hinted_l);
+	toml.put_child ("hinted_scheduler", hinted_l);
+
+	nano::tomlconfig priority_l;
+	priority_scheduler.serialize (priority_l);
+	toml.put_child ("priority_scheduler", priority_l);
+
 	nano::tomlconfig priority_bucket_l;
 	priority_bucket.serialize (priority_bucket_l);
 	toml.put_child ("priority_bucket", priority_bucket_l);
@@ -283,6 +290,14 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	vote_rebroadcaster.serialize (vote_rebroadcaster_l);
 	toml.put_child ("vote_rebroadcaster", vote_rebroadcaster_l);
 
+	nano::tomlconfig cementing_set_l;
+	cementing_set.serialize (cementing_set_l);
+	toml.put_child ("cementing_set", cementing_set_l);
+
+	nano::tomlconfig local_block_broadcaster_l;
+	local_block_broadcaster.serialize (local_block_broadcaster_l);
+	toml.put_child ("local_block_broadcaster", local_block_broadcaster_l);
+
 	return toml.get_error ();
 }
 
@@ -344,6 +359,12 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 			hinted_scheduler.deserialize (config_l);
 		}
 
+		if (toml.has_key ("priority_scheduler"))
+		{
+			auto config_l = toml.get_required_child ("priority_scheduler");
+			priority_scheduler.deserialize (config_l);
+		}
+
 		if (toml.has_key ("priority_bucket"))
 		{
 			auto config_l = toml.get_required_child ("priority_bucket");
@@ -452,6 +473,18 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 			vote_rebroadcaster.deserialize (config_l);
 		}
 
+		if (toml.has_key ("cementing_set"))
+		{
+			auto config_l = toml.get_required_child ("cementing_set");
+			cementing_set.deserialize (config_l);
+		}
+
+		if (toml.has_key ("local_block_broadcaster"))
+		{
+			auto config_l = toml.get_required_child ("local_block_broadcaster");
+			local_block_broadcaster.deserialize (config_l);
+		}
+
 		/*
 		 * Values
 		 */
@@ -605,10 +638,6 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 
 		toml.get<bool> ("backup_before_upgrade", backup_before_upgrade);
 
-		auto cementing_set_batch_time_l (cementing_set_batch_time.count ());
-		toml.get ("cementing_set_batch_time", cementing_set_batch_time_l);
-		cementing_set_batch_time = std::chrono::milliseconds (cementing_set_batch_time_l);
-
 		toml.get<double> ("max_work_generate_multiplier", max_work_generate_multiplier);
 
 		toml.get<uint32_t> ("max_queued_requests", max_queued_requests);

nano/node/nodeconfig.hpp

@@ -133,7 +133,6 @@ public:
 	double bootstrap_bandwidth_burst_ratio{ 1. };
 	nano::bootstrap_config bootstrap;
 	nano::bootstrap_server_config bootstrap_server;
-	std::chrono::milliseconds cementing_set_batch_time{ 250 };
 	bool backup_before_upgrade{ false };
 	double max_work_generate_multiplier{ 64. };
 	uint32_t max_queued_requests{ 512 };

nano/node/repcrawler.cpp

@@ -519,17 +519,14 @@ nano::rep_crawler_config::rep_crawler_config (nano::network_constants const & ne
 
 nano::error nano::rep_crawler_config::serialize (nano::tomlconfig & toml) const
 {
-	// TODO: Descriptions
+	toml.put ("query_timeout", query_timeout.count (), "Timeout for rep crawler queries.\ntype:milliseconds");
-	toml.put ("query_timeout", query_timeout.count ());
 
 	return toml.get_error ();
 }
 
 nano::error nano::rep_crawler_config::deserialize (nano::tomlconfig & toml)
 {
-	auto query_timeout_l = query_timeout.count ();
+	toml.get_duration ("query_timeout", query_timeout);
-	toml.get ("query_timeout", query_timeout_l);
-	query_timeout = std::chrono::milliseconds{ query_timeout_l };
 
 	return toml.get_error ();
 }

nano/node/scheduler/hinted.cpp

@@ -275,13 +275,8 @@ nano::error nano::scheduler::hinted_config::deserialize (nano::tomlconfig & toml
 	toml.get ("enable", enable);
 	toml.get ("hinting_threshold", hinting_threshold_percent);
 
-	auto check_interval_l = check_interval.count ();
+	toml.get_duration ("check_interval", check_interval);
-	toml.get ("check_interval", check_interval_l);
+	toml.get_duration ("block_cooldown", block_cooldown);
-	check_interval = std::chrono::milliseconds{ check_interval_l };
-
-	auto block_cooldown_l = block_cooldown.count ();
-	toml.get ("block_cooldown", block_cooldown_l);
-	block_cooldown = std::chrono::milliseconds{ block_cooldown_l };
 
 	toml.get ("vacancy_threshold", vacancy_threshold_percent);
 

nano/node/scheduler/priority.cpp

@@ -25,6 +25,11 @@ nano::scheduler::priority::priority (nano::node_config & node_config, nano::node
 		buckets[index] = std::make_unique<scheduler::bucket> (index, node_config.priority_bucket, active, stats);
 	}
 
+	if (!config.enable)
+	{
+		return;
+	}
+
 	// Activate accounts with fresh blocks
 	ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
 		auto transaction = ledger.tx_begin_read ();
@@ -275,4 +280,22 @@ nano::container_info nano::scheduler::priority::container_info () const
 	info.add ("blocks", collect_blocks ());
 	info.add ("elections", collect_elections ());
 	return info;
+}
+
+/*
+ * priority_config
+ */
+
+nano::error nano::scheduler::priority_config::serialize (nano::tomlconfig & toml) const
+{
+	toml.put ("enable", enable, "Enable or disable priority scheduler\ntype:bool");
+
+	return toml.get_error ();
+}
+
+nano::error nano::scheduler::priority_config::deserialize (nano::tomlconfig & toml)
+{
+	toml.get ("enable", enable);
+
+	return toml.get_error ();
 }

nano/node/scheduler/priority.hpp

@@ -18,7 +18,8 @@ class buckets;
 class priority_config
 {
 public:
-	// TODO: Serialization & deserialization
+	nano::error deserialize (nano::tomlconfig &);
+	nano::error serialize (nano::tomlconfig &) const;
 
 public:
 	bool enable{ true };