[Pruning] Pruning configuration settings (#2947)

* [Pruning] Pruning configuration settings
Splitting https://github.com/nanocurrency/nano-node/pull/2881

* Enable optional ledger pruning with node launch flag `--enable_pruning` for automatic node background actions.
* Pruning rules can be configured in [node.experimental] config with “max_pruning_age” (Time in seconds to limit for blocks age after pruning. Also period for automatic node pruning task, default 1 day, 5 minutes for beta network) & “max_pruning_depth” (Limit for full blocks in chain after pruning, default 0 - unlimited). By default all confirmed blocks older than 1 day can be pruned, except genesis block & last confirmed block in each account chain.
* `--enable_pruning` flag & config option “enable_voting” are mutually exclusive to prevent representatives start with pruned ledger. Pruning is not available for any representatives.

* Update ledger.pruning status if there are any pruned blocks in ledger

* Remove unused variables

nano/core_test/toml.cpp

@@ -513,6 +513,8 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 
 	[node.experimental]
 	secondary_work_peers = ["dev.org:998"]
+	max_pruning_age = 999
+	max_pruning_depth = 999
 
 	[opencl]
 	device = 999
@@ -567,6 +569,8 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.frontiers_confirmation, defaults.node.frontiers_confirmation);
 	ASSERT_NE (conf.node.network_threads, defaults.node.network_threads);
 	ASSERT_NE (conf.node.secondary_work_peers, defaults.node.secondary_work_peers);
+	ASSERT_NE (conf.node.max_pruning_age, defaults.node.max_pruning_age);
+	ASSERT_NE (conf.node.max_pruning_depth, defaults.node.max_pruning_depth);
 	ASSERT_NE (conf.node.work_watcher_period, defaults.node.work_watcher_period);
 	ASSERT_NE (conf.node.online_weight_minimum, defaults.node.online_weight_minimum);
 	ASSERT_NE (conf.node.online_weight_quorum, defaults.node.online_weight_quorum);

nano/nano_node/daemon.cpp

@@ -2,6 +2,7 @@
 #include <nano/lib/threading.hpp>
 #include <nano/lib/utility.hpp>
 #include <nano/nano_node/daemon.hpp>
+#include <nano/node/cli.hpp>
 #include <nano/node/daemonconfig.hpp>
 #include <nano/node/ipc/ipc_server.hpp>
 #include <nano/node/json_handler.hpp>
@@ -42,6 +43,10 @@ void nano_daemon::daemon::run (boost::filesystem::path const & data_path, nano::
 	auto error = nano::read_node_config_toml (data_path, config, flags.config_overrides);
 	nano::set_use_memory_pools (config.node.use_memory_pools);
 	if (!error)
+	{
+		error = nano::flags_config_conflicts (flags, config.node);
+	}
+	if (!error)
 	{
 		config.node.logging.init (data_path);
 		nano::logger_mt logger{ config.node.logging.min_time_between_log_output };

nano/nano_wallet/entry.cpp

@@ -87,6 +87,11 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost
 		error = read_wallet_config (wallet_config, data_path);
 	}
 
+	if (!error)
+	{
+		error = nano::flags_config_conflicts (flags, config.node);
+	}
+
 	if (!error)
 	{
 		nano::set_use_memory_pools (config.node.use_memory_pools);

nano/node/cli.cpp

@@ -33,6 +33,8 @@ std::string nano::error_cli_messages::message (int ev) const
 			return "Flags --disable_tcp_realtime and --disable_udp cannot be used together";
 		case nano::error_cli::ambiguous_udp_options:
 			return "Flags --disable_udp and --enable_udp cannot be used together";
+		case nano::error_cli::ambiguous_pruning_voting_options:
+			return "Flag --enable_pruning and enable_voting in node config cannot be used together";
 	}
 
 	return "Invalid error code";
@@ -97,6 +99,7 @@ void nano::add_node_flag_options (boost::program_options::options_description &
 		("disable_unchecked_drop", "Disables drop of unchecked table at startup")
 		("disable_providing_telemetry_metrics", "Disable using any node information in the telemetry_ack messages.")
 		("disable_block_processor_unchecked_deletion", "Disable deletion of unchecked blocks after processing")
+		("enable_pruning", "Enable experimental ledger pruning")
 		("allow_bootstrap_peers_duplicates", "Allow multiple connections to same peer in bootstrap attempts")
 		("fast_bootstrap", "Increase bootstrap speed for high end nodes with higher limits")
 		("block_processor_batch_size", boost::program_options::value<std::size_t>(), "Increase block processor transaction batch write size, default 0 (limited by config block_processor_batch_max_time), 256k for fast_bootstrap")
@@ -133,6 +136,7 @@ std::error_code nano::update_flags (nano::node_flags & flags_a, boost::program_o
 	flags_a.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
 	flags_a.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
 	flags_a.disable_block_processor_unchecked_deletion = (vm.count ("disable_block_processor_unchecked_deletion") > 0);
+	flags_a.enable_pruning = (vm.count ("enable_pruning") > 0);
 	flags_a.allow_bootstrap_peers_duplicates = (vm.count ("allow_bootstrap_peers_duplicates") > 0);
 	flags_a.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
 	if (flags_a.fast_bootstrap)
@@ -176,6 +180,16 @@ std::error_code nano::update_flags (nano::node_flags & flags_a, boost::program_o
 	return ec;
 }
 
+std::error_code nano::flags_config_conflicts (nano::node_flags const & flags_a, nano::node_config const & config_a)
+{
+	std::error_code ec;
+	if (flags_a.enable_pruning && config_a.enable_voting)
+	{
+		ec = nano::error_cli::ambiguous_pruning_voting_options;
+	}
+	return ec;
+}
+
 namespace
 {
 void database_write_lock_error (std::error_code & ec)

nano/node/cli.hpp

@@ -17,12 +17,14 @@ enum class error_cli
 	database_write_error = 5,
 	reading_config = 6,
 	disable_all_network = 7,
-	ambiguous_udp_options = 8
+	ambiguous_udp_options = 8,
+	ambiguous_pruning_voting_options = 9
 };
 
 void add_node_options (boost::program_options::options_description &);
 void add_node_flag_options (boost::program_options::options_description &);
 std::error_code update_flags (nano::node_flags &, boost::program_options::variables_map const &);
+std::error_code flags_config_conflicts (nano::node_flags const &, nano::node_config const &);
 std::error_code handle_node_options (boost::program_options::variables_map const &);
 }
 

nano/node/node.cpp

@@ -424,6 +424,26 @@ node_seq (seq)
 				logger.always_log ("Dropping unchecked blocks");
 			}
 		}
+
+		ledger.pruning = flags.enable_pruning || store.pruned_count (store.tx_begin_read ()) > 0;
+
+		if (ledger.pruning)
+		{
+			if (config.enable_voting && !flags.inactive_node)
+			{
+				std::string str = "Incompatibility detected between config node.enable_voting and existing pruned blocks";
+				logger.always_log (str);
+				std::cerr << str << std::endl;
+				std::exit (1);
+			}
+			else if (!flags.enable_pruning && !flags.inactive_node)
+			{
+				std::string str = "To start node with existing pruned blocks use launch flag --enable_pruning";
+				logger.always_log (str);
+				std::cerr << str << std::endl;
+				std::exit (1);
+			}
+		}
 	}
 	node_initialized_latch.count_down ();
 }

nano/node/nodeconfig.cpp

@@ -137,6 +137,8 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	{
 		secondary_work_peers_l->push_back (boost::str (boost::format ("%1%:%2%") % i->first % i->second));
 	}
+	experimental_l.put ("max_pruning_age", max_pruning_age.count (), "Time limit for blocks age after pruning.\ntype:seconds");
+	experimental_l.put ("max_pruning_depth", max_pruning_depth, "Limit for full blocks in chain after pruning.\ntype:uint64");
 	toml.put_child ("experimental", experimental_l);
 
 	nano::tomlconfig callback_l;
@@ -389,6 +391,10 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 					this->deserialize_address (entry_a, this->secondary_work_peers);
 				});
 			}
+			auto max_pruning_age_l (max_pruning_age.count ());
+			experimental_config_l.get ("max_pruning_age", max_pruning_age_l);
+			max_pruning_age = std::chrono::seconds (max_pruning_age_l);
+			experimental_config_l.get<uint64_t> ("max_pruning_depth", max_pruning_depth);
 		}
 
 		// Validate ranges
@@ -437,6 +443,10 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 		{
 			toml.get_error ().set ((boost::format ("block_processor_batch_max_time value must be equal or larger than %1%ms") % network_params.node.process_confirmed_interval.count ()).str ());
 		}
+		if (max_pruning_age < std::chrono::seconds (5 * 60) && !network.is_dev_network ())
+		{
+			toml.get_error ().set ("max_pruning_age must be greater than or equal to 5 minutes");
+		}
 	}
 	catch (std::runtime_error const & ex)
 	{

nano/node/nodeconfig.hpp

@@ -100,6 +100,8 @@ public:
 	std::chrono::seconds work_watcher_period{ std::chrono::seconds (5) };
 	double max_work_generate_multiplier{ 64. };
 	uint32_t max_queued_requests{ 512 };
+	std::chrono::seconds max_pruning_age{ !network_params.network.is_beta_network () ? std::chrono::seconds (24 * 60 * 60) : std::chrono::seconds (5 * 60) }; // 1 day; 5 minutes for beta network
+	uint64_t max_pruning_depth{ 0 };
 	nano::rocksdb_config rocksdb_config;
 	nano::lmdb_config lmdb_config;
 	nano::frontiers_confirmation_mode frontiers_confirmation{ nano::frontiers_confirmation_mode::automatic };
@@ -139,6 +141,7 @@ public:
 	bool allow_bootstrap_peers_duplicates{ false };
 	bool disable_max_peers_per_ip{ false }; // For testing only
 	bool force_use_write_database_queue{ false }; // For testing only. RocksDB does not use the database queue, but some tests rely on it being used.
+	bool enable_pruning{ false };
 	bool fast_bootstrap{ false };
 	bool read_only{ false };
 	nano::confirmation_height_mode confirmation_height_processor_mode{ nano::confirmation_height_mode::automatic };

nano/secure/ledger.hpp

@@ -67,6 +67,7 @@ public:
 	std::atomic<size_t> bootstrap_weights_size{ 0 };
 	uint64_t bootstrap_weight_max_blocks{ 1 };
 	std::atomic<bool> check_bootstrap_weights;
+	bool pruning{ false };
 	std::function<void()> epoch_2_started_cb;
 
 private: