Node configuration auto updater (#4579)

This pull request introduces a new command line feature --update_config , which updates the nodes current configuration by merging custom settings into the latest configuration template. Custom settings will be uncommented in the new configuration, while default values will be commented.
This feature simplifies the process for node operators to maintain an up-to-date configuration without manually merging the current configuration with the new one.

Features:

Updates config file with new entries, tables and documentation
Removes entries that are no longer referenced in code
Preserves custom values from current configuration
Formatted output
Limitations:

Currently only works with node configuration file but could easily be extended to rpc an log configs

nano/core_test/toml.cpp

@@ -1066,3 +1066,47 @@ TEST (toml, log_config_no_required)
 
 	ASSERT_FALSE (toml.get_error ()) << toml.get_error ().get_message ();
 }
+
+TEST (toml, merge_config_files)
+{
+	nano::network_params network_params{ nano::network_constants::active_network };
+	nano::tomlconfig default_toml;
+	nano::tomlconfig current_toml;
+	nano::tomlconfig merged_toml;
+	nano::daemon_config default_config{ ".", network_params };
+	nano::daemon_config current_config{ ".", network_params };
+	nano::daemon_config merged_config{ ".", network_params };
+
+	std::stringstream ss;
+
+	ss << R"toml(
+	[node]
+	 active_elections_size = 999
+	 # backlog_scan_batch_size = 7777
+	[node.bootstrap_ascending]
+	 block_wait_count = 33333
+	 old_entry = 34
+	)toml";
+
+	current_toml.read (ss);
+	current_config.deserialize_toml (current_toml);
+
+	current_config.serialize_toml (current_toml);
+	default_config.serialize_toml (default_toml);
+
+	auto merged_config_string = current_toml.merge_defaults (current_toml, default_toml);
+
+	// Configs have been merged. Let's read and parse the new config file and verify the values
+
+	std::stringstream ss2;
+	ss2 << merged_config_string;
+
+	merged_toml.read (ss2);
+	merged_config.deserialize_toml (merged_toml);
+
+	ASSERT_NE (merged_config.node.active_elections_size, default_config.node.active_elections_size);
+	ASSERT_EQ (merged_config.node.active_elections_size, 999);
+	ASSERT_NE (merged_config.node.backlog_scan_batch_size, 7777);
+	ASSERT_EQ (merged_config.node.bootstrap_ascending.block_wait_count, 33333);
+	ASSERT_TRUE (merged_config_string.find ("old_entry") == std::string::npos);
+}

nano/lib/tomlconfig.cpp

@@ -186,6 +186,38 @@ void nano::tomlconfig::erase_default_values (tomlconfig & defaults_a)
 	erase_defaults (defaults_l.get_tree (), self.get_tree (), get_tree ());
 }
 
+// Merges two TOML configurations and commenting values that are identical
+std::string nano::tomlconfig::merge_defaults (nano::tomlconfig & current_config, nano::tomlconfig & default_config)
+{
+	// Serialize both configs to commented strings
+	std::string defaults_str = default_config.to_string (true);
+	std::string current_str = current_config.to_string (true);
+
+	// Read both configs line by line
+	std::istringstream stream_defaults (defaults_str);
+	std::istringstream stream_current (current_str);
+	std::string line_defaults, line_current, result;
+
+	while (std::getline (stream_defaults, line_defaults) && std::getline (stream_current, line_current))
+	{
+		if (line_defaults == line_current)
+		{
+			// Use default value
+			result += line_defaults + "\n";
+		}
+		else
+		{
+			// Non default value. Removing the # to uncomment
+			size_t pos = line_current.find ('#');
+			debug_assert (pos != std::string::npos);
+			debug_assert (pos < line_current.length ());
+			result += line_current.substr (0, pos) + line_current.substr (pos + 1) + "\n";
+		}
+	}
+
+	return result;
+}
+
 std::string nano::tomlconfig::to_string (bool comment_values)
 {
 	std::stringstream ss, ss_processed;

nano/lib/tomlconfig.hpp

@@ -48,6 +48,7 @@ public:
 	std::shared_ptr<cpptoml::array> create_array (std::string const & key, boost::optional<char const *> documentation_a);
 	void erase_default_values (tomlconfig & defaults_a);
 	std::string to_string (bool comment_values);
+	std::string merge_defaults (nano::tomlconfig & current_config, nano::tomlconfig & default_config);
 
 	/** Set value for the given key. Any existing value will be overwritten. */
 	template <typename T>

nano/node/cli.cpp

@@ -62,6 +62,7 @@ void nano::add_node_options (boost::program_options::options_description & descr
 	("migrate_database_lmdb_to_rocksdb", "Migrates LMDB database to RocksDB")
 	("diagnostics", "Run internal diagnostics")
 	("generate_config", boost::program_options::value<std::string> (), "Write configuration to stdout, populated with defaults suitable for this system. Pass the configuration type node, rpc or log. See also use_defaults.")
+	("update_config", "Reads the current node configuration and updates it with missing keys and values and delete keys that are no longer used. Updated configuration is written to stdout.")
 	("key_create", "Generates a adhoc random keypair and prints it to stdout")
 	("key_expand", "Derive public key and account number from <key>")
 	("wallet_add_adhoc", "Insert <key> in to <wallet>")
@@ -713,6 +714,31 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
 			}
 		}
 	}
+	else if (vm.count ("update_config"))
+	{
+		nano::network_params network_params{ nano::network_constants::active_network };
+		nano::tomlconfig default_toml;
+		nano::tomlconfig current_toml;
+		nano::daemon_config default_config{ data_path, network_params };
+		nano::daemon_config current_config{ data_path, network_params };
+
+		std::vector<std::string> config_overrides;
+		auto error = nano::read_node_config_toml (data_path, current_config, config_overrides);
+		if (error)
+		{
+			std::cerr << "Could not read existing config file\n";
+			ec = nano::error_cli::reading_config;
+		}
+		else
+		{
+			current_config.serialize_toml (current_toml);
+			default_config.serialize_toml (default_toml);
+
+			auto output = current_toml.merge_defaults (current_toml, default_toml);
+
+			std::cout << output;
+		}
+	}
 	else if (vm.count ("diagnostics"))
 	{
 		auto inactive_node = nano::default_inactive_node (data_path, vm);