From 20b52accf14ea0fc9e21d7c3fb55bb0b7aef8146 Mon Sep 17 00:00:00 2001 From: cryptocode Date: Fri, 23 Aug 2019 18:49:32 +0200 Subject: [PATCH] TOML config file support and migration (#2221) * TOML config file support and migration * Add config override support via a CLI option * Incorporate config changes from #2214 * Remove deprecation as this currently fails to compile on CI * Add default empty vector to read_node_config_toml * Make sure override streams are not empty * Adjust unit tests with changed base/override order * Incorporate config changes from #2198 and reorder some methods * httpcallback section, and documentation improvements * Clarify data type for wallet account, formatting * CI formatting * Improved node config unit test coverage, address review feedback * Full node and rpc config file coverage. Fix httpcallback deserialization. * Improve toml diff test * Use const ref in toml config, fix schema type for rpc_path * Incorporate work watcher period from #2222 * Add work_watcher_period validation as well as unit test --- .gitmodules | 3 + CMakeLists.txt | 1 + cpptoml | 1 + nano/core_test/CMakeLists.txt | 1 + nano/core_test/toml.cpp | 487 ++++++++++++++++++++++++++++ nano/core_test/uint256_union.cpp | 1 - nano/lib/CMakeLists.txt | 4 + nano/lib/config.hpp | 15 + nano/lib/configbase.hpp | 97 ++++++ nano/lib/jsonconfig.hpp | 99 +----- nano/lib/rpcconfig.cpp | 131 ++++++++ nano/lib/rpcconfig.hpp | 6 + nano/lib/stats.cpp | 50 +++ nano/lib/stats.hpp | 4 +- nano/lib/tomlconfig.hpp | 540 +++++++++++++++++++++++++++++++ nano/lib/walletconfig.cpp | 52 +++ nano/lib/walletconfig.hpp | 26 ++ nano/load_test/entry.cpp | 36 ++- nano/nano_node/daemon.cpp | 7 +- nano/nano_node/entry.cpp | 7 + nano/nano_rpc/entry.cpp | 2 +- nano/nano_wallet/entry.cpp | 243 +++----------- nano/node/cli.cpp | 36 ++- nano/node/daemonconfig.cpp | 191 +++++++++-- nano/node/daemonconfig.hpp | 16 +- nano/node/diagnosticsconfig.cpp | 31 ++ nano/node/diagnosticsconfig.hpp | 4 +- nano/node/ipcconfig.cpp | 52 +++ nano/node/ipcconfig.hpp | 4 +- nano/node/logging.cpp | 62 ++++ nano/node/logging.hpp | 3 + nano/node/node_rpc_config.cpp | 34 ++ nano/node/node_rpc_config.hpp | 4 + nano/node/nodeconfig.cpp | 276 ++++++++++++++++ nano/node/nodeconfig.hpp | 4 + nano/node/openclconfig.cpp | 24 ++ nano/node/openclconfig.hpp | 5 +- nano/node/websocketconfig.cpp | 17 + nano/node/websocketconfig.hpp | 3 + 39 files changed, 2214 insertions(+), 365 deletions(-) create mode 160000 cpptoml create mode 100644 nano/core_test/toml.cpp create mode 100644 nano/lib/configbase.hpp create mode 100644 nano/lib/tomlconfig.hpp create mode 100644 nano/lib/walletconfig.cpp create mode 100644 nano/lib/walletconfig.hpp diff --git a/.gitmodules b/.gitmodules index 37ef9b39..c0340b2a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,3 +15,6 @@ path = gtest url = https://github.com/google/googletest.git branch = v1.8.x +[submodule "cpptoml"] + path = cpptoml + url = https://github.com/cryptocode/cpptoml.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 47f03a43..fb71a03a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,6 +193,7 @@ else () set (BOOST_PROCESS_SUPPORTED 1) endif () +include_directories(cpptoml/include) add_subdirectory(crypto/ed25519-donna) set (UPNPC_BUILD_SHARED OFF CACHE BOOL "") diff --git a/cpptoml b/cpptoml new file mode 160000 index 00000000..53996500 --- /dev/null +++ b/cpptoml @@ -0,0 +1 @@ +Subproject commit 539965005606a481ae29b723e21aa254d3c29c62 diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index d692f3eb..0b6b2e1e 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable (core_test ${rocksdb_test} signing.cpp socket.cpp + toml.cpp timer.cpp uint256_union.cpp utility.cpp diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp new file mode 100644 index 00000000..6645895a --- /dev/null +++ b/nano/core_test/toml.cpp @@ -0,0 +1,487 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +using namespace std::chrono_literals; + +/** Ensure only different values survive a toml diff */ +TEST (toml, diff) +{ + nano::tomlconfig defaults, other; + + // Defaults + std::stringstream ss; + ss << R"toml( + a = false + b = false + )toml"; + + defaults.read (ss); + + // User file. The rpc section is the same and doesn't need to be emitted + std::stringstream ss_override; + ss_override << R"toml( + a = true + b = false + )toml"; + + other.read (ss_override); + other.erase_default_values (defaults); + + ASSERT_TRUE (other.has_key ("a")); + ASSERT_FALSE (other.has_key ("b")); +} + +/** Diff on equal toml files leads to an empty result */ +TEST (toml, diff_equal) +{ + nano::tomlconfig defaults, other; + + std::stringstream ss; + ss << R"toml( + [node] + allow_local_peers = false + )toml"; + + defaults.read (ss); + + std::stringstream ss_override; + ss_override << R"toml( + [node] + allow_local_peers = false + )toml"; + + other.read (ss_override); + other.erase_default_values (defaults); + ASSERT_TRUE (other.empty ()); +} + +TEST (toml, daemon_config_update_array) +{ + nano::tomlconfig t; + boost::filesystem::path data_path ("."); + nano::daemon_config c (data_path); + c.node.preconfigured_peers.push_back ("test-peer.org"); + c.serialize_toml (t); + c.deserialize_toml (t); + ASSERT_EQ (c.node.preconfigured_peers[0], "test-peer.org"); +} + +/** Empty config file should match a default config object */ +TEST (toml, daemon_config_deserialize_defaults) +{ + std::stringstream ss; + ss << R"toml( + )toml"; + + nano::tomlconfig t; + t.read (ss); + nano::daemon_config c; + nano::daemon_config defaults; + c.deserialize_toml (t); + ASSERT_EQ (c.opencl_enable, defaults.opencl_enable); + ASSERT_EQ (c.opencl.device, defaults.opencl.device); + ASSERT_EQ (c.opencl.platform, defaults.opencl.platform); + ASSERT_EQ (c.opencl.threads, defaults.opencl.threads); + ASSERT_EQ (c.rpc.enable_sign_hash, false); + ASSERT_EQ (c.rpc.max_work_generate_difficulty, 0xffffffffc0000000); + ASSERT_EQ (c.rpc.child_process.enable, false); +} + +TEST (toml, optional_child) +{ + std::stringstream ss; + ss << R"toml( + [child] + val=1 + )toml"; + + nano::tomlconfig t; + t.read (ss); + auto c1 = t.get_required_child ("child"); + int val = 0; + c1.get_required ("val", val); + ASSERT_EQ (val, 1); + auto c2 = t.get_optional_child ("child2"); + ASSERT_FALSE (c2); +} + +/** Config settings passed via CLI overrides the config file settings. This is solved +using an override stream. */ +TEST (toml, dot_child_syntax) +{ + std::stringstream ss_override; + ss_override << R"toml( + node.a = 1 + node.b = 2 + )toml"; + + std::stringstream ss; + ss << R"toml( + [node] + b=5 + c=3 + )toml"; + + nano::tomlconfig t; + t.read (ss_override, ss); + + auto node = t.get_required_child ("node"); + uint16_t a, b, c; + node.get ("a", a); + ASSERT_EQ (a, 1); + node.get ("b", b); + ASSERT_EQ (b, 2); + node.get ("c", c); + ASSERT_EQ (c, 3); +} + +TEST (toml, base_override) +{ + std::stringstream ss_base; + ss_base << R"toml( + node.peering_port=7075 + )toml"; + + std::stringstream ss_override; + ss_override << R"toml( + node.peering_port=8075 + node.too_big=70000 + )toml"; + + nano::tomlconfig t; + t.read (ss_override, ss_base); + + // Query optional existent value + uint16_t port = 0; + t.get_optional ("node.peering_port", port); + ASSERT_EQ (port, 8075); + ASSERT_FALSE (t.get_error ()); + + // Query optional non-existent value, make sure we get default and no errors + port = 65535; + t.get_optional ("node.peering_port_non_existent", port); + ASSERT_EQ (port, 65535); + ASSERT_FALSE (t.get_error ()); + t.get_error ().clear (); + + // Query required non-existent value, make sure it errors + t.get_required ("node.peering_port_not_existent", port); + ASSERT_EQ (port, 65535); + ASSERT_TRUE (t.get_error ()); + ASSERT_TRUE (t.get_error () == nano::error_config::missing_value); + t.get_error ().clear (); + + // Query uint16 that's too big, make sure we have an error + t.get_required ("node.too_big", port); + ASSERT_TRUE (t.get_error ()); + ASSERT_TRUE (t.get_error () == nano::error_config::invalid_value); +} + +TEST (toml, put) +{ + nano::tomlconfig config; + nano::tomlconfig config_node; + // Overwrite value and add to child node + config_node.put ("port", "7074"); + config_node.put ("port", "7075"); + config.put_child ("node", config_node); + uint16_t port; + config.get_required ("node.port", port); + ASSERT_EQ (port, 7075); + ASSERT_FALSE (config.get_error ()); +} + +TEST (toml, array) +{ + nano::tomlconfig config; + nano::tomlconfig config_node; + config.put_child ("node", config_node); + config_node.push ("items", "item 1"); + config_node.push ("items", "item 2"); + int i = 1; + config_node.array_entries_required ("items", [&i](std::string item) { + ASSERT_TRUE (item == std::string ("item ") + std::to_string (i)); + i++; + }); +} + +/** Deserialize a node config with non-default values */ +TEST (toml, daemon_config_deserialize_no_defaults) +{ + std::stringstream ss; + + // A config file with values that differs from test-net defaults + ss << R"toml( + [node] + active_elections_size = 999 + allow_local_peers = false + backup_before_upgrade = true + bandwidth_limit = 999 + block_processor_batch_max_time = 999 + bootstrap_connections = 999 + bootstrap_connections_max = 999 + bootstrap_fraction_numerator = 999 + confirmation_history_size = 999 + enable_voting = false + external_address = "0:0:0:0:0:ffff:7f01:101" + external_port = 999 + io_threads = 999 + lmdb_max_dbs = 999 + network_threads = 999 + online_weight_minimum = "999" + online_weight_quorum = 99 + password_fanout = 999 + peering_port = 999 + pow_sleep_interval= 999 + preconfigured_peers = ["test.org"] + preconfigured_representatives = ["nano_3arg3asgtigae3xckabaaewkx3bzsh7nwz7jkmjos79ihyaxwphhm6qgjps4"] + receive_minimum = "999" + signature_checker_threads = 999 + tcp_incoming_connections_max = 999 + tcp_io_timeout = 999 + unchecked_cutoff_time = 999 + use_memory_pools = false + vote_generator_delay = 999 + vote_generator_threshold = 9 + vote_minimum = "999" + work_peers = ["test.org:999"] + work_threads = 999 + work_watcher_period = 999 + [node.diagnostics.txn_tracking] + enable = true + ignore_writes_below_block_processor_max_time = false + min_read_txn_time = 999 + min_write_txn_time = 999 + + [node.httpcallback] + address = "test.org" + port = 999 + target = "/test" + + [node.ipc.local] + allow_unsafe = true + enable = true + io_timeout = 999 + path = "/tmp/test" + + [node.ipc.tcp] + enable = true + io_timeout = 999 + port = 999 + + [node.logging] + bulk_pull = true + flush = false + insufficient_work = false + ledger = true + ledger_duplicate = true + log_ipc = false + log_to_cerr = true + max_size = 999 + min_time_between_output = 999 + network = false + network_keepalive = true + network_message = true + network_node_id_handshake = true + network_packet = true + network_publish = true + network_timeout = true + node_lifetime_tracing = true + rotation_size = 999 + single_line_record = true + timing = true + upnp_details = true + vote = true + work_generation_time = false + + [node.statistics.log] + filename_counters = "testcounters.stat" + filename_samples = "testsamples.stat" + headers = false + interval_counters = 999 + interval_samples = 999 + rotation_count = 999 + + [node.statistics.sampling] + capacity = 999 + enable = true + interval = 999 + + [node.websocket] + address = "0:0:0:0:0:ffff:7f01:101" + enable = true + port = 999 + + [opencl] + device = 999 + enable = true + platform = 999 + threads = 999 + + [rpc] + enable = true + enable_sign_hash = true + max_work_generate_difficulty = "ffffffffc9999999" + + [rpc.child_process] + enable = true + rpc_path = "/test/nano_rpc" + )toml"; + + nano::tomlconfig toml; + toml.read (ss); + nano::daemon_config conf; + nano::daemon_config defaults; + conf.deserialize_toml (toml); + + ASSERT_FALSE (toml.get_error ()) << toml.get_error ().get_message (); + + ASSERT_NE (conf.opencl_enable, defaults.opencl_enable); + ASSERT_NE (conf.opencl.device, defaults.opencl.device); + ASSERT_NE (conf.opencl.platform, defaults.opencl.platform); + ASSERT_NE (conf.opencl.threads, defaults.opencl.threads); + ASSERT_NE (conf.rpc_enable, defaults.rpc_enable); + ASSERT_NE (conf.rpc.enable_sign_hash, defaults.rpc.enable_sign_hash); + ASSERT_NE (conf.rpc.max_work_generate_difficulty, defaults.rpc.max_work_generate_difficulty); + ASSERT_NE (conf.rpc.child_process.enable, defaults.rpc.child_process.enable); + ASSERT_NE (conf.rpc.child_process.rpc_path, defaults.rpc.child_process.rpc_path); + + ASSERT_NE (conf.node.active_elections_size, defaults.node.active_elections_size); + ASSERT_NE (conf.node.allow_local_peers, defaults.node.allow_local_peers); + ASSERT_NE (conf.node.backup_before_upgrade, defaults.node.backup_before_upgrade); + ASSERT_NE (conf.node.bandwidth_limit, defaults.node.bandwidth_limit); + ASSERT_NE (conf.node.block_processor_batch_max_time, defaults.node.block_processor_batch_max_time); + ASSERT_NE (conf.node.bootstrap_connections, defaults.node.bootstrap_connections); + ASSERT_NE (conf.node.bootstrap_connections_max, defaults.node.bootstrap_connections_max); + ASSERT_NE (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator); + ASSERT_NE (conf.node.confirmation_history_size, defaults.node.confirmation_history_size); + ASSERT_NE (conf.node.enable_voting, defaults.node.enable_voting); + ASSERT_NE (conf.node.external_address, defaults.node.external_address); + ASSERT_NE (conf.node.external_port, defaults.node.external_port); + ASSERT_NE (conf.node.io_threads, defaults.node.io_threads); + ASSERT_NE (conf.node.lmdb_max_dbs, defaults.node.lmdb_max_dbs); + ASSERT_NE (conf.node.network_threads, defaults.node.network_threads); + ASSERT_NE (conf.node.work_watcher_period, defaults.node.work_watcher_period); + ASSERT_NE (conf.node.online_weight_minimum, defaults.node.online_weight_minimum); + ASSERT_NE (conf.node.online_weight_quorum, defaults.node.online_weight_quorum); + ASSERT_NE (conf.node.password_fanout, defaults.node.password_fanout); + ASSERT_NE (conf.node.peering_port, defaults.node.peering_port); + ASSERT_NE (conf.node.pow_sleep_interval, defaults.node.pow_sleep_interval); + ASSERT_NE (conf.node.preconfigured_peers, defaults.node.preconfigured_peers); + ASSERT_NE (conf.node.preconfigured_representatives, defaults.node.preconfigured_representatives); + ASSERT_NE (conf.node.receive_minimum, defaults.node.receive_minimum); + ASSERT_NE (conf.node.signature_checker_threads, defaults.node.signature_checker_threads); + ASSERT_NE (conf.node.tcp_incoming_connections_max, defaults.node.tcp_incoming_connections_max); + ASSERT_NE (conf.node.tcp_io_timeout, defaults.node.tcp_io_timeout); + ASSERT_NE (conf.node.unchecked_cutoff_time, defaults.node.unchecked_cutoff_time); + ASSERT_NE (conf.node.use_memory_pools, defaults.node.use_memory_pools); + ASSERT_NE (conf.node.vote_generator_delay, defaults.node.vote_generator_delay); + ASSERT_NE (conf.node.vote_generator_threshold, defaults.node.vote_generator_threshold); + ASSERT_NE (conf.node.vote_minimum, defaults.node.vote_minimum); + ASSERT_NE (conf.node.work_peers, defaults.node.work_peers); + ASSERT_NE (conf.node.work_threads, defaults.node.work_threads); + + ASSERT_NE (conf.node.logging.bulk_pull_logging_value, defaults.node.logging.bulk_pull_logging_value); + ASSERT_NE (conf.node.logging.flush, defaults.node.logging.flush); + ASSERT_NE (conf.node.logging.insufficient_work_logging_value, defaults.node.logging.insufficient_work_logging_value); + ASSERT_NE (conf.node.logging.ledger_logging_value, defaults.node.logging.ledger_logging_value); + ASSERT_NE (conf.node.logging.ledger_duplicate_logging_value, defaults.node.logging.ledger_duplicate_logging_value); + ASSERT_NE (conf.node.logging.log_ipc_value, defaults.node.logging.log_ipc_value); + ASSERT_NE (conf.node.logging.log_to_cerr_value, defaults.node.logging.log_to_cerr_value); + ASSERT_NE (conf.node.logging.max_size, defaults.node.logging.max_size); + ASSERT_NE (conf.node.logging.min_time_between_log_output.count (), defaults.node.logging.min_time_between_log_output.count ()); + ASSERT_NE (conf.node.logging.network_logging_value, defaults.node.logging.network_logging_value); + ASSERT_NE (conf.node.logging.network_keepalive_logging_value, defaults.node.logging.network_keepalive_logging_value); + ASSERT_NE (conf.node.logging.network_message_logging_value, defaults.node.logging.network_message_logging_value); + ASSERT_NE (conf.node.logging.network_node_id_handshake_logging_value, defaults.node.logging.network_node_id_handshake_logging_value); + ASSERT_NE (conf.node.logging.network_packet_logging_value, defaults.node.logging.network_packet_logging_value); + ASSERT_NE (conf.node.logging.network_publish_logging_value, defaults.node.logging.network_publish_logging_value); + ASSERT_NE (conf.node.logging.network_timeout_logging_value, defaults.node.logging.network_timeout_logging_value); + ASSERT_NE (conf.node.logging.node_lifetime_tracing_value, defaults.node.logging.node_lifetime_tracing_value); + ASSERT_NE (conf.node.logging.rotation_size, defaults.node.logging.rotation_size); + ASSERT_NE (conf.node.logging.single_line_record_value, defaults.node.logging.single_line_record_value); + ASSERT_NE (conf.node.logging.timing_logging_value, defaults.node.logging.timing_logging_value); + ASSERT_NE (conf.node.logging.upnp_details_logging_value, defaults.node.logging.upnp_details_logging_value); + ASSERT_NE (conf.node.logging.vote_logging_value, defaults.node.logging.vote_logging_value); + ASSERT_NE (conf.node.logging.work_generation_time_value, defaults.node.logging.work_generation_time_value); + + ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); + ASSERT_NE (conf.node.websocket_config.address, defaults.node.websocket_config.address); + ASSERT_NE (conf.node.websocket_config.port, defaults.node.websocket_config.port); + + ASSERT_NE (conf.node.callback_address, defaults.node.callback_address); + ASSERT_NE (conf.node.callback_port, defaults.node.callback_port); + ASSERT_NE (conf.node.callback_target, defaults.node.callback_target); + + ASSERT_NE (conf.node.ipc_config.transport_domain.allow_unsafe, defaults.node.ipc_config.transport_domain.allow_unsafe); + ASSERT_NE (conf.node.ipc_config.transport_domain.enabled, defaults.node.ipc_config.transport_domain.enabled); + ASSERT_NE (conf.node.ipc_config.transport_domain.io_timeout, defaults.node.ipc_config.transport_domain.io_timeout); + ASSERT_NE (conf.node.ipc_config.transport_domain.path, defaults.node.ipc_config.transport_domain.path); + ASSERT_NE (conf.node.ipc_config.transport_tcp.enabled, defaults.node.ipc_config.transport_tcp.enabled); + ASSERT_NE (conf.node.ipc_config.transport_tcp.io_timeout, defaults.node.ipc_config.transport_tcp.io_timeout); + ASSERT_NE (conf.node.ipc_config.transport_tcp.port, defaults.node.ipc_config.transport_tcp.port); + + ASSERT_NE (conf.node.diagnostics_config.txn_tracking.enable, defaults.node.diagnostics_config.txn_tracking.enable); + ASSERT_NE (conf.node.diagnostics_config.txn_tracking.ignore_writes_below_block_processor_max_time, defaults.node.diagnostics_config.txn_tracking.ignore_writes_below_block_processor_max_time); + ASSERT_NE (conf.node.diagnostics_config.txn_tracking.min_read_txn_time, defaults.node.diagnostics_config.txn_tracking.min_read_txn_time); + ASSERT_NE (conf.node.diagnostics_config.txn_tracking.min_write_txn_time, defaults.node.diagnostics_config.txn_tracking.min_write_txn_time); + + ASSERT_NE (conf.node.stat_config.sampling_enabled, defaults.node.stat_config.sampling_enabled); + ASSERT_NE (conf.node.stat_config.interval, defaults.node.stat_config.interval); + ASSERT_NE (conf.node.stat_config.capacity, defaults.node.stat_config.capacity); + ASSERT_NE (conf.node.stat_config.log_rotation_count, defaults.node.stat_config.log_rotation_count); + ASSERT_NE (conf.node.stat_config.log_interval_samples, defaults.node.stat_config.log_interval_samples); + ASSERT_NE (conf.node.stat_config.log_interval_counters, defaults.node.stat_config.log_interval_counters); + ASSERT_NE (conf.node.stat_config.log_headers, defaults.node.stat_config.log_headers); + ASSERT_NE (conf.node.stat_config.log_counters_filename, defaults.node.stat_config.log_counters_filename); + ASSERT_NE (conf.node.stat_config.log_samples_filename, defaults.node.stat_config.log_samples_filename); +} + +/** Deserialize an rpc config with non-default values */ +TEST (toml, rpc_config_deserialize_no_defaults) +{ + std::stringstream ss; + + // A config file with values that differs from test-net defaults + ss << R"toml( + address = "0:0:0:0:0:ffff:7f01:101" + enable_control = true + max_json_depth = 9 + max_request_size = 999 + port = 999 + [process] + io_threads = 999 + ipc_address = "0:0:0:0:0:ffff:7f01:101" + ipc_port = 999 + num_ipc_connections = 999 + )toml"; + + nano::tomlconfig toml; + toml.read (ss); + nano::rpc_config conf; + nano::rpc_config defaults; + conf.deserialize_toml (toml); + + ASSERT_FALSE (toml.get_error ()) << toml.get_error ().get_message (); + + ASSERT_NE (conf.address, defaults.address); + ASSERT_NE (conf.enable_control, defaults.enable_control); + ASSERT_NE (conf.max_json_depth, defaults.max_json_depth); + ASSERT_NE (conf.max_request_size, defaults.max_request_size); + ASSERT_NE (conf.port, defaults.port); + + ASSERT_NE (conf.rpc_process.io_threads, defaults.rpc_process.io_threads); + ASSERT_NE (conf.rpc_process.ipc_address, defaults.rpc_process.ipc_address); + ASSERT_NE (conf.rpc_process.ipc_port, defaults.rpc_process.ipc_port); + ASSERT_NE (conf.rpc_process.num_ipc_connections, defaults.rpc_process.num_ipc_connections); +} diff --git a/nano/core_test/uint256_union.cpp b/nano/core_test/uint256_union.cpp index 75d5e673..92084d66 100644 --- a/nano/core_test/uint256_union.cpp +++ b/nano/core_test/uint256_union.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index 7a47f3c7..44ed9877 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -20,6 +20,7 @@ add_library (nano_lib blocks.cpp config.hpp config.cpp + configbase.hpp errors.hpp errors.cpp ipc.hpp @@ -39,8 +40,11 @@ add_library (nano_lib stats.hpp stats.cpp timer.hpp + tomlconfig.hpp utility.hpp utility.cpp + walletconfig.hpp + walletconfig.cpp work.hpp work.cpp) diff --git a/nano/lib/config.hpp b/nano/lib/config.hpp index 47f2df24..22585761 100644 --- a/nano/lib/config.hpp +++ b/nano/lib/config.hpp @@ -162,6 +162,21 @@ inline boost::filesystem::path get_rpc_config_path (boost::filesystem::path cons return data_path / "rpc_config.json"; } +inline boost::filesystem::path get_node_toml_config_path (boost::filesystem::path const & data_path) +{ + return data_path / "config-node.toml"; +} + +inline boost::filesystem::path get_rpc_toml_config_path (boost::filesystem::path const & data_path) +{ + return data_path / "config-rpc.toml"; +} + +inline boost::filesystem::path get_qtwallet_toml_config_path (boost::filesystem::path const & data_path) +{ + return data_path / "config-qtwallet.toml"; +} + /** Called by gtest_main to enforce test network */ void force_nano_test_network (); diff --git a/nano/lib/configbase.hpp b/nano/lib/configbase.hpp new file mode 100644 index 00000000..78bcc76b --- /dev/null +++ b/nano/lib/configbase.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include + +namespace nano +{ +/** Type trait to determine if T is compatible with boost's lexical_cast */ +template +struct is_lexical_castable : std::integral_constant::value && (boost::has_right_shift, T>::value || boost::has_right_shift, T>::value))> +{ +}; + +/* Type descriptions are used to automatically construct configuration error messages */ +// clang-format off +template inline std::string type_desc (void) { return "an unknown type"; } +template <> inline std::string type_desc (void) { return "an integer between -128 and 127"; } +template <> inline std::string type_desc (void) { return "an integer between 0 and 255"; } +template <> inline std::string type_desc (void) { return "an integer between -32768 and 32767"; } +template <> inline std::string type_desc (void) { return "an integer between 0 and 65535"; } +template <> inline std::string type_desc (void) { return "a 32-bit signed integer"; } +template <> inline std::string type_desc (void) { return "a 32-bit unsigned integer"; } +template <> inline std::string type_desc (void) { return "a 64-bit signed integer"; } +template <> inline std::string type_desc (void) { return "a 64-bit unsigned integer"; } +template <> inline std::string type_desc (void) { return "a single precision floating point number"; } +template <> inline std::string type_desc (void) { return "a double precison floating point number"; } +template <> inline std::string type_desc (void) { return "a character"; } +template <> inline std::string type_desc (void) { return "a string"; } +template <> inline std::string type_desc (void) { return "a boolean"; } +template <> inline std::string type_desc (void) { return "an IP address"; } +// clang-format on + +/** Base type for configuration wrappers */ +class configbase : public nano::error_aware<> +{ +public: + configbase () = default; + configbase (std::shared_ptr const & error_a) : + error (error_a) + { + } + + /** Returns the current error */ + nano::error & get_error () override + { + return *error; + } + + /** Turn on or off automatic error message generation */ + void set_auto_error_message (bool auto_a) + { + auto_error_message = auto_a; + } + +protected: + template + void construct_error_message (bool optional, std::string const & key) + { + if (auto_error_message && *error) + { + if (optional) + { + error->set_message (key + " is not " + type_desc ()); + } + else + { + error->set_message (key + " is required and must be " + type_desc ()); + } + } + } + + /** Set error if not already set. That is, first error remains until get_error().clear() is called. */ + template + void conditionally_set_error (V error_a, bool optional, std::string const & key) + { + if (!*error) + { + *error = error_a; + construct_error_message (optional, key); + } + } + + /** We're a nano::error_aware type. Child nodes share the error state. */ + std::shared_ptr error; + + /** If set, automatically construct error messages based on parameters and type information. */ + bool auto_error_message{ true }; +}; +} diff --git a/nano/lib/jsonconfig.hpp b/nano/lib/jsonconfig.hpp index 8d31d8c9..ce9d47ab 100644 --- a/nano/lib/jsonconfig.hpp +++ b/nano/lib/jsonconfig.hpp @@ -1,46 +1,20 @@ #pragma once #include +#include #include #include #include #include -#include #include #include namespace nano { -/** Type trait to determine if T is compatible with boost's lexical_cast */ -template -struct is_lexical_castable : std::integral_constant::value && (boost::has_right_shift, T>::value || boost::has_right_shift, T>::value))> -{ -}; - -/* Type descriptions are used to automatically construct configuration error messages */ -// clang-format off -template inline std::string type_desc (void) { return "an unknown type"; } -template <> inline std::string type_desc (void) { return "an integer between -128 and 127"; } -template <> inline std::string type_desc (void) { return "an integer between 0 and 255"; } -template <> inline std::string type_desc (void) { return "an integer between -32768 and 32767"; } -template <> inline std::string type_desc (void) { return "an integer between 0 and 65535"; } -template <> inline std::string type_desc (void) { return "a 32-bit signed integer"; } -template <> inline std::string type_desc (void) { return "a 32-bit unsigned integer"; } -template <> inline std::string type_desc (void) { return "a 64-bit signed integer"; } -template <> inline std::string type_desc (void) { return "a 64-bit unsigned integer"; } -template <> inline std::string type_desc (void) { return "a single precision floating point number"; } -template <> inline std::string type_desc (void) { return "a double precison floating point number"; } -template <> inline std::string type_desc (void) { return "a character"; } -template <> inline std::string type_desc (void) { return "a string"; } -template <> inline std::string type_desc (void) { return "a boolean"; } -template <> inline std::string type_desc (void) { return "an IP address"; } -// clang-format on - /** Manages a node in a boost configuration tree. */ -class jsonconfig : public nano::error_aware<> +class jsonconfig : public nano::configbase { public: jsonconfig () : @@ -50,7 +24,7 @@ public: } jsonconfig (boost::property_tree::ptree & tree_a, std::shared_ptr error_a = nullptr) : - tree (tree_a), error (error_a) + nano::configbase (error_a), tree (tree_a) { if (!error) { @@ -234,17 +208,6 @@ public: return *this; } - /** Iterate array entries */ - template - jsonconfig & array_entries (std::function callback) - { - for (auto & entry : tree) - { - callback (entry.second.get ("")); - } - return *this; - } - /** Returns true if \p key_a is present */ bool has_key (std::string const & key_a) { @@ -258,6 +221,17 @@ public: return *this; } + /** Iterate array entries */ + template + jsonconfig & array_entries (std::function callback) + { + for (auto & entry : tree) + { + callback (entry.second.get ("")); + } + return *this; + } + /** Get optional, using \p default_value if \p key is missing. */ template jsonconfig & get_optional (std::string const & key, T & target, T default_value) @@ -300,8 +274,7 @@ public: } /** - * Get value of required key - * @note May set nano::error_config::missing_value if \p key is missing, nano::error_config::invalid_value if value is invalid. + * Get value of optional key. Use default value of data type if missing. */ template T get (std::string const & key) @@ -322,44 +295,7 @@ public: return *this; } - /** Turn on or off automatic error message generation */ - void set_auto_error_message (bool auto_a) - { - auto_error_message = auto_a; - } - - nano::error & get_error () override - { - return *error; - } - protected: - template - void construct_error_message (bool optional, std::string const & key) - { - if (auto_error_message && *error) - { - if (optional) - { - error->set_message (key + " is not " + type_desc ()); - } - else - { - error->set_message (key + " is required and must be " + type_desc ()); - } - } - } - - /** Set error if not already set. That is, first error remains until get_error().clear() is called. */ - template - void conditionally_set_error (V error_a, bool optional, std::string const & key) - { - if (!*error) - { - *error = error_a; - construct_error_message (optional, key); - } - } template ::value>> jsonconfig & get_config (bool optional, std::string key, T & target, T default_value = T ()) { @@ -495,10 +431,5 @@ private: /** The property node being managed */ boost::property_tree::ptree & tree; boost::property_tree::ptree tree_default; - /** If set, automatically construct error messages based on parameters and type information. */ - bool auto_error_message{ true }; - - /** We're a nano::error_aware type. Child nodes share the error state. */ - std::shared_ptr error; }; } diff --git a/nano/lib/rpcconfig.cpp b/nano/lib/rpcconfig.cpp index c2839232..7ef2477c 100644 --- a/nano/lib/rpcconfig.cpp +++ b/nano/lib/rpcconfig.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include @@ -28,6 +29,30 @@ nano::error nano::rpc_secure_config::deserialize_json (nano::jsonconfig & json) return json.get_error (); } +nano::error nano::rpc_secure_config::serialize_toml (nano::tomlconfig & toml) const +{ + toml.put ("enable", enable, "Enable or disable TLS support\ntype:bool"); + toml.put ("verbose_logging", verbose_logging, "Enable or disable verbose logging\ntype:bool"); + toml.put ("server_key_passphrase", server_key_passphrase, "Server key passphrase\ntype:string"); + toml.put ("server_cert_path", server_cert_path, "Directory containing certificates\ntype:string,path"); + toml.put ("server_key_path", server_key_path, "Path to server key PEM file\ntype:string,path"); + toml.put ("server_dh_path", server_dh_path, "Path to Diffie-Hellman params file\ntype:string,path"); + toml.put ("client_certs_path", client_certs_path, "Directory containing client certificates\ntype:string"); + return toml.get_error (); +} + +nano::error nano::rpc_secure_config::deserialize_toml (nano::tomlconfig & toml) +{ + toml.get_required ("enable", enable); + toml.get_required ("verbose_logging", verbose_logging); + toml.get_required ("server_key_passphrase", server_key_passphrase); + toml.get_required ("server_cert_path", server_cert_path); + toml.get_required ("server_key_path", server_key_path); + toml.get_required ("server_dh_path", server_dh_path); + toml.get_required ("client_certs_path", client_certs_path); + return toml.get_error (); +} + nano::rpc_config::rpc_config (bool enable_control_a) : enable_control (enable_control_a) { @@ -114,8 +139,114 @@ nano::error nano::rpc_config::deserialize_json (bool & upgraded_a, nano::jsoncon return json.get_error (); } +nano::error nano::rpc_config::serialize_toml (nano::tomlconfig & toml) const +{ + toml.put ("address", address.to_string (), "Bind address for the RPC server\ntype:string,ip"); + toml.put ("port", port, "Listening port for the RPC server\ntype:uint16"); + toml.put ("enable_control", enable_control, "Enable or disable control-level requests\ntype:bool"); + toml.put ("max_json_depth", max_json_depth, "Maximum number of levels in JSON requests\ntype:uint8"); + toml.put ("max_request_size", max_request_size, "Maximum number of bytes allowed in request bodies\ntype:uint64"); + + nano::tomlconfig rpc_process_l; + rpc_process_l.put ("io_threads", rpc_process.io_threads, "Number of threads used to serve IO\ntype:uint32"); + rpc_process_l.put ("ipc_address", rpc_process.ipc_address.to_string (), "Address of IPC server\ntype:string,ip"); + rpc_process_l.put ("ipc_port", rpc_process.ipc_port, "Listening port of IPC server\ntype:uint16"); + rpc_process_l.put ("num_ipc_connections", rpc_process.num_ipc_connections, "Number of IPC connections to establish\ntype:uint32"); + toml.put_child ("process", rpc_process_l); + return toml.get_error (); +} + +nano::error nano::rpc_config::deserialize_toml (nano::tomlconfig & toml) +{ + if (!toml.empty ()) + { + auto rpc_secure_l (toml.get_optional_child ("secure")); + if (rpc_secure_l) + { + secure.deserialize_toml (*rpc_secure_l); + } + + toml.get_optional ("address", address); + toml.get_optional ("port", port); + toml.get_optional ("enable_control", enable_control); + toml.get_optional ("max_json_depth", max_json_depth); + toml.get_optional ("max_request_size", max_request_size); + + auto rpc_process_l (toml.get_optional_child ("process")); + if (rpc_process_l) + { + rpc_process_l->get_optional ("io_threads", rpc_process.io_threads); + rpc_process_l->get_optional ("ipc_port", rpc_process.ipc_port); + rpc_process_l->get_optional ("ipc_address", rpc_process.ipc_address); + rpc_process_l->get_optional ("num_ipc_connections", rpc_process.num_ipc_connections); + } + } + + return toml.get_error (); +} + namespace nano { +nano::error read_rpc_config_toml (boost::filesystem::path const & data_path_a, nano::rpc_config & config_a) +{ + nano::error error; + auto json_config_path = nano::get_rpc_config_path (data_path_a); + auto toml_config_path = nano::get_rpc_toml_config_path (data_path_a); + if (boost::filesystem::exists (json_config_path)) + { + if (boost::filesystem::exists (toml_config_path)) + { + error = "Both json and toml rpc configuration files exists. " + "Either remove the config.json file and restart, or remove " + "the config-rpc.toml file to start migration on next launch."; + } + else + { + // Migrate + nano::rpc_config config_json_l; + error = read_and_update_rpc_config (data_path_a, config_json_l); + + if (!error) + { + nano::tomlconfig toml_l; + config_json_l.serialize_toml (toml_l); + + // Only write out non-default values + nano::rpc_config config_defaults_l; + nano::tomlconfig toml_defaults_l; + config_defaults_l.serialize_toml (toml_defaults_l); + + toml_l.erase_default_values (toml_defaults_l); + if (!toml_l.empty ()) + { + toml_l.write (toml_config_path); + boost::system::error_code error_chmod; + nano::set_secure_perm_file (toml_config_path, error_chmod); + } + + auto backup_path = data_path_a / "rpc_config_backup_toml_migration.json"; + boost::filesystem::rename (json_config_path, backup_path); + } + } + } + + // Parse and deserialize + nano::tomlconfig toml; + + // Make sure we don't create an empty toml file if it doesn't exist. Running without a toml file is the default. + if (!error && boost::filesystem::exists (toml_config_path)) + { + error = toml.read (toml_config_path); + } + + if (!error) + { + error = config_a.deserialize_toml (toml); + } + + return error; +} + nano::error read_and_update_rpc_config (boost::filesystem::path const & data_path, nano::rpc_config & config_a) { boost::system::error_code error_chmod; diff --git a/nano/lib/rpcconfig.hpp b/nano/lib/rpcconfig.hpp index f444256f..d371c740 100644 --- a/nano/lib/rpcconfig.hpp +++ b/nano/lib/rpcconfig.hpp @@ -12,6 +12,7 @@ namespace nano { class jsonconfig; +class tomlconfig; /** Configuration options for RPC TLS */ class rpc_secure_config final @@ -19,6 +20,8 @@ class rpc_secure_config final public: nano::error serialize_json (nano::jsonconfig &) const; nano::error deserialize_json (nano::jsonconfig &); + nano::error serialize_toml (nano::tomlconfig &) const; + nano::error deserialize_toml (nano::tomlconfig &); /** If true, enable TLS */ bool enable{ false }; @@ -56,6 +59,8 @@ public: explicit rpc_config (bool = false); nano::error serialize_json (nano::jsonconfig &) const; nano::error deserialize_json (bool & upgraded_a, nano::jsonconfig &); + nano::error serialize_toml (nano::tomlconfig &) const; + nano::error deserialize_toml (nano::tomlconfig &); nano::rpc_process_config rpc_process; boost::asio::ip::address_v6 address{ boost::asio::ip::address_v6::loopback () }; @@ -70,6 +75,7 @@ public: } }; +nano::error read_rpc_config_toml (boost::filesystem::path const & data_path_a, nano::rpc_config & config_a); nano::error read_and_update_rpc_config (boost::filesystem::path const & data_path, nano::rpc_config & config_a); std::string get_default_rpc_filepath (); diff --git a/nano/lib/stats.cpp b/nano/lib/stats.cpp index 452f576f..6a814164 100644 --- a/nano/lib/stats.cpp +++ b/nano/lib/stats.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -40,6 +41,55 @@ nano::error nano::stat_config::deserialize_json (nano::jsonconfig & json) return json.get_error (); } +nano::error nano::stat_config::deserialize_toml (nano::tomlconfig & toml) +{ + auto sampling_l (toml.get_optional_child ("sampling")); + if (sampling_l) + { + sampling_l->get ("enable", sampling_enabled); + sampling_l->get ("capacity", capacity); + sampling_l->get ("interval", interval); + } + + auto log_l (toml.get_optional_child ("log")); + if (log_l) + { + log_l->get ("headers", log_headers); + log_l->get ("interval_counters", log_interval_counters); + log_l->get ("interval_samples", log_interval_samples); + log_l->get ("rotation_count", log_rotation_count); + log_l->get ("filename_counters", log_counters_filename); + log_l->get ("filename_samples", log_samples_filename); + + // Don't allow specifying the same file name for counter and samples logs + if (log_counters_filename == log_samples_filename) + { + toml.get_error ().set ("The statistics counter and samples config values must be different"); + } + } + + return toml.get_error (); +} + +nano::error nano::stat_config::serialize_toml (nano::tomlconfig & toml) const +{ + nano::tomlconfig sampling_l; + sampling_l.put ("enable", sampling_enabled, "Enable or disable samling.\ntype:bool"); + sampling_l.put ("capacity", capacity, "How many sample intervals to keep in the ring buffer.\ntype:uint64"); + sampling_l.put ("interval", interval, "Sample interval.\ntype:milliseconds"); + toml.put_child ("sampling", sampling_l); + + nano::tomlconfig log_l; + log_l.put ("headers", log_headers, "If true, write headers on each counter or samples writeout.\nThe header contains log type and the current wall time.\ntype:bool"); + log_l.put ("interval_counters", log_interval_counters, "How often to log counters. 0 disables logging.\ntype:milliseconds"); + log_l.put ("interval_samples", log_interval_samples, "How often to log samples. 0 disables logging.\ntype:milliseconds"); + log_l.put ("rotation_count", log_rotation_count, "Maximum number of log outputs before rotating the file.\ntype:uint64"); + log_l.put ("filename_counters", log_counters_filename, "Log file name for counters.\ntype:string"); + log_l.put ("filename_samples", log_samples_filename, "Log file name for samples.\ntype:string"); + toml.put_child ("log", log_l); + return toml.get_error (); +} + std::string nano::stat_log_sink::tm_to_string (tm & tm) { return (boost::format ("%04d.%02d.%02d %02d:%02d:%02d") % (1900 + tm.tm_year) % (tm.tm_mon + 1) % tm.tm_mday % tm.tm_hour % tm.tm_min % tm.tm_sec).str (); diff --git a/nano/lib/stats.hpp b/nano/lib/stats.hpp index 7011198f..bd8191b7 100644 --- a/nano/lib/stats.hpp +++ b/nano/lib/stats.hpp @@ -18,7 +18,7 @@ namespace nano { class node; - +class tomlconfig; /** * Serialize and deserialize the 'statistics' node from config.json * All configuration values have defaults. In particular, file logging of statistics @@ -29,6 +29,8 @@ class stat_config final public: /** Reads the JSON statistics node */ nano::error deserialize_json (nano::jsonconfig & json); + nano::error deserialize_toml (nano::tomlconfig & toml); + nano::error serialize_toml (nano::tomlconfig & toml) const; /** If true, sampling of counters is enabled */ bool sampling_enabled{ false }; diff --git a/nano/lib/tomlconfig.hpp b/nano/lib/tomlconfig.hpp new file mode 100644 index 00000000..26cb3ffa --- /dev/null +++ b/nano/lib/tomlconfig.hpp @@ -0,0 +1,540 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include + +namespace nano +{ +/** Manages a table in a toml configuration table hierarchy */ +class tomlconfig : public nano::configbase +{ +public: + tomlconfig () : + tree (cpptoml::make_table ()) + { + error = std::make_shared (); + } + + tomlconfig (std::shared_ptr const & tree_a, std::shared_ptr const & error_a = nullptr) : + nano::configbase (error_a), tree (tree_a) + { + if (!error) + { + error = std::make_shared (); + } + } + + void doc (std::string const & key, std::string const & doc) + { + tree->document (key, doc); + } + + /** + * Reads a json object from the stream + * @return nano::error&, including a descriptive error message if the config file is malformed. + */ + nano::error & read (boost::filesystem::path const & path_a) + { + std::stringstream stream_override_empty; + stream_override_empty << std::endl; + return read (stream_override_empty, path_a); + } + + nano::error & read (std::istream & stream_overrides, boost::filesystem::path const & path_a) + { + std::fstream stream; + open_or_create (stream, path_a.string ()); + if (!stream.fail ()) + { + try + { + read (stream_overrides, stream); + } + catch (std::runtime_error const & ex) + { + auto pos (stream.tellg ()); + if (pos != std::streampos (0)) + { + *error = ex; + } + } + stream.close (); + } + return *error; + } + + /** Read from two streams where keys in the first will take precedence over those in the second stream. */ + void read (std::istream & stream_first_a, std::istream & stream_second_a) + { + tree = cpptoml::parse_base_and_override_files (stream_first_a, stream_second_a, cpptoml::parser::merge_type::ignore, true); + } + + void read (std::istream & stream_a) + { + std::stringstream stream_override_empty; + stream_override_empty << std::endl; + tree = cpptoml::parse_base_and_override_files (stream_override_empty, stream_a, cpptoml::parser::merge_type::ignore, true); + } + + void write (boost::filesystem::path const & path_a) + { + std::fstream stream; + open_or_create (stream, path_a.string ()); + write (stream); + } + + void write (std::ostream & stream_a) const + { + cpptoml::toml_writer writer{ stream_a, "" }; + tree->accept (writer); + } + + /** Open configuration file, create if necessary */ + void open_or_create (std::fstream & stream_a, std::string const & path_a) + { + if (!boost::filesystem::exists (path_a)) + { + // Create temp stream to first create the file + std::ofstream stream (path_a); + + // Set permissions before opening otherwise Windows only has read permissions + nano::set_secure_perm_file (path_a); + } + + stream_a.open (path_a); + } + + /** Returns the table managed by this instance */ + std::shared_ptr get_tree () + { + return tree; + } + + /** Returns true if the toml table is empty */ + bool empty () const + { + return tree->empty (); + } + + boost::optional get_optional_child (std::string const & key_a) + { + boost::optional child_config; + if (tree->contains (key_a)) + { + return tomlconfig (tree->get_table (key_a), error); + } + return child_config; + } + + tomlconfig get_required_child (std::string const & key_a) + { + if (!tree->contains (key_a)) + { + *error = nano::error_config::missing_value; + error->set_message ("Missing configuration node: " + key_a); + return *this; + } + else + { + return tomlconfig (tree->get_table (key_a), error); + } + } + + tomlconfig & put_child (std::string const & key_a, nano::tomlconfig & conf_a) + { + tree->insert (key_a, conf_a.get_tree ()); + return *this; + } + + tomlconfig & replace_child (std::string const & key_a, nano::tomlconfig & conf_a) + { + tree->erase (key_a); + put_child (key_a, conf_a); + return *this; + } + + /** Set value for the given key. Any existing value will be overwritten. */ + template + tomlconfig & put (std::string const & key, T const & value, boost::optional documentation_a = boost::none) + { + tree->insert (key, value); + if (documentation_a) + { + doc (key, *documentation_a); + } + return *this; + } + + /** Returns true if \p key_a is present */ + bool has_key (std::string const & key_a) + { + return tree->contains (key_a); + } + + /** Erase the property of given key */ + tomlconfig & erase (std::string const & key_a) + { + tree->erase (key_a); + return *this; + } + + /** + * Push array element + * @param key Array element key. Qualified (dotted) keys are not supported for arrays so this must be called on the correct tomlconfig node. + */ + template + tomlconfig & push (std::string const & key, T const & value) + { + if (!has_key (key)) + { + auto arr = cpptoml::make_array (); + tree->insert (key, arr); + } + auto arr = tree->get_qualified (key)->as_array (); + arr->push_back (value); + return *this; + } + + auto create_array (std::string const & key, boost::optional documentation_a) + { + if (!has_key (key)) + { + auto arr = cpptoml::make_array (); + tree->insert (key, arr); + if (documentation_a) + { + doc (key, *documentation_a); + } + } + + return tree->get_qualified (key)->as_array (); + } + + /** + * Iterate array entries. + * @param key Array element key. Qualified (dotted) keys are not supported for arrays so this must be called on the correct tomlconfig node. + */ + template + tomlconfig & array_entries_required (std::string const & key, std::function callback) + { + if (tree->contains_qualified (key)) + { + auto items = tree->get_qualified_array_of (key); + for (auto & item : *items) + { + callback (item); + } + } + else + { + conditionally_set_error (nano::error_config::missing_value, false, key); + } + return *this; + } + + /** Get optional, using \p default_value if \p key is missing. */ + template + tomlconfig & get_optional (std::string const & key, T & target, T default_value) + { + get_config (true, key, target, default_value); + return *this; + } + + /** + * Get optional value, using the current value of \p target as the default if \p key is missing. + * @return May return nano::error_config::invalid_value + */ + template + tomlconfig & get_optional (std::string const & key, T & target) + { + get_config (true, key, target, target); + return *this; + } + + /** Return a boost::optional for the given key */ + template + boost::optional get_optional (std::string const & key) + { + boost::optional res; + if (has_key (key)) + { + T target{}; + get_config (true, key, target, target); + res = target; + } + return res; + } + + /** Get value, using the current value of \p target as the default if \p key is missing. */ + template + tomlconfig & get (std::string const & key, T & target) + { + get_config (true, key, target, target); + return *this; + } + + /** + * Get value of optional key. Use default value of data type if missing. + */ + template + T get (std::string const & key) + { + T target{}; + get_config (true, key, target, target); + return target; + } + + /** + * Get required value. + * @note May set nano::error_config::missing_value if \p key is missing, nano::error_config::invalid_value if value is invalid. + */ + template + tomlconfig & get_required (std::string const & key, T & target) + { + get_config (false, key, target); + return *this; + } + + /** + * Erase keys whose values are equal to the one in \p defaults + */ + void erase_default_values (tomlconfig & defaults_a) + { + std::shared_ptr clone = std::dynamic_pointer_cast (tree->clone ()); + tomlconfig self (clone); + + // The toml library doesn't offer a general way to compare values, so let the diff run on a stringified parse + std::stringstream ss_self; + write (ss_self); + self.read (ss_self); + + tomlconfig defaults_l; + std::stringstream ss; + defaults_a.write (ss); + defaults_l.read (ss); + + erase_defaults (defaults_l.get_tree (), self.get_tree (), get_tree ()); + } + + std::string to_string () + { + std::stringstream ss; + cpptoml::toml_writer writer{ ss, "" }; + tree->accept (writer); + return ss.str (); + } + +protected: + template ::value>> + tomlconfig & get_config (bool optional, std::string const & key, T & target, T default_value = T ()) + { + try + { + if (tree->contains_qualified (key)) + { + auto val (tree->get_qualified_as (key)); + if (!boost::conversion::try_lexical_convert (*val, target)) + { + conditionally_set_error (nano::error_config::invalid_value, optional, key); + } + } + else if (!optional) + { + conditionally_set_error (nano::error_config::missing_value, optional, key); + } + else + { + target = default_value; + } + } + catch (std::runtime_error & ex) + { + conditionally_set_error (ex, optional, key); + } + + return *this; + } + + // boost's lexical cast doesn't handle (u)int8_t + template ::value>> + tomlconfig & get_config (bool optional, std::string const & key, uint8_t & target, uint8_t default_value = T ()) + { + try + { + if (tree->contains_qualified (key)) + { + int64_t tmp; + auto val (tree->get_qualified_as (key)); + if (!boost::conversion::try_lexical_convert (*val, tmp) || tmp < 0 || tmp > 255) + { + conditionally_set_error (nano::error_config::invalid_value, optional, key); + } + else + { + target = static_cast (tmp); + } + } + else if (!optional) + { + conditionally_set_error (nano::error_config::missing_value, optional, key); + } + else + { + target = default_value; + } + } + catch (std::runtime_error & ex) + { + conditionally_set_error (ex, optional, key); + } + + return *this; + } + + template ::value>> + tomlconfig & get_config (bool optional, std::string const & key, bool & target, bool default_value = false) + { + auto bool_conv = [this, &target, &key, optional](std::string val) { + if (val == "true") + { + target = true; + } + else if (val == "false") + { + target = false; + } + else if (!*error) + { + conditionally_set_error (nano::error_config::invalid_value, optional, key); + } + }; + try + { + if (tree->contains_qualified (key)) + { + auto val (tree->get_qualified_as (key)); + bool_conv (*val); + } + else if (!optional) + { + conditionally_set_error (nano::error_config::missing_value, optional, key); + } + else + { + target = default_value; + } + } + catch (std::runtime_error & ex) + { + conditionally_set_error (ex, optional, key); + } + return *this; + } + + template ::value>> + tomlconfig & get_config (bool optional, std::string key, boost::asio::ip::address_v6 & target, boost::asio::ip::address_v6 default_value = T ()) + { + try + { + if (tree->contains_qualified (key)) + { + auto address_l (tree->get_qualified_as (key)); + boost::system::error_code bec; + target = boost::asio::ip::address_v6::from_string (address_l.value_or (""), bec); + if (bec) + { + conditionally_set_error (nano::error_config::invalid_value, optional, key); + } + } + else if (!optional) + { + conditionally_set_error (nano::error_config::missing_value, optional, key); + } + else + { + target = default_value; + } + } + catch (std::runtime_error & ex) + { + conditionally_set_error (ex, optional, key); + } + + return *this; + } + +private: + /** The config node being managed */ + std::shared_ptr tree; + + /** Compare two stringified configs, remove keys where values are equal */ + void erase_defaults (std::shared_ptr base, std::shared_ptr other, std::shared_ptr update_target) + { + std::vector erased; + assert (other != nullptr); + for (auto & item : *other) + { + std::string const & key = item.first; + if (other->contains (key) && base->contains (key)) + { + auto value = item.second; + if (value->is_table ()) + { + auto child_base = base->get_table (key); + auto child_other = other->get_table (key); + auto child_target = update_target->get_table (key); + erase_defaults (child_base, child_other, child_target); + if (child_target->empty ()) + { + erased.push_back (key); + } + } + else if (value->is_array ()) + { + auto arr_other = other->get_array (key)->get (); + auto arr_base = base->get_array (key)->get (); + + if (arr_other.size () == arr_base.size ()) + { + bool equal = std::equal (arr_other.begin (), arr_other.end (), arr_base.begin (), + [](auto const & item1, auto const & item2) -> bool { + return (item1->template as ()->get () == item2->template as ()->get ()); + }); + + if (equal) + { + erased.push_back (key); + } + } + } + else if (value->is_value ()) + { + auto val_other = std::dynamic_pointer_cast> (other->get (key)); + auto val_base = std::dynamic_pointer_cast> (base->get (key)); + + if (val_other->get () == val_base->get ()) + { + erased.push_back (key); + } + } + } + } + for (auto & key : erased) + { + update_target->erase (key); + } + } +}; +} diff --git a/nano/lib/walletconfig.cpp b/nano/lib/walletconfig.cpp new file mode 100644 index 00000000..b1c24697 --- /dev/null +++ b/nano/lib/walletconfig.cpp @@ -0,0 +1,52 @@ +#include +#include + +nano::wallet_config::wallet_config () +{ + nano::random_pool::generate_block (wallet.bytes.data (), wallet.bytes.size ()); + assert (!wallet.is_zero ()); +} + +nano::error nano::wallet_config::parse (std::string wallet_a, std::string account_a) +{ + nano::error error; + if (wallet.decode_hex (wallet_a)) + { + error.set ("Invalid wallet id"); + } + else if (account.decode_account (account_a)) + { + error.set ("Invalid account format"); + } + return error; +} + +nano::error nano::wallet_config::serialize_toml (nano::tomlconfig & toml) const +{ + std::string wallet_string; + wallet.encode_hex (wallet_string); + + toml.put ("wallet", wallet_string, "Wallet identifier\ntype:string,hex"); + toml.put ("account", account.to_account (), "Current wallet account\ntype:string,account"); + return toml.get_error (); +} + +nano::error nano::wallet_config::deserialize_toml (nano::tomlconfig & toml) +{ + std::string wallet_l; + std::string account_l; + + toml.get ("wallet", wallet_l); + toml.get ("account", account_l); + + if (wallet.decode_hex (wallet_l)) + { + toml.get_error ().set ("Invalid wallet id. Did you open a node daemon config?"); + } + else if (account.decode_account (account_l)) + { + toml.get_error ().set ("Invalid account"); + } + + return toml.get_error (); +} diff --git a/nano/lib/walletconfig.hpp b/nano/lib/walletconfig.hpp new file mode 100644 index 00000000..f191d186 --- /dev/null +++ b/nano/lib/walletconfig.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class jsonconfig; +class tomlconfig; + +/** Configuration options for the Qt wallet */ +class wallet_config final +{ +public: + wallet_config (); + /** Update this instance by parsing the given wallet and account */ + nano::error parse (std::string wallet_a, std::string account_a); + nano::error serialize_toml (nano::tomlconfig & toml_a) const; + nano::error deserialize_toml (nano::tomlconfig & toml_a); + nano::uint256_union wallet; + nano::account account{ 0 }; +}; +} diff --git a/nano/load_test/entry.cpp b/nano/load_test/entry.cpp index 2b76cdfb..47247708 100644 --- a/nano/load_test/entry.cpp +++ b/nano/load_test/entry.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -37,24 +38,27 @@ constexpr auto ipc_port_start = 62000; void write_config_files (boost::filesystem::path const & data_path, int index) { nano::daemon_config daemon_config (data_path); - nano::jsonconfig json; - json.read_and_update (daemon_config, data_path / "config.json"); - auto node_l = json.get_required_child ("node"); - node_l.put ("peering_port", peering_port_start + index); + daemon_config.node.peering_port = peering_port_start + index; + daemon_config.node.ipc_config.transport_tcp.enabled = true; + daemon_config.node.ipc_config.transport_tcp.port = ipc_port_start + index; + // Alternate use of memory pool - node_l.put ("use_memory_pools", (index % 2) == 0); - auto tcp = node_l.get_required_child ("ipc").get_required_child ("tcp"); - tcp.put ("enable", true); - tcp.put ("port", ipc_port_start + index); - json.write (data_path / "config.json"); + daemon_config.node.use_memory_pools = (index % 2) == 0; + + // Write daemon config + nano::tomlconfig toml; + daemon_config.serialize_toml (toml); + toml.write (nano::get_node_toml_config_path (data_path)); nano::rpc_config rpc_config; - nano::jsonconfig json1; - json1.read_and_update (rpc_config, data_path / "rpc_config.json"); - json1.put ("port", rpc_port_start + index); - json1.put ("enable_control", true); - json1.get_required_child ("process").put ("ipc_port", ipc_port_start + index); - json1.write (data_path / "rpc_config.json"); + rpc_config.port = rpc_port_start + index; + rpc_config.enable_control = true; + rpc_config.rpc_process.ipc_port = ipc_port_start + index; + + // Write rpc config + nano::tomlconfig toml_rpc; + rpc_config.serialize_toml (toml_rpc); + toml_rpc.write (nano::get_rpc_toml_config_path (data_path)); } // Report a failure @@ -365,7 +369,7 @@ int main (int argc, char * const * argv) ("simultaneous_process_calls", boost::program_options::value ()->default_value (20), "Number of simultaneous rpc sends to do") ("destination_count", boost::program_options::value ()->default_value (2), "How many destination accounts to choose between") ("node_path", boost::program_options::value (), "The path to the nano_node to test") - ("rpc_path", boost::program_options::value (), "The path to do nano_rpc to test"); + ("rpc_path", boost::program_options::value (), "The path to the nano_rpc to test"); // clang-format on boost::program_options::variables_map vm; diff --git a/nano/nano_node/daemon.cpp b/nano/nano_node/daemon.cpp index 1a7f6eb6..ac4879a2 100644 --- a/nano/nano_node/daemon.cpp +++ b/nano/nano_node/daemon.cpp @@ -41,7 +41,7 @@ void nano_daemon::daemon::run (boost::filesystem::path const & data_path, nano:: nano::set_secure_perm_directory (data_path, error_chmod); std::unique_ptr runner; nano::daemon_config config (data_path); - auto error = nano::read_and_update_daemon_config (data_path, config); + auto error = nano::read_node_config_toml (data_path, config, flags.config_overrides); nano::set_use_memory_pools (config.node.use_memory_pools); if (!error) { @@ -79,10 +79,11 @@ void nano_daemon::daemon::run (boost::filesystem::path const & data_path, nano:: { // Launch rpc in-process nano::rpc_config rpc_config; - auto error = nano::read_and_update_rpc_config (data_path, rpc_config); + auto error = nano::read_rpc_config_toml (data_path, rpc_config); if (error) { - throw std::runtime_error ("Could not deserialize rpc_config file"); + std::cout << error.get_message () << std::endl; + std::exit (1); } rpc_handler = std::make_unique (*node, config.rpc, [&ipc_server, &alarm, &io_ctx]() { ipc_server.stop (); diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 601ca94e..535bb52a 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -90,6 +90,7 @@ int main (int argc, char * const * argv) description.add_options () ("help", "Print out options") ("version", "Prints out version") + ("config", boost::program_options::value>()->multitoken(), "Pass node configuration values. This takes precedence over any values in the configuration file. This option can be repeated multiple times.") ("daemon", "Start node daemon") ("disable_backup", "Disable wallet automatic backups") ("disable_lazy_bootstrap", "Disables lazy bootstrap") @@ -181,6 +182,12 @@ int main (int argc, char * const * argv) nano_daemon::daemon daemon; nano::node_flags flags; update_flags (flags, vm); + + auto config (vm.find ("config")); + if (config != vm.end ()) + { + flags.config_overrides = config->second.as> (); + } daemon.run (data_path, flags); } else if (vm.count ("debug_block_count")) diff --git a/nano/nano_rpc/entry.cpp b/nano/nano_rpc/entry.cpp index 5151a9ee..7ac76067 100644 --- a/nano/nano_rpc/entry.cpp +++ b/nano/nano_rpc/entry.cpp @@ -43,7 +43,7 @@ void run (boost::filesystem::path const & data_path) std::unique_ptr runner; nano::rpc_config rpc_config; - auto error = nano::read_and_update_rpc_config (data_path, rpc_config); + auto error = nano::read_rpc_config_toml (data_path, rpc_config); if (!error) { logging_init (data_path); diff --git a/nano/nano_wallet/entry.cpp b/nano/nano_wallet/entry.cpp index 2aa0de72..c8a8f844 100644 --- a/nano/nano_wallet/entry.cpp +++ b/nano/nano_wallet/entry.cpp @@ -1,11 +1,13 @@ #include #include #include -#include #include +#include #include +#include #include #include +#include #include #include #include @@ -18,167 +20,6 @@ #include #include -class qt_wallet_config final -{ -public: - explicit qt_wallet_config (boost::filesystem::path const & data_path_a) - { - nano::random_pool::generate_block (wallet.bytes.data (), wallet.bytes.size ()); - assert (!wallet.is_zero ()); - } - bool upgrade_json (unsigned version_a, nano::jsonconfig & json) - { - json.put ("version", json_version ()); - switch (version_a) - { - case 1: - { - nano::account account; - account.decode_account (json.get ("account")); - json.erase ("account"); - json.put ("account", account.to_account ()); - json.erase ("version"); - } - case 2: - { - nano::jsonconfig rpc_l; - rpc.serialize_json (rpc_l); - json.put ("rpc_enable", "false"); - json.put_child ("rpc", rpc_l); - json.erase ("version"); - } - case 3: - { - auto opencl_enable_l (json.get_optional ("opencl_enable")); - if (!opencl_enable_l) - { - json.put ("opencl_enable", "false"); - } - auto opencl_l (json.get_optional_child ("opencl")); - if (!opencl_l) - { - nano::jsonconfig opencl_l; - opencl.serialize_json (opencl_l); - json.put_child ("opencl", opencl_l); - } - } - case 4: - break; - default: - throw std::runtime_error ("Unknown qt_wallet_config version"); - } - return version_a < json_version (); - } - - nano::error deserialize_json (bool & upgraded_a, nano::jsonconfig & json) - { - if (!json.empty ()) - { - auto version_l (json.get_optional ("version")); - if (!version_l) - { - version_l = 1; - json.put ("version", version_l.get ()); - upgraded_a = true; - } - - upgraded_a |= upgrade_json (version_l.get (), json); - auto wallet_l (json.get ("wallet")); - auto account_l (json.get ("account")); - auto node_l (json.get_required_child ("node")); - auto rpc_l (json.get_required_child ("rpc")); - rpc_enable = json.get ("rpc_enable"); - opencl_enable = json.get ("opencl_enable"); - auto opencl_l (json.get_required_child ("opencl")); - - if (wallet.decode_hex (wallet_l)) - { - json.get_error ().set ("Invalid wallet id. Did you open a node daemon config?"); - } - else if (account.decode_account (account_l)) - { - json.get_error ().set ("Invalid account"); - } - if (!node_l.get_error ()) - { - node.deserialize_json (upgraded_a, node_l); - } - if (!rpc_l.get_error ()) - { - rpc.deserialize_json (upgraded_a, rpc_l, data_path); - } - if (!opencl_l.get_error ()) - { - opencl.deserialize_json (opencl_l); - } - if (wallet.is_zero ()) - { - nano::random_pool::generate_block (wallet.bytes.data (), wallet.bytes.size ()); - upgraded_a = true; - } - } - else - { - serialize_json (json); - upgraded_a = true; - } - return json.get_error (); - } - - void serialize_json (nano::jsonconfig & json) - { - std::string wallet_string; - wallet.encode_hex (wallet_string); - json.put ("version", json_version ()); - json.put ("wallet", wallet_string); - json.put ("account", account.to_account ()); - nano::jsonconfig node_l; - node.enable_voting = false; - node.bootstrap_connections_max = 4; - node.serialize_json (node_l); - json.put_child ("node", node_l); - json.put ("rpc_enable", rpc_enable); - nano::jsonconfig rpc_l; - rpc.serialize_json (rpc_l); - json.put_child ("rpc", rpc_l); - json.put ("opencl_enable", opencl_enable); - nano::jsonconfig opencl_l; - opencl.serialize_json (opencl_l); - json.put_child ("opencl", opencl_l); - } - - bool serialize_json_stream (std::ostream & stream_a) - { - auto result (false); - stream_a.seekp (0); - try - { - nano::jsonconfig json; - serialize_json (json); - json.write (stream_a); - } - catch (std::runtime_error const & ex) - { - std::cerr << ex.what () << std::endl; - result = true; - } - return result; - } - - nano::uint256_union wallet; - nano::account account{ 0 }; - nano::node_config node; - bool rpc_enable{ false }; - nano::node_rpc_config rpc; - bool opencl_enable{ false }; - nano::opencl_config opencl; - boost::filesystem::path data_path; - unsigned json_version () const - { - return 4; - } -}; - namespace { void show_error (std::string const & message_a) @@ -188,33 +29,23 @@ void show_error (std::string const & message_a) message.show (); message.exec (); } -bool update_config (qt_wallet_config & config_a, boost::filesystem::path const & config_path_a) -{ - auto account (config_a.account); - auto wallet (config_a.wallet); - auto error (false); - nano::jsonconfig config; - if (!config.read_and_update (config_a, config_path_a)) - { - if (account != config_a.account || wallet != config_a.wallet) - { - config_a.account = account; - config_a.wallet = wallet; - // Update json file with new account and/or wallet values - std::fstream config_file; - config_file.open (config_path_a.string (), std::ios_base::out | std::ios_base::trunc); - boost::system::error_code error_chmod; - nano::set_secure_perm_file (config_path_a, error_chmod); - error = config_a.serialize_json_stream (config_file); - } - } - return error; +nano::error read_and_update_wallet_config (nano::wallet_config & config_a, boost::filesystem::path const & data_path_a) +{ + nano::tomlconfig wallet_config_toml; + auto wallet_path (nano::get_qtwallet_toml_config_path (data_path_a)); + wallet_config_toml.read (nano::get_qtwallet_toml_config_path (data_path_a)); + config_a.serialize_toml (wallet_config_toml); + + // Write wallet config. If missing, the file is created and permissions are set. + wallet_config_toml.write (wallet_path); + return wallet_config_toml.get_error (); } } int run_wallet (QApplication & application, int argc, char * const * argv, boost::filesystem::path const & data_path) { + int result (0); nano_qt::eventloop_processor processor; boost::system::error_code error_chmod; boost::filesystem::create_directories (data_path); @@ -225,15 +56,20 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost application.processEvents (); splash->showMessage (QSplashScreen::tr ("Remember - Back Up Your Wallet Seed"), Qt::AlignBottom | Qt::AlignHCenter, Qt::darkGray); application.processEvents (); - qt_wallet_config config (data_path); - auto config_path (nano::get_config_path (data_path)); - int result (0); - nano::jsonconfig json; - auto error (json.read_and_update (config, config_path)); - nano::set_use_memory_pools (config.node.use_memory_pools); - nano::set_secure_perm_file (config_path, error_chmod); + + nano::daemon_config config (data_path); + nano::wallet_config wallet_config; + + auto error = nano::read_node_config_toml (data_path, config); if (!error) { + error = read_and_update_wallet_config (wallet_config, data_path); + } + + if (!error) + { + nano::set_use_memory_pools (config.node.use_memory_pools); + config.node.logging.init (data_path); nano::logger_mt logger{ config.node.logging.min_time_between_log_output }; @@ -255,36 +91,36 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost node = std::make_shared (init, io_ctx, data_path, alarm, config.node, work, flags); if (!init.error ()) { - auto wallet (node->wallets.open (config.wallet)); + auto wallet (node->wallets.open (wallet_config.wallet)); if (wallet == nullptr) { auto existing (node->wallets.items.begin ()); if (existing != node->wallets.items.end ()) { wallet = existing->second; - config.wallet = existing->first; + wallet_config.wallet = existing->first; } else { - wallet = node->wallets.create (config.wallet); + wallet = node->wallets.create (wallet_config.wallet); } } - if (config.account.is_zero () || !wallet->exists (config.account)) + if (wallet_config.account.is_zero () || !wallet->exists (wallet_config.account)) { auto transaction (wallet->wallets.tx_begin_write ()); auto existing (wallet->store.begin (transaction)); if (existing != wallet->store.end ()) { nano::uint256_union account (existing->first); - config.account = account; + wallet_config.account = account; } else { - config.account = wallet->deterministic_insert (transaction); + wallet_config.account = wallet->deterministic_insert (transaction); } } - assert (wallet->exists (config.account)); - update_config (config, config_path); + assert (wallet->exists (wallet_config.account)); + read_and_update_wallet_config (wallet_config, data_path); node->start (); nano::ipc::ipc_server ipc (*node, config.rpc); @@ -299,10 +135,11 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost { // Launch rpc in-process nano::rpc_config rpc_config; - auto error = nano::read_and_update_rpc_config (data_path, rpc_config); + auto error = nano::read_rpc_config_toml (data_path, rpc_config); if (error) { - throw std::runtime_error ("Could not deserialize rpc_config file"); + std::cout << error.get_message () << std::endl; + std::exit (1); } rpc_handler = std::make_unique (*node, config.rpc); rpc = nano::get_rpc (io_ctx, rpc_config, *rpc_handler); @@ -340,7 +177,7 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost runner.stop_event_processing (); }); application.postEvent (&processor, new nano_qt::eventloop_event ([&]() { - gui = std::make_shared (application, processor, *node, wallet, config.account); + gui = std::make_shared (application, processor, *node, wallet, wallet_config.account); splash->close (); gui->start (); gui->client_window->show (); @@ -353,12 +190,12 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost splash->hide (); show_error ("Error initializing node"); } - update_config (config, config_path); + read_and_update_wallet_config (wallet_config, data_path); } else { splash->hide (); - show_error ("Error deserializing config: " + json.get_error ().get_message ()); + show_error ("Error deserializing config: " + error.get_message ()); } return result; } diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index f011231e..0e617a43 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -43,6 +44,7 @@ void nano::add_node_options (boost::program_options::options_description & descr ("unchecked_clear", "Clear unchecked blocks") ("confirmation_height_clear", "Clear confirmation height") ("diagnostics", "Run internal diagnostics") + ("generate_config", boost::program_options::value (), "Write configuration to stdout, populated with defaults suitable for this system. Pass the configuration type node or rpc.") ("key_create", "Generates a adhoc random keypair and prints it to stdout") ("key_expand", "Derive public key and account number from ") ("wallet_add_adhoc", "Insert in to ") @@ -346,20 +348,32 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map std::cout << "Confirmation heights of all accounts (except genesis) are set to 0" << std::endl; } } + else if (vm.count ("generate_config")) + { + auto type = vm["generate_config"].as (); + + if (type == "node") + { + nano::daemon_config config (data_path); + nano::tomlconfig toml; + config.serialize_toml (toml); + std::cout << toml.to_string () << std::endl; + } + else if (type == "rpc") + { + nano::rpc_config config (false); + nano::tomlconfig toml; + config.serialize_toml (toml); + std::cout << toml.to_string () << std::endl; + } + else + { + std::cerr << "Invalid configuration type " << type << ". Must be node or rpc." << std::endl; + } + } else if (vm.count ("diagnostics")) { inactive_node node (data_path); - - // Check/upgrade the config.json file. - { - nano::daemon_config config (data_path); - auto error = nano::read_and_update_daemon_config (data_path, config); - if (error) - { - std::cerr << "Error deserializing config: " << error.get_message () << std::endl; - } - } - std::cout << "Testing hash function" << std::endl; nano::raw_key key; key.data.clear (); diff --git a/nano/node/daemonconfig.cpp b/nano/node/daemonconfig.cpp index 140ecc6c..97bf43ec 100644 --- a/nano/node/daemonconfig.cpp +++ b/nano/node/daemonconfig.cpp @@ -1,11 +1,68 @@ #include +#include +#include +#include #include +#include +#include + nano::daemon_config::daemon_config (boost::filesystem::path const & data_path_a) : data_path (data_path_a) { } +nano::error nano::daemon_config::serialize_toml (nano::tomlconfig & toml) +{ + nano::tomlconfig rpc_l; + rpc.serialize_toml (rpc_l); + rpc_l.doc ("enable", "Enable or disable RPC\ntype:bool"); + rpc_l.put ("enable", rpc_enable); + toml.put_child ("rpc", rpc_l); + + nano::tomlconfig node_l; + node.serialize_toml (node_l); + nano::tomlconfig node (node_l); + toml.put_child ("node", node); + + nano::tomlconfig opencl_l; + opencl.serialize_toml (opencl_l); + opencl_l.doc ("enable", "Enable or disable OpenCL work generation\ntype:bool"); + opencl_l.put ("enable", opencl_enable); + toml.put_child ("opencl", opencl_l); + + return toml.get_error (); +} + +nano::error nano::daemon_config::deserialize_toml (nano::tomlconfig & toml) +{ + auto rpc_l (toml.get_optional_child ("rpc")); + + if (!toml.get_error () && rpc_l) + { + rpc_l->get_optional ("enable", rpc_enable); + rpc.deserialize_toml (*rpc_l); + } + + auto node_l (toml.get_optional_child ("node")); + if (!toml.get_error () && node_l) + { + node.deserialize_toml (*node_l); + } + + if (!toml.get_error ()) + { + auto opencl_l (toml.get_optional_child ("opencl")); + if (!toml.get_error () && opencl_l) + { + opencl_l->get_optional ("enable", opencl_enable); + opencl.deserialize_toml (*opencl_l); + } + } + + return toml.get_error (); +} + nano::error nano::daemon_config::serialize_json (nano::jsonconfig & json) { json.put ("version", json_version ()); @@ -35,9 +92,6 @@ nano::error nano::daemon_config::deserialize_json (bool & upgraded_a, nano::json { int version_l; json.get_optional ("version", version_l); - - upgraded_a |= upgrade_json (version_l, json); - json.get_optional ("rpc_enable", rpc_enable); auto rpc_l (json.get_required_child ("rpc")); @@ -74,43 +128,110 @@ nano::error nano::daemon_config::deserialize_json (bool & upgraded_a, nano::json return json.get_error (); } -bool nano::daemon_config::upgrade_json (unsigned version_a, nano::jsonconfig & json) -{ - json.put ("version", json_version ()); - switch (version_a) - { - case 1: - { - bool opencl_enable_l{ false }; - json.get_optional ("opencl_enable", opencl_enable_l); - if (!opencl_enable_l) - { - json.put ("opencl_enable", false); - } - auto opencl_l (json.get_optional_child ("opencl")); - if (!opencl_l) - { - nano::jsonconfig opencl_l; - opencl.serialize_json (opencl_l); - json.put_child ("opencl", opencl_l); - } - } - case 2: - break; - default: - throw std::runtime_error ("Unknown daemon_config version"); - } - return version_a < json_version (); -} - namespace nano { -nano::error read_and_update_daemon_config (boost::filesystem::path const & data_path, nano::daemon_config & config_a) +nano::error read_node_config_toml (boost::filesystem::path const & data_path_a, nano::daemon_config & config_a, std::vector const & config_overrides) +{ + nano::error error; + auto json_config_path = nano::get_config_path (data_path_a); + auto toml_config_path = nano::get_node_toml_config_path (data_path_a); + auto toml_qt_config_path = nano::get_qtwallet_toml_config_path (data_path_a); + if (boost::filesystem::exists (json_config_path)) + { + if (boost::filesystem::exists (toml_config_path)) + { + error = "Both json and toml node configuration files exists. " + "Either remove the config.json file and restart, or remove " + "the config-node.toml file to start migration on next launch."; + } + else + { + // Migrate + nano::daemon_config config_old_l; + nano::jsonconfig json; + read_and_update_daemon_config (data_path_a, config_old_l, json); + error = json.get_error (); + + // Move qt wallet entries to wallet config file + if (!error && json.has_key ("wallet") && json.has_key ("account")) + { + if (!boost::filesystem::exists (toml_config_path)) + { + nano::wallet_config wallet_conf; + error = wallet_conf.parse (json.get ("wallet"), json.get ("account")); + if (!error) + { + nano::tomlconfig wallet_toml_l; + wallet_conf.serialize_toml (wallet_toml_l); + wallet_toml_l.write (toml_qt_config_path); + + boost::system::error_code error_chmod; + nano::set_secure_perm_file (toml_qt_config_path, error_chmod); + } + } + else + { + std::cout << "Not migrating wallet and account as wallet config file already exists" << std::endl; + } + } + + if (!error) + { + nano::tomlconfig toml_l; + config_old_l.serialize_toml (toml_l); + + // Only write out non-default values + nano::daemon_config config_defaults_l; + nano::tomlconfig toml_defaults_l; + config_defaults_l.serialize_toml (toml_defaults_l); + + toml_l.erase_default_values (toml_defaults_l); + if (!toml_l.empty ()) + { + toml_l.write (toml_config_path); + boost::system::error_code error_chmod; + nano::set_secure_perm_file (toml_config_path, error_chmod); + } + + auto backup_path = data_path_a / "config_backup_toml_migration.json"; + boost::filesystem::rename (json_config_path, backup_path); + } + } + } + + // Parse and deserialize + nano::tomlconfig toml; + + std::stringstream config_stream; + for (auto const & entry : config_overrides) + { + config_stream << entry << std::endl; + } + config_stream << std::endl; + + // Make sure we don't create an empty toml file if it doesn't exist. Running without a toml file is the default. + if (!error && boost::filesystem::exists (toml_config_path)) + { + toml.read (config_stream, toml_config_path); + } + else if (!error) + { + toml.read (config_stream); + } + + if (!error) + { + error = config_a.deserialize_toml (toml); + } + + return error; +} + +nano::error read_and_update_daemon_config (boost::filesystem::path const & data_path, nano::daemon_config & config_a, nano::jsonconfig & json_a) { boost::system::error_code error_chmod; - nano::jsonconfig json; auto config_path = nano::get_config_path (data_path); - auto error (json.read_and_update (config_a, config_path)); + auto error (json_a.read_and_update (config_a, config_path)); nano::set_secure_perm_file (config_path, error_chmod); return error; } diff --git a/nano/node/daemonconfig.hpp b/nano/node/daemonconfig.hpp index 5ca5c30c..37736f07 100644 --- a/nano/node/daemonconfig.hpp +++ b/nano/node/daemonconfig.hpp @@ -5,20 +5,21 @@ #include #include +#include + namespace nano { +class jsonconfig; +class tomlconfig; class daemon_config { public: + daemon_config () = default; daemon_config (boost::filesystem::path const & data_path); nano::error deserialize_json (bool &, nano::jsonconfig &); nano::error serialize_json (nano::jsonconfig &); - /** - * Returns true if an upgrade occurred - * @param version The version to upgrade to. - * @param config Configuration to upgrade. - */ - bool upgrade_json (unsigned version, nano::jsonconfig & config); + nano::error deserialize_toml (nano::tomlconfig &); + nano::error serialize_toml (nano::tomlconfig &); bool rpc_enable{ false }; nano::node_rpc_config rpc; nano::node_config node; @@ -31,5 +32,6 @@ public: } }; -nano::error read_and_update_daemon_config (boost::filesystem::path const &, nano::daemon_config & config_a); +nano::error read_node_config_toml (boost::filesystem::path const &, nano::daemon_config & config_a, std::vector const & config_overrides = std::vector ()); +nano::error read_and_update_daemon_config (boost::filesystem::path const &, nano::daemon_config & config_a, nano::jsonconfig & json_a); } diff --git a/nano/node/diagnosticsconfig.cpp b/nano/node/diagnosticsconfig.cpp index b71d8fe9..a4e23c68 100644 --- a/nano/node/diagnosticsconfig.cpp +++ b/nano/node/diagnosticsconfig.cpp @@ -1,4 +1,5 @@ #include +#include #include nano::error nano::diagnostics_config::serialize_json (nano::jsonconfig & json) const @@ -30,3 +31,33 @@ nano::error nano::diagnostics_config::deserialize_json (nano::jsonconfig & json) } return json.get_error (); } + +nano::error nano::diagnostics_config::serialize_toml (nano::tomlconfig & toml) const +{ + nano::tomlconfig txn_tracking_l; + txn_tracking_l.put ("enable", txn_tracking.enable, "Enable or disable database transaction tracing\ntype:bool"); + txn_tracking_l.put ("min_read_txn_time", txn_tracking.min_read_txn_time.count (), "Log stacktrace when read transactions are held longer than this duration\ntype:milliseconds"); + txn_tracking_l.put ("min_write_txn_time", txn_tracking.min_write_txn_time.count (), "Log stacktrace when write transactions are held longer than this duration\ntype:milliseconds"); + txn_tracking_l.put ("ignore_writes_below_block_processor_max_time", txn_tracking.ignore_writes_below_block_processor_max_time, "Ignore any block processor writes less than block_processor_batch_max_time\ntype:bool"); + toml.put_child ("txn_tracking", txn_tracking_l); + return toml.get_error (); +} + +nano::error nano::diagnostics_config::deserialize_toml (nano::tomlconfig & toml) +{ + auto txn_tracking_l (toml.get_optional_child ("txn_tracking")); + if (txn_tracking_l) + { + txn_tracking_l->get_optional ("enable", txn_tracking.enable); + auto min_read_txn_time_l = static_cast (txn_tracking.min_read_txn_time.count ()); + txn_tracking_l->get_optional ("min_read_txn_time", min_read_txn_time_l); + txn_tracking.min_read_txn_time = std::chrono::milliseconds (min_read_txn_time_l); + + auto min_write_txn_time_l = static_cast (txn_tracking.min_write_txn_time.count ()); + txn_tracking_l->get_optional ("min_write_txn_time", min_write_txn_time_l); + txn_tracking.min_write_txn_time = std::chrono::milliseconds (min_write_txn_time_l); + + txn_tracking_l->get_optional ("ignore_writes_below_block_processor_max_time", txn_tracking.ignore_writes_below_block_processor_max_time); + } + return toml.get_error (); +} diff --git a/nano/node/diagnosticsconfig.hpp b/nano/node/diagnosticsconfig.hpp index 28ef1d48..3e6537eb 100644 --- a/nano/node/diagnosticsconfig.hpp +++ b/nano/node/diagnosticsconfig.hpp @@ -7,7 +7,7 @@ namespace nano { class jsonconfig; - +class tomlconfig; class txn_tracking_config final { public: @@ -24,6 +24,8 @@ class diagnostics_config final public: nano::error serialize_json (nano::jsonconfig &) const; nano::error deserialize_json (nano::jsonconfig &); + nano::error serialize_toml (nano::tomlconfig &) const; + nano::error deserialize_toml (nano::tomlconfig &); txn_tracking_config txn_tracking; }; diff --git a/nano/node/ipcconfig.cpp b/nano/node/ipcconfig.cpp index 7a5e337b..48c4e038 100644 --- a/nano/node/ipcconfig.cpp +++ b/nano/node/ipcconfig.cpp @@ -1,6 +1,58 @@ #include +#include #include +nano::error nano::ipc::ipc_config::serialize_toml (nano::tomlconfig & toml) const +{ + nano::tomlconfig tcp_l; + // Only write out experimental config values if they're previously set explicitly in the config file + if (transport_tcp.io_threads >= 0) + { + tcp_l.put ("io_threads", transport_tcp.io_threads); + } + tcp_l.put ("enable", transport_tcp.enabled); + tcp_l.put ("port", transport_tcp.port); + tcp_l.put ("io_timeout", transport_tcp.io_timeout); + toml.put_child ("tcp", tcp_l); + + nano::tomlconfig domain_l; + if (transport_domain.io_threads >= 0) + { + domain_l.put ("io_threads", transport_domain.io_threads); + } + domain_l.put ("enable", transport_domain.enabled); + domain_l.put ("allow_unsafe", transport_domain.allow_unsafe); + domain_l.put ("path", transport_domain.path); + domain_l.put ("io_timeout", transport_domain.io_timeout); + toml.put_child ("local", domain_l); + return toml.get_error (); +} + +nano::error nano::ipc::ipc_config::deserialize_toml (nano::tomlconfig & toml) +{ + auto tcp_l (toml.get_optional_child ("tcp")); + if (tcp_l) + { + tcp_l->get_optional ("io_threads", transport_tcp.io_threads, -1); + tcp_l->get ("allow_unsafe", transport_tcp.allow_unsafe); + tcp_l->get ("enable", transport_tcp.enabled); + tcp_l->get ("port", transport_tcp.port); + tcp_l->get ("io_timeout", transport_tcp.io_timeout); + } + + auto domain_l (toml.get_optional_child ("local")); + if (domain_l) + { + domain_l->get_optional ("io_threads", transport_domain.io_threads, -1); + domain_l->get ("allow_unsafe", transport_domain.allow_unsafe); + domain_l->get ("enable", transport_domain.enabled); + domain_l->get ("path", transport_domain.path); + domain_l->get ("io_timeout", transport_domain.io_timeout); + } + + return toml.get_error (); +} + nano::error nano::ipc::ipc_config::serialize_json (nano::jsonconfig & json) const { nano::jsonconfig tcp_l; diff --git a/nano/node/ipcconfig.hpp b/nano/node/ipcconfig.hpp index 3a7a9df9..7abf8f3a 100644 --- a/nano/node/ipcconfig.hpp +++ b/nano/node/ipcconfig.hpp @@ -8,7 +8,7 @@ namespace nano { class jsonconfig; - +class tomlconfig; namespace ipc { /** Base class for transport configurations */ @@ -57,6 +57,8 @@ namespace ipc public: nano::error deserialize_json (bool & upgraded_a, nano::jsonconfig & json_a); nano::error serialize_json (nano::jsonconfig & json) const; + nano::error deserialize_toml (nano::tomlconfig & toml_a); + nano::error serialize_toml (nano::tomlconfig & toml) const; ipc_config_domain_socket transport_domain; ipc_config_tcp_socket transport_tcp; }; diff --git a/nano/node/logging.cpp b/nano/node/logging.cpp index fdbffd1f..c9a11787 100644 --- a/nano/node/logging.cpp +++ b/nano/node/logging.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -80,6 +82,66 @@ void nano::logging::release_file_sink () } } +nano::error nano::logging::serialize_toml (nano::tomlconfig & toml) const +{ + toml.put ("ledger", ledger_logging_value, "Log ledger related messages\ntype:bool"); + toml.put ("ledger_duplicate", ledger_duplicate_logging_value, "Log when a duplicate block is attempted inserted into the ledger\ntype:bool"); + toml.put ("vote", vote_logging_value, "Vote logging. Enabling this option leads to a high volume\nof log messages which may affect node performance\ntype:bool"); + toml.put ("network", network_logging_value, "Log network related messages\ntype:bool"); + toml.put ("network_timeout", network_timeout_logging_value, "Log TCP timeouts\ntype:bool"); + toml.put ("network_message", network_message_logging_value, "Log network errors and message details\ntype:bool"); + toml.put ("network_publish", network_publish_logging_value, "Log publish related network messages\ntype:bool"); + toml.put ("network_packet", network_packet_logging_value, "Log network packet activity\ntype:bool"); + toml.put ("network_keepalive", network_keepalive_logging_value, "Log keepalive related messages\ntype:bool"); + toml.put ("network_node_id_handshake", network_node_id_handshake_logging_value, "Log node-id handshake related messages\ntype:bool"); + toml.put ("node_lifetime_tracing", node_lifetime_tracing_value, "Log node startup and shutdown messages\ntype:bool"); + toml.put ("insufficient_work", insufficient_work_logging_value, "Log if insufficient work is detected\ntype:bool"); + toml.put ("log_ipc", log_ipc_value, "Log IPC related activity\ntype:bool"); + toml.put ("bulk_pull", bulk_pull_logging_value, "Log bulk pull errors and messages\ntype:bool"); + toml.put ("work_generation_time", work_generation_time_value, "Log work generation timing information\ntype:bool"); + toml.put ("upnp_details", upnp_details_logging_value, "Log UPNP discovery details. WARNING: this may include information\nabout discovered devices, such as product identification. Please review before sharing logs.\ntype:bool"); + toml.put ("timing", timing_logging_value, "Log detailed timing information for various node operations\ntype:bool"); + toml.put ("log_to_cerr", log_to_cerr_value, "Log to standard error in addition to the log file\ntype:bool"); + toml.put ("max_size", max_size, "Maximum log file size in bytes\ntype:uint64"); + toml.put ("rotation_size", rotation_size, "Log file rotation size in character count\ntype:uint64"); + toml.put ("flush", flush, "If enabled, immediately flush new entries to log file. This may negatively affect logging performance.\ntype:bool"); + toml.put ("min_time_between_output", min_time_between_log_output.count (), "Minimum time that must pass for low priority entries to be logged\ntype:milliseconds"); + toml.put ("single_line_record", single_line_record_value, "Keep log entries on single lines\ntype:bool"); + + return toml.get_error (); +} + +nano::error nano::logging::deserialize_toml (nano::tomlconfig & toml) +{ + toml.get ("ledger", ledger_logging_value); + toml.get ("ledger_duplicate", ledger_duplicate_logging_value); + toml.get ("vote", vote_logging_value); + toml.get ("network", network_logging_value); + toml.get ("network_timeout", network_timeout_logging_value); + toml.get ("network_message", network_message_logging_value); + toml.get ("network_publish", network_publish_logging_value); + toml.get ("network_packet", network_packet_logging_value); + toml.get ("network_keepalive", network_keepalive_logging_value); + toml.get ("network_node_id_handshake", network_node_id_handshake_logging_value); + toml.get ("node_lifetime_tracing", node_lifetime_tracing_value); + toml.get ("insufficient_work", insufficient_work_logging_value); + toml.get ("log_ipc", log_ipc_value); + toml.get ("bulk_pull", bulk_pull_logging_value); + toml.get ("work_generation_time", work_generation_time_value); + toml.get ("upnp_details", upnp_details_logging_value); + toml.get ("timing", timing_logging_value); + toml.get ("log_to_cerr", log_to_cerr_value); + toml.get ("flush", flush); + toml.get ("single_line_record", single_line_record_value); + toml.get ("max_size", max_size); + toml.get ("rotation_size", rotation_size); + uintmax_t min_time_between_log_output_raw; + toml.get ("min_time_between_output", min_time_between_log_output_raw); + min_time_between_log_output = std::chrono::milliseconds (min_time_between_log_output_raw); + + return toml.get_error (); +} + nano::error nano::logging::serialize_json (nano::jsonconfig & json) const { json.put ("version", json_version ()); diff --git a/nano/node/logging.hpp b/nano/node/logging.hpp index 834551e6..78aab8fc 100644 --- a/nano/node/logging.hpp +++ b/nano/node/logging.hpp @@ -15,11 +15,14 @@ namespace nano { +class tomlconfig; class logging final { public: nano::error serialize_json (nano::jsonconfig &) const; nano::error deserialize_json (bool &, nano::jsonconfig &); + nano::error serialize_toml (nano::tomlconfig &) const; + nano::error deserialize_toml (nano::tomlconfig &); bool upgrade_json (unsigned, nano::jsonconfig &); bool ledger_logging () const; bool ledger_duplicate_logging () const; diff --git a/nano/node/node_rpc_config.cpp b/nano/node/node_rpc_config.cpp index 68ade0ee..af9814e4 100644 --- a/nano/node/node_rpc_config.cpp +++ b/nano/node/node_rpc_config.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include nano::error nano::node_rpc_config::serialize_json (nano::jsonconfig & json) const @@ -17,6 +18,39 @@ nano::error nano::node_rpc_config::serialize_json (nano::jsonconfig & json) cons return json.get_error (); } +nano::error nano::node_rpc_config::serialize_toml (nano::tomlconfig & toml) const +{ + toml.put ("enable_sign_hash", enable_sign_hash, "Allow or disallow signing of hashes\ntype:bool"); + toml.put ("max_work_generate_difficulty", nano::to_string_hex (max_work_generate_difficulty), "Maximum allowed difficulty request for work generation\ntype:string,hex"); + + nano::tomlconfig child_process_l; + child_process_l.put ("enable", child_process.enable, "Enable or disable RPC child process. If false, an in-process RPC server is used.\ntype:bool"); + child_process_l.put ("rpc_path", child_process.rpc_path, "Path to the nano_rpc executable. Must be set if child process is enabled.\ntype:string,path"); + toml.put_child ("child_process", child_process_l); + return toml.get_error (); +} + +nano::error nano::node_rpc_config::deserialize_toml (nano::tomlconfig & toml) +{ + toml.get_optional ("enable_sign_hash", enable_sign_hash); + toml.get_optional ("enable_sign_hash", enable_sign_hash); + std::string max_work_generate_difficulty_text; + toml.get_optional ("max_work_generate_difficulty", max_work_generate_difficulty_text); + if (!max_work_generate_difficulty_text.empty ()) + { + nano::from_string_hex (max_work_generate_difficulty_text, max_work_generate_difficulty); + } + + auto child_process_l (toml.get_optional_child ("child_process")); + if (child_process_l) + { + child_process_l->get_optional ("enable", child_process.enable); + child_process_l->get_optional ("rpc_path", child_process.rpc_path); + } + + return toml.get_error (); +} + nano::error nano::node_rpc_config::deserialize_json (bool & upgraded_a, nano::jsonconfig & json, boost::filesystem::path const & data_path) { auto version_l (json.get_optional ("version")); diff --git a/nano/node/node_rpc_config.hpp b/nano/node/node_rpc_config.hpp index d519b044..c88297aa 100644 --- a/nano/node/node_rpc_config.hpp +++ b/nano/node/node_rpc_config.hpp @@ -8,6 +8,7 @@ namespace nano { +class tomlconfig; class rpc_child_process_config final { public: @@ -20,6 +21,9 @@ class node_rpc_config final public: nano::error serialize_json (nano::jsonconfig &) const; nano::error deserialize_json (bool & upgraded_a, nano::jsonconfig &, boost::filesystem::path const & data_path); + nano::error serialize_toml (nano::tomlconfig & toml) const; + nano::error deserialize_toml (nano::tomlconfig & toml); + bool enable_sign_hash{ false }; uint64_t max_work_generate_difficulty{ 0xffffffffc0000000 }; nano::rpc_child_process_config child_process; diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index 1e005706..34db24f7 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include // NOTE: to reduce compile times, this include can be replaced by more narrow includes // once nano::network is factored out of node.{c|h}pp @@ -62,6 +63,281 @@ logging (logging_a) } } +nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const +{ + toml.put ("peering_port", peering_port, "Node peering port\ntype:uint16"); + toml.put ("bootstrap_fraction_numerator", bootstrap_fraction_numerator, "Change bootstrap threshold (online stake / 256 * bootstrap_fraction_numerator)\ntype:uint32"); + toml.put ("receive_minimum", receive_minimum.to_string_dec (), "Minimum receive amount\ntype:string,amount,raw"); + toml.put ("online_weight_minimum", online_weight_minimum.to_string_dec (), "Online weight minimum required to confirm block\ntype:string,amount,raw"); + toml.put ("online_weight_quorum", online_weight_quorum, "Percentage of votes required to rollback blocks\ntype:uint64"); + toml.put ("password_fanout", password_fanout, "Password fanout factor\ntype:uint64"); + toml.put ("io_threads", io_threads, "Number of threads dedicated to I/O opeations\ntype:uint64"); + toml.put ("network_threads", network_threads, "Number of threads dedicated to processing network messages\ntype:uint64"); + toml.put ("work_threads", work_threads, "Number of threads dedicated to CPU generated work. Defaults to all available CPU threads.\ntype:uint64"); + toml.put ("signature_checker_threads", signature_checker_threads, "Number of additional threads dedicated to signature verification\ntype:uint64"); + toml.put ("enable_voting", enable_voting, "Enable or disable voting. Enabling voting requires additional system resources.\ntype:bool"); + toml.put ("bootstrap_connections", bootstrap_connections, "Number of outbound bootstrap connections. Must be a power of 2.\ntype:uint64"); + toml.put ("bootstrap_connections_max", bootstrap_connections_max, "Maximum number of inbound bootstrap connections\ntype:uint64"); + toml.put ("lmdb_max_dbs", lmdb_max_dbs, "Maximum open lmdb databases. Increase default if more than 100 wallets is required.\ntype:uint64"); + toml.put ("block_processor_batch_max_time", block_processor_batch_max_time.count (), "The maximum time the block processor can process blocks at a time\ntype:milliseconds"); + toml.put ("allow_local_peers", allow_local_peers, "Enable or disable local host peering\ntype:bool"); + toml.put ("vote_minimum", vote_minimum.to_string_dec (), "Do not vote if delegated weight is under this threshold\ntype:string,amount,raw"); + toml.put ("vote_generator_delay", vote_generator_delay.count (), "Delay before votes are sent to allow for better bundling of hashes in votes.\nHigh performance nodes may need slightly higher values to optimize vote bandwidth.\ntype:milliseconds"); + toml.put ("vote_generator_threshold", vote_generator_threshold, "Number of bundled hashes required for an additional generator delay\ntype:uint64,[1..11]"); + toml.put ("unchecked_cutoff_time", unchecked_cutoff_time.count (), "Number of seconds unchecked entry survives before being cleaned\ntype:seconds"); + toml.put ("tcp_io_timeout", tcp_io_timeout.count (), "Timeout for TCP connect-, read- and write operations\ntype:seconds"); + toml.put ("pow_sleep_interval", pow_sleep_interval.count (), "The amount to sleep after each batch of POW calculations. Reduces max CPU usage at the expensive of a longer.\ntype:nanoseconds"); + toml.put ("external_address", external_address.to_string (), "The external address of this node (NAT). If not set, the node will request this information via UPnP.\ntype:string,ip"); + toml.put ("external_port", external_port, "The external port number of this node (NAT). If not set, the node will request this information via UPnP.\ntype:uint16"); + toml.put ("tcp_incoming_connections_max", tcp_incoming_connections_max, "Maximum number of incoming TCP connections\ntype:uint64"); + toml.put ("use_memory_pools", use_memory_pools, "If true, allocate memory from memory pools. Enabling this may improve performance. Memory is never released to the OS.\ntype:bool"); + toml.put ("confirmation_history_size", confirmation_history_size, "Maximum confirmation history size\ntype:uint64"); + toml.put ("active_elections_size", active_elections_size, "Limits number of active elections before dropping will be considered (other conditions must also be satisfied)\ntype:uint64,[250..]"); + toml.put ("bandwidth_limit", bandwidth_limit, "Outbound traffic limit in bytes/sec after which messages will be dropped\ntype:uint64"); + toml.put ("backup_before_upgrade", backup_before_upgrade, "Backup the ledger database before performing upgrades\ntype:bool"); + toml.put ("work_watcher_period", work_watcher_period.count (), "Time between checks for confirmation and re-generating higher difficulty work if unconfirmed, for blocks in the work watcher.\ntype:seconds"); + + auto work_peers_l (toml.create_array ("work_peers", "A list of \"address:port\" entries to identify work peers")); + for (auto i (work_peers.begin ()), n (work_peers.end ()); i != n; ++i) + { + work_peers_l->push_back (boost::str (boost::format ("%1%:%2%") % i->first % i->second)); + } + + auto preconfigured_peers_l (toml.create_array ("preconfigured_peers", "A list of \"address:port\" entries to identify preconfigured peers")); + for (auto i (preconfigured_peers.begin ()), n (preconfigured_peers.end ()); i != n; ++i) + { + preconfigured_peers_l->push_back (*i); + } + + auto preconfigured_representatives_l (toml.create_array ("preconfigured_representatives", "A list of representative account addresses")); + for (auto i (preconfigured_representatives.begin ()), n (preconfigured_representatives.end ()); i != n; ++i) + { + preconfigured_representatives_l->push_back (i->to_account ()); + } + nano::tomlconfig callback_l; + callback_l.put ("address", callback_address, "Callback address\ntype:string,ip"); + callback_l.put ("port", callback_port, "Callback port number\ntype:uint16"); + callback_l.put ("target", callback_target, "Callback target path\ntype:string,uri"); + toml.put_child ("httpcallback", callback_l); + + nano::tomlconfig logging_l; + logging.serialize_toml (logging_l); + toml.put_child ("logging", logging_l); + + nano::tomlconfig websocket_l; + websocket_config.serialize_toml (websocket_l); + toml.put_child ("websocket", websocket_l); + + nano::tomlconfig ipc_l; + ipc_config.serialize_toml (ipc_l); + toml.put_child ("ipc", ipc_l); + + nano::tomlconfig diagnostics_l; + diagnostics_config.serialize_toml (diagnostics_l); + toml.put_child ("diagnostics", diagnostics_l); + + nano::tomlconfig stat_l; + stat_config.serialize_toml (stat_l); + toml.put_child ("statistics", stat_l); + + return toml.get_error (); +} + +nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) +{ + try + { + if (toml.has_key ("httpcallback")) + { + auto callback_l (toml.get_required_child ("httpcallback")); + callback_l.get ("address", callback_address); + callback_l.get ("port", callback_port); + callback_l.get ("target", callback_target); + } + + if (toml.has_key ("logging")) + { + auto logging_l (toml.get_required_child ("logging")); + logging.deserialize_toml (logging_l); + } + + if (toml.has_key ("websocket")) + { + auto websocket_config_l (toml.get_required_child ("websocket")); + websocket_config.deserialize_toml (websocket_config_l); + } + + if (toml.has_key ("ipc")) + { + auto ipc_config_l (toml.get_required_child ("ipc")); + ipc_config.deserialize_toml (ipc_config_l); + } + + if (toml.has_key ("diagnostics")) + { + auto diagnostics_config_l (toml.get_required_child ("diagnostics")); + diagnostics_config.deserialize_toml (diagnostics_config_l); + } + + if (toml.has_key ("statistics")) + { + auto stat_config_l (toml.get_required_child ("statistics")); + stat_config.deserialize_toml (stat_config_l); + } + + if (toml.has_key ("work_peers")) + { + work_peers.clear (); + toml.array_entries_required ("work_peers", [this](std::string const & entry) { + auto port_position (entry.rfind (':')); + bool result = port_position == -1; + if (!result) + { + auto port_str (entry.substr (port_position + 1)); + uint16_t port; + result |= parse_port (port_str, port); + if (!result) + { + auto address (entry.substr (0, port_position)); + this->work_peers.emplace_back (address, port); + } + } + }); + } + + if (toml.has_key (preconfigured_peers_key)) + { + preconfigured_peers.clear (); + toml.array_entries_required (preconfigured_peers_key, [this](std::string entry) { + preconfigured_peers.push_back (entry); + }); + } + + if (toml.has_key ("preconfigured_representatives")) + { + preconfigured_representatives.clear (); + toml.array_entries_required ("preconfigured_representatives", [this, &toml](std::string entry) { + nano::account representative (0); + if (representative.decode_account (entry)) + { + toml.get_error ().set ("Invalid representative account: " + entry); + } + preconfigured_representatives.push_back (representative); + }); + } + + if (preconfigured_representatives.empty ()) + { + toml.get_error ().set ("At least one representative account must be set"); + } + + auto receive_minimum_l (toml.get ("receive_minimum")); + if (receive_minimum.decode_dec (receive_minimum_l)) + { + toml.get_error ().set ("receive_minimum contains an invalid decimal amount"); + } + + auto online_weight_minimum_l (toml.get ("online_weight_minimum")); + if (online_weight_minimum.decode_dec (online_weight_minimum_l)) + { + toml.get_error ().set ("online_weight_minimum contains an invalid decimal amount"); + } + + auto vote_minimum_l (toml.get ("vote_minimum")); + if (vote_minimum.decode_dec (vote_minimum_l)) + { + toml.get_error ().set ("vote_minimum contains an invalid decimal amount"); + } + + auto delay_l = vote_generator_delay.count (); + toml.get ("vote_generator_delay", delay_l); + vote_generator_delay = std::chrono::milliseconds (delay_l); + + toml.get ("vote_generator_threshold", vote_generator_threshold); + + auto block_processor_batch_max_time_l (toml.get ("block_processor_batch_max_time")); + block_processor_batch_max_time = std::chrono::milliseconds (block_processor_batch_max_time_l); + auto unchecked_cutoff_time_l = static_cast (unchecked_cutoff_time.count ()); + toml.get ("unchecked_cutoff_time", unchecked_cutoff_time_l); + unchecked_cutoff_time = std::chrono::seconds (unchecked_cutoff_time_l); + + auto tcp_io_timeout_l = static_cast (tcp_io_timeout.count ()); + toml.get ("tcp_io_timeout", tcp_io_timeout_l); + tcp_io_timeout = std::chrono::seconds (tcp_io_timeout_l); + + toml.get ("peering_port", peering_port); + toml.get ("bootstrap_fraction_numerator", bootstrap_fraction_numerator); + toml.get ("online_weight_quorum", online_weight_quorum); + toml.get ("password_fanout", password_fanout); + toml.get ("io_threads", io_threads); + toml.get ("work_threads", work_threads); + toml.get ("network_threads", network_threads); + toml.get ("bootstrap_connections", bootstrap_connections); + toml.get ("bootstrap_connections_max", bootstrap_connections_max); + toml.get ("lmdb_max_dbs", lmdb_max_dbs); + toml.get ("enable_voting", enable_voting); + toml.get ("allow_local_peers", allow_local_peers); + toml.get (signature_checker_threads_key, signature_checker_threads); + toml.get ("external_address", external_address); + toml.get ("external_port", external_port); + toml.get ("tcp_incoming_connections_max", tcp_incoming_connections_max); + + auto pow_sleep_interval_l (pow_sleep_interval.count ()); + toml.get (pow_sleep_interval_key, pow_sleep_interval_l); + pow_sleep_interval = std::chrono::nanoseconds (pow_sleep_interval_l); + toml.get ("use_memory_pools", use_memory_pools); + toml.get ("confirmation_history_size", confirmation_history_size); + toml.get ("active_elections_size", active_elections_size); + toml.get ("bandwidth_limit", bandwidth_limit); + toml.get ("backup_before_upgrade", backup_before_upgrade); + + auto work_watcher_period_l = work_watcher_period.count (); + toml.get ("work_watcher_period", work_watcher_period_l); + work_watcher_period = std::chrono::seconds (work_watcher_period_l); + + auto conf_height_processor_batch_min_time_l (conf_height_processor_batch_min_time.count ()); + toml.get ("conf_height_processor_batch_min_time", conf_height_processor_batch_min_time_l); + conf_height_processor_batch_min_time = std::chrono::milliseconds (conf_height_processor_batch_min_time_l); + + nano::network_constants network; + // Validate ranges + if (online_weight_quorum > 100) + { + toml.get_error ().set ("online_weight_quorum must be less than 100"); + } + if (password_fanout < 16 || password_fanout > 1024 * 1024) + { + toml.get_error ().set ("password_fanout must be a number between 16 and 1048576"); + } + if (io_threads == 0) + { + toml.get_error ().set ("io_threads must be non-zero"); + } + if (active_elections_size <= 250 && !network.is_test_network ()) + { + toml.get_error ().set ("active_elections_size must be greater than 250"); + } + if (bandwidth_limit > std::numeric_limits::max ()) + { + toml.get_error ().set ("bandwidth_limit unbounded = 0, default = 5242880, max = 18446744073709551615"); + } + if (vote_generator_threshold < 1 || vote_generator_threshold > 11) + { + toml.get_error ().set ("vote_generator_threshold must be a number between 1 and 11"); + } + if (work_watcher_period < std::chrono::seconds (1)) + { + toml.get_error ().set ("work_watcher_period must be equal or larger than 1"); + } + } + catch (std::runtime_error const & ex) + { + toml.get_error ().set (ex.what ()); + } + + return toml.get_error (); +} + nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const { json.put ("version", json_version ()); diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index fe905254..eb4922c4 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -16,6 +16,7 @@ namespace nano { +class tomlconfig; /** * Node configuration */ @@ -26,6 +27,8 @@ public: node_config (uint16_t, nano::logging const &); nano::error serialize_json (nano::jsonconfig &) const; nano::error deserialize_json (bool &, nano::jsonconfig &); + nano::error serialize_toml (nano::tomlconfig &) const; + nano::error deserialize_toml (nano::tomlconfig &); bool upgrade_json (unsigned, nano::jsonconfig &); nano::account random_representative (); nano::network_params network_params; @@ -88,6 +91,7 @@ public: class node_flags final { public: + std::vector config_overrides; bool disable_backup{ false }; bool disable_lazy_bootstrap{ false }; bool disable_legacy_bootstrap{ false }; diff --git a/nano/node/openclconfig.cpp b/nano/node/openclconfig.cpp index bb88254e..4af49333 100644 --- a/nano/node/openclconfig.cpp +++ b/nano/node/openclconfig.cpp @@ -1,3 +1,5 @@ +#include +#include #include nano::opencl_config::opencl_config (unsigned platform_a, unsigned device_a, unsigned threads_a) : @@ -22,3 +24,25 @@ nano::error nano::opencl_config::deserialize_json (nano::jsonconfig & json) json.get_optional ("threads", threads); return json.get_error (); } + +nano::error nano::opencl_config::serialize_toml (nano::tomlconfig & toml) const +{ + toml.put ("platform", platform); + toml.put ("device", device); + toml.put ("threads", threads); + + // Add documentation + toml.doc ("platform", "OpenCL platform identifier"); + toml.doc ("device", "OpenCL device identifier"); + toml.doc ("threads", "OpenCL thread count"); + + return toml.get_error (); +} + +nano::error nano::opencl_config::deserialize_toml (nano::tomlconfig & toml) +{ + toml.get_optional ("platform", platform); + toml.get_optional ("device", device); + toml.get_optional ("threads", threads); + return toml.get_error (); +} diff --git a/nano/node/openclconfig.hpp b/nano/node/openclconfig.hpp index 12715870..6fe79e04 100644 --- a/nano/node/openclconfig.hpp +++ b/nano/node/openclconfig.hpp @@ -1,10 +1,11 @@ #pragma once #include -#include namespace nano { +class jsonconfig; +class tomlconfig; class opencl_config { public: @@ -12,6 +13,8 @@ public: opencl_config (unsigned, unsigned, unsigned); nano::error serialize_json (nano::jsonconfig &) const; nano::error deserialize_json (nano::jsonconfig &); + nano::error serialize_toml (nano::tomlconfig &) const; + nano::error deserialize_toml (nano::tomlconfig &); unsigned platform{ 0 }; unsigned device{ 0 }; unsigned threads{ 1024 * 1024 }; diff --git a/nano/node/websocketconfig.cpp b/nano/node/websocketconfig.cpp index 4a1aded8..61151999 100644 --- a/nano/node/websocketconfig.cpp +++ b/nano/node/websocketconfig.cpp @@ -1,4 +1,5 @@ #include +#include #include nano::websocket::config::config () : @@ -6,6 +7,22 @@ port (network_constants.default_websocket_port) { } +nano::error nano::websocket::config::serialize_toml (nano::tomlconfig & toml) const +{ + toml.put ("enable", enabled, "Enable or disable WebSocket server\ntype:bool"); + toml.put ("address", address.to_string (), "WebSocket server bind address\ntype:string,ip"); + toml.put ("port", port, "WebSocket server listening port\ntype:uint16"); + return toml.get_error (); +} + +nano::error nano::websocket::config::deserialize_toml (nano::tomlconfig & toml) +{ + toml.get ("enable", enabled); + toml.get_required ("address", address); + toml.get ("port", port); + return toml.get_error (); +} + nano::error nano::websocket::config::serialize_json (nano::jsonconfig & json) const { json.put ("enable", enabled); diff --git a/nano/node/websocketconfig.hpp b/nano/node/websocketconfig.hpp index 42a454b6..de994b10 100644 --- a/nano/node/websocketconfig.hpp +++ b/nano/node/websocketconfig.hpp @@ -7,6 +7,7 @@ namespace nano { class jsonconfig; +class tomlconfig; namespace websocket { /** websocket configuration */ @@ -16,6 +17,8 @@ namespace websocket config (); nano::error deserialize_json (nano::jsonconfig & json_a); nano::error serialize_json (nano::jsonconfig & json) const; + nano::error deserialize_toml (nano::tomlconfig & toml_a); + nano::error serialize_toml (nano::tomlconfig & toml) const; nano::network_constants network_constants; bool enabled{ false }; uint16_t port;