Splitting wallets from ledger database. (#1610)

* Splitting wallets from ledger database.

* Fixing race condition between setting store version and splitting wallets.
Fixing initialization order of wallet store.

nano/core_test/wallets.cpp

@@ -4,6 +4,8 @@
 #include <nano/node/node.hpp>
 #include <nano/node/testing.hpp>
 
+#include <boost/polymorphic_cast.hpp>
+
 using namespace std::chrono_literals;
 
 TEST (wallets, open_create)
@@ -73,6 +75,38 @@ TEST (wallets, remove)
 	}
 }
 
+TEST (wallets, upgrade)
+{
+	nano::system system (24000, 1);
+	auto path (nano::unique_path ());
+	nano::keypair id;
+	{
+		nano::node_init init1;
+		auto node1 (std::make_shared<nano::node> (init1, system.io_ctx, 24001, path, system.alarm, system.logging, system.work));
+		ASSERT_FALSE (init1.error ());
+		node1->wallets.create (id.pub);
+		auto & mdb_store (dynamic_cast<nano::mdb_store &> (node1->store));
+		auto transaction_destination (mdb_store.tx_begin_write ());
+		MDB_txn * tx_destination (*boost::polymorphic_downcast<nano::mdb_txn *> (transaction_destination.impl.get ()));
+		auto transaction_source (node1->wallets.env.tx_begin (true));
+		MDB_txn * tx_source (*boost::polymorphic_downcast<nano::mdb_txn *> (transaction_source.impl.get ()));
+		node1->wallets.move_table (id.pub.to_string (), tx_source, tx_destination);
+		node1->store.version_put (transaction_destination, 11);
+	}
+	nano::node_init init1;
+	auto node1 (std::make_shared<nano::node> (init1, system.io_ctx, 24001, path, system.alarm, system.logging, system.work));
+	ASSERT_EQ (1, node1->wallets.items.size ());
+	ASSERT_EQ (id.pub, node1->wallets.items.begin ()->first);
+	auto transaction_old (node1->store.tx_begin_write ());
+	MDB_txn * tx_old (*boost::polymorphic_downcast<nano::mdb_txn *> (transaction_old.impl.get ()));
+	auto transaction_new (node1->wallets.env.tx_begin (true));
+	MDB_txn * tx_new (*boost::polymorphic_downcast<nano::mdb_txn *> (transaction_new.impl.get ()));
+	MDB_dbi old_handle;
+	ASSERT_EQ (MDB_NOTFOUND, mdb_dbi_open (tx_old, id.pub.to_string ().c_str (), 0, &old_handle));
+	MDB_dbi new_handle;
+	ASSERT_EQ (0, mdb_dbi_open (tx_new, id.pub.to_string ().c_str (), 0, &new_handle));
+}
+
 // Keeps breaking whenever we add new DBs
 TEST (wallets, DISABLED_wallet_create_max)
 {

nano/node/node.cpp

@@ -778,7 +778,7 @@ wallet_init (false)
 
 bool nano::node_init::error ()
 {
-	return block_store_init || wallet_init;
+	return block_store_init || wallet_init || wallets_store_init;
 }
 
 nano::vote_processor::vote_processor (nano::node & node_a) :
@@ -1630,6 +1630,8 @@ alarm (alarm_a),
 work (work_a),
 store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, config.logging, application_path_a / "data.ldb", config_a.lmdb_max_dbs)),
 store (*store_impl),
+wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (init_a.wallets_store_init, application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)),
+wallets_store (*wallets_store_impl),
 gap_cache (*this),
 ledger (store, stats, config.epoch_block_link, config.epoch_block_signer),
 active (*this),
@@ -1638,7 +1640,7 @@ bootstrap_initiator (*this),
 bootstrap (io_ctx_a, config.peering_port, *this),
 peers (network.endpoint ()),
 application_path (application_path_a),
-wallets (init_a.block_store_init, *this),
+wallets (init_a.wallet_init, *this),
 port_mapping (*this),
 vote_processor (*this),
 warmed_up (0),
@@ -2292,7 +2294,7 @@ void nano::node::bootstrap_wallet ()
 	std::deque<nano::account> accounts;
 	{
 		std::lock_guard<std::mutex> lock (wallets.mutex);
-		auto transaction (store.tx_begin_read ());
+		auto transaction (wallets.tx_begin_read ());
 		for (auto i (wallets.items.begin ()), n (wallets.items.end ()); i != n && accounts.size () < 128; ++i)
 		{
 			auto & wallet (*i->second);

nano/node/node.hpp

@@ -330,6 +330,7 @@ public:
 	node_init ();
 	bool error ();
 	bool block_store_init;
+	bool wallets_store_init;
 	bool wallet_init;
 };
 class node_observers
@@ -512,6 +513,8 @@ public:
 	boost::log::sources::logger_mt log;
 	std::unique_ptr<nano::block_store> store_impl;
 	nano::block_store & store;
+	std::unique_ptr<nano::wallets_store> wallets_store_impl;
+	nano::wallets_store & wallets_store;
 	nano::gap_cache gap_cache;
 	nano::ledger ledger;
 	nano::active_transactions active;

nano/node/wallet.cpp

@@ -1292,7 +1292,7 @@ void nano::wallet::work_cache_blocking (nano::account const & account_a, nano::b
 nano::wallets::wallets (bool & error_a, nano::node & node_a) :
 observer ([](bool) {}),
 node (node_a),
-env (boost::polymorphic_downcast<nano::mdb_store *> (node_a.store_impl.get ())->env),
+env (boost::polymorphic_downcast<nano::mdb_wallets_store *> (node_a.wallets_store_impl.get ())->environment),
 stopped (false),
 thread ([this]() {
 	nano::thread_role::set (nano::thread_role::name::wallet_actions);
@@ -1304,6 +1304,7 @@ thread ([this]() {
 	{
 		auto transaction (tx_begin_write ());
 		auto status (mdb_dbi_open (env.tx (transaction), nullptr, MDB_CREATE, &handle));
+		split_if_needed (transaction, node.store);
 		status |= mdb_dbi_open (env.tx (transaction), "send_action_ids", MDB_CREATE, &send_action_ids);
 		assert (status == 0);
 		std::string beginning (nano::uint256_union (0).to_string ());
@@ -1328,9 +1329,9 @@ thread ([this]() {
 			}
 		}
 	}
-	for (auto i (items.begin ()), n (items.end ()); i != n; ++i)
+	for (auto & item : items)
 	{
-		i->second->enter_initial_password ();
+		item.second->enter_initial_password ();
 	}
 }
 
@@ -1562,6 +1563,57 @@ void nano::wallets::clear_send_ids (nano::transaction const & transaction_a)
 	assert (status == 0);
 }
 
+void nano::wallets::split_if_needed (nano::transaction & transaction_destination, nano::block_store & store_a)
+{
+	auto store_l (dynamic_cast<nano::mdb_store *> (&store_a));
+	if (store_l != nullptr)
+	{
+		auto transaction_source (store_l->tx_begin_write ());
+		MDB_txn * tx_source (*boost::polymorphic_downcast<nano::mdb_txn *> (transaction_source.impl.get ()));
+		if (items.empty ())
+		{
+			MDB_txn * tx_destination (*boost::polymorphic_downcast<nano::mdb_txn *> (transaction_destination.impl.get ()));
+			std::string beginning (nano::uint256_union (0).to_string ());
+			std::string end ((nano::uint256_union (nano::uint256_t (0) - nano::uint256_t (1))).to_string ());
+			nano::store_iterator<std::array<char, 64>, nano::mdb_val::no_value> i (std::make_unique<nano::mdb_iterator<std::array<char, 64>, nano::mdb_val::no_value>> (transaction_source, handle, nano::mdb_val (beginning.size (), const_cast<char *> (beginning.c_str ()))));
+			nano::store_iterator<std::array<char, 64>, nano::mdb_val::no_value> n (std::make_unique<nano::mdb_iterator<std::array<char, 64>, nano::mdb_val::no_value>> (transaction_source, handle, nano::mdb_val (end.size (), const_cast<char *> (end.c_str ()))));
+			for (; i != n; ++i)
+			{
+				nano::uint256_union id;
+				std::string text (i->first.data (), i->first.size ());
+				auto error1 (id.decode_hex (text));
+				assert (!error1);
+				assert (strlen (text.c_str ()) == text.size ());
+				move_table (text, tx_source, tx_destination);
+			}
+		}
+	}
+}
+
+void nano::wallets::move_table (std::string const & name_a, MDB_txn * tx_source, MDB_txn * tx_destination)
+{
+	MDB_dbi handle_source;
+	auto error2 (mdb_dbi_open (tx_source, name_a.c_str (), MDB_CREATE, &handle_source));
+	assert (!error2);
+	MDB_dbi handle_destination;
+	auto error3 (mdb_dbi_open (tx_destination, name_a.c_str (), MDB_CREATE, &handle_destination));
+	assert (!error3);
+	MDB_cursor * cursor;
+	auto error4 (mdb_cursor_open (tx_source, handle_source, &cursor));
+	assert (!error4);
+	MDB_val val_key;
+	MDB_val val_value;
+	auto cursor_status (mdb_cursor_get (cursor, &val_key, &val_value, MDB_FIRST));
+	while (cursor_status == MDB_SUCCESS)
+	{
+		auto error5 (mdb_put (tx_destination, handle_destination, &val_key, &val_value, 0));
+		assert (!error5);
+		cursor_status = mdb_cursor_get (cursor, &val_key, &val_value, MDB_NEXT);
+	}
+	auto error6 (mdb_drop (tx_source, handle_source, 1));
+	assert (!error6);
+}
+
 nano::uint128_t const nano::wallets::generate_priority = std::numeric_limits<nano::uint128_t>::max ();
 nano::uint128_t const nano::wallets::high_priority = std::numeric_limits<nano::uint128_t>::max () - 1;
 
@@ -1603,7 +1655,10 @@ nano::store_iterator<nano::uint256_union, nano::wallet_value> nano::wallet_store
 {
 	return nano::store_iterator<nano::uint256_union, nano::wallet_value> (nullptr);
 }
-
+nano::mdb_wallets_store::mdb_wallets_store (bool & error_a, boost::filesystem::path const & path_a, int lmdb_max_dbs) :
+environment (error_a, path_a, lmdb_max_dbs, 1ULL * 1024 * 1024 * 1024)
+{
+}
 MDB_txn * nano::wallet_store::tx (nano::transaction const & transaction_a) const
 {
 	auto result (boost::polymorphic_downcast<nano::mdb_txn *> (transaction_a.impl.get ()));

nano/node/wallet.hpp

@@ -178,6 +178,8 @@ public:
 	bool exists (nano::transaction const &, nano::public_key const &);
 	void stop ();
 	void clear_send_ids (nano::transaction const &);
+	void split_if_needed (nano::transaction &, nano::block_store &);
+	void move_table (std::string const &, MDB_txn *, MDB_txn *);
 	std::function<void(bool)> observer;
 	std::unordered_map<nano::uint256_union, std::shared_ptr<nano::wallet>> items;
 	std::multimap<nano::uint128_t, std::pair<std::shared_ptr<nano::wallet>, std::function<void(nano::wallet &)>>, std::greater<nano::uint128_t>> actions;
@@ -205,4 +207,15 @@ public:
 	 */
 	nano::transaction tx_begin (bool write = false);
 };
+class wallets_store
+{
+public:
+	virtual ~wallets_store () = default;
+};
+class mdb_wallets_store : public wallets_store
+{
+public:
+	mdb_wallets_store (bool &, boost::filesystem::path const &, int lmdb_max_dbs = 128);
+	nano::mdb_env environment;
+};
 }