Moving resizing outside of creating a transaction.

rai/config.hpp

@@ -15,6 +15,6 @@ enum class rai_networks
 	rai_live_network
 };
 rai::rai_networks const rai_network = rai_networks::ACTIVE_NETWORK;
-int const database_check_interval = rai_network == rai::rai_networks::rai_test_network ? 4 : 1024;
+int const database_check_interval = rai_network == rai::rai_networks::rai_test_network ? 64 : 4096;
 size_t const database_size_increment = rai_network == rai::rai_networks::rai_test_network ? 2 * 1024 * 1024 : 256 * 1024 * 1024;
-}
+}

rai/node/node.cpp

@@ -1043,6 +1043,14 @@ port_mapping (*this),
 vote_processor (*this),
 warmed_up (0)
 {
+	store.environment.sizing_action = [this] ()
+	{
+		auto this_l (shared_from_this ());
+		background ([this_l] ()
+		{
+			this_l->store.environment.handle_environment_sizing ();
+		});
+	};
 	wallets.observer = [this] (rai::account const & account_a, bool active)
 	{
 		observers.wallet (account_a, active);

rai/node/testing.cpp

@@ -161,16 +161,22 @@ void rai::system::generate_rollback (rai::node & node_a, std::vector <rai::accou
 
 void rai::system::generate_receive (rai::node & node_a)
 {
-	rai::transaction transaction (node_a.store.environment, nullptr, false);
-    rai::uint256_union random_block;
-    random_pool.GenerateBlock (random_block.bytes.data (), sizeof (random_block.bytes));
-	auto i (node_a.store.pending_begin (transaction, rai::pending_key (random_block, 0)));
-	if (i != node_a.store.pending_end ())
+	std::unique_ptr <rai::block> send_block;
+	{
+		rai::transaction transaction (node_a.store.environment, nullptr, false);
+		rai::uint256_union random_block;
+		random_pool.GenerateBlock (random_block.bytes.data (), sizeof (random_block.bytes));
+		auto i (node_a.store.pending_begin (transaction, rai::pending_key (random_block, 0)));
+		if (i != node_a.store.pending_end ())
+		{
+			rai::pending_key send_hash (i->first);
+			rai::pending_info info (i->second);
+			send_block = node_a.store.block_get (transaction, send_hash.hash);
+			assert (send_block != nullptr);
+		}
+	}
+	if (send_block != nullptr)
 	{
-		rai::pending_key send_hash (i->first);
-		rai::pending_info info (i->second);
-		auto send_block (node_a.store.block_get (transaction, send_hash.hash));
-		assert (send_block != nullptr);
 		auto receive_error (wallet (0)->receive_sync (static_cast <rai::send_block &> (*send_block), rai::genesis_account, std::numeric_limits<rai::uint128_t>::max ()));
 		(void) receive_error;
 	}

rai/utility.cpp

@@ -45,7 +45,8 @@ bool rai::from_string_hex (std::string const & value_a, uint64_t & target_a)
 rai::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a) :
 open_transactions (0),
 transaction_iteration (0),
-resizing (false)
+resizing (false),
+sizing_action ([this] () { handle_environment_sizing (); })
 {
 	boost::system::error_code error;
 	if (path_a.has_parent_path ())
@@ -57,7 +58,7 @@ resizing (false)
 			assert (status1 == 0);
 			auto status2 (mdb_env_set_maxdbs (environment, 128));
 			assert (status2 == 0);
-			auto status3 (mdb_env_set_mapsize (environment, 2 * database_size_increment));
+			auto status3 (mdb_env_set_mapsize (environment, 0));
 			assert (status3 == 0);
 			auto status4 (mdb_env_open (environment, path_a.string ().c_str (), MDB_NOSUBDIR, 00600));
 			error_a = status4 != 0;
@@ -90,38 +91,45 @@ rai::mdb_env::operator MDB_env * () const
 
 void rai::mdb_env::add_transaction ()
 {
+	// Minimize I/O from checking if we need to resive
+	// Too high and the enviroment might overflow
+	// Too low and we're making lots of trips to the disk
+	if ((transaction_iteration++ % rai::database_check_interval) == 0)
+	{
+		sizing_action ();
+	}
 	std::unique_lock <std::mutex> lock_l (lock);
+	// Wait for any resizing operations to complete
 	while (resizing)
 	{
 		resize_notify.wait (lock_l);
 	}
-	if ((transaction_iteration % rai::database_check_interval) == 0)
+	++open_transactions;
+}
+
+void rai::mdb_env::handle_environment_sizing ()
+{
+	MDB_stat stats;
+	mdb_env_stat (environment, &stats);
+	MDB_envinfo info;
+	mdb_env_info (environment, &info);
+	size_t load (info.me_last_pgno * stats.ms_psize);
+	auto slack (info.me_mapsize - load);
+	if (slack < (rai::database_size_increment / 4))
 	{
-		MDB_stat stats;
-		mdb_env_stat (environment, &stats);
-		MDB_envinfo info;
-		mdb_env_info (environment, &info);
-		size_t load (info.me_last_pgno * stats.ms_psize);
-		auto slack (info.me_mapsize - load);
-		if (slack < (rai::database_size_increment / 4))
+		if (!resizing.exchange (true))
 		{
-			resizing = true;
-			auto done (std::chrono::system_clock::now () + std::chrono::milliseconds (500));
-			while (std::chrono::system_clock::now () < done && open_transactions > 0)
+			std::unique_lock <std::mutex> lock_l (lock);
+			while (open_transactions > 0)
 			{
-				open_notify.wait_for (lock_l, std::chrono::milliseconds (50));
-			}
-			if (open_transactions == 0)
-			{
-				auto next_size (((info.me_mapsize / database_size_increment) + 1) * database_size_increment);
-				mdb_env_set_mapsize (environment, next_size);
+				open_notify.wait (lock_l);
 			}
+			auto next_size (((info.me_mapsize / database_size_increment) + 1) * database_size_increment);
+			mdb_env_set_mapsize (environment, next_size);
 			resizing = false;
 			resize_notify.notify_all ();
 		}
 	}
-	++transaction_iteration;
-	++open_transactions;
 }
 
 void rai::mdb_env::remove_transaction ()

rai/utility.hpp

@@ -138,13 +138,15 @@ public:
 	operator MDB_env * () const;
 	void add_transaction ();
 	void remove_transaction ();
+	void handle_environment_sizing ();
 	MDB_env * environment;
 	std::mutex lock;
 	std::condition_variable open_notify;
 	unsigned open_transactions;
-	unsigned transaction_iteration;
+	std::atomic_uint transaction_iteration;
 	std::condition_variable resize_notify;
-	bool resizing;
+	std::atomic_bool resizing;
+	std::function <void ()> sizing_action;
 };
 class mdb_val
 {