Add wallet_history with timestamps (#994)

* process_local

* process_local for wallet actions

* Fix

* Add timestamps table

* Add timestamp in process_local

* Timestamp in RPC history & blocks_info

* Timestamps core_test

* Add timestamps to RPC history core_test

* RPC wallet_history

* Wallet_history sorting

* wallet_history test

* Typo

* Formatting

* Updates

* Typo

* Fix

* Add sideband information to database.
This commit adds block-type-specific sideband information which eliminates expensive value computations.
This information is stored in-line with the block itself, instead of in a separate table, for efficient retrieval.
Since this involves rewriting all block entries the process is run iteratively in the background.
Once the upgrade is complete the blocks_info table is dropped from the database.

* Sideband RPC & open blocks adjustment (#1555)

* Disable sideband account & height for open blocks

* Add sideband information to RPC block_info

* Replacing RPC block with PRC block_info

* Add local timestamp to RPC account_history

* Fix

* Rolling back changes from different PR

* Correct account & balance with sideband

* Update rpc.blocks_info test

* Improving log message for sideband upgrade process.
Initializing timestamp to sentinal value during the upgrade process.

* fix merge artifacts

* Adding test to show that blocks are put in the correct epoch.

* Fix test_response

* Update test

* Use decode_unsigned & correct error code for timestamps

* Bump the version number on start of sideband upgrade in addition to at the end.

* Proper transactions

nano/core_test/rpc.cpp

@@ -4150,3 +4150,67 @@ TEST (rpc, uptime)
 	ASSERT_EQ (200, response.status);
 	ASSERT_LE (1, response.json.get<int> ("seconds"));
 }
+
+TEST (rpc, wallet_history)
+{
+	nano::system system (24000, 1);
+	auto node0 (system.nodes[0]);
+	nano::genesis genesis;
+	system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv);
+	auto timestamp1 (nano::seconds_since_epoch ());
+	auto send (system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, node0->config.receive_minimum.number ()));
+	ASSERT_NE (nullptr, send);
+	std::this_thread::sleep_for (std::chrono::milliseconds (1000));
+	auto timestamp2 (nano::seconds_since_epoch ());
+	auto receive (system.wallet (0)->receive_action (static_cast<nano::send_block &> (*send), nano::test_genesis_key.pub, node0->config.receive_minimum.number ()));
+	ASSERT_NE (nullptr, receive);
+	nano::keypair key;
+	std::this_thread::sleep_for (std::chrono::milliseconds (1000));
+	auto timestamp3 (nano::seconds_since_epoch ());
+	auto send2 (system.wallet (0)->send_action (nano::test_genesis_key.pub, key.pub, node0->config.receive_minimum.number ()));
+	ASSERT_NE (nullptr, send2);
+	system.deadline_set (10s);
+	nano::rpc rpc (system.io_ctx, *node0, nano::rpc_config (true));
+	rpc.start ();
+	boost::property_tree::ptree request;
+	request.put ("action", "wallet_history");
+	request.put ("wallet", node0->wallets.items.begin ()->first.to_string ());
+	test_response response (request, rpc, system.io_ctx);
+	system.deadline_set (5s);
+	while (response.status == 0)
+	{
+		system.poll ();
+	}
+	ASSERT_EQ (200, response.status);
+	std::vector<std::tuple<std::string, std::string, std::string, std::string, std::string, std::string>> history_l;
+	auto & history_node (response.json.get_child ("history"));
+	for (auto i (history_node.begin ()), n (history_node.end ()); i != n; ++i)
+	{
+		history_l.push_back (std::make_tuple (i->second.get<std::string> ("type"), i->second.get<std::string> ("account"), i->second.get<std::string> ("amount"), i->second.get<std::string> ("hash"), i->second.get<std::string> ("wallet_account"), i->second.get<std::string> ("local_timestamp")));
+	}
+	ASSERT_EQ (4, history_l.size ());
+	ASSERT_EQ ("send", std::get<0> (history_l[0]));
+	ASSERT_EQ (key.pub.to_account (), std::get<1> (history_l[0]));
+	ASSERT_EQ (node0->config.receive_minimum.to_string_dec (), std::get<2> (history_l[0]));
+	ASSERT_EQ (send2->hash ().to_string (), std::get<3> (history_l[0]));
+	ASSERT_EQ (nano::test_genesis_key.pub.to_account (), std::get<4> (history_l[0]));
+	ASSERT_EQ (std::to_string (timestamp3), std::get<5> (history_l[0]));
+	ASSERT_EQ ("receive", std::get<0> (history_l[1]));
+	ASSERT_EQ (nano::test_genesis_key.pub.to_account (), std::get<1> (history_l[1]));
+	ASSERT_EQ (node0->config.receive_minimum.to_string_dec (), std::get<2> (history_l[1]));
+	ASSERT_EQ (receive->hash ().to_string (), std::get<3> (history_l[1]));
+	ASSERT_EQ (nano::test_genesis_key.pub.to_account (), std::get<4> (history_l[1]));
+	ASSERT_EQ (std::to_string (timestamp2), std::get<5> (history_l[1]));
+	ASSERT_EQ ("send", std::get<0> (history_l[2]));
+	ASSERT_EQ (nano::test_genesis_key.pub.to_account (), std::get<1> (history_l[2]));
+	ASSERT_EQ (node0->config.receive_minimum.to_string_dec (), std::get<2> (history_l[2]));
+	ASSERT_EQ (send->hash ().to_string (), std::get<3> (history_l[2]));
+	ASSERT_EQ (nano::test_genesis_key.pub.to_account (), std::get<4> (history_l[2]));
+	ASSERT_EQ (std::to_string (timestamp1), std::get<5> (history_l[2]));
+	// Genesis block
+	ASSERT_EQ ("receive", std::get<0> (history_l[3]));
+	ASSERT_EQ (nano::test_genesis_key.pub.to_account (), std::get<1> (history_l[3]));
+	ASSERT_EQ (nano::genesis_amount.convert_to<std::string> (), std::get<2> (history_l[3]));
+	ASSERT_EQ (genesis.hash ().to_string (), std::get<3> (history_l[3]));
+	ASSERT_EQ (nano::test_genesis_key.pub.to_account (), std::get<4> (history_l[3]));
+}

nano/lib/errors.cpp

@@ -158,6 +158,8 @@ std::string nano::error_rpc_messages::message (int ev) const
 			return "Invalid root hash";
 		case nano::error_rpc::invalid_sources:
 			return "Invalid sources number";
+		case nano::error_rpc::invalid_timestamp:
+			return "Invalid timestamp";
 		case nano::error_rpc::payment_account_balance:
 			return "Account has non-zero balance";
 		case nano::error_rpc::payment_unable_create_account:

nano/lib/errors.hpp

@@ -95,6 +95,7 @@ enum class error_rpc
 	invalid_missing_type,
 	invalid_root,
 	invalid_sources,
+	invalid_timestamp,
 	payment_account_balance,
 	payment_unable_create_account,
 	rpc_control_disabled,

nano/node/rpc.cpp

@@ -1981,7 +1981,10 @@ void nano::rpc_handler::ledger ()
 		boost::optional<std::string> modified_since_text (request.get_optional<std::string> ("modified_since"));
 		if (modified_since_text.is_initialized ())
 		{
-			modified_since = strtoul (modified_since_text.get ().c_str (), NULL, 10);
+			if (decode_unsigned (modified_since_text.get (), modified_since))
+			{
+				ec = nano::error_rpc::invalid_timestamp;
+			}
 		}
 		const bool sorting = request.get<bool> ("sorting", false);
 		const bool representative = request.get<bool> ("representative", false);
@@ -3381,6 +3384,64 @@ void nano::rpc_handler::wallet_frontiers ()
 	response_errors ();
 }
 
+void nano::rpc_handler::wallet_history ()
+{
+	uint64_t modified_since (1);
+	boost::optional<std::string> modified_since_text (request.get_optional<std::string> ("modified_since"));
+	if (modified_since_text.is_initialized ())
+	{
+		if (decode_unsigned (modified_since_text.get (), modified_since))
+		{
+			ec = nano::error_rpc::invalid_timestamp;
+		}
+	}
+	auto wallet (wallet_impl ());
+	if (!ec)
+	{
+		std::multimap<uint64_t, boost::property_tree::ptree, std::greater<uint64_t>> entries;
+		auto transaction (node.wallets.tx_begin_read ());
+		auto block_transaction (node.store.tx_begin_read ());
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		{
+			nano::account account (i->first);
+			nano::account_info info;
+			if (!node.store.account_get (block_transaction, account, info))
+			{
+				auto timestamp (info.modified);
+				auto hash (info.head);
+				while (timestamp >= modified_since && timestamp != std::numeric_limits<uint32_t>::max () && !hash.is_zero ())
+				{
+					nano::block_sideband sideband;
+					auto block (node.store.block_get (block_transaction, hash, &sideband));
+					timestamp = sideband.timestamp;
+					if (block != nullptr && timestamp >= modified_since && timestamp != std::numeric_limits<uint64_t>::max ())
+					{
+						boost::property_tree::ptree entry;
+						entry.put ("wallet_account", account.to_account ());
+						entry.put ("hash", hash.to_string ());
+						history_visitor visitor (*this, false, block_transaction, entry, hash);
+						block->visit (visitor);
+						entry.put ("local_timestamp", std::to_string (timestamp));
+						entries.insert (std::make_pair (timestamp, entry));
+						hash = block->previous ();
+					}
+					else
+					{
+						hash.clear ();
+					}
+				}
+			}
+		}
+		boost::property_tree::ptree history;
+		for (auto i (entries.begin ()), n (entries.end ()); i != n; ++i)
+		{
+			history.push_back (std::make_pair ("", i->second));
+		}
+		response_l.add_child ("history", history);
+	}
+	response_errors ();
+}
+
 void nano::rpc_handler::wallet_key_valid ()
 {
 	auto wallet (wallet_impl ());
@@ -4346,6 +4407,10 @@ void nano::rpc_handler::process_request ()
 			{
 				wallet_frontiers ();
 			}
+			else if (action == "wallet_history")
+			{
+				wallet_history ();
+			}
 			else if (action == "wallet_info")
 			{
 				wallet_info ();

nano/node/rpc.hpp

@@ -207,6 +207,7 @@ public:
 	void wallet_destroy ();
 	void wallet_export ();
 	void wallet_frontiers ();
+	void wallet_history ();
 	void wallet_info ();
 	void wallet_key_valid ();
 	void wallet_ledger ();