From cf724fd58ca174e1a3e01df79e171ec591b7c1ac Mon Sep 17 00:00:00 2001 From: SergiySW Date: Sat, 14 Oct 2017 23:04:12 +0300 Subject: [PATCH 1/5] Adding blocks_info subdatabase for faster block_account & block_balance --- rai/secure.cpp | 202 +++++++++++++++++++++++++++++++++++++++++++++++-- rai/secure.hpp | 26 +++++++ 2 files changed, 221 insertions(+), 7 deletions(-) diff --git a/rai/secure.cpp b/rai/secure.cpp index 6b775e08..622a5d02 100755 --- a/rai/secure.cpp +++ b/rai/secure.cpp @@ -1515,6 +1515,7 @@ receive_blocks (0), open_blocks (0), change_blocks (0), pending (0), +blocks_info (0), representation (0), unchecked (0), unsynced (0), @@ -1530,6 +1531,7 @@ checksum (0) error_a |= mdb_dbi_open (transaction, "open", MDB_CREATE, &open_blocks) != 0; error_a |= mdb_dbi_open (transaction, "change", MDB_CREATE, &change_blocks) != 0; error_a |= mdb_dbi_open (transaction, "pending", MDB_CREATE, &pending) != 0; + error_a |= mdb_dbi_open (transaction, "blocks_info", MDB_CREATE, &blocks_info) != 0; error_a |= mdb_dbi_open (transaction, "representation", MDB_CREATE, &representation) != 0; error_a |= mdb_dbi_open (transaction, "unchecked", MDB_CREATE | MDB_DUPSORT, &unchecked) != 0; error_a |= mdb_dbi_open (transaction, "unsynced", MDB_CREATE, &unsynced) != 0; @@ -1592,6 +1594,8 @@ void rai::block_store::do_upgrades (MDB_txn * transaction_a) case 8: upgrade_v8_to_v9 (transaction_a); case 9: + upgrade_v9_to_v10 (transaction_a); + case 10: break; default: assert (false); @@ -1801,6 +1805,35 @@ void rai::block_store::upgrade_v8_to_v9 (MDB_txn * transaction_a) mdb_drop (transaction_a, sequence, 1); } +void rai::block_store::upgrade_v9_to_v10 (MDB_txn * transaction_a) +{ + version_put (transaction_a, 10); + mdb_dbi_open (transaction_a, "blocks_info", MDB_CREATE, &blocks_info); + for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) + { + rai::account account (i->first); + rai::account_info info (i->second); + if (info.block_count >= block_info_max) + { + uint64_t block_count (1); + auto hash (info.open_block); + while (!hash.is_zero ()) + { + if (!(block_count % block_info_max)) + { + rai::block_info block_info; + block_info.account = account; + rai::amount balance (block_balance (transaction_a, hash)); + block_info.balance = balance; + block_info_put (transaction_a, hash, block_info); + } + hash = block_successor (transaction_a, hash); + ++block_count; + } + } + } +} + void rai::block_store::clear (MDB_dbi db_a) { rai::transaction transaction (environment, nullptr, true); @@ -2304,6 +2337,110 @@ rai::mdb_val rai::pending_key::val () const return rai::mdb_val (sizeof (*this), const_cast (this)); } +void rai::block_store::block_info_put (MDB_txn * transaction_a, rai::block_hash const & hash_a, rai::block_info const & block_info_a) +{ + auto status (mdb_put (transaction_a, blocks_info, hash_a.val (), block_info_a.val (), 0)); + assert (status == 0); +} + +void rai::block_store::block_info_del (MDB_txn * transaction_a, rai::block_hash const & hash_a) +{ + auto status (mdb_del (transaction_a, blocks_info, hash_a.val (), nullptr)); + assert (status == 0); +} + +bool rai::block_store::block_info_exists (MDB_txn * transaction_a, rai::block_hash const & hash_a) +{ + auto iterator (block_info_begin (transaction_a, hash_a)); + return iterator != rai::store_iterator (nullptr) && rai::block_hash (iterator->first) == hash_a; +} + +bool rai::block_store::block_info_get (MDB_txn * transaction_a, rai::block_hash const & hash_a, rai::block_info & block_info_a) +{ + MDB_val value; + auto status (mdb_get (transaction_a, blocks_info, hash_a.val (), &value)); + assert (status == 0 || status == MDB_NOTFOUND); + bool result; + if (status == MDB_NOTFOUND) + { + result = true; + } + else + { + result = false; + assert (value.mv_size == sizeof (block_info_a.account.bytes) + sizeof (block_info_a.balance.bytes)); + rai::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); + auto error1 (rai::read (stream, block_info_a.account)); + assert (!error1); + auto error2 (rai::read (stream, block_info_a.balance)); + assert (!error2); + } + return result; +} + +rai::store_iterator rai::block_store::block_info_begin (MDB_txn * transaction_a, rai::block_hash const & hash_a) +{ + rai::store_iterator result (transaction_a, blocks_info, hash_a.val ()); + return result; +} + +rai::store_iterator rai::block_store::block_info_begin (MDB_txn * transaction_a) +{ + rai::store_iterator result (transaction_a, blocks_info); + return result; +} + +rai::store_iterator rai::block_store::block_info_end () +{ + rai::store_iterator result (nullptr); + return result; +} + +rai::block_info::block_info () : +account (0), +balance (0) +{ +} + +rai::block_info::block_info (MDB_val const & val_a) +{ + assert(val_a.mv_size == sizeof (*this)); + static_assert (sizeof (account) + sizeof (balance) == sizeof (*this), "Packed class"); + std::copy (reinterpret_cast (val_a.mv_data), reinterpret_cast (val_a.mv_data) + sizeof (*this), reinterpret_cast (this)); +} + +rai::block_info::block_info (rai::account const & account_a, rai::amount const & balance_a) : +account (account_a), +balance (balance_a) +{ +} + +void rai::block_info::serialize (rai::stream & stream_a) const +{ + rai::write (stream_a, account.bytes); + rai::write (stream_a, balance.bytes); +} + +bool rai::block_info::deserialize (rai::stream & stream_a) +{ + auto result (rai::read (stream_a, account.bytes)); + if (!result) + { + result = rai::read (stream_a, balance.bytes); + } + return result; +} + +bool rai::block_info::operator == (rai::block_info const & other_a) const +{ + return account == other_a.account && balance == other_a.balance; +} + +rai::mdb_val rai::block_info::val () const +{ + return rai::mdb_val (sizeof (*this), const_cast (this)); +} + rai::uint128_t rai::block_store::representation_get (MDB_txn * transaction_a, rai::account const & account_a) { MDB_val value; @@ -2808,10 +2945,18 @@ void balance_visitor::send_block (rai::send_block const & block_a) void balance_visitor::receive_block (rai::receive_block const & block_a) { - amount_visitor source (transaction, store); - source.compute (block_a.hashables.source); - result += source.result; - current = block_a.hashables.previous; + amount_visitor source (transaction, store); + source.compute (block_a.hashables.source); + rai::block_info block_info; + if (!store.block_info_get (transaction, block_a.hash (), block_info)) + { + result += block_info.balance.number (); + current = 0; + } + else { + result += source.result; + current = block_a.hashables.previous; + } } void balance_visitor::open_block (rai::open_block const & block_a) @@ -2824,7 +2969,15 @@ void balance_visitor::open_block (rai::open_block const & block_a) void balance_visitor::change_block (rai::change_block const & block_a) { - current = block_a.hashables.previous; + rai::block_info block_info; + if (!store.block_info_get (transaction, block_a.hash (), block_info)) + { + result += block_info.balance.number (); + current = 0; + } + else { + current = block_a.hashables.previous; + } } // Rollback this block @@ -2855,6 +3008,10 @@ public: ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, pending.source); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); + if (!(info.block_count % ledger.store.block_info_max)) + { + ledger.store.block_info_del (transaction, hash); + } } void receive_block (rai::receive_block const & block_a) override { @@ -2872,6 +3029,10 @@ public: ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, destination_account); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); + if (!(info.block_count % ledger.store.block_info_max)) + { + ledger.store.block_info_del (transaction, hash); + } } void open_block (rai::open_block const & block_a) override { @@ -2901,6 +3062,10 @@ public: ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, account); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); + if (!(info.block_count % ledger.store.block_info_max)) + { + ledger.store.block_info_del (transaction, hash); + } } MDB_txn * transaction; rai::ledger & ledger; @@ -2947,6 +3112,13 @@ rai::uint128_t rai::ledger::balance (MDB_txn * transaction_a, rai::block_hash co return visitor.result; } +rai::uint128_t rai::block_store::block_balance (MDB_txn * transaction_a, rai::block_hash const & hash_a) +{ + balance_visitor visitor (transaction_a, *this); + visitor.compute (hash_a); + return visitor.result; +} + // Balance for an account by account number rai::uint128_t rai::ledger::account_balance (MDB_txn * transaction_a, rai::account const & account_a) { @@ -3054,7 +3226,8 @@ rai::account rai::ledger::account (MDB_txn * transaction_a, rai::block_hash cons assert (store.block_exists (transaction_a, hash_a)); auto hash (hash_a); rai::block_hash successor (1); - while (!successor.is_zero ()) + rai::block_info block_info; + while (!successor.is_zero () && store.block_info_get (transaction_a, successor, block_info)) { successor = store.block_successor (transaction_a, hash); if (!successor.is_zero ()) @@ -3062,7 +3235,15 @@ rai::account rai::ledger::account (MDB_txn * transaction_a, rai::block_hash cons hash = successor; } } - auto result (store.frontier_get (transaction_a, hash)); + rai::account result; + if (successor.is_zero ()) + { + result = store.frontier_get (transaction_a, hash); + } + else + { + result = block_info.account; + } assert (!result.is_zero ()); return result; } @@ -3161,6 +3342,13 @@ void rai::ledger::change_latest (MDB_txn * transaction_a, rai::account const & a info.modified = store.now (); info.block_count = block_count_a; store.account_put (transaction_a, account_a, info); + if (!(block_count_a % store.block_info_max)) + { + rai::block_info block_info; + block_info.account = account_a; + block_info.balance = balance_a; + store.block_info_put (transaction_a, hash_a, block_info); + } checksum_update (transaction_a, hash_a); } else diff --git a/rai/secure.hpp b/rai/secure.hpp index 2a992e14..cff35980 100644 --- a/rai/secure.hpp +++ b/rai/secure.hpp @@ -303,6 +303,19 @@ public: rai::account account; rai::block_hash hash; }; +class block_info +{ +public: + block_info (); + block_info (MDB_val const &); + block_info (rai::account const &, rai::amount const &); + void serialize (rai::stream &) const; + bool deserialize (rai::stream &); + bool operator == (rai::block_info const &) const; + rai::mdb_val val () const; + rai::account account; + rai::amount balance; +}; class block_counts { public: @@ -387,6 +400,16 @@ public: rai::store_iterator pending_begin (MDB_txn *); rai::store_iterator pending_end (); + void block_info_put (MDB_txn *, rai::block_hash const &, rai::block_info const &); + void block_info_del (MDB_txn *, rai::block_hash const &); + bool block_info_get (MDB_txn *, rai::block_hash const &, rai::block_info &); + bool block_info_exists (MDB_txn *, rai::block_hash const &); + rai::store_iterator block_info_begin (MDB_txn *, rai::block_hash const &); + rai::store_iterator block_info_begin (MDB_txn *); + rai::store_iterator block_info_end (); + rai::uint128_t block_balance (MDB_txn *, rai::block_hash const &); + static size_t const block_info_max = 32; + rai::uint128_t representation_get (MDB_txn *, rai::account const &); void representation_put (MDB_txn *, rai::account const &, rai::uint128_t const &); void representation_add (MDB_txn *, rai::account const &, rai::uint128_t const &); @@ -448,6 +471,7 @@ public: void upgrade_v6_to_v7 (MDB_txn *); void upgrade_v7_to_v8 (MDB_txn *); void upgrade_v8_to_v9 (MDB_txn *); + void upgrade_v9_to_v10 (MDB_txn *); void clear (MDB_dbi); @@ -466,6 +490,8 @@ public: MDB_dbi change_blocks; // block_hash -> sender, amount, destination // Pending blocks to sender account, amount, destination account MDB_dbi pending; + // block_hash -> account, balance // Blocks info + MDB_dbi blocks_info; // account -> weight // Representation MDB_dbi representation; // block_hash -> block // Unchecked bootstrap blocks From 6fc6490ff3ab1dca55c2c4b5f08b907b8220365d Mon Sep 17 00:00:00 2001 From: SergiySW Date: Mon, 23 Oct 2017 01:38:58 +0300 Subject: [PATCH 2/5] Adding test for block_info table upgrade --- rai/core_test/block_store.cpp | 39 ++++++++++++++++++++++++++++++++++- rai/core_test/rpc.cpp | 2 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/rai/core_test/block_store.cpp b/rai/core_test/block_store.cpp index 9bbf2bbb..d3837e78 100644 --- a/rai/core_test/block_store.cpp +++ b/rai/core_test/block_store.cpp @@ -921,8 +921,45 @@ TEST (block_store, upgrade_v8_v9) rai::block_store store (init, path); ASSERT_FALSE (init); rai::transaction transaction (store.environment, nullptr, false); - ASSERT_EQ (9, store.version_get (transaction)); + ASSERT_LT (8, store.version_get (transaction)); auto vote (store.vote_get (transaction, key.pub)); ASSERT_NE (nullptr, vote); ASSERT_EQ (10, vote->sequence); } + +TEST (block_store, upgrade_v9_v10) +{ + auto path (rai::unique_path ()); + rai::block_hash hash (0); + { + bool init (false); + rai::block_store store (init, path); + ASSERT_FALSE (init); + rai::transaction transaction (store.environment, nullptr, true); + rai::genesis genesis;; + genesis.initialize (transaction, store); + rai::ledger ledger (store); + store.version_put (transaction, 9); + rai::account_info info; + store.account_get (transaction, rai::test_genesis_key.pub, info); + rai::keypair key0; + rai::uint128_t balance (rai::genesis_amount); + hash = info.head; + for (auto i (1); i < 32; ++i) // Making 31 send blocks (+ 1 open = 32 total) + { + balance = balance - rai::Gxrb_ratio; + rai::send_block block0 (hash, key0.pub, balance, rai::test_genesis_key.prv, rai::test_genesis_key.pub, 0); + ASSERT_EQ (rai::process_result::progress, ledger.process (transaction, block0).code); + hash = block0.hash (); + } + } + bool init (false); + rai::block_store store (init, path); + ASSERT_FALSE (init); + rai::transaction transaction (store.environment, nullptr, false); + ASSERT_LT (9, store.version_get (transaction)); + rai::block_info block_info; + store.block_info_get (transaction, hash, block_info); + ASSERT_EQ (block_info.account, rai::test_genesis_key.pub); + ASSERT_EQ (block_info.balance.number (), rai::genesis_amount - rai::Gxrb_ratio * 31); +} diff --git a/rai/core_test/rpc.cpp b/rai/core_test/rpc.cpp index 7501a318..6c6673bc 100644 --- a/rai/core_test/rpc.cpp +++ b/rai/core_test/rpc.cpp @@ -1359,7 +1359,7 @@ TEST (rpc, version) ASSERT_EQ (200, response1.status); ASSERT_EQ ("1", response1.json.get ("rpc_version")); ASSERT_EQ (200, response1.status); - ASSERT_EQ ("9", response1.json.get ("store_version")); + ASSERT_EQ ("10", response1.json.get ("store_version")); ASSERT_EQ (boost::str (boost::format ("RaiBlocks %1%.%2%") % RAIBLOCKS_VERSION_MAJOR % RAIBLOCKS_VERSION_MINOR), response1.json.get ("node_vendor")); auto headers (response1.resp.find ("Access-Control-Allow-Origin")); ASSERT_NE (response1.resp.end (), headers); From 3a2589938741bbe557d6328de0aba0a4147011b3 Mon Sep 17 00:00:00 2001 From: SergiySW Date: Tue, 24 Oct 2017 10:28:05 +0300 Subject: [PATCH 3/5] Adding automatic block_info creation test --- rai/core_test/block_store.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rai/core_test/block_store.cpp b/rai/core_test/block_store.cpp index d3837e78..00de66f2 100644 --- a/rai/core_test/block_store.cpp +++ b/rai/core_test/block_store.cpp @@ -952,6 +952,13 @@ TEST (block_store, upgrade_v9_v10) ASSERT_EQ (rai::process_result::progress, ledger.process (transaction, block0).code); hash = block0.hash (); } + rai::block_info block_info_auto; // Checking automatic block_info creation for block 32 + store.block_info_get (transaction, hash, block_info_auto); + ASSERT_EQ (block_info_auto.account, rai::test_genesis_key.pub); + ASSERT_EQ (block_info_auto.balance.number (), balance); + ASSERT_EQ (0, mdb_drop (transaction, store.blocks_info, 0)); // Cleaning blocks_info subdatabase + bool block_info_exists (store.block_info_exists (transaction, hash)); + ASSERT_EQ (block_info_exists, 0); // Checking if automatic block_info is deleted } bool init (false); rai::block_store store (init, path); From af63bb5f575e2ac67555e3c53f21392852f21b84 Mon Sep 17 00:00:00 2001 From: clemahieu Date: Wed, 1 Nov 2017 21:44:30 -0500 Subject: [PATCH 4/5] Database has already been opened in main constructor. --- rai/secure.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/rai/secure.cpp b/rai/secure.cpp index 622a5d02..0d659c30 100755 --- a/rai/secure.cpp +++ b/rai/secure.cpp @@ -1808,7 +1808,6 @@ void rai::block_store::upgrade_v8_to_v9 (MDB_txn * transaction_a) void rai::block_store::upgrade_v9_to_v10 (MDB_txn * transaction_a) { version_put (transaction_a, 10); - mdb_dbi_open (transaction_a, "blocks_info", MDB_CREATE, &blocks_info); for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) { rai::account account (i->first); From 787dede51b2dffb77019af6598d5740698158f2d Mon Sep 17 00:00:00 2001 From: clemahieu Date: Thu, 2 Nov 2017 15:57:36 -0500 Subject: [PATCH 5/5] Checking block count in the beginning is redundant with checking block_count % block_info_max --- rai/secure.cpp | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/rai/secure.cpp b/rai/secure.cpp index 0d659c30..4fc6b589 100755 --- a/rai/secure.cpp +++ b/rai/secure.cpp @@ -1812,23 +1812,20 @@ void rai::block_store::upgrade_v9_to_v10 (MDB_txn * transaction_a) { rai::account account (i->first); rai::account_info info (i->second); - if (info.block_count >= block_info_max) + size_t block_count (1); + auto hash (info.open_block); + while (!hash.is_zero ()) { - uint64_t block_count (1); - auto hash (info.open_block); - while (!hash.is_zero ()) + if ((block_count % block_info_max) == 0) { - if (!(block_count % block_info_max)) - { - rai::block_info block_info; - block_info.account = account; - rai::amount balance (block_balance (transaction_a, hash)); - block_info.balance = balance; - block_info_put (transaction_a, hash, block_info); - } - hash = block_successor (transaction_a, hash); - ++block_count; + rai::block_info block_info; + block_info.account = account; + rai::amount balance (block_balance (transaction_a, hash)); + block_info.balance = balance; + block_info_put (transaction_a, hash, block_info); } + hash = block_successor (transaction_a, hash); + ++block_count; } } }