dn724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Packed timestamp (#2879)

Browse source 
* Adding unique timestamp class which packs a timestamp + counter in to a 64-bit number.

* Using only 44 bits for time portion and 20 bits for counter portion.
Fixing some initialization and stepping issue.

* Removing possible race condition in test.

* Formatting.

* Updating comments.

* Renaming timestamp_now to now.

* Rewriting tests in terms of vote counts instead of sequence numbers.

* Rewriting tests in terms of vote counts instead of sequence numbers.

* Replacing vote sequence numbers with timestamps.

* Removing vote table from database.

* Removing vote functionality from block_store class.

* Removing plist file.

* Substituting references to sequence with timestamp.

* Using nano::lock_guard.

* Using ASSERT() instead of assert()

* Modernizing syntax and renaming timestamp functions to better represent what they're doing.

* Add forward-declaration header and templatize the timestamp class to be based on different clocks.

* Formatting.

* Fuse branches in to a single cmpxchg + conditional move.
Track by last timestamp rather than next.

* Cleanup, commenting, and using std::chrono::milliseconds instead of the associated count to give type assurances.

* Correct docker/node/entry.sh permissions

* Disabling dropping the vote table for beta.

* Timestamp_generator isn't used in request aggregator.

* Preferring to use votes() result when applicable.

* Outputting both sequence and timestamp to allow deprecation of "sequence" in favor of "timestamp" by users.

* Revert "Preferring to use votes() result when applicable."

This reverts commit 9bf4073602a1a2d978c9ebdfa07a633ed2d55030.

* Appeasing VS bugs.

* Fix merge database migrate path

* Dropping the vote table during upgrade.

Co-authored-by: Sergey Kroshnin <sergiysw@gmail.com>
This commit is contained in:
clemahieu 2020-10-28 11:31:27 +01:00 • committed by GitHub
commit 59772acc02
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 235 additions and 431 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nano/core_test/CMakeLists.txt
View file

@ -34,6 +34,7 @@ add_executable (core_test
telemetry.cpp
toml.cpp
timer.cpp
timestamp.cpp
uint256_union.cpp
utility.cpp
vote_processor.cpp

4
nano/core_test/active_transactions.cpp
View file

@ -269,7 +269,7 @@ TEST (active_transactions, inactive_votes_cache_existing_vote)
ASSERT_EQ (1, node.stats.count (nano::stat::type::election, nano::stat::detail::vote_new));
auto last_vote1 (election->votes ()[key.pub]);
ASSERT_EQ (send->hash (), last_vote1.hash);
ASSERT_EQ (1, last_vote1.sequence);
ASSERT_EQ (1, last_vote1.timestamp);
// Attempt to change vote with inactive_votes_cache
nano::unique_lock<std::mutex> active_lock (node.active.mutex);
node.active.add_inactive_votes_cache (active_lock, send->hash (), key.pub);
@ -282,7 +282,7 @@ TEST (active_transactions, inactive_votes_cache_existing_vote)
ASSERT_EQ (2, election->votes ().size ());
auto last_vote2 (election->votes ()[key.pub]);
ASSERT_EQ (last_vote1.hash, last_vote2.hash);
ASSERT_EQ (last_vote1.sequence, last_vote2.sequence);
ASSERT_EQ (last_vote1.timestamp, last_vote2.timestamp);
ASSERT_EQ (last_vote1.time, last_vote2.time);
ASSERT_EQ (0, node.stats.count (nano::stat::type::election, nano::stat::detail::vote_cached));
}

88
nano/core_test/block_store.cpp
View file

@ -497,7 +497,6 @@ TEST (block_store, one_bootstrap)
auto block1 (std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5));
auto transaction (store->tx_begin_write ());
store->unchecked_put (transaction, block1->hash (), block1);
store->flush (transaction);
auto begin (store->unchecked_begin (transaction));
auto end (store->unchecked_end ());
ASSERT_NE (end, begin);
@ -876,53 +875,6 @@ TEST (block_store, cemented_count_cache)
ASSERT_EQ (1, ledger_cache.cemented_count);
}
TEST (block_store, pruned_count)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path ());
ASSERT_TRUE (!store->init_error ());
{
auto transaction (store->tx_begin_write ());
nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0);
block.sideband_set ({});
auto hash1 (block.hash ());
store->block_put (transaction, hash1, block);
store->pruned_put (transaction, hash1);
}
auto transaction (store->tx_begin_read ());
ASSERT_EQ (1, store->pruned_count (transaction));
ASSERT_EQ (1, store->block_count (transaction));
}
TEST (block_store, sequence_increment)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path ());
ASSERT_TRUE (!store->init_error ());
nano::keypair key1;
nano::keypair key2;
auto block1 (std::make_shared<nano::open_block> (0, 1, 0, nano::keypair ().prv, 0, 0));
auto transaction (store->tx_begin_write ());
auto vote1 (store->vote_generate (transaction, key1.pub, key1.prv, block1));
ASSERT_EQ (1, vote1->sequence);
auto vote2 (store->vote_generate (transaction, key1.pub, key1.prv, block1));
ASSERT_EQ (2, vote2->sequence);
auto vote3 (store->vote_generate (transaction, key2.pub, key2.prv, block1));
ASSERT_EQ (1, vote3->sequence);
auto vote4 (store->vote_generate (transaction, key2.pub, key2.prv, block1));
ASSERT_EQ (2, vote4->sequence);
vote1->sequence = 20;
auto seq5 (store->vote_max (transaction, vote1));
ASSERT_EQ (20, seq5->sequence);
vote3->sequence = 30;
auto seq6 (store->vote_max (transaction, vote3));
ASSERT_EQ (30, seq6->sequence);
auto vote5 (store->vote_generate (transaction, key1.pub, key1.prv, block1));
ASSERT_EQ (21, vote5->sequence);
auto vote6 (store->vote_generate (transaction, key2.pub, key2.prv, block1));
ASSERT_EQ (31, vote6->sequence);
}
TEST (block_store, block_random)
{
nano::logger_mt logger;
@ -974,7 +926,6 @@ TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort ta
ASSERT_NE (send1->hash (), send2->hash ());
store.unchecked_put (transaction, send1->hash (), send1);
store.unchecked_put (transaction, send1->hash (), send2);
store.flush (transaction);
{
auto iterator1 (store.unchecked_begin (transaction));
++iterator1;
@ -985,7 +936,6 @@ TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort ta
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked));
store.unchecked_put (transaction, send1->hash (), send1);
store.unchecked_put (transaction, send1->hash (), send2);
store.flush (transaction);
{
auto iterator1 (store.unchecked_begin (transaction));
++iterator1;
@ -995,7 +945,6 @@ TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort ta
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked));
store.unchecked_put (transaction, send1->hash (), send1);
store.unchecked_put (transaction, send1->hash (), send2);
store.flush (transaction);
{
auto iterator1 (store.unchecked_begin (transaction));
++iterator1;
@ -1005,43 +954,6 @@ TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort ta
}
}
TEST (block_store, sequence_flush)
{
auto path (nano::unique_path ());
nano::logger_mt logger;
auto store = nano::make_store (logger, path);
ASSERT_FALSE (store->init_error ());
auto transaction (store->tx_begin_write ());
nano::keypair key1;
auto send1 (std::make_shared<nano::send_block> (0, 0, 0, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, 0));
auto vote1 (store->vote_generate (transaction, key1.pub, key1.prv, send1));
auto seq2 (store->vote_get (transaction, vote1->account));
ASSERT_EQ (nullptr, seq2);
store->flush (transaction);
auto seq3 (store->vote_get (transaction, vote1->account));
ASSERT_EQ (*seq3, *vote1);
}
TEST (block_store, sequence_flush_by_hash)
{
auto path (nano::unique_path ());
nano::logger_mt logger;
auto store = nano::make_store (logger, path);
ASSERT_FALSE (store->init_error ());
auto transaction (store->tx_begin_write ());
nano::keypair key1;
std::vector<nano::block_hash> blocks1;
blocks1.push_back (nano::genesis_hash);
blocks1.push_back (1234);
blocks1.push_back (5678);
auto vote1 (store->vote_generate (transaction, key1.pub, key1.prv, blocks1));
auto seq2 (store->vote_get (transaction, vote1->account));
ASSERT_EQ (nullptr, seq2);
store->flush (transaction);
auto seq3 (store->vote_get (transaction, vote1->account));
ASSERT_EQ (*seq3, *vote1);
}
TEST (block_store, state_block)
{
nano::logger_mt logger;

18
nano/core_test/ledger.cpp
View file

@ -832,7 +832,7 @@ TEST (votes, add_two)
namespace nano
{
// Higher sequence numbers change the vote
// Higher timestamps change the vote
TEST (votes, add_existing)
{
nano::system system;
@ -853,7 +853,7 @@ TEST (votes, add_existing)
ASSERT_EQ (nano::vote_code::vote, node1.active.vote (vote1));
// Block is already processed from vote
ASSERT_TRUE (node1.active.publish (send1));
ASSERT_EQ (1, election1.election->votes ()[nano::dev_genesis_key.pub].sequence);
ASSERT_EQ (1, election1.election->last_votes[nano::dev_genesis_key.pub].timestamp);
nano::keypair key2;
auto send2 (std::make_shared<nano::send_block> (genesis.hash (), key2.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, 0));
node1.work_generate_blocking (*send2);
@ -864,13 +864,13 @@ TEST (votes, add_existing)
lock.unlock ();
ASSERT_EQ (nano::vote_code::vote, node1.active.vote (vote2));
ASSERT_FALSE (node1.active.publish (send2));
ASSERT_EQ (2, election1.election->votes ()[nano::dev_genesis_key.pub].sequence);
// Also resend the old vote, and see if we respect the sequence number
ASSERT_EQ (2, election1.election->last_votes[nano::dev_genesis_key.pub].timestamp);
// Also resend the old vote, and see if we respect the timestamp
lock.lock ();
election1.election->last_votes[nano::dev_genesis_key.pub].time = std::chrono::steady_clock::now () - std::chrono::seconds (20);
lock.unlock ();
ASSERT_EQ (nano::vote_code::replay, node1.active.vote (vote1));
ASSERT_EQ (2, election1.election->votes ()[nano::dev_genesis_key.pub].sequence);
ASSERT_EQ (2, election1.election->votes ()[nano::dev_genesis_key.pub].timestamp);
auto votes (election1.election->votes ());
ASSERT_EQ (2, votes.size ());
ASSERT_NE (votes.end (), votes.find (nano::dev_genesis_key.pub));
@ -878,7 +878,7 @@ TEST (votes, add_existing)
ASSERT_EQ (*send2, *election1.election->tally ().begin ()->second);
}
// Lower sequence numbers are ignored
// Lower timestamps are ignored
TEST (votes, add_old)
{
nano::system system (1);
@ -910,7 +910,7 @@ TEST (votes, add_old)
}
}
// Lower sequence numbers are accepted for different accounts
// Lower timestamps are accepted for different accounts
TEST (votes, add_old_different_account)
{
nano::system system (1);
@ -3794,8 +3794,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
.work (*pool.generate (nano::genesis_hash))
.build_shared ();
auto vote (std::make_shared<nano::vote> (nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, 0, std::vector<nano::block_hash> (1, send->hash ())));
nano::endpoint_key endpoint_key (address.to_bytes (), port);
auto version = 99;
@ -3817,7 +3815,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
store.version_put (transaction, version);
send->sideband_set ({});
store.block_put (transaction, send->hash (), *send);
store.vote_put (transaction, nano::account (5), vote);
}
auto error = ledger.migrate_lmdb_to_rocksdb (path);
@ -3840,7 +3837,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
auto block1 = rocksdb_store.block_get (rocksdb_transaction, send->hash ());
ASSERT_EQ (*send, *block1);
ASSERT_EQ (*vote, *rocksdb_store.vote_get (rocksdb_transaction, nano::account (5)));
ASSERT_TRUE (rocksdb_store.peer_exists (rocksdb_transaction, endpoint_key));
ASSERT_EQ (rocksdb_store.version_get (rocksdb_transaction), version);
ASSERT_EQ (rocksdb_store.frontier_get (rocksdb_transaction, 2), 5);

75
nano/core_test/node.cpp
View file

@ -2343,40 +2343,6 @@ TEST (node, send_callback)
ASSERT_EQ (std::numeric_limits<nano::uint128_t>::max () - node0.config.receive_minimum.number (), node0.balance (nano::dev_genesis_key.pub));
}
// Check that votes get replayed back to nodes if they sent an old sequence number.
// This helps representatives continue from their last sequence number if their node is reinitialized and the old sequence number is lost
TEST (node, vote_replay)
{
nano::system system (1);
auto & node1 (*system.nodes[0]);
nano::keypair key;
nano::genesis genesis;
for (auto i (0); i < 11000; ++i)
{
auto transaction (node1.store.tx_begin_read ());
auto vote (node1.store.vote_generate (transaction, nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, genesis.open));
}
auto node2 = system.add_node ();
{
auto transaction (node2->store.tx_begin_read ());
nano::lock_guard<std::mutex> lock (node2->store.get_cache_mutex ());
auto vote (node2->store.vote_current (transaction, nano::dev_genesis_key.pub));
ASSERT_EQ (nullptr, vote);
}
system.wallet (1)->insert_adhoc (nano::dev_genesis_key.prv);
auto done (false);
system.deadline_set (20s);
while (!done)
{
auto ec = system.poll ();
auto transaction (node2->store.tx_begin_read ());
nano::lock_guard<std::mutex> lock (node2->store.get_cache_mutex ());
auto vote (node2->store.vote_current (transaction, nano::dev_genesis_key.pub));
done = vote && (vote->sequence >= 10000);
ASSERT_NO_ERROR (ec);
}
}
TEST (node, balance_observer)
{
nano::system system (1);
@ -2654,23 +2620,9 @@ TEST (node, local_votes_cache)
nano::confirm_req message2 (send2);
auto channel (node.network.udp_channels.create (node.network.endpoint ()));
node.network.process_message (message1, channel);
auto wait_vote_sequence = [&node, &system](unsigned sequence) {
std::shared_ptr<nano::vote> current_vote;
system.deadline_set (5s);
while (current_vote == nullptr || current_vote->sequence < sequence)
{
{
nano::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
auto transaction (node.store.tx_begin_read ());
current_vote = node.store.vote_current (transaction, nano::dev_genesis_key.pub);
}
ASSERT_NO_ERROR (system.poll ());
}
ASSERT_EQ (sequence, current_vote->sequence);
};
wait_vote_sequence (1);
ASSERT_TIMELY (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes) == 1);
node.network.process_message (message2, channel);
wait_vote_sequence (2);
ASSERT_TIMELY (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes) == 2);
for (auto i (0); i < 100; ++i)
{
node.network.process_message (message1, channel);
@ -2681,10 +2633,7 @@ TEST (node, local_votes_cache)
ASSERT_NO_ERROR (system.poll (node.aggregator.max_delay));
}
// Make sure a new vote was not generated
{
nano::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
ASSERT_EQ (2, node.store.vote_current (node.store.tx_begin_read (), nano::dev_genesis_key.pub)->sequence);
}
ASSERT_TIMELY (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes) == 2);
// Max cache
{
auto transaction (node.store.tx_begin_write ());
@ -2699,7 +2648,7 @@ TEST (node, local_votes_cache)
{
ASSERT_NO_ERROR (system.poll (node.aggregator.max_delay));
}
wait_vote_sequence (3);
ASSERT_TIMELY (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes) == 3);
ASSERT_FALSE (node.history.votes (send1->root (), send1->hash ()).empty ());
ASSERT_FALSE (node.history.votes (send2->root (), send2->hash ()).empty ());
ASSERT_FALSE (node.history.votes (send3->root (), send3->hash ()).empty ());
@ -2772,13 +2721,7 @@ TEST (node, local_votes_cache_generate_new_vote)
ASSERT_EQ (1, votes1.size ());
ASSERT_EQ (1, votes1[0]->blocks.size ());
ASSERT_EQ (genesis.open->hash (), boost::get<nano::block_hash> (votes1[0]->blocks[0]));
{
nano::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
auto transaction (node.store.tx_begin_read ());
auto current_vote (node.store.vote_current (transaction, nano::dev_genesis_key.pub));
ASSERT_EQ (current_vote->sequence, 1);
ASSERT_EQ (current_vote, votes1[0]);
}
ASSERT_TIMELY (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes) == 1);
auto send1 = nano::state_block_builder ()
.account (nano::dev_genesis_key.pub)
.previous (genesis.hash ())
@ -2797,13 +2740,7 @@ TEST (node, local_votes_cache_generate_new_vote)
auto votes2 (node.history.votes (send1->root (), send1->hash ()));
ASSERT_EQ (1, votes2.size ());
ASSERT_EQ (1, votes2[0]->blocks.size ());
{
nano::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
auto transaction (node.store.tx_begin_read ());
auto current_vote (node.store.vote_current (transaction, nano::dev_genesis_key.pub));
ASSERT_EQ (current_vote->sequence, 2);
ASSERT_EQ (current_vote, votes2[0]);
}
ASSERT_TIMELY (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes) == 2);
ASSERT_FALSE (node.history.votes (genesis.open->root (), genesis.open->hash ()).empty ());
ASSERT_FALSE (node.history.votes (send1->root (), send1->hash ()).empty ());
// First generated + again cached + new generated

75
nano/core_test/timestamp.cpp Normal file
View file

@ -0,0 +1,75 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/timestamp.hpp>
#include <gtest/gtest.h>
#include <boost/format.hpp>
#include <thread>
#include <unordered_set>
TEST (timestamp, now)
{
nano::timestamp_generator generator;
ASSERT_FALSE (nano::timestamp_generator::is_steady);
auto before_ms = std::chrono::duration_cast<std::chrono::milliseconds> (std::chrono::system_clock::now ().time_since_epoch ());
auto before = generator.timestamp_from_ms (before_ms);
ASSERT_EQ (before_ms, generator.ms_from_timestamp (before));
auto now = generator.now ();
auto after_ms = std::chrono::duration_cast<std::chrono::milliseconds> (std::chrono::system_clock::now ().time_since_epoch ());
auto after (generator.timestamp_from_ms (after_ms));
ASSERT_EQ (after_ms, generator.ms_from_timestamp (after));
ASSERT_LE (before, now);
ASSERT_LE (now, after);
}
TEST (timestamp, basic)
{
nano::timestamp_generator generator;
auto one = generator.now ();
ASSERT_NE (0, generator.mask_time (one));
auto two = generator.now ();
ASSERT_NE (0, generator.mask_time (two));
ASSERT_LT (one, two);
}
TEST (timestamp, count)
{
nano::timestamp_generator generator;
auto one = generator.now ();
auto two = generator.now ();
while (generator.mask_time (one) != generator.mask_time (two))
{
one = two;
two = generator.now ();
}
ASSERT_EQ (one + 1, two);
}
TEST (timestamp, parallel)
{
auto constexpr thread_count = 100;
auto iteration_count = 1000;
std::mutex mutex;
std::unordered_set<uint64_t> timestamps;
timestamps.reserve (thread_count * iteration_count);
nano::timestamp_generator generator;
std::vector<std::thread> threads;
for (auto i = 0; i < thread_count; ++i)
{
threads.emplace_back ([&timestamps, &generator, &mutex, &iteration_count]() {
for (auto i = 0; i < iteration_count; ++i)
{
auto stamp = generator.now ();
nano::lock_guard<std::mutex> lock (mutex);
auto inserted = timestamps.insert (stamp);
ASSERT_TRUE (inserted.second);
}
});
}
for (auto & i : threads)
{
i.join ();
}
}

18
nano/core_test/vote_processor.cpp
View file

@ -38,8 +38,8 @@ TEST (vote_processor, codes)
// Invalid takes precedence
ASSERT_EQ (nano::vote_code::invalid, node.vote_processor.vote_blocking (vote_invalid, channel));
// A higher sequence is not a replay
++vote->sequence;
// A higher timestamp is not a replay
++vote->timestamp;
ASSERT_EQ (nano::vote_code::invalid, node.vote_processor.vote_blocking (vote, channel));
vote->signature = nano::sign_message (key.prv, key.pub, vote->hash ());
ASSERT_EQ (nano::vote_code::vote, node.vote_processor.vote_blocking (vote, channel));
@ -60,7 +60,7 @@ TEST (vote_processor, flush)
{
auto new_vote (std::make_shared<nano::vote> (*vote));
node.vote_processor.vote (new_vote, channel);
++vote->sequence; // invalidates votes without signing again
++vote->timestamp; // invalidates votes without signing again
}
node.vote_processor.flush ();
ASSERT_TRUE (node.vote_processor.empty ());
@ -208,7 +208,7 @@ TEST (vote_processor, no_broadcast_local)
ASSERT_TRUE (node.wallets.reps ().exists (nano::dev_genesis_key.pub));
ASSERT_FALSE (node.wallets.reps ().have_half_rep ());
// Process a vote
auto vote (node.store.vote_generate (node.store.tx_begin_read (), nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, { send->hash () }));
auto vote = std::make_shared<nano::vote> (nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, node.timestamps.now (), std::vector<nano::block_hash>{ send->hash () });
ASSERT_EQ (nano::vote_code::vote, node.active.vote (vote));
// Make sure the vote was processed
auto election (node.active.election (send->qualified_root ()));
@ -216,7 +216,7 @@ TEST (vote_processor, no_broadcast_local)
auto votes (election->votes ());
auto existing (votes.find (nano::dev_genesis_key.pub));
ASSERT_NE (votes.end (), existing);
ASSERT_EQ (vote->sequence, existing->second.sequence);
ASSERT_EQ (vote->timestamp, existing->second.timestamp);
// Ensure the vote, from a local representative, was not broadcast on processing - it should be flooded on generation instead
ASSERT_EQ (0, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
ASSERT_EQ (1, node.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));
@ -241,7 +241,7 @@ TEST (vote_processor, no_broadcast_local)
ASSERT_EQ (node.config.vote_minimum, node.weight (nano::dev_genesis_key.pub));
node.block_confirm (send2);
// Process a vote
auto vote2 (node.store.vote_generate (node.store.tx_begin_read (), nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, { send2->hash () }));
auto vote2 = std::make_shared<nano::vote> (nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, node.timestamps.now (), std::vector<nano::block_hash>{ send2->hash () });
ASSERT_EQ (nano::vote_code::vote, node.active.vote (vote2));
// Make sure the vote was processed
auto election2 (node.active.election (send2->qualified_root ()));
@ -249,7 +249,7 @@ TEST (vote_processor, no_broadcast_local)
auto votes2 (election2->votes ());
auto existing2 (votes2.find (nano::dev_genesis_key.pub));
ASSERT_NE (votes2.end (), existing2);
ASSERT_EQ (vote2->sequence, existing2->second.sequence);
ASSERT_EQ (vote2->timestamp, existing2->second.timestamp);
// Ensure the vote was broadcast
ASSERT_EQ (1, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
ASSERT_EQ (2, node.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));
@ -275,7 +275,7 @@ TEST (vote_processor, no_broadcast_local)
ASSERT_TRUE (node.wallets.reps ().exists (nano::dev_genesis_key.pub));
ASSERT_TRUE (node.wallets.reps ().have_half_rep ());
// Process a vote
auto vote3 (node.store.vote_generate (node.store.tx_begin_read (), nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, { open->hash () }));
auto vote3 = std::make_shared<nano::vote> (nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, node.timestamps.now (), std::vector<nano::block_hash>{ open->hash () });
ASSERT_EQ (nano::vote_code::vote, node.active.vote (vote3));
// Make sure the vote was processed
auto election3 (node.active.election (open->qualified_root ()));
@ -283,7 +283,7 @@ TEST (vote_processor, no_broadcast_local)
auto votes3 (election3->votes ());
auto existing3 (votes3.find (nano::dev_genesis_key.pub));
ASSERT_NE (votes3.end (), existing3);
ASSERT_EQ (vote3->sequence, existing3->second.sequence);
ASSERT_EQ (vote3->timestamp, existing3->second.timestamp);
// Ensure the vote wass not broadcasst
ASSERT_EQ (1, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
ASSERT_EQ (3, node.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));

3
nano/lib/CMakeLists.txt
View file

@ -64,6 +64,9 @@ add_library (nano_lib
threading.cpp
timer.hpp
timer.cpp
timestamp_fwd.hpp
timestamp.hpp
timestamp.cpp
tomlconfig.hpp
tomlconfig.cpp
utility.hpp

0
nano/lib/timestamp.cpp Normal file
View file

76
nano/lib/timestamp.hpp Normal file
View file

@ -0,0 +1,76 @@
#pragma once
#include <atomic>
#include <chrono>
namespace nano
{
/**
* Returns seconds passed since unix epoch (posix time)
*/
inline uint64_t seconds_since_epoch ()
{
return std::chrono::duration_cast<std::chrono::seconds> (std::chrono::system_clock::now ().time_since_epoch ()).count ();
}
/**
Creates a unique 64-bit timestamp each time timestamp_now is called.
The upper 44-bits are the number of milliseconds since unix epoch
The lower 20 bits are a monotonically increasing counter from 0, each millisecond
*/
template <typename CLOCK>
class timestamp_generator_base
{
public:
// If CLOCK::is_steady, this class will be a steady
static bool constexpr is_steady = CLOCK::is_steady;
static uint64_t mask_time (uint64_t timestamp)
{
auto result = timestamp & time_mask;
return result;
}
static uint64_t mask_count (uint64_t timestamp)
{
auto result = timestamp & count_mask;
return result;
}
// Return a timestamp based on `ms' the number of milliseconds since the UTC epoch
static uint64_t timestamp_from_ms (std::chrono::milliseconds ms)
{
auto result = static_cast<uint64_t> (ms.count ()) << count_bits;
return result;
}
// Return the number of milliseconds since the UTC epoch, represented in `timestamp'
static std::chrono::milliseconds ms_from_timestamp (uint64_t timestamp)
{
auto result = timestamp >> count_bits;
return std::chrono::milliseconds{ result };
}
// If CLOCK::is_steady, now is guaranteed to produce monotonically increasing timestamps
static uint64_t now ()
{
uint64_t stored;
uint64_t result;
do
{
stored = last.load ();
std::chrono::milliseconds delta = std::chrono::duration_cast<std::chrono::milliseconds> (CLOCK::now ().time_since_epoch ());
auto now_l = timestamp_from_ms (delta);
// If `delta` hasn't changed since the last call, increment the counter, otherwise use the current value with a zero counter.
result = mask_time (stored) == now_l ? stored + 1 : now_l;
} while (!last.compare_exchange_weak (stored, result));
return result;
}
private:
static inline std::atomic<uint64_t> last{ 0 };
static int constexpr time_bits{ 44 }; // 44 bits for milliseconds = 17,592,186,044,416 ~ 545 years.
static int constexpr count_bits{ 20 }; // 20-bit monotonic counter, 1,048,576 samples per ms
static_assert (time_bits + count_bits == 64);
static uint64_t constexpr time_mask{ ~0ULL << count_bits }; // Portion associated with timer
static uint64_t constexpr count_mask{ ~0ULL >> time_bits }; // Portion associated with counter
};
using timestamp_generator = timestamp_generator_base<std::chrono::system_clock>;
} // namespace nano

10
nano/lib/timestamp_fwd.hpp Normal file
View file

@ -0,0 +1,10 @@
#pragma once
#include <chrono>
namespace nano
{
template <typename>
class timestamp_generator_base;
using timestamp_generator = timestamp_generator_base<std::chrono::system_clock>;
}

8
nano/lib/utility.hpp
View file

@ -113,14 +113,6 @@ void dump_crash_stacktrace ();
*/
std::string generate_stacktrace ();
/**
* Returns seconds passed since unix epoch (posix time)
*/
inline uint64_t seconds_since_epoch ()
{
return std::chrono::duration_cast<std::chrono::seconds> (std::chrono::system_clock::now ().time_since_epoch ()).count ();
}
template <typename... T>
class observer_set final
{

4
nano/node/active_transactions.cpp
View file

@ -24,7 +24,7 @@ confirmation_height_processor (confirmation_height_processor_a),
node (node_a),
multipliers_cb (20, 1.),
trended_active_multiplier (1.0),
generator (node_a.config, node_a.ledger, node_a.wallets, node_a.vote_processor, node_a.history, node_a.network, node_a.stats),
generator (node_a.timestamps, node_a.config, node_a.ledger, node_a.wallets, node_a.vote_processor, node_a.history, node_a.network, node_a.stats),
check_all_elections_period (node_a.network_params.network.is_dev_network () ? 10ms : 5s),
election_time_to_live (node_a.network_params.network.is_dev_network () ? 0s : 2s),
prioritized_cutoff (std::max<size_t> (1, node_a.config.active_elections_size / 10)),
@ -913,7 +913,7 @@ nano::vote_code nano::active_transactions::vote (std::shared_ptr<nano::vote> vot
bool processed (false);
for (auto const & [election, block_hash] : process)
{
auto const result_l = election->vote (vote_a->account, vote_a->sequence, block_hash);
auto const result_l = election->vote (vote_a->account, vote_a->timestamp, block_hash);
processed = processed || result_l.processed;
replay = replay || result_l.replay;
}

12
nano/node/cli.cpp
View file

@ -72,7 +72,6 @@ void nano::add_node_options (boost::program_options::options_description & descr
("wallet_remove", "Remove <account> from <wallet>")
("wallet_representative_get", "Prints default representative for <wallet>")
("wallet_representative_set", "Set <account> as default representative for <wallet>")
("vote_dump", "Dump most recent votes from representatives")
("account", boost::program_options::value<std::string> (), "Defines <account> for other commands")
("file", boost::program_options::value<std::string> (), "Defines <file> for other commands")
("key", boost::program_options::value<std::string> (), "Defines the <key> for other commands, hex")
@ -1236,17 +1235,6 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
ec = nano::error_cli::invalid_arguments;
}
}
else if (vm.count ("vote_dump") == 1)
{
auto inactive_node = nano::default_inactive_node (data_path, vm);
auto node = inactive_node->node;
auto transaction (node->store.tx_begin_read ());
for (auto i (node->store.vote_begin (transaction)), n (node->store.vote_end ()); i != n; ++i)
{
auto const & vote (i->second);
std::cerr << boost::str (boost::format ("%1%\n") % vote->to_json ());
}
}
else
{
ec = nano::error_cli::unknown_command;

8
nano/node/election.cpp
View file

@ -313,7 +313,7 @@ void nano::election::log_votes (nano::tally_t const & tally_a, std::string const
{
if (i->first != node.network_params.random.not_an_account)
{
tally << boost::str (boost::format ("%1%%2% %3% %4%") % line_end % i->first.to_account () % std::to_string (i->second.sequence) % i->second.hash.to_string ());
tally << boost::str (boost::format ("%1%%2% %3% %4%") % line_end % i->first.to_account () % std::to_string (i->second.timestamp) % i->second.hash.to_string ());
}
}
node.logger.try_log (tally.str ());
@ -330,7 +330,7 @@ std::shared_ptr<nano::block> nano::election::find (nano::block_hash const & hash
return result;
}
nano::election_vote_result nano::election::vote (nano::account const & rep, uint64_t sequence, nano::block_hash const & block_hash)
nano::election_vote_result nano::election::vote (nano::account const & rep, uint64_t timestamp_a, nano::block_hash const & block_hash)
{
// see republish_vote documentation for an explanation of these rules
auto replay (false);
@ -363,7 +363,7 @@ nano::election_vote_result nano::election::vote (nano::account const & rep, uint
else
{
auto last_vote_l (last_vote_it->second);
if (last_vote_l.sequence < sequence || (last_vote_l.sequence == sequence && last_vote_l.hash < block_hash))
if (last_vote_l.timestamp < timestamp_a || (last_vote_l.timestamp == timestamp_a && last_vote_l.hash < block_hash))
{
if (last_vote_l.time <= std::chrono::steady_clock::now () - std::chrono::seconds (cooldown))
{
@ -378,7 +378,7 @@ nano::election_vote_result nano::election::vote (nano::account const & rep, uint
if (should_process)
{
node.stats.inc (nano::stat::type::election, nano::stat::detail::vote_new);
last_votes[rep] = { std::chrono::steady_clock::now (), sequence, block_hash };
last_votes[rep] = { std::chrono::steady_clock::now (), timestamp_a, block_hash };
if (!confirmed ())
{
confirm_if_quorum (lock);

2
nano/node/election.hpp
View file

@ -20,7 +20,7 @@ class vote_info final
{
public:
std::chrono::steady_clock::time_point time;
uint64_t sequence;
uint64_t timestamp;
nano::block_hash hash;
};
class election_vote_result final

9
nano/node/lmdb/lmdb.cpp
View file

@ -190,7 +190,6 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const &
{
error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "vote", flags, &vote) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0;
@ -725,6 +724,10 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa
auto count_post (count (transaction_a, blocks));
release_assert (count_pre == count_post);
MDB_dbi vote{ 0 };
release_assert (!mdb_dbi_open (env.tx (transaction_a), "vote", MDB_CREATE, &vote));
release_assert (!mdb_drop (env.tx (transaction_a), vote, 1));
version_put (transaction_a, 19);
logger.always_log ("Finished upgrading all blocks to new blocks database");
}
@ -847,8 +850,6 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const
return pending;
case tables::unchecked:
return unchecked;
case tables::vote:
return vote;
case tables::online_weight:
return online_weight;
case tables::meta:
@ -888,7 +889,7 @@ bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file)
void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a)
{
// Tables with uint256_union key
std::vector<MDB_dbi> tables = { accounts, blocks, vote, pruned, confirmation_height };
std::vector<MDB_dbi> tables = { accounts, blocks, pruned, confirmation_height };
for (auto const & table : tables)
{
MDB_dbi temp;

6
nano/node/lmdb/lmdb.hpp
View file

@ -156,12 +156,6 @@ public:
*/
MDB_dbi unchecked{ 0 };
/**
* Highest vote observed for account.
* nano::account -> uint64_t
*/
MDB_dbi vote{ 0 };
/**
* Samples of online vote weight
* uint64_t -> nano::amount

2
nano/node/network.cpp
View file

@ -449,7 +449,7 @@ public:
{
if (node.config.logging.network_message_logging ())
{
node.logger.try_log (boost::str (boost::format ("Received confirm_ack message from %1% for %2%sequence %3%") % channel->to_string () % message_a.vote->hashes_string () % std::to_string (message_a.vote->sequence)));
node.logger.try_log (boost::str (boost::format ("Received confirm_ack message from %1% for %2% timestamp %3%") % channel->to_string () % message_a.vote->hashes_string () % std::to_string (message_a.vote->timestamp)));
}
node.stats.inc (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::in);
if (!message_a.vote->account.is_zero ())

18
nano/node/node.cpp
View file

@ -650,7 +650,6 @@ void nano::node::start ()
this_l->ongoing_unchecked_cleanup ();
});
}
ongoing_store_flush ();
if (!flags.disable_rep_crawler)
{
rep_crawler.start ();
@ -848,23 +847,6 @@ void nano::node::ongoing_bootstrap ()
});
}
void nano::node::ongoing_store_flush ()
{
{
auto transaction (store.tx_begin_write ({ tables::vote }));
store.flush (transaction);
}
std::weak_ptr<nano::node> node_w (shared_from_this ());
alarm.add (std::chrono::steady_clock::now () + std::chrono::seconds (5), [node_w]() {
if (auto node_l = node_w.lock ())
{
node_l->worker.push_task ([node_l]() {
node_l->ongoing_store_flush ();
});
}
});
}
void nano::node::ongoing_peer_store ()
{
bool stored (network.tcp_channels.store_all (true));

3
nano/node/node.hpp
View file

@ -4,6 +4,7 @@
#include <nano/lib/alarm.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/timestamp.hpp>
#include <nano/lib/work.hpp>
#include <nano/lib/worker.hpp>
#include <nano/node/active_transactions.hpp>
@ -125,7 +126,6 @@ public:
nano::uint128_t minimum_principal_weight (nano::uint128_t const &);
void ongoing_rep_calculation ();
void ongoing_bootstrap ();
void ongoing_store_flush ();
void ongoing_peer_store ();
void ongoing_unchecked_cleanup ();
void backup_wallet ();
@ -156,6 +156,7 @@ public:
bool init_error () const;
bool epoch_upgrader (nano::private_key const &, nano::epoch, uint64_t, uint64_t);
std::pair<uint64_t, decltype (nano::ledger::bootstrap_weights)> get_bootstrap_weights () const;
nano::timestamp_generator timestamps;
nano::worker worker;
nano::write_database_queue write_database_queue;
boost::asio::io_context & io_ctx;

10
nano/node/repcrawler.cpp
View file

@ -32,7 +32,6 @@ void nano::rep_crawler::validate ()
nano::lock_guard<std::mutex> lock (active_mutex);
responses_l.swap (responses);
}
auto transaction (node.store.tx_begin_read ());
auto minimum = node.minimum_principal_weight ();
for (auto const & i : responses_l)
{
@ -73,15 +72,6 @@ void nano::rep_crawler::validate ()
node.logger.try_log (boost::str (boost::format ("Found a representative at %1%") % channel->to_string ()));
}
}
// This tries to assist rep nodes that have lost track of their highest sequence number by replaying our highest known vote back to them
// Only do this if the sequence number is significantly different to account for network reordering
// Amplify attack considerations: We're sending out a confirm_ack in response to a confirm_ack for no net traffic increase
auto max_vote (node.store.vote_max (transaction, vote));
if (max_vote->sequence > vote->sequence + 10000)
{
nano::confirm_ack confirm (max_vote);
channel->send (confirm); // this is non essential traffic as it will be resolicited if not received
}
}
}
}

1
nano/node/request_aggregator.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/stats.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/timestamp.hpp>
#include <nano/node/active_transactions.hpp>
#include <nano/node/common.hpp>
#include <nano/node/network.hpp>

1
nano/node/request_aggregator.hpp
View file

@ -2,6 +2,7 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/timestamp_fwd.hpp>
#include <nano/node/transport/transport.hpp>
#include <boost/multi_index/hashed_index.hpp>

2
nano/node/vote_processor.cpp
View file

@ -196,7 +196,7 @@ nano::vote_code nano::vote_processor::vote_blocking (std::shared_ptr<nano::vote>
}
if (config.logging.vote_logging ())
{
logger.try_log (boost::str (boost::format ("Vote from: %1% sequence: %2% block(s): %3%status: %4%") % vote_a->account.to_account () % std::to_string (vote_a->sequence) % vote_a->hashes_string () % status));
logger.try_log (boost::str (boost::format ("Vote from: %1% timestamp: %2% block(s): %3%status: %4%") % vote_a->account.to_account () % std::to_string (vote_a->timestamp) % vote_a->hashes_string () % status));
}
return result;
}

12
nano/node/voting.cpp
View file

@ -111,8 +111,9 @@ std::unique_ptr<nano::container_info_component> nano::collect_container_info (na
return composite;
}
nano::vote_generator::vote_generator (nano::node_config const & config_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::vote_processor & vote_processor_a, nano::local_vote_history & history_a, nano::network & network_a, nano::stat & stats_a) :
nano::vote_generator::vote_generator (nano::timestamp_generator & timestamps_a, nano::node_config const & config_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::vote_processor & vote_processor_a, nano::local_vote_history & history_a, nano::network & network_a, nano::stat & stats_a) :
config (config_a),
timestamps{ timestamps_a },
ledger (ledger_a),
wallets (wallets_a),
vote_processor (vote_processor_a),
@ -280,12 +281,9 @@ void nano::vote_generator::vote (std::vector<nano::block_hash> const & hashes_a,
{
debug_assert (hashes_a.size () == roots_a.size ());
std::vector<std::shared_ptr<nano::vote>> votes_l;
{
auto transaction (ledger.store.tx_begin_read ());
wallets.foreach_representative ([this, &hashes_a, &transaction, &votes_l](nano::public_key const & pub_a, nano::raw_key const & prv_a) {
votes_l.emplace_back (this->ledger.store.vote_generate (transaction, pub_a, prv_a, hashes_a));
});
}
wallets.foreach_representative ([this, &hashes_a, &votes_l](nano::public_key const & pub_a, nano::raw_key const & prv_a) {
votes_l.emplace_back (std::make_shared<nano::vote> (pub_a, prv_a, timestamps.now (), hashes_a));
});
for (auto const & vote_l : votes_l)
{
for (size_t i (0), n (hashes_a.size ()); i != n; ++i)

4
nano/node/voting.hpp
View file

@ -2,6 +2,7 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/timestamp_fwd.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/wallet.hpp>
#include <nano/secure/common.hpp>
@ -86,7 +87,7 @@ private:
using request_t = std::pair<std::vector<candidate_t>, std::shared_ptr<nano::transport::channel>>;
public:
vote_generator (nano::node_config const & config_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::vote_processor & vote_processor_a, nano::local_vote_history & history_a, nano::network & network_a, nano::stat & stats_a);
vote_generator (nano::timestamp_generator & timestamps_a, nano::node_config const & config_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::vote_processor & vote_processor_a, nano::local_vote_history & history_a, nano::network & network_a, nano::stat & stats_a);
/** Queue items for vote generation, or broadcast votes already in cache */
void add (nano::root const &, nano::block_hash const &);
/** Queue blocks for vote generation, returning the number of successful candidates.*/
@ -102,6 +103,7 @@ private:
void broadcast_action (std::shared_ptr<nano::vote> const &) const;
std::function<void(std::shared_ptr<nano::vote> const &, std::shared_ptr<nano::transport::channel> &)> reply_action; // must be set only during initialization by using set_reply_action
nano::node_config const & config;
nano::timestamp_generator & timestamps;
nano::ledger & ledger;
nano::wallets & wallets;
nano::vote_processor & vote_processor;

17
nano/secure/blockstore.hpp
View file

@ -659,21 +659,6 @@ public:
virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> unchecked_end () const = 0;
virtual size_t unchecked_count (nano::transaction const &) = 0;
// Return latest vote for an account from store
virtual std::shared_ptr<nano::vote> vote_get (nano::transaction const &, nano::account const &) = 0;
// Populate vote with the next sequence number
virtual std::shared_ptr<nano::vote> vote_generate (nano::transaction const &, nano::account const &, nano::raw_key const &, std::shared_ptr<nano::block>) = 0;
virtual std::shared_ptr<nano::vote> vote_generate (nano::transaction const &, nano::account const &, nano::raw_key const &, std::vector<nano::block_hash>) = 0;
// Return either vote or the stored vote with a higher sequence number
virtual std::shared_ptr<nano::vote> vote_max (nano::transaction const &, std::shared_ptr<nano::vote>) = 0;
// Return latest vote for an account considering the vote cache
virtual std::shared_ptr<nano::vote> vote_current (nano::transaction const &, nano::account const &) = 0;
virtual void flush (nano::write_transaction const &) = 0;
virtual void vote_put (nano::write_transaction const &, nano::account const &, std::shared_ptr<nano::vote> const &) = 0;
virtual nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_begin (nano::transaction const &, nano::account const &) const = 0;
virtual nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_end () const = 0;
virtual void online_weight_put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0;
virtual void online_weight_del (nano::write_transaction const &, uint64_t) = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const &) const = 0;
@ -719,10 +704,8 @@ public:
virtual void pruned_for_each_par (std::function<void(nano::read_transaction const &, nano::store_iterator<nano::block_hash, std::nullptr_t>, nano::store_iterator<nano::block_hash, std::nullptr_t>)> const & action_a) const = 0;
virtual void blocks_for_each_par (std::function<void(nano::read_transaction const &, nano::store_iterator<nano::block_hash, block_w_sideband>, nano::store_iterator<nano::block_hash, block_w_sideband>)> const & action_a) const = 0;
virtual void frontiers_for_each_par (std::function<void(nano::read_transaction const &, nano::store_iterator<nano::block_hash, nano::account>, nano::store_iterator<nano::block_hash, nano::account>)> const & action_a) const = 0;
virtual void votes_for_each_par (std::function<void(nano::read_transaction const &, nano::store_iterator<nano::account, std::shared_ptr<nano::vote>>, nano::store_iterator<nano::account, std::shared_ptr<nano::vote>>)> const & action_a) const = 0;
virtual uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0;
virtual std::mutex & get_cache_mutex () = 0;
virtual unsigned max_block_write_batch_num () const = 0;

134
nano/secure/blockstore_partial.hpp
View file

@ -3,6 +3,7 @@
#include <nano/lib/config.hpp>
#include <nano/lib/rep_weights.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/timestamp.hpp>
#include <nano/secure/blockstore.hpp>
#include <nano/secure/buffer.hpp>
@ -31,8 +32,6 @@ public:
friend class nano::block_predecessor_set<Val, Derived_Store>;
std::mutex cache_mutex;
/**
* If using a different store version than the latest then you may need
* to modify some of the objects in the store to be appropriate for the version before an upgrade.
@ -248,76 +247,11 @@ public:
unchecked_put (transaction_a, key, info);
}
std::shared_ptr<nano::vote> vote_current (nano::transaction const & transaction_a, nano::account const & account_a) override
{
debug_assert (!cache_mutex.try_lock ());
std::shared_ptr<nano::vote> result;
auto existing (vote_cache_l1.find (account_a));
auto have_existing (true);
if (existing == vote_cache_l1.end ())
{
existing = vote_cache_l2.find (account_a);
if (existing == vote_cache_l2.end ())
{
have_existing = false;
}
}
if (have_existing)
{
result = existing->second;
}
else
{
result = vote_get (transaction_a, account_a);
}
return result;
}
std::shared_ptr<nano::vote> vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::shared_ptr<nano::block> block_a) override
{
debug_assert (nano::network_constants ().is_dev_network () || nano::thread_role::get () == nano::thread_role::name::voting);
nano::lock_guard<std::mutex> lock (cache_mutex);
auto result (vote_current (transaction_a, account_a));
uint64_t sequence ((result ? result->sequence : 0) + 1);
result = std::make_shared<nano::vote> (account_a, key_a, sequence, block_a);
vote_cache_l1[account_a] = result;
return result;
}
std::shared_ptr<nano::vote> vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::vector<nano::block_hash> blocks_a) override
{
debug_assert (nano::network_constants ().is_dev_network () || nano::thread_role::get () == nano::thread_role::name::voting);
nano::lock_guard<std::mutex> lock (cache_mutex);
auto result (vote_current (transaction_a, account_a));
uint64_t sequence ((result ? result->sequence : 0) + 1);
result = std::make_shared<nano::vote> (account_a, key_a, sequence, blocks_a);
vote_cache_l1[account_a] = result;
return result;
}
std::shared_ptr<nano::vote> vote_max (nano::transaction const & transaction_a, std::shared_ptr<nano::vote> vote_a) override
{
nano::lock_guard<std::mutex> lock (cache_mutex);
auto current (vote_current (transaction_a, vote_a->account));
auto result (vote_a);
if (current != nullptr && current->sequence > result->sequence)
{
result = current;
}
vote_cache_l1[vote_a->account] = result;
return result;
}
nano::store_iterator<nano::unchecked_key, nano::unchecked_info> unchecked_end () const override
{
return nano::store_iterator<nano::unchecked_key, nano::unchecked_info> (nullptr);
}
nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_end () const override
{
return nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> (nullptr);
}
nano::store_iterator<nano::endpoint_key, nano::no_value> peers_end () const override
{
return nano::store_iterator<nano::endpoint_key, nano::no_value> (nullptr);
@ -358,11 +292,6 @@ public:
return nano::store_iterator<nano::block_hash, nano::account> (nullptr);
}
std::mutex & get_cache_mutex () override
{
return cache_mutex;
}
void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override
{
auto status = del (transaction_a, tables::blocks, hash_a);
@ -469,46 +398,6 @@ public:
release_assert (success (status));
}
std::shared_ptr<nano::vote> vote_get (nano::transaction const & transaction_a, nano::account const & account_a) override
{
nano::db_val<Val> value;
auto status (get (transaction_a, tables::vote, nano::db_val<Val> (account_a), value));
release_assert (success (status) || not_found (status));
if (success (status))
{
std::shared_ptr<nano::vote> result (value);
debug_assert (result != nullptr);
return result;
}
return nullptr;
}
void vote_put (nano::write_transaction const & transaction_a, nano::account const & account_a, std::shared_ptr<nano::vote> const & vote_a) override
{
std::vector<uint8_t> vector;
{
nano::vectorstream stream (vector);
vote_a->serialize (stream);
}
nano::db_val<Val> value (vector.size (), vector.data ());
auto status1 (put (transaction_a, tables::vote, account_a, value));
release_assert (success (status1));
}
void flush (nano::write_transaction const & transaction_a) override
{
{
nano::lock_guard<std::mutex> lock (cache_mutex);
vote_cache_l1.swap (vote_cache_l2);
vote_cache_l1.clear ();
}
for (auto i (vote_cache_l2.begin ()), n (vote_cache_l2.end ()); i != n; ++i)
{
vote_put (transaction_a, i->first, i->second);
}
}
void online_weight_put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override
{
nano::db_val<Val> value (amount_a);
@ -759,16 +648,6 @@ public:
return make_iterator<nano::unchecked_key, nano::unchecked_info> (transaction_a, tables::unchecked, nano::db_val<Val> (key_a));
}
nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_begin (nano::transaction const & transaction_a, nano::account const & account_a) const override
{
return make_iterator<nano::account, std::shared_ptr<nano::vote>> (transaction_a, tables::vote, nano::db_val<Val> (account_a));
}
nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_begin (nano::transaction const & transaction_a) const override
{
return make_iterator<nano::account, std::shared_ptr<nano::vote>> (transaction_a, tables::vote);
}
nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const & transaction_a) const override
{
return make_iterator<uint64_t, nano::amount> (transaction_a, tables::online_weight);
@ -875,21 +754,10 @@ public:
});
}
void votes_for_each_par (std::function<void(nano::read_transaction const &, nano::store_iterator<nano::account, std::shared_ptr<nano::vote>>, nano::store_iterator<nano::account, std::shared_ptr<nano::vote>>)> const & action_a) const override
{
parallel_traversal<nano::uint256_t> (
[&action_a, this](nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) {
auto transaction (this->tx_begin_read ());
action_a (transaction, this->vote_begin (transaction, start), !is_last ? this->vote_begin (transaction, end) : this->vote_end ());
});
}
int const minimum_version{ 14 };
protected:
nano::network_params network_params;
std::unordered_map<nano::account, std::shared_ptr<nano::vote>> vote_cache_l1;
std::unordered_map<nano::account, std::shared_ptr<nano::vote>> vote_cache_l2;
int const version{ 20 };
template <typename Key, typename Value>

25
nano/secure/common.cpp
View file

@ -450,7 +450,7 @@ bool nano::vote::operator== (nano::vote const & other_a) const
}
}
}
return sequence == other_a.sequence && blocks_equal && account == other_a.account && signature == other_a.signature;
return timestamp == other_a.timestamp && blocks_equal && account == other_a.account && signature == other_a.signature;
}
bool nano::vote::operator!= (nano::vote const & other_a) const
@ -462,7 +462,8 @@ void nano::vote::serialize_json (boost::property_tree::ptree & tree) const
{
tree.put ("account", account.to_account ());
tree.put ("signature", signature.number ());
tree.put ("sequence", std::to_string (sequence));
tree.put ("sequence", std::to_string (timestamp));
tree.put ("timestamp", std::to_string (timestamp));
boost::property_tree::ptree blocks_tree;
for (auto block : blocks)
{
@ -490,7 +491,7 @@ std::string nano::vote::to_json () const
}
nano::vote::vote (nano::vote const & other_a) :
sequence (other_a.sequence),
timestamp{ other_a.timestamp },
blocks (other_a.blocks),
account (other_a.account),
signature (other_a.signature)
@ -508,7 +509,7 @@ nano::vote::vote (bool & error_a, nano::stream & stream_a, nano::block_type type
{
nano::read (stream_a, account.bytes);
nano::read (stream_a, signature.bytes);
nano::read (stream_a, sequence);
nano::read (stream_a, timestamp);
while (stream_a.in_avail () > 0)
{
@ -540,16 +541,16 @@ nano::vote::vote (bool & error_a, nano::stream & stream_a, nano::block_type type
}
}
nano::vote::vote (nano::account const & account_a, nano::raw_key const & prv_a, uint64_t sequence_a, std::shared_ptr<nano::block> block_a) :
sequence (sequence_a),
nano::vote::vote (nano::account const & account_a, nano::raw_key const & prv_a, uint64_t timestamp_a, std::shared_ptr<nano::block> block_a) :
timestamp{ timestamp_a },
blocks (1, block_a),
account (account_a),
signature (nano::sign_message (prv_a, account_a, hash ()))
{
}
nano::vote::vote (nano::account const & account_a, nano::raw_key const & prv_a, uint64_t sequence_a, std::vector<nano::block_hash> const & blocks_a) :
sequence (sequence_a),
nano::vote::vote (nano::account const & account_a, nano::raw_key const & prv_a, uint64_t timestamp_a, std::vector<nano::block_hash> const & blocks_a) :
timestamp{ timestamp_a },
account (account_a)
{
debug_assert (!blocks_a.empty ());
@ -590,7 +591,7 @@ nano::block_hash nano::vote::hash () const
uint64_t qword;
std::array<uint8_t, 8> bytes;
};
qword = sequence;
qword = timestamp;
blake2b_update (&hash, bytes.data (), sizeof (bytes));
blake2b_final (&hash, result.bytes.data (), sizeof (result.bytes));
return result;
@ -612,7 +613,7 @@ void nano::vote::serialize (nano::stream & stream_a, nano::block_type type) cons
{
write (stream_a, account);
write (stream_a, signature);
write (stream_a, sequence);
write (stream_a, timestamp);
for (auto const & block : blocks)
{
if (block.which ())
@ -638,7 +639,7 @@ void nano::vote::serialize (nano::stream & stream_a) const
{
write (stream_a, account);
write (stream_a, signature);
write (stream_a, sequence);
write (stream_a, timestamp);
for (auto const & block : blocks)
{
if (block.which ())
@ -660,7 +661,7 @@ bool nano::vote::deserialize (nano::stream & stream_a, nano::block_uniquer * uni
{
nano::read (stream_a, account);
nano::read (stream_a, signature);
nano::read (stream_a, sequence);
nano::read (stream_a, timestamp);
nano::block_type type;

10
nano/secure/common.hpp
View file

@ -263,13 +263,13 @@ public:
boost::transform_iterator<nano::iterate_vote_blocks_as_hash, nano::vote_blocks_vec_iter> begin () const;
boost::transform_iterator<nano::iterate_vote_blocks_as_hash, nano::vote_blocks_vec_iter> end () const;
std::string to_json () const;
// Vote round sequence number
uint64_t sequence;
// Vote timestamp
uint64_t timestamp;
// The blocks, or block hashes, that this vote is for
std::vector<boost::variant<std::shared_ptr<nano::block>, nano::block_hash>> blocks;
// Account that's voting
nano::account account;
// Signature of sequence + block hashes
// Signature of timestamp + block hashes
nano::signature signature;
static const std::string hash_prefix;
};
@ -297,8 +297,8 @@ std::unique_ptr<container_info_component> collect_container_info (vote_uniquer &
enum class vote_code
{
invalid, // Vote is not signed correctly
replay, // Vote does not have the highest sequence number, it's a replay
vote, // Vote has the highest sequence number
replay, // Vote does not have the highest timestamp, it's a replay
vote, // Vote has the highest timestamp
indeterminate // Unknown if replay or vote
};

10
nano/secure/ledger.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/rep_weights.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/timestamp.hpp>
#include <nano/lib/utility.hpp>
#include <nano/lib/work.hpp>
#include <nano/secure/blockstore.hpp>
@ -1481,15 +1482,6 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
}
});
store.votes_for_each_par (
[&rocksdb_store](nano::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::vote }));
rocksdb_store->vote_put (rocksdb_transaction, i->first, i->second);
}
});
auto lmdb_transaction (store.tx_begin_read ());
auto version = store.version_get (lmdb_transaction);
auto rocksdb_transaction (rocksdb_store->tx_begin_write ());