Merge branch 'develop' into frontier-scan-5

# Conflicts:
#	nano/core_test/utility.cpp
#	nano/lib/thread_pool.hpp
#	nano/node/bootstrap_ascending/database_scan.cpp
#	nano/store/account.hpp
#	nano/store/iterator.hpp

.github/actions/restore-git-mtimes/action.yml

@@ -3,7 +3,7 @@ description: "Restore file modification timestamps from git commit timestamps"
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         repository: "MestreLion/git-tools"
         ref: a42a069bc755f0fa81316965335cb33dbf22a968 # pin latest commit

.github/workflows/analyzers.yml

@@ -8,7 +8,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+        uses: actions/checkout@v4
 
       - name: Installing clang-format
         env:
@@ -28,7 +28,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 #v4.3.0
@@ -50,6 +50,6 @@ jobs:
     runs-on: ubuntu-20.04
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - uses: actions/checkout@v4
       - name: Run code-inspector
         run: ci/code-inspector-check.sh

.github/workflows/build_deploy.yml

@@ -1,7 +1,7 @@
 name: Build & Deploy
 on:
   schedule:
-    - cron: "0 0 * * 3,6"
+    - cron: "0 0 * * 3,6" # Every Wednesday and Saturday at 00:00
   workflow_dispatch:
     inputs:
         is_release_build:
@@ -14,12 +14,15 @@ env:
 
 jobs:
   prepare_build:
+    name: Prepare Build
     runs-on: ubuntu-22.04
     outputs:
       ci_tag: ${{ steps.set_vars.outputs.ci_tag }}
       tag_created: ${{ steps.set_vars.outputs.tag_created }}
     steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Set the tag and version
         id: tag_set
         run: |
@@ -27,14 +30,15 @@ jobs:
         env:
           GITHUB_ACTOR: ${{ github.actor }}
           IS_RELEASE_BUILD: ${{ github.event.inputs.is_release_build || 'false' }}
+
       - name: Set output
         id: set_vars
         run: |
           echo "ci_tag=$CI_TAG" >> $GITHUB_OUTPUT
           echo "tag_created=$TAG_CREATED" >> $GITHUB_OUTPUT
 
-
   osx_job:
+    name: macOS [${{ matrix.network }}]
     needs: prepare_build
     if: ${{ needs.prepare_build.outputs.tag_created == 'true' }}
     runs-on: macOS-14
@@ -43,19 +47,22 @@ jobs:
       matrix:
         network: ["TEST", "BETA", "LIVE"]
     steps:
-
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
           ref: ${{ needs.prepare_build.outputs.ci_tag }}
           repository: ${{ github.repository }}
+
       - name: Prepare
         run: ci/prepare/macos/prepare.sh
+        
       - name: Build Artifact
         run: ci/build-deploy.sh "/tmp/qt/lib/cmake/Qt5";
         env:
           NETWORK: ${{ matrix.network }}
           CI_TAG: ${{ needs.prepare_build.outputs.ci_tag }}
+
       - name: Deploy Artifact
         run: ci/actions/deploy.sh
         env:
@@ -66,9 +73,9 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: us-east-2
-  
 
   linux_job:
+    name: Linux [${{ matrix.network }}]
     needs: prepare_build
     if: ${{ needs.prepare_build.outputs.tag_created == 'true' }}
     runs-on: ubuntu-22.04
@@ -77,13 +84,16 @@ jobs:
       matrix:
         network: ["TEST", "BETA", "LIVE"]
     steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
           ref: ${{ needs.prepare_build.outputs.ci_tag }}
           repository: ${{ github.repository }}
+
       - name: Prepare
         run: sudo -E ci/prepare/linux/prepare.sh
+
       - name: Build Artifact
         run: ci/build-deploy.sh "/usr/lib/x86_64-linux-gnu/cmake/Qt5"
         env:
@@ -101,8 +111,8 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: us-east-2
 
-
   linux_docker_job:
+    name: Docker [${{ matrix.network }}]
     needs: prepare_build
     if: ${{ needs.prepare_build.outputs.tag_created == 'true' }}
     runs-on: ubuntu-22.04
@@ -117,21 +127,25 @@ jobs:
       DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
       IS_RELEASE_BUILD: ${{ github.event.inputs.is_release_build || 'false' }}
     steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
           ref: ${{ needs.prepare_build.outputs.ci_tag }}
           repository: ${{ github.repository }}
+
       - name: Build Docker
         run: ci/actions/linux/docker-build.sh
         env:
           NETWORK: ${{ matrix.network }}
-      - name: Deploy Docker Hub
+
+      - name: Deploy Docker (Docker Hub)
         if: env.DOCKER_PASSWORD != ''
         run: ci/actions/linux/docker-deploy.sh
         env:
           NETWORK: ${{ matrix.network }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Deploy Docker (ghcr.io)
         run: ci/actions/linux/ghcr-deploy.sh
         env:
@@ -140,8 +154,8 @@ jobs:
           DOCKER_USER: ${{ github.repository_owner }}
           DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
 
-
   windows_job:
+    name: Windows [${{ matrix.network }}]
     needs: prepare_build
     if: ${{ needs.prepare_build.outputs.tag_created == 'true' }}
     runs-on: windows-latest
@@ -150,18 +164,22 @@ jobs:
       matrix:
         network: ["TEST", "BETA", "LIVE"]
     steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
           ref: ${{ needs.prepare_build.outputs.ci_tag }}
           repository: ${{ github.repository }}
+
       - name: Prepare
         run: ci/prepare/windows/prepare.ps1
+
       - name: Build Artifact
         run: ci/actions/windows/build.ps1
         env:
           CSC_LINK: ${{ secrets.CSC_LINK }}
           CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
+
       - name: Deploy Artifact
         run: ci/actions/windows/deploy.ps1
         env:

.github/workflows/changelog.yml

@@ -37,7 +37,7 @@ jobs:
           echo "REPO_TO_RUN=${{ github.event.inputs.repo }}" >> $GITHUB_ENV
           echo "MODE=${{ github.event.inputs.mode }}" >> $GITHUB_ENV
       - name: Checks out the required workflow files (workflow repo)
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.ref }}
           repository: ${{ github.repository }}

.github/workflows/code_sanitizers.yml

@@ -16,7 +16,7 @@ jobs:
           - { name: ASAN, ignore_errors: false, leak_check: false }
           - { name: ASAN_INT, ignore_errors: true, leak_check: false }
           - { name: TSAN, ignore_errors: false }
-          - { name: LEAK, ignore_errors: true, leak_check: true }
+          - { name: LEAK, ignore_errors: false, leak_check: true }
         exclude:
           # Bug when running with TSAN: "ThreadSanitizer: CHECK failed: sanitizer_deadlock_detector"
           - BACKEND: rocksdb
@@ -35,7 +35,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
 
@@ -91,7 +91,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
 

.github/workflows/coverage.yml

@@ -24,7 +24,7 @@ jobs:
     continue-on-error: true
     timeout-minutes: 60
     steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - uses: actions/checkout@v4
         with:
           submodules: "recursive"
       - name: Fetch Deps

.github/workflows/develop_branch_dockers_deploy.yml

@@ -9,7 +9,7 @@ jobs:
   linux_job:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+      - uses: actions/checkout@v4
         with:
           submodules: "recursive"
       - name: Fetch Deps

.github/workflows/flamegraphs.yml

@@ -22,7 +22,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
 

.github/workflows/prepare_release.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3.1.0
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.ref }}
           fetch-depth: 0

.github/workflows/unit_tests.yml

@@ -20,7 +20,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
 
@@ -66,7 +66,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
 
@@ -115,7 +115,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
 

CMakeLists.txt

@@ -466,6 +466,8 @@ set(BOOST_MODULE_LIBS
     ratio
     rational
     regex
+    scope
+    scope_exit
     serialization
     smart_ptr
     spirit

ci/actions/deploy.sh

@@ -20,8 +20,6 @@ case "${NETWORK}" in
 esac
 
 if [[ -n "${S3_BUILD_DIRECTORY}" ]]; then
-
-
     DIRECTORY="${S3_BUILD_DIRECTORY}/${BUILD}"
 else
     DIRECTORY=$BUILD
@@ -33,7 +31,7 @@ if [[ "$OS" == 'Linux' && "$IS_RPM_DEPLOY" -eq "1" ]]; then
 
     for rpm in $RPMS; do
         SHA=$(sha256sum ${rpm})
-        echo "Hash: $SHA"
+        echo "::notice::Hash: $SHA"
         echo $SHA > ${GITHUB_WORKSPACE}/$(basename "${rpm}.sha256")
 
         aws s3 cp ${rpm} s3://${S3_BUCKET_NAME}/$DIRECTORY/binaries/$(basename "${rpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
@@ -42,7 +40,7 @@ if [[ "$OS" == 'Linux' && "$IS_RPM_DEPLOY" -eq "1" ]]; then
     
     for srpm in $SRPMS; do
         SHA=$(sha256sum ${srpm})
-        echo "Hash: $SHA"
+        echo "::notice::Hash: $SHA"
         echo $SHA > ${GITHUB_WORKSPACE}/$(basename "${srpm}).sha256")
 
         aws s3 cp ${srpm} s3://${S3_BUCKET_NAME}/$DIRECTORY/source/$(basename "${srpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
@@ -50,11 +48,11 @@ if [[ "$OS" == 'Linux' && "$IS_RPM_DEPLOY" -eq "1" ]]; then
     done
 elif [[ "$OS" == 'Linux' ]]; then
     SHA=$(sha256sum $GITHUB_WORKSPACE/build/nano-node-*-Linux.tar.bz2)
-    echo "Hash: $SHA"
+    echo "::notice::Hash: $SHA"
     echo $SHA >$GITHUB_WORKSPACE/nano-node-$TAG-Linux.tar.bz2.sha256
 
     SHA=$(sha256sum $GITHUB_WORKSPACE/build/nano-node-*-Linux.deb)
-    echo "Hash: $SHA"
+    echo "::notice::Hash: $SHA"
     echo $SHA >$GITHUB_WORKSPACE/nano-node-$TAG-Linux.deb.sha256
 
     aws s3 cp $GITHUB_WORKSPACE/build/nano-node-*-Linux.tar.bz2 s3://${S3_BUCKET_NAME}/$DIRECTORY/binaries/nano-node-$TAG-Linux.tar.bz2 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
@@ -63,7 +61,7 @@ elif [[ "$OS" == 'Linux' ]]; then
     aws s3 cp $GITHUB_WORKSPACE/nano-node-$TAG-Linux.deb.sha256 s3://${S3_BUCKET_NAME}/$DIRECTORY/binaries/nano-node-$TAG-Linux.deb.sha256 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
 else
     SHA=$(sha256sum $GITHUB_WORKSPACE/build/nano-node-*-Darwin.dmg)
-    echo "Hash: $SHA"
+    echo "::notice::Hash: $SHA"
     echo $SHA >$GITHUB_WORKSPACE/build/nano-node-$TAG-Darwin.dmg.sha256
 
     aws s3 cp $GITHUB_WORKSPACE/build/nano-node-*-Darwin.dmg s3://${S3_BUCKET_NAME}/$DIRECTORY/binaries/nano-node-$TAG-Darwin.dmg --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers

ci/actions/linux/docker-impl/docker-common.sh

@@ -134,7 +134,22 @@ docker_login()
 push_docker_image()
 {
     local image_name=$1
+
+    # Log the image name before pushing
+    echo "Pushing image: $image_name"
+
+    # Push the image
     "$scripts"/custom-timeout.sh 30 docker push "$image_name"
+
+    # After pushing, get the digest from the local image manifest
+    local digest
+    digest=$(docker image inspect --format='{{index .RepoDigests 0}}' "$image_name")
+
+    if [ -n "$digest" ]; then
+        echo "::notice::Hash: $digest $image_name"
+    else
+        echo "::error::Could not retrieve digest for image $image_name"
+    fi
 }
 
 deploy_env_images()
@@ -154,7 +169,7 @@ deploy_tags()
     local exclude_pattern=$2
     local tags=$(docker images --format '{{.Repository}}:{{.Tag }}' | grep "$repo" | grep -vE "$exclude_pattern")
 
-    #Debug list all tags
+    # Debug list all tags
     docker images --format '{{.Repository}}:{{.Tag }}'
 
     for tag in $tags; do

ci/actions/windows/deploy.ps1

@@ -27,8 +27,8 @@ $zip_hash = ((Get-FileHash $zip).hash)+" "+(split-path -Path $zip -Resolve -leaf
 $exe_hash | Out-file -FilePath "$exe.sha256"
 $zip_hash | Out-file -FilePath "$zip.sha256"
 
-Write-Output "Hash: $exe_hash"
-Write-Output "Hash: $zip_hash"
+Write-Output "::notice::Hash: $exe_hash"
+Write-Output "::notice::Hash: $zip_hash"
 
 aws s3 cp "$exe" s3://$env:S3_BUCKET_NAME/$directory/binaries/nano-node-$env:TAG-win64.exe --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
 aws s3 cp "$exe.sha256" s3://$env:S3_BUCKET_NAME/$directory/binaries/nano-node-$env:TAG-win64.exe.sha256 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers

ci/build-ci.sh

@@ -48,7 +48,6 @@ cmake \
 -DNANO_WARN_TO_ERR=ON \
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE:-Debug} \
 -DQt5_DIR=${qt_dir} \
-${BACKTRACE:-} \
 ${SANITIZERS:-} \
 ..
 

ci/build.sh

@@ -66,7 +66,6 @@ cmake \
 -DCI_VERSION_PRE_RELEASE=${CI_VERSION_PRE_RELEASE:-OFF} \
 ${CMAKE_SANITIZER:-} \
 ${CMAKE_QT_DIR:-} \
-${CMAKE_BACKTRACE:-} \
 ${SRC}
 
 number_of_processors() {

ci/prepare/macos/prepare.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euox pipefail
 
-#Homebrew randomly fails to update. Retry 5 times with 15s interval
+# Homebrew randomly fails to update. Retry 5 times with 15s interval
 for i in {1..5}; do brew update && break || { echo "Update failed, retrying..."; sleep 15; }; done
 
 brew install coreutils
@@ -10,4 +10,4 @@ brew install qt@5
 brew link qt@5
 
 # Workaround: https://github.com/Homebrew/homebrew-core/issues/8392
-echo "$(brew --prefix qt5)/bin" >> $GITHUB_PATH
+echo "$(brew --prefix qt@5)/bin" >> $GITHUB_PATH

ci/tests/run-system-tests.sh

@@ -44,7 +44,7 @@ for script in ${NANO_SYSTEST_DIR}/*.sh; do
 done
 
 if [ $overall_status -eq 0 ]; then
-    echo "::notice::All systests passed"
+    echo "All systests passed"
 else
     echo "::error::Some systests failed"
     exit 1

ci/tests/show-sanitizer-reports.sh

@@ -19,7 +19,7 @@ if [[ ${#reports[@]} -gt 0 ]]; then
         issue_reported=true
     done
 else
-    echo "::notice::No report has been generated."
+    echo "No report has been generated."
 fi
 
 echo "issue_reported=${issue_reported}" >> $GITHUB_OUTPUT
@@ -28,6 +28,6 @@ if $issue_reported; then
     echo "::error::Issues were reported in the sanitizer report."
     exit 1
 else
-    echo "::notice::No issues found in the sanitizer reports."
+    echo "No issues found in the sanitizer reports."
     exit 0
 fi

docker/node/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:22.04 AS builder
 
 ARG COMPILER=gcc
 ARG NANO_NETWORK=live
@@ -10,7 +10,7 @@ RUN /tmp/prepare/prepare.sh
 COPY ./ /tmp/src
 WORKDIR /tmp/src
 
-#Define ARGs for ci/build-node.sh
+# Define ARGs for ci/build-node.sh
 ARG BUILD_TYPE=RelWithDebInfo
 ARG NANO_TEST=OFF
 ARG NANO_TRACING=OFF

nano/core_test/CMakeLists.txt

@@ -35,10 +35,12 @@ add_executable(
   network_filter.cpp
   network_functions.cpp
   node.cpp
+  numbers.cpp
   object_stream.cpp
   optimistic_scheduler.cpp
   processing_queue.cpp
   processor_service.cpp
+  random_pool.cpp
   rep_crawler.cpp
   receivable.cpp
   peer_history.cpp
@@ -51,10 +53,10 @@ add_executable(
   socket.cpp
   system.cpp
   telemetry.cpp
+  thread_pool.cpp
   throttle.cpp
   toml.cpp
   timer.cpp
-  uint256_union.cpp
   unchecked_map.cpp
   utility.cpp
   vote_cache.cpp

nano/core_test/active_elections.cpp

@@ -1224,7 +1224,7 @@ TEST (active_elections, activate_inactive)
 	ASSERT_NE (nullptr, election);
 	election->force_confirm ();
 
-	ASSERT_TIMELY (5s, !node.confirming_set.exists (send2->hash ()));
+	ASSERT_TIMELY (5s, !node.confirming_set.contains (send2->hash ()));
 	ASSERT_TIMELY (5s, node.block_confirmed (send2->hash ()));
 	ASSERT_TIMELY (5s, node.block_confirmed (send->hash ()));
 
@@ -1232,8 +1232,8 @@ TEST (active_elections, activate_inactive)
 	ASSERT_TIMELY_EQ (5s, 1, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
 	ASSERT_ALWAYS_EQ (50ms, 0, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
 
-	// The first block was not active so no activation takes place
-	ASSERT_FALSE (node.active.active (open->qualified_root ()) || node.block_confirmed_or_being_confirmed (open->hash ()));
+	// Cementing of send should activate open
+	ASSERT_TIMELY (5s, node.active.active (open->qualified_root ()));
 }
 
 TEST (active_elections, list_active)
@@ -1308,7 +1308,9 @@ TEST (active_elections, vacancy)
 				.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
 				.work (*system.work.generate (nano::dev::genesis->hash ()))
 				.build ();
-	node.active.vacancy_update = [&updated] () { updated = true; };
+	node.active.vacancy_updated.add ([&updated] () {
+		updated = true;
+	});
 	ASSERT_EQ (nano::block_status::progress, node.process (send));
 	ASSERT_EQ (1, node.active.vacancy (nano::election_behavior::priority));
 	ASSERT_EQ (0, node.active.size ());
@@ -1398,6 +1400,8 @@ TEST (active_elections, bound_election_winners)
 	nano::node_config config = system.default_config ();
 	// Set election winner limit to a low value
 	config.active_elections.max_election_winners = 5;
+	// Large batch size would complicate this testcase
+	config.confirming_set.batch_size = 1;
 	auto & node = *system.add_node (config);
 
 	// Start elections for a couple of blocks, number of elections is larger than the election winner set limit
@@ -1411,22 +1415,12 @@ TEST (active_elections, bound_election_winners)
 		auto guard = node.ledger.tx_begin_write (nano::store::writer::testing);
 
 		// Ensure that when the number of election winners reaches the limit, AEC vacancy reflects that
+		// Confirming more elections should make the vacancy negative
 		ASSERT_TRUE (node.active.vacancy (nano::election_behavior::priority) > 0);
 
-		int index = 0;
-		for (; index < config.active_elections.max_election_winners; ++index)
+		for (auto const & block : blocks)
 		{
-			auto election = node.vote_router.election (blocks[index]->hash ());
-			ASSERT_TRUE (election);
-			election->force_confirm ();
-		}
-
-		ASSERT_TIMELY_EQ (5s, node.active.vacancy (nano::election_behavior::priority), 0);
-
-		// Confirming more elections should make the vacancy negative
-		for (; index < blocks.size (); ++index)
-		{
-			auto election = node.vote_router.election (blocks[index]->hash ());
+			auto election = node.vote_router.election (block->hash ());
 			ASSERT_TRUE (election);
 			election->force_confirm ();
 		}

nano/core_test/block_store.cpp

@@ -314,10 +314,10 @@ TEST (block_store, pending_iterator)
 	auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
 	ASSERT_TRUE (!store->init_error ());
 	auto transaction (store->tx_begin_write ());
-	ASSERT_EQ (store->pending.end (), store->pending.begin (transaction));
+	ASSERT_EQ (store->pending.end (transaction), store->pending.begin (transaction));
 	store->pending.put (transaction, nano::pending_key (1, 2), { 2, 3, nano::epoch::epoch_1 });
 	auto current (store->pending.begin (transaction));
-	ASSERT_NE (store->pending.end (), current);
+	ASSERT_NE (store->pending.end (transaction), current);
 	nano::pending_key key1 (current->first);
 	ASSERT_EQ (nano::account (1), key1.account);
 	ASSERT_EQ (nano::block_hash (2), key1.hash);
@@ -411,7 +411,7 @@ TEST (block_store, empty_accounts)
 	ASSERT_TRUE (!store->init_error ());
 	auto transaction (store->tx_begin_read ());
 	auto begin (store->account.begin (transaction));
-	auto end (store->account.end ());
+	auto end (store->account.end (transaction));
 	ASSERT_EQ (end, begin);
 }
 
@@ -498,7 +498,7 @@ TEST (block_store, one_account)
 	store->confirmation_height.put (transaction, account, { 20, nano::block_hash (15) });
 	store->account.put (transaction, account, { hash, account, hash, 42, 100, 200, nano::epoch::epoch_0 });
 	auto begin (store->account.begin (transaction));
-	auto end (store->account.end ());
+	auto end (store->account.end (transaction));
 	ASSERT_NE (end, begin);
 	ASSERT_EQ (account, nano::account (begin->first));
 	nano::account_info info (begin->second);
@@ -567,7 +567,7 @@ TEST (block_store, two_account)
 	store->confirmation_height.put (transaction, account2, { 30, nano::block_hash (20) });
 	store->account.put (transaction, account2, { hash2, account2, hash2, 84, 200, 400, nano::epoch::epoch_0 });
 	auto begin (store->account.begin (transaction));
-	auto end (store->account.end ());
+	auto end (store->account.end (transaction));
 	ASSERT_NE (end, begin);
 	ASSERT_EQ (account1, nano::account (begin->first));
 	nano::account_info info1 (begin->second);
@@ -787,7 +787,7 @@ TEST (block_store, large_iteration)
 	std::unordered_set<nano::account> accounts2;
 	nano::account previous{};
 	auto transaction (store->tx_begin_read ());
-	for (auto i (store->account.begin (transaction, 0)), n (store->account.end ()); i != n; ++i)
+	for (auto i (store->account.begin (transaction, 0)), n (store->account.end (transaction)); i != n; ++i)
 	{
 		nano::account current (i->first);
 		ASSERT_GT (current.number (), previous.number ());
@@ -798,7 +798,7 @@ TEST (block_store, large_iteration)
 	// Reverse iteration
 	std::unordered_set<nano::account> accounts3;
 	previous = std::numeric_limits<nano::uint256_t>::max ();
-	for (auto i (store->account.rbegin (transaction)), n (store->account.end ()); i != n; --i)
+	for (auto i (store->account.rbegin (transaction)), n (store->account.rend (transaction)); i != n; ++i)
 	{
 		nano::account current (i->first);
 		ASSERT_LT (current.number (), previous.number ());
@@ -1253,8 +1253,8 @@ TEST (block_store, online_weight)
 	{
 		auto transaction (store->tx_begin_write ());
 		ASSERT_EQ (0, store->online_weight.count (transaction));
-		ASSERT_EQ (store->online_weight.end (), store->online_weight.begin (transaction));
-		ASSERT_EQ (store->online_weight.end (), store->online_weight.rbegin (transaction));
+		ASSERT_EQ (store->online_weight.end (transaction), store->online_weight.begin (transaction));
+		ASSERT_EQ (store->online_weight.rend (transaction), store->online_weight.rbegin (transaction));
 		store->online_weight.put (transaction, 1, 2);
 		store->online_weight.put (transaction, 3, 4);
 	}
@@ -1262,22 +1262,22 @@ TEST (block_store, online_weight)
 		auto transaction (store->tx_begin_write ());
 		ASSERT_EQ (2, store->online_weight.count (transaction));
 		auto item (store->online_weight.begin (transaction));
-		ASSERT_NE (store->online_weight.end (), item);
+		ASSERT_NE (store->online_weight.end (transaction), item);
 		ASSERT_EQ (1, item->first);
 		ASSERT_EQ (2, item->second.number ());
 		auto item_last (store->online_weight.rbegin (transaction));
-		ASSERT_NE (store->online_weight.end (), item_last);
+		ASSERT_NE (store->online_weight.rend (transaction), item_last);
 		ASSERT_EQ (3, item_last->first);
 		ASSERT_EQ (4, item_last->second.number ());
 		store->online_weight.del (transaction, 1);
 		ASSERT_EQ (1, store->online_weight.count (transaction));
-		ASSERT_EQ (store->online_weight.begin (transaction), store->online_weight.rbegin (transaction));
+		ASSERT_EQ (*store->online_weight.begin (transaction), *store->online_weight.rbegin (transaction));
 		store->online_weight.del (transaction, 3);
 	}
 	auto transaction (store->tx_begin_read ());
 	ASSERT_EQ (0, store->online_weight.count (transaction));
-	ASSERT_EQ (store->online_weight.end (), store->online_weight.begin (transaction));
-	ASSERT_EQ (store->online_weight.end (), store->online_weight.rbegin (transaction));
+	ASSERT_EQ (store->online_weight.end (transaction), store->online_weight.begin (transaction));
+	ASSERT_EQ (store->online_weight.rend (transaction), store->online_weight.rbegin (transaction));
 }
 
 TEST (block_store, pruned_blocks)

nano/core_test/bootstrap_server.cpp

@@ -92,7 +92,7 @@ TEST (bootstrap_server, serve_account_blocks)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -137,7 +137,7 @@ TEST (bootstrap_server, serve_hash)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -182,7 +182,7 @@ TEST (bootstrap_server, serve_hash_one)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -221,7 +221,7 @@ TEST (bootstrap_server, serve_end_of_chain)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -260,7 +260,7 @@ TEST (bootstrap_server, serve_missing)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -303,7 +303,7 @@ TEST (bootstrap_server, serve_multiple)
 			request.payload = request_payload;
 			request.update_header ();
 
-			node.network.inbound (request, nano::test::fake_channel (node));
+			node.inbound (request, nano::test::fake_channel (node));
 		}
 	}
 
@@ -359,7 +359,7 @@ TEST (bootstrap_server, serve_account_info)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -405,7 +405,7 @@ TEST (bootstrap_server, serve_account_info_missing)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -450,7 +450,7 @@ TEST (bootstrap_server, serve_frontiers)
 	request.payload = request_payload;
 	request.update_header ();
 
-	node.network.inbound (request, nano::test::fake_channel (node));
+	node.inbound (request, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY_EQ (5s, responses.size (), 1);
 
@@ -503,7 +503,7 @@ TEST (bootstrap_server, serve_frontiers_invalid_count)
 		request.payload = request_payload;
 		request.update_header ();
 
-		node.network.inbound (request, nano::test::fake_channel (node));
+		node.inbound (request, nano::test::fake_channel (node));
 	}
 
 	ASSERT_TIMELY_EQ (5s, node.stats.count (nano::stat::type::bootstrap_server, nano::stat::detail::invalid), 1);
@@ -521,7 +521,7 @@ TEST (bootstrap_server, serve_frontiers_invalid_count)
 		request.payload = request_payload;
 		request.update_header ();
 
-		node.network.inbound (request, nano::test::fake_channel (node));
+		node.inbound (request, nano::test::fake_channel (node));
 	}
 
 	ASSERT_TIMELY_EQ (5s, node.stats.count (nano::stat::type::bootstrap_server, nano::stat::detail::invalid), 2);
@@ -539,7 +539,7 @@ TEST (bootstrap_server, serve_frontiers_invalid_count)
 		request.payload = request_payload;
 		request.update_header ();
 
-		node.network.inbound (request, nano::test::fake_channel (node));
+		node.inbound (request, nano::test::fake_channel (node));
 	}
 
 	ASSERT_TIMELY_EQ (5s, node.stats.count (nano::stat::type::bootstrap_server, nano::stat::detail::invalid), 3);

nano/core_test/confirming_set.cpp

@@ -20,24 +20,24 @@ TEST (confirming_set, construction)
 {
 	auto ctx = nano::test::ledger_empty ();
 	nano::confirming_set_config config{};
-	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () };
+	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats (), ctx.logger () };
 }
 
 TEST (confirming_set, add_exists)
 {
 	auto ctx = nano::test::ledger_send_receive ();
 	nano::confirming_set_config config{};
-	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () };
+	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats (), ctx.logger () };
 	auto send = ctx.blocks ()[0];
 	confirming_set.add (send->hash ());
-	ASSERT_TRUE (confirming_set.exists (send->hash ()));
+	ASSERT_TRUE (confirming_set.contains (send->hash ()));
 }
 
 TEST (confirming_set, process_one)
 {
 	auto ctx = nano::test::ledger_send_receive ();
 	nano::confirming_set_config config{};
-	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () };
+	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats (), ctx.logger () };
 	std::atomic<int> count = 0;
 	std::mutex mutex;
 	std::condition_variable condition;
@@ -54,7 +54,7 @@ TEST (confirming_set, process_multiple)
 {
 	auto ctx = nano::test::ledger_send_receive ();
 	nano::confirming_set_config config{};
-	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () };
+	nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats (), ctx.logger () };
 	std::atomic<int> count = 0;
 	std::mutex mutex;
 	std::condition_variable condition;
@@ -111,7 +111,6 @@ TEST (confirmation_callback, observer_callbacks)
 
 	ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
 	ASSERT_EQ (3, node->ledger.cemented_count ());
-	ASSERT_EQ (0, node->active.election_winner_details_size ());
 }
 
 // The callback and confirmation history should only be updated after confirmation height is set (and not just after voting)
@@ -171,8 +170,7 @@ TEST (confirmation_callback, confirmed_history)
 
 	ASSERT_TIMELY (10s, !node->store.write_queue.contains (nano::store::writer::confirmation_height));
 
-	auto transaction = node->ledger.tx_begin_read ();
-	ASSERT_TRUE (node->ledger.confirmed.block_exists (transaction, send->hash ()));
+	ASSERT_TIMELY (5s, node->ledger.confirmed.block_exists (node->ledger.tx_begin_read (), send->hash ()));
 
 	ASSERT_TIMELY_EQ (10s, node->active.size (), 0);
 	ASSERT_TIMELY_EQ (10s, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out), 1);
@@ -186,7 +184,6 @@ TEST (confirmation_callback, confirmed_history)
 	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
 	ASSERT_TIMELY_EQ (5s, 2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
 	ASSERT_EQ (3, node->ledger.cemented_count ());
-	ASSERT_EQ (0, node->active.election_winner_details_size ());
 }
 
 TEST (confirmation_callback, dependent_election)
@@ -242,35 +239,10 @@ TEST (confirmation_callback, dependent_election)
 	// Wait for blocks to be confirmed in ledger, callbacks will happen after
 	ASSERT_TIMELY_EQ (5s, 3, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
 	// Once the item added to the confirming set no longer exists, callbacks have completed
-	ASSERT_TIMELY (5s, !node->confirming_set.exists (send2->hash ()));
+	ASSERT_TIMELY (5s, !node->confirming_set.contains (send2->hash ()));
 
 	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
 	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
 	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
 	ASSERT_EQ (4, node->ledger.cemented_count ());
-
-	ASSERT_EQ (0, node->active.election_winner_details_size ());
-}
-
-TEST (confirmation_callback, election_winner_details_clearing_node_process_confirmed)
-{
-	// Make sure election_winner_details is also cleared if the block never enters the confirmation height processor from node::process_confirmed
-	nano::test::system system (1);
-	auto node = system.nodes.front ();
-
-	nano::block_builder builder;
-	auto send = builder
-				.send ()
-				.previous (nano::dev::genesis->hash ())
-				.destination (nano::dev::genesis_key.pub)
-				.balance (nano::dev::constants.genesis_amount - nano::Knano_ratio)
-				.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
-				.work (*system.work.generate (nano::dev::genesis->hash ()))
-				.build ();
-	// Add to election_winner_details. Use an unrealistic iteration so that it should fall into the else case and do a cleanup
-	node->active.add_election_winner_details (send->hash (), nullptr);
-	nano::election_status election;
-	election.winner = send;
-	node->process_confirmed (election, 1000000);
-	ASSERT_EQ (0, node->active.election_winner_details_size ());
 }

nano/core_test/election_scheduler.cpp

@@ -195,7 +195,7 @@ TEST (election_scheduler, no_vacancy)
 				.work (*system.work.generate (nano::dev::genesis->hash ()))
 				.build ();
 	ASSERT_EQ (nano::block_status::progress, node.process (send));
-	node.process_confirmed (nano::election_status{ send });
+	node.process_confirmed (send->hash ());
 
 	auto receive = builder.make_block ()
 				   .account (key.pub)
@@ -207,7 +207,7 @@ TEST (election_scheduler, no_vacancy)
 				   .work (*system.work.generate (key.pub))
 				   .build ();
 	ASSERT_EQ (nano::block_status::progress, node.process (receive));
-	node.process_confirmed (nano::election_status{ receive });
+	node.process_confirmed (receive->hash ());
 
 	ASSERT_TIMELY (5s, nano::test::confirmed (node, { send, receive }));
 

nano/core_test/ledger.cpp

@@ -2406,7 +2406,7 @@ TEST (ledger, state_account)
 				 .work (*pool.generate (nano::dev::genesis->hash ()))
 				 .build ();
 	ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send1));
-	ASSERT_EQ (nano::dev::genesis_key.pub, ledger.any.block_account (transaction, send1->hash ()));
+	ASSERT_EQ (nano::dev::genesis_key.pub, ledger.any.block_account (transaction, send1->hash ()).value ());
 }
 
 TEST (ledger, state_send_receive)
@@ -5464,7 +5464,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
 
 	ASSERT_TRUE (rocksdb_store.pending.get (rocksdb_transaction, nano::pending_key (nano::dev::genesis_key.pub, send->hash ())));
 
-	for (auto i = rocksdb_store.online_weight.begin (rocksdb_transaction); i != rocksdb_store.online_weight.end (); ++i)
+	for (auto i = rocksdb_store.online_weight.begin (rocksdb_transaction); i != rocksdb_store.online_weight.end (rocksdb_transaction); ++i)
 	{
 		ASSERT_EQ (i->first, 100);
 		ASSERT_EQ (i->second, 2);

nano/core_test/ledger_confirm.cpp

@@ -752,29 +752,6 @@ TEST (ledger_confirm, observers)
 	ASSERT_EQ (2, node1->ledger.cemented_count ());
 }
 
-TEST (ledger_confirm, election_winner_details_clearing_node_process_confirmed)
-{
-	// Make sure election_winner_details is also cleared if the block never enters the confirmation height processor from node::process_confirmed
-	nano::test::system system (1);
-	auto node = system.nodes.front ();
-
-	nano::block_builder builder;
-	auto send = builder
-				.send ()
-				.previous (nano::dev::genesis->hash ())
-				.destination (nano::dev::genesis_key.pub)
-				.balance (nano::dev::constants.genesis_amount - nano::Knano_ratio)
-				.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
-				.work (*system.work.generate (nano::dev::genesis->hash ()))
-				.build ();
-	// Add to election_winner_details. Use an unrealistic iteration so that it should fall into the else case and do a cleanup
-	node->active.add_election_winner_details (send->hash (), nullptr);
-	nano::election_status election;
-	election.winner = send;
-	node->process_confirmed (election, 1000000);
-	ASSERT_EQ (0, node->active.election_winner_details_size ());
-}
-
 TEST (ledger_confirm, pruned_source)
 {
 	nano::test::system system;

nano/core_test/network.cpp

@@ -377,7 +377,7 @@ TEST (receivable_processor, confirm_insufficient_pos)
 	nano::confirm_ack con1{ nano::dev::network_params.network, vote };
 	auto channel1 = std::make_shared<nano::transport::inproc::channel> (node1, node1);
 	ASSERT_EQ (1, election->votes ().size ());
-	node1.network.inbound (con1, channel1);
+	node1.inbound (con1, channel1);
 	ASSERT_TIMELY_EQ (5s, 2, election->votes ().size ())
 	ASSERT_FALSE (election->confirmed ());
 }
@@ -402,7 +402,7 @@ TEST (receivable_processor, confirm_sufficient_pos)
 	nano::confirm_ack con1{ nano::dev::network_params.network, vote };
 	auto channel1 = std::make_shared<nano::transport::inproc::channel> (node1, node1);
 	ASSERT_EQ (1, election->votes ().size ());
-	node1.network.inbound (con1, channel1);
+	node1.inbound (con1, channel1);
 	ASSERT_TIMELY_EQ (5s, 2, election->votes ().size ())
 	ASSERT_TRUE (election->confirmed ());
 }
@@ -743,10 +743,10 @@ TEST (network, duplicate_revert_publish)
 	auto channel = nano::test::establish_tcp (system, *other_node, node.network.endpoint ());
 	ASSERT_NE (nullptr, channel);
 	ASSERT_EQ (0, publish.digest);
-	node.network.inbound (publish, channel);
+	node.inbound (publish, nano::test::fake_channel (node));
 	ASSERT_TRUE (node.network.filter.apply (bytes.data (), bytes.size ()));
 	publish.digest = digest;
-	node.network.inbound (publish, channel);
+	node.inbound (publish, nano::test::fake_channel (node));
 	ASSERT_FALSE (node.network.filter.apply (bytes.data (), bytes.size ()));
 }
 
@@ -1024,14 +1024,13 @@ TEST (network, loopback_channel)
 	auto & node2 = *system.nodes[1];
 	nano::transport::inproc::channel channel1 (node1, node1);
 	ASSERT_EQ (channel1.get_type (), nano::transport::transport_type::loopback);
-	ASSERT_EQ (channel1.get_endpoint (), node1.network.endpoint ());
-	ASSERT_EQ (channel1.get_tcp_endpoint (), nano::transport::map_endpoint_to_tcp (node1.network.endpoint ()));
+	ASSERT_EQ (channel1.get_remote_endpoint (), node1.network.endpoint ());
 	ASSERT_EQ (channel1.get_network_version (), node1.network_params.network.protocol_version);
 	ASSERT_EQ (channel1.get_node_id (), node1.node_id.pub);
 	ASSERT_EQ (channel1.get_node_id_optional ().value_or (0), node1.node_id.pub);
 	nano::transport::inproc::channel channel2 (node2, node2);
 	++node1.network.port;
-	ASSERT_NE (channel1.get_endpoint (), node1.network.endpoint ());
+	ASSERT_NE (channel1.get_remote_endpoint (), node1.network.endpoint ());
 }
 
 // Ensure the network filters messages with the incorrect magic number

nano/core_test/node.cpp

@@ -688,16 +688,15 @@ TEST (node, fork_flip)
 				 .work (*system.work.generate (nano::dev::genesis->hash ()))
 				 .build ();
 	nano::publish publish2{ nano::dev::network_params.network, send2 };
-	auto ignored_channel = nano::test::fake_channel (node1);
-	node1.network.inbound (publish1, ignored_channel);
-	node2.network.inbound (publish2, ignored_channel);
+	node1.inbound (publish1, nano::test::fake_channel (node1));
+	node2.inbound (publish2, nano::test::fake_channel (node2));
 	ASSERT_TIMELY_EQ (5s, 1, node1.active.size ());
 	ASSERT_TIMELY_EQ (5s, 1, node2.active.size ());
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	// Fill nodes with forked blocks
-	node1.network.inbound (publish2, ignored_channel);
+	node1.inbound (publish2, nano::test::fake_channel (node1));
 	ASSERT_TIMELY (5s, node1.active.active (*send2));
-	node2.network.inbound (publish1, ignored_channel);
+	node2.inbound (publish1, nano::test::fake_channel (node2));
 	ASSERT_TIMELY (5s, node2.active.active (*send1));
 	auto election1 (node2.active.election (nano::qualified_root (nano::dev::genesis->hash (), nano::dev::genesis->hash ())));
 	ASSERT_NE (nullptr, election1);
@@ -829,7 +828,7 @@ TEST (node, fork_open)
 				 .build ();
 	nano::publish publish1{ nano::dev::network_params.network, send1 };
 	auto channel1 = std::make_shared<nano::transport::fake::channel> (node);
-	node.network.inbound (publish1, channel1);
+	node.inbound (publish1, channel1);
 	ASSERT_TIMELY (5s, (election = node.active.election (publish1.block->qualified_root ())) != nullptr);
 	election->force_confirm ();
 	ASSERT_TIMELY (5s, node.active.empty () && node.block_confirmed (publish1.block->hash ()));
@@ -848,7 +847,7 @@ TEST (node, fork_open)
 				 .work (*system.work.generate (key1.pub))
 				 .build ();
 	nano::publish publish2{ nano::dev::network_params.network, open1 };
-	node.network.inbound (publish2, channel1);
+	node.inbound (publish2, channel1);
 	ASSERT_TIMELY_EQ (5s, 1, node.active.size ());
 
 	// create 2nd open block, which is a fork of open1 block
@@ -860,7 +859,7 @@ TEST (node, fork_open)
 				 .work (*system.work.generate (key1.pub))
 				 .build ();
 	nano::publish publish3{ nano::dev::network_params.network, open2 };
-	node.network.inbound (publish3, channel1);
+	node.inbound (publish3, channel1);
 	ASSERT_TIMELY (5s, (election = node.active.election (publish3.block->qualified_root ())) != nullptr);
 
 	// we expect to find 2 blocks in the election and we expect the first block to be the winner just because it was first
@@ -1856,14 +1855,14 @@ TEST (node, DISABLED_local_votes_cache)
 	nano::confirm_req message1{ nano::dev::network_params.network, send1->hash (), send1->root () };
 	nano::confirm_req message2{ nano::dev::network_params.network, send2->hash (), send2->root () };
 	auto channel = std::make_shared<nano::transport::fake::channel> (node);
-	node.network.inbound (message1, channel);
+	node.inbound (message1, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes), 1);
-	node.network.inbound (message2, channel);
+	node.inbound (message2, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes), 2);
 	for (auto i (0); i < 100; ++i)
 	{
-		node.network.inbound (message1, channel);
-		node.network.inbound (message2, channel);
+		node.inbound (message1, channel);
+		node.inbound (message2, channel);
 	}
 	// Make sure a new vote was not generated
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes), 2);
@@ -1873,7 +1872,7 @@ TEST (node, DISABLED_local_votes_cache)
 		ASSERT_EQ (nano::block_status::progress, node.ledger.process (transaction, send3));
 	}
 	nano::confirm_req message3{ nano::dev::network_params.network, send3->hash (), send3->root () };
-	node.network.inbound (message3, channel);
+	node.inbound (message3, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes), 3);
 	ASSERT_TIMELY (3s, !node.history.votes (send1->root (), send1->hash ()).empty ());
 	ASSERT_TIMELY (3s, !node.history.votes (send2->root (), send2->hash ()).empty ());
@@ -1881,7 +1880,7 @@ TEST (node, DISABLED_local_votes_cache)
 	// All requests should be served from the cache
 	for (auto i (0); i < 100; ++i)
 	{
-		node.network.inbound (message3, channel);
+		node.inbound (message3, channel);
 	}
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_generated_votes), 3);
 }
@@ -1935,26 +1934,26 @@ TEST (node, DISABLED_local_votes_cache_batch)
 	nano::confirm_req message{ nano::dev::network_params.network, batch };
 	auto channel = std::make_shared<nano::transport::fake::channel> (node);
 	// Generates and sends one vote for both hashes which is then cached
-	node.network.inbound (message, channel);
+	node.inbound (message, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out), 1);
 	ASSERT_EQ (1, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
 	ASSERT_FALSE (node.history.votes (send2->root (), send2->hash ()).empty ());
 	ASSERT_FALSE (node.history.votes (receive1->root (), receive1->hash ()).empty ());
 	// Only one confirm_ack should be sent if all hashes are part of the same vote
-	node.network.inbound (message, channel);
+	node.inbound (message, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out), 2);
 	ASSERT_EQ (2, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
 	// Test when votes are different
 	node.history.erase (send2->root ());
 	node.history.erase (receive1->root ());
-	node.network.inbound (nano::confirm_req{ nano::dev::network_params.network, send2->hash (), send2->root () }, channel);
+	node.inbound (nano::confirm_req{ nano::dev::network_params.network, send2->hash (), send2->root () }, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out), 3);
 	ASSERT_EQ (3, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
-	node.network.inbound (nano::confirm_req{ nano::dev::network_params.network, receive1->hash (), receive1->root () }, channel);
+	node.inbound (nano::confirm_req{ nano::dev::network_params.network, receive1->hash (), receive1->root () }, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out), 4);
 	ASSERT_EQ (4, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
 	// There are two different votes, so both should be sent in response
-	node.network.inbound (message, channel);
+	node.inbound (message, channel);
 	ASSERT_TIMELY_EQ (3s, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out), 6);
 	ASSERT_EQ (6, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
 }
@@ -1975,7 +1974,7 @@ TEST (node, DISABLED_local_votes_cache_generate_new_vote)
 	// Send a confirm req for genesis block to node
 	nano::confirm_req message1{ nano::dev::network_params.network, nano::dev::genesis->hash (), nano::dev::genesis->root () };
 	auto channel = std::make_shared<nano::transport::fake::channel> (node);
-	node.network.inbound (message1, channel);
+	node.inbound (message1, channel);
 
 	// check that the node generated a vote for the genesis block and that it is stored in the local vote cache and it is the only vote
 	ASSERT_TIMELY (5s, !node.history.votes (nano::dev::genesis->root (), nano::dev::genesis->hash ()).empty ());
@@ -1998,7 +1997,7 @@ TEST (node, DISABLED_local_votes_cache_generate_new_vote)
 	// One of the hashes is cached
 	std::vector<std::pair<nano::block_hash, nano::root>> roots_hashes{ std::make_pair (nano::dev::genesis->hash (), nano::dev::genesis->root ()), std::make_pair (send1->hash (), send1->root ()) };
 	nano::confirm_req message2{ nano::dev::network_params.network, roots_hashes };
-	node.network.inbound (message2, channel);
+	node.inbound (message2, channel);
 	ASSERT_TIMELY (3s, !node.history.votes (send1->root (), send1->hash ()).empty ());
 	auto votes2 (node.history.votes (send1->root (), send1->hash ()));
 	ASSERT_EQ (1, votes2.size ());
@@ -2441,13 +2440,13 @@ TEST (node, fork_election_invalid_block_signature)
 				 .build ();
 
 	auto channel1 = std::make_shared<nano::transport::fake::channel> (node1);
-	node1.network.inbound (nano::publish{ nano::dev::network_params.network, send1 }, channel1);
+	node1.inbound (nano::publish{ nano::dev::network_params.network, send1 }, channel1);
 	ASSERT_TIMELY (5s, node1.active.active (send1->qualified_root ()));
 	auto election (node1.active.election (send1->qualified_root ()));
 	ASSERT_NE (nullptr, election);
 	ASSERT_EQ (1, election->blocks ().size ());
-	node1.network.inbound (nano::publish{ nano::dev::network_params.network, send3 }, channel1);
-	node1.network.inbound (nano::publish{ nano::dev::network_params.network, send2 }, channel1);
+	node1.inbound (nano::publish{ nano::dev::network_params.network, send3 }, channel1);
+	node1.inbound (nano::publish{ nano::dev::network_params.network, send2 }, channel1);
 	ASSERT_TIMELY (3s, election->blocks ().size () > 1);
 	ASSERT_EQ (election->blocks ()[send2->hash ()]->block_signature (), send2->block_signature ());
 }
@@ -2704,7 +2703,7 @@ TEST (node, peer_history_restart)
 		ASSERT_TIMELY (10s, !node2->network.empty ());
 		// Confirm that the peers match with the endpoints we are expecting
 		auto list (node2->network.list (2));
-		ASSERT_EQ (node1->network.endpoint (), list[0]->get_endpoint ());
+		ASSERT_EQ (node1->network.endpoint (), list[0]->get_remote_endpoint ());
 		ASSERT_EQ (1, node2->network.size ());
 		system.stop_node (*node2);
 	}
@@ -2727,7 +2726,7 @@ TEST (node, peer_history_restart)
 		ASSERT_TIMELY (10s, !node3->network.empty ());
 		// Confirm that the peers match with the endpoints we are expecting
 		auto list (node3->network.list (2));
-		ASSERT_EQ (node1->network.endpoint (), list[0]->get_endpoint ());
+		ASSERT_EQ (node1->network.endpoint (), list[0]->get_remote_endpoint ());
 		ASSERT_EQ (1, node3->network.size ());
 		system.stop_node (*node3);
 	}
@@ -2789,11 +2788,11 @@ TEST (node, bidirectional_tcp)
 	ASSERT_EQ (1, node2->network.size ());
 	auto list1 (node1->network.list (1));
 	ASSERT_EQ (nano::transport::transport_type::tcp, list1[0]->get_type ());
-	ASSERT_NE (node2->network.endpoint (), list1[0]->get_endpoint ()); // Ephemeral port
+	ASSERT_NE (node2->network.endpoint (), list1[0]->get_remote_endpoint ()); // Ephemeral port
 	ASSERT_EQ (node2->node_id.pub, list1[0]->get_node_id ());
 	auto list2 (node2->network.list (1));
 	ASSERT_EQ (nano::transport::transport_type::tcp, list2[0]->get_type ());
-	ASSERT_EQ (node1->network.endpoint (), list2[0]->get_endpoint ());
+	ASSERT_EQ (node1->network.endpoint (), list2[0]->get_remote_endpoint ());
 	ASSERT_EQ (node1->node_id.pub, list2[0]->get_node_id ());
 	// Test block propagation from node 1
 	nano::keypair key;
@@ -3564,9 +3563,9 @@ TEST (node, pruning_automatic)
 	ASSERT_TIMELY (5s, node1.block (send2->hash ()) != nullptr);
 
 	// Force-confirm both blocks
-	node1.process_confirmed (nano::election_status{ send1 });
+	node1.process_confirmed (send1->hash ());
 	ASSERT_TIMELY (5s, node1.block_confirmed (send1->hash ()));
-	node1.process_confirmed (nano::election_status{ send2 });
+	node1.process_confirmed (send2->hash ());
 	ASSERT_TIMELY (5s, node1.block_confirmed (send2->hash ()));
 
 	// Check pruning result
@@ -3615,9 +3614,9 @@ TEST (node, DISABLED_pruning_age)
 	node1.process_active (send2);
 
 	// Force-confirm both blocks
-	node1.process_confirmed (nano::election_status{ send1 });
+	node1.process_confirmed (send1->hash ());
 	ASSERT_TIMELY (5s, node1.block_confirmed (send1->hash ()));
-	node1.process_confirmed (nano::election_status{ send2 });
+	node1.process_confirmed (send2->hash ());
 	ASSERT_TIMELY (5s, node1.block_confirmed (send2->hash ()));
 
 	// Three blocks in total, nothing pruned yet
@@ -3676,9 +3675,9 @@ TEST (node, DISABLED_pruning_depth)
 	node1.process_active (send2);
 
 	// Force-confirm both blocks
-	node1.process_confirmed (nano::election_status{ send1 });
+	node1.process_confirmed (send1->hash ());
 	ASSERT_TIMELY (5s, node1.block_confirmed (send1->hash ()));
-	node1.process_confirmed (nano::election_status{ send2 });
+	node1.process_confirmed (send2->hash ());
 	ASSERT_TIMELY (5s, node1.block_confirmed (send2->hash ()));
 
 	// Three blocks in total, nothing pruned yet

nano/core_test/numbers.cpp

@@ -1,27 +1,192 @@
-#include <nano/crypto_lib/random_pool.hpp>
+#include <nano/lib/numbers.hpp>
 #include <nano/secure/common.hpp>
-#include <nano/test_common/testutil.hpp>
 
 #include <gtest/gtest.h>
 
+#include <boost/container_hash/hash.hpp>
+
 #include <thread>
+#include <unordered_set>
+
+TEST (numbers, identity)
+{
+	ASSERT_EQ (1, nano::uint128_union (1).number ().convert_to<uint8_t> ());
+	ASSERT_EQ (1, nano::uint256_union (1).number ().convert_to<uint8_t> ());
+	ASSERT_EQ (1, nano::uint512_union (1).number ().convert_to<uint8_t> ());
+}
 
 namespace
 {
-template <typename Union, typename Bound>
-void assert_union_types ();
+template <typename Type>
+void check_operator_less_than (Type lhs, Type rhs)
+{
+	ASSERT_TRUE (lhs < rhs);
+	ASSERT_FALSE (rhs < lhs);
+	ASSERT_FALSE (lhs < lhs);
+	ASSERT_FALSE (rhs < rhs);
+}
 
-template <typename Union, typename Bound>
-void test_union_operator_less_than ();
+template <typename Type>
+void test_operator_less_than ()
+{
+	using underlying_t = typename Type::underlying_type;
 
-template <typename Num>
-void check_operator_less_than (Num lhs, Num rhs);
+	// Small
+	check_operator_less_than (Type{ 123 }, Type{ 124 });
+	check_operator_less_than (Type{ 124 }, Type{ 125 });
 
-template <typename Union, typename Bound>
-void test_union_operator_greater_than ();
+	// Medium
+	check_operator_less_than (Type{ std::numeric_limits<uint16_t>::max () - 1 }, Type{ std::numeric_limits<uint16_t>::max () + 1 });
+	check_operator_less_than (Type{ std::numeric_limits<uint32_t>::max () - 12345678 }, Type{ std::numeric_limits<uint32_t>::max () - 123456 });
 
-template <typename Num>
-void check_operator_greater_than (Num lhs, Num rhs);
+	// Large
+	check_operator_less_than (Type{ std::numeric_limits<uint64_t>::max () - 555555555555 }, Type{ std::numeric_limits<uint64_t>::max () - 1 });
+
+	// Boundary values
+	check_operator_less_than (Type{ std::numeric_limits<underlying_t>::min () }, Type{ std::numeric_limits<underlying_t>::max () });
+}
+
+template <typename Type>
+void check_operator_greater_than (Type lhs, Type rhs)
+{
+	ASSERT_TRUE (lhs > rhs);
+	ASSERT_FALSE (rhs > lhs);
+	ASSERT_FALSE (lhs > lhs);
+	ASSERT_FALSE (rhs > rhs);
+}
+
+template <typename Type>
+void test_operator_greater_than ()
+{
+	using underlying_t = typename Type::underlying_type;
+
+	// Small
+	check_operator_greater_than (Type{ 124 }, Type{ 123 });
+	check_operator_greater_than (Type{ 125 }, Type{ 124 });
+
+	// Medium
+	check_operator_greater_than (Type{ std::numeric_limits<uint16_t>::max () + 1 }, Type{ std::numeric_limits<uint16_t>::max () - 1 });
+	check_operator_greater_than (Type{ std::numeric_limits<uint32_t>::max () - 123456 }, Type{ std::numeric_limits<uint32_t>::max () - 12345678 });
+
+	// Large
+	check_operator_greater_than (Type{ std::numeric_limits<uint64_t>::max () - 1 }, Type{ std::numeric_limits<uint64_t>::max () - 555555555555 });
+
+	// Boundary values
+	check_operator_greater_than (Type{ std::numeric_limits<underlying_t>::max () }, Type{ std::numeric_limits<underlying_t>::min () });
+}
+
+template <typename Type>
+void test_comparison ()
+{
+	test_operator_less_than<Type> ();
+	test_operator_greater_than<Type> ();
+}
+}
+
+TEST (numbers, comparison)
+{
+	test_comparison<nano::uint128_union> ();
+	test_comparison<nano::uint256_union> ();
+	test_comparison<nano::uint512_union> ();
+	test_comparison<nano::block_hash> ();
+	test_comparison<nano::public_key> ();
+	test_comparison<nano::hash_or_account> ();
+	test_comparison<nano::link> ();
+	test_comparison<nano::root> ();
+	test_comparison<nano::raw_key> ();
+	test_comparison<nano::wallet_id> ();
+	test_comparison<nano::qualified_root> ();
+}
+
+namespace
+{
+template <typename Type, template <typename> class Hash>
+void test_hashing ()
+{
+	Hash<Type> hash;
+	using underlying_t = typename Type::underlying_type;
+
+	// Basic equality tests
+	ASSERT_EQ (hash (Type{}), hash (Type{}));
+	ASSERT_EQ (hash (Type{ 123 }), hash (Type{ 123 }));
+
+	// Basic inequality tests
+	ASSERT_NE (hash (Type{ 123 }), hash (Type{ 124 }));
+	ASSERT_NE (hash (Type{ 0 }), hash (Type{ 1 }));
+
+	// Boundary value tests
+	constexpr auto min_val = std::numeric_limits<underlying_t>::min ();
+	constexpr auto max_val = std::numeric_limits<underlying_t>::max ();
+
+	// Min/Max tests
+	ASSERT_EQ (hash (Type{ min_val }), hash (Type{ min_val }));
+	ASSERT_EQ (hash (Type{ max_val }), hash (Type{ max_val }));
+	ASSERT_NE (hash (Type{ min_val }), hash (Type{ max_val }));
+
+	// Near boundary tests
+	ASSERT_NE (hash (Type{ min_val }), hash (Type{ min_val + 1 }));
+	ASSERT_NE (hash (Type{ max_val }), hash (Type{ max_val - 1 }));
+	ASSERT_NE (hash (Type{ min_val + 1 }), hash (Type{ max_val }));
+	ASSERT_NE (hash (Type{ max_val - 1 }), hash (Type{ min_val }));
+
+	// Common value tests
+	std::vector<underlying_t> common_values = {
+		0, // Zero
+		1, // One
+		42, // Common test value
+		0xFF, // Byte boundary
+		0xFFFF, // Word boundary
+		min_val, // Minimum
+		max_val, // Maximum
+		max_val / 2, // Middle value
+		min_val + (max_val / 2) // Offset middle
+	};
+
+	// Test all common values against each other
+	for (size_t i = 0; i < common_values.size (); ++i)
+	{
+		for (size_t j = i + 1; j < common_values.size (); ++j)
+		{
+			if (common_values[i] != common_values[j])
+			{
+				ASSERT_NE (hash (Type{ common_values[i] }), hash (Type{ common_values[j] }));
+			}
+			else
+			{
+				ASSERT_EQ (hash (Type{ common_values[i] }), hash (Type{ common_values[j] }));
+			}
+		}
+	}
+}
+}
+
+TEST (numbers, hashing)
+{
+	// Using std::hash
+	test_hashing<nano::uint128_union, std::hash> ();
+	test_hashing<nano::uint256_union, std::hash> ();
+	test_hashing<nano::uint512_union, std::hash> ();
+	test_hashing<nano::block_hash, std::hash> ();
+	test_hashing<nano::public_key, std::hash> ();
+	test_hashing<nano::hash_or_account, std::hash> ();
+	test_hashing<nano::link, std::hash> ();
+	test_hashing<nano::root, std::hash> ();
+	test_hashing<nano::raw_key, std::hash> ();
+	test_hashing<nano::wallet_id, std::hash> ();
+	test_hashing<nano::qualified_root, std::hash> ();
+
+	// Using boost::hash
+	test_hashing<nano::uint128_union, boost::hash> ();
+	test_hashing<nano::uint256_union, boost::hash> ();
+	test_hashing<nano::uint512_union, boost::hash> ();
+	test_hashing<nano::block_hash, boost::hash> ();
+	test_hashing<nano::public_key, boost::hash> ();
+	test_hashing<nano::hash_or_account, boost::hash> ();
+	test_hashing<nano::link, boost::hash> ();
+	test_hashing<nano::root, boost::hash> ();
+	test_hashing<nano::raw_key, boost::hash> ();
+	test_hashing<nano::wallet_id, boost::hash> ();
+	test_hashing<nano::qualified_root, boost::hash> ();
 }
 
 TEST (uint128_union, decode_dec)
@@ -64,16 +229,6 @@ TEST (uint128_union, decode_dec_overflow)
 	ASSERT_TRUE (error);
 }
 
-TEST (uint128_union, operator_less_than)
-{
-	test_union_operator_less_than<nano::uint128_union, nano::uint128_t> ();
-}
-
-TEST (uint128_union, operator_greater_than)
-{
-	test_union_operator_greater_than<nano::uint128_union, nano::uint128_t> ();
-}
-
 struct test_punct : std::moneypunct<char>
 {
 	pattern do_pos_format () const
@@ -149,13 +304,6 @@ TEST (uint128_union, decode_decimal)
 	ASSERT_EQ (1230 * nano::Knano_ratio, amount.number ());
 }
 
-TEST (unions, identity)
-{
-	ASSERT_EQ (1, nano::uint128_union (1).number ().convert_to<uint8_t> ());
-	ASSERT_EQ (1, nano::uint256_union (1).number ().convert_to<uint8_t> ());
-	ASSERT_EQ (1, nano::uint512_union (1).number ().convert_to<uint8_t> ());
-}
-
 TEST (uint256_union, key_encryption)
 {
 	nano::keypair key1;
@@ -449,11 +597,6 @@ TEST (uint256_union, bounds)
 	ASSERT_TRUE (key.decode_account (bad2));
 }
 
-TEST (uint256_union, operator_less_than)
-{
-	test_union_operator_less_than<nano::uint256_union, nano::uint256_t> ();
-}
-
 TEST (uint64_t, parse)
 {
 	uint64_t value0 (1);
@@ -504,114 +647,4 @@ TEST (uint512_union, hash)
 			ASSERT_NE (h (x1), h (x2));
 		}
 	}
-}
-
-namespace
-{
-template <typename Union, typename Bound>
-void assert_union_types ()
-{
-	static_assert ((std::is_same<Union, nano::uint128_union>::value && std::is_same<Bound, nano::uint128_t>::value) || (std::is_same<Union, nano::uint256_union>::value && std::is_same<Bound, nano::uint256_t>::value) || (std::is_same<Union, nano::uint512_union>::value && std::is_same<Bound, nano::uint512_t>::value),
-	"Union type needs to be consistent with the lower/upper Bound type");
-}
-
-template <typename Union, typename Bound>
-void test_union_operator_less_than ()
-{
-	assert_union_types<Union, Bound> ();
-
-	// Small
-	check_operator_less_than (Union (123), Union (124));
-	check_operator_less_than (Union (124), Union (125));
-
-	// Medium
-	check_operator_less_than (Union (std::numeric_limits<uint16_t>::max () - 1), Union (std::numeric_limits<uint16_t>::max () + 1));
-	check_operator_less_than (Union (std::numeric_limits<uint32_t>::max () - 12345678), Union (std::numeric_limits<uint32_t>::max () - 123456));
-
-	// Large
-	check_operator_less_than (Union (std::numeric_limits<uint64_t>::max () - 555555555555), Union (std::numeric_limits<uint64_t>::max () - 1));
-
-	// Boundary values
-	check_operator_less_than (Union (std::numeric_limits<Bound>::min ()), Union (std::numeric_limits<Bound>::max ()));
-}
-
-template <typename Num>
-void check_operator_less_than (Num lhs, Num rhs)
-{
-	ASSERT_TRUE (lhs < rhs);
-	ASSERT_FALSE (rhs < lhs);
-	ASSERT_FALSE (lhs < lhs);
-	ASSERT_FALSE (rhs < rhs);
-}
-
-template <typename Union, typename Bound>
-void test_union_operator_greater_than ()
-{
-	assert_union_types<Union, Bound> ();
-
-	// Small
-	check_operator_greater_than (Union (124), Union (123));
-	check_operator_greater_than (Union (125), Union (124));
-
-	// Medium
-	check_operator_greater_than (Union (std::numeric_limits<uint16_t>::max () + 1), Union (std::numeric_limits<uint16_t>::max () - 1));
-	check_operator_greater_than (Union (std::numeric_limits<uint32_t>::max () - 123456), Union (std::numeric_limits<uint32_t>::max () - 12345678));
-
-	// Large
-	check_operator_greater_than (Union (std::numeric_limits<uint64_t>::max () - 1), Union (std::numeric_limits<uint64_t>::max () - 555555555555));
-
-	// Boundary values
-	check_operator_greater_than (Union (std::numeric_limits<Bound>::max ()), Union (std::numeric_limits<Bound>::min ()));
-}
-
-template <typename Num>
-void check_operator_greater_than (Num lhs, Num rhs)
-{
-	ASSERT_TRUE (lhs > rhs);
-	ASSERT_FALSE (rhs > lhs);
-	ASSERT_FALSE (lhs > lhs);
-	ASSERT_FALSE (rhs > rhs);
-}
-}
-
-TEST (random_pool, multithreading)
-{
-	std::vector<std::thread> threads;
-	for (auto i = 0; i < 100; ++i)
-	{
-		threads.emplace_back ([] () {
-			nano::uint256_union number;
-			nano::random_pool::generate_block (number.bytes.data (), number.bytes.size ());
-		});
-	}
-	for (auto & i : threads)
-	{
-		i.join ();
-	}
-}
-
-// Test that random 64bit numbers are within the given range
-TEST (random_pool, generate_word64)
-{
-	int occurrences[10] = { 0 };
-	for (auto i = 0; i < 1000; ++i)
-	{
-		auto random = nano::random_pool::generate_word64 (1, 9);
-		ASSERT_TRUE (random >= 1 && random <= 9);
-		occurrences[random] += 1;
-	}
-
-	for (auto i = 1; i < 10; ++i)
-	{
-		ASSERT_GT (occurrences[i], 0);
-	}
-}
-
-// Test random numbers > uint32 max
-TEST (random_pool, generate_word64_big_number)
-{
-	uint64_t min = static_cast<uint64_t> (std::numeric_limits<uint32_t>::max ()) + 1;
-	uint64_t max = std::numeric_limits<uint64_t>::max ();
-	auto big_random = nano::random_pool::generate_word64 (min, max);
-	ASSERT_GE (big_random, min);
-}
+}

nano/core_test/peer_container.cpp

@@ -79,16 +79,12 @@ TEST (peer_container, tcp_channel_cleanup_works)
 	ASSERT_NE (nullptr, channel1);
 	// set the last packet sent for channel1 only to guarantee it contains a value.
 	// it won't be necessarily the same use by the cleanup cutoff time
-	node1.network.tcp_channels.modify (channel1, [&now] (auto channel) {
-		channel->set_last_packet_sent (now - std::chrono::seconds (5));
-	});
+	channel1->set_last_packet_sent (now - std::chrono::seconds (5));
 	auto channel2 = nano::test::establish_tcp (system, node1, outer_node2->network.endpoint ());
 	ASSERT_NE (nullptr, channel2);
 	// set the last packet sent for channel2 only to guarantee it contains a value.
 	// it won't be necessarily the same use by the cleanup cutoff time
-	node1.network.tcp_channels.modify (channel2, [&now] (auto channel) {
-		channel->set_last_packet_sent (now + std::chrono::seconds (1));
-	});
+	channel2->set_last_packet_sent (now + std::chrono::seconds (1));
 	ASSERT_EQ (2, node1.network.size ());
 	ASSERT_EQ (2, node1.network.tcp_channels.size ());
 

nano/core_test/random_pool.cpp

@@ -0,0 +1,48 @@
+#include <nano/crypto_lib/random_pool.hpp>
+#include <nano/lib/numbers.hpp>
+
+#include <gtest/gtest.h>
+
+#include <thread>
+
+TEST (random_pool, multithreading)
+{
+	std::vector<std::thread> threads;
+	for (auto i = 0; i < 100; ++i)
+	{
+		threads.emplace_back ([] () {
+			nano::uint256_union number;
+			nano::random_pool::generate_block (number.bytes.data (), number.bytes.size ());
+		});
+	}
+	for (auto & i : threads)
+	{
+		i.join ();
+	}
+}
+
+// Test that random 64bit numbers are within the given range
+TEST (random_pool, generate_word64)
+{
+	int occurrences[10] = { 0 };
+	for (auto i = 0; i < 1000; ++i)
+	{
+		auto random = nano::random_pool::generate_word64 (1, 9);
+		ASSERT_TRUE (random >= 1 && random <= 9);
+		occurrences[random] += 1;
+	}
+
+	for (auto i = 1; i < 10; ++i)
+	{
+		ASSERT_GT (occurrences[i], 0);
+	}
+}
+
+// Test random numbers > uint32 max
+TEST (random_pool, generate_word64_big_number)
+{
+	uint64_t min = static_cast<uint64_t> (std::numeric_limits<uint32_t>::max ()) + 1;
+	uint64_t max = std::numeric_limits<uint64_t>::max ();
+	auto big_random = nano::random_pool::generate_word64 (min, max);
+	ASSERT_GE (big_random, min);
+}

nano/core_test/request_aggregator.cpp

@@ -230,7 +230,7 @@ TEST (request_aggregator, two_endpoints)
 
 	auto dummy_channel1 = std::make_shared<nano::transport::inproc::channel> (node1, node1);
 	auto dummy_channel2 = std::make_shared<nano::transport::inproc::channel> (node2, node2);
-	ASSERT_NE (nano::transport::map_endpoint_to_v6 (dummy_channel1->get_endpoint ()), nano::transport::map_endpoint_to_v6 (dummy_channel2->get_endpoint ()));
+	ASSERT_NE (nano::transport::map_endpoint_to_v6 (dummy_channel1->get_remote_endpoint ()), nano::transport::map_endpoint_to_v6 (dummy_channel2->get_remote_endpoint ()));
 
 	std::vector<std::pair<nano::block_hash, nano::root>> request{ { send1->hash (), send1->root () } };
 

nano/core_test/system.cpp

@@ -95,9 +95,9 @@ TEST (system, DISABLED_generate_send_new)
 	{
 		auto transaction (node1.store.tx_begin_read ());
 		auto iterator1 (node1.store.account.begin (transaction));
-		ASSERT_NE (node1.store.account.end (), iterator1);
+		ASSERT_NE (node1.store.account.end (transaction), iterator1);
 		++iterator1;
-		ASSERT_EQ (node1.store.account.end (), iterator1);
+		ASSERT_EQ (node1.store.account.end (transaction), iterator1);
 	}
 	nano::keypair stake_preserver;
 	auto send_block (system.wallet (0)->send_action (nano::dev::genesis_key.pub, stake_preserver.pub, nano::dev::constants.genesis_amount / 3 * 2, true));
@@ -130,13 +130,13 @@ TEST (system, DISABLED_generate_send_new)
 			new_account = iterator2->first;
 		}
 		++iterator2;
-		ASSERT_NE (system.wallet (0)->store.end (), iterator2);
+		ASSERT_NE (system.wallet (0)->store.end (transaction), iterator2);
 		if (iterator2->first != nano::dev::genesis_key.pub)
 		{
 			new_account = iterator2->first;
 		}
 		++iterator2;
-		ASSERT_EQ (system.wallet (0)->store.end (), iterator2);
+		ASSERT_EQ (system.wallet (0)->store.end (transaction), iterator2);
 		ASSERT_FALSE (new_account.is_zero ());
 	}
 	ASSERT_TIMELY (10s, node1.balance (new_account) != 0);

nano/core_test/telemetry.cpp

@@ -68,18 +68,18 @@ TEST (telemetry, basic)
 	ASSERT_NE (nullptr, channel);
 
 	std::optional<nano::telemetry_data> telemetry_data;
-	ASSERT_TIMELY (5s, telemetry_data = node_client->telemetry.get_telemetry (channel->get_endpoint ()));
+	ASSERT_TIMELY (5s, telemetry_data = node_client->telemetry.get_telemetry (channel->get_remote_endpoint ()));
 	ASSERT_EQ (node_server->get_node_id (), telemetry_data->node_id);
 
 	// Check the metrics are correct
 	ASSERT_TRUE (nano::test::compare_telemetry (*telemetry_data, *node_server));
 
 	// Call again straight away
-	auto telemetry_data_2 = node_client->telemetry.get_telemetry (channel->get_endpoint ());
+	auto telemetry_data_2 = node_client->telemetry.get_telemetry (channel->get_remote_endpoint ());
 	ASSERT_TRUE (telemetry_data_2);
 
 	// Call again straight away
-	auto telemetry_data_3 = node_client->telemetry.get_telemetry (channel->get_endpoint ());
+	auto telemetry_data_3 = node_client->telemetry.get_telemetry (channel->get_remote_endpoint ());
 	ASSERT_TRUE (telemetry_data_3);
 
 	// we expect at least one consecutive repeat of telemetry
@@ -89,7 +89,7 @@ TEST (telemetry, basic)
 	WAIT (3s);
 
 	std::optional<nano::telemetry_data> telemetry_data_4;
-	ASSERT_TIMELY (5s, telemetry_data_4 = node_client->telemetry.get_telemetry (channel->get_endpoint ()));
+	ASSERT_TIMELY (5s, telemetry_data_4 = node_client->telemetry.get_telemetry (channel->get_remote_endpoint ()));
 	ASSERT_NE (*telemetry_data, *telemetry_data_4);
 }
 
@@ -120,13 +120,13 @@ TEST (telemetry, disconnected)
 	ASSERT_NE (nullptr, channel);
 
 	// Ensure telemetry is available before disconnecting
-	ASSERT_TIMELY (5s, node_client->telemetry.get_telemetry (channel->get_endpoint ()));
+	ASSERT_TIMELY (5s, node_client->telemetry.get_telemetry (channel->get_remote_endpoint ()));
 
 	system.stop_node (*node_server);
 	ASSERT_TRUE (channel);
 
 	// Ensure telemetry from disconnected peer is removed
-	ASSERT_TIMELY (5s, !node_client->telemetry.get_telemetry (channel->get_endpoint ()));
+	ASSERT_TIMELY (5s, !node_client->telemetry.get_telemetry (channel->get_remote_endpoint ()));
 }
 
 TEST (telemetry, dos_tcp)
@@ -185,14 +185,14 @@ TEST (telemetry, disable_metrics)
 
 	node_client->telemetry.trigger ();
 
-	ASSERT_NEVER (1s, node_client->telemetry.get_telemetry (channel->get_endpoint ()));
+	ASSERT_NEVER (1s, node_client->telemetry.get_telemetry (channel->get_remote_endpoint ()));
 
 	// It should still be able to receive metrics though
 	auto channel1 = node_server->network.find_node_id (node_client->get_node_id ());
 	ASSERT_NE (nullptr, channel1);
 
 	std::optional<nano::telemetry_data> telemetry_data;
-	ASSERT_TIMELY (5s, telemetry_data = node_server->telemetry.get_telemetry (channel1->get_endpoint ()));
+	ASSERT_TIMELY (5s, telemetry_data = node_server->telemetry.get_telemetry (channel1->get_remote_endpoint ()));
 
 	ASSERT_TRUE (nano::test::compare_telemetry (*telemetry_data, *node_client));
 }
@@ -237,7 +237,7 @@ TEST (telemetry, maker_pruning)
 	ASSERT_NE (nullptr, channel);
 
 	std::optional<nano::telemetry_data> telemetry_data;
-	ASSERT_TIMELY (5s, telemetry_data = node_client->telemetry.get_telemetry (channel->get_endpoint ()));
+	ASSERT_TIMELY (5s, telemetry_data = node_client->telemetry.get_telemetry (channel->get_remote_endpoint ()));
 	ASSERT_EQ (node_server->get_node_id (), telemetry_data->node_id);
 
 	// Ensure telemetry response indicates pruned node
@@ -253,7 +253,7 @@ TEST (telemetry, invalid_signature)
 	telemetry.block_count = 9999; // Change data so signature is no longer valid
 
 	auto message = nano::telemetry_ack{ nano::dev::network_params.network, telemetry };
-	node.network.inbound (message, nano::test::fake_channel (node));
+	node.inbound (message, nano::test::fake_channel (node));
 
 	ASSERT_TIMELY (5s, node.stats.count (nano::stat::type::telemetry, nano::stat::detail::invalid_signature) > 0);
 	ASSERT_ALWAYS (1s, node.stats.count (nano::stat::type::telemetry, nano::stat::detail::process) == 0)
@@ -267,7 +267,7 @@ TEST (telemetry, mismatched_node_id)
 	auto telemetry = node.local_telemetry ();
 
 	auto message = nano::telemetry_ack{ nano::dev::network_params.network, telemetry };
-	node.network.inbound (message, nano::test::fake_channel (node, /* node id */ { 123 }));
+	node.inbound (message, nano::test::fake_channel (node, /* node id */ { 123 }));
 
 	ASSERT_TIMELY (5s, node.stats.count (nano::stat::type::telemetry, nano::stat::detail::node_id_mismatch) > 0);
 	ASSERT_ALWAYS (1s, node.stats.count (nano::stat::type::telemetry, nano::stat::detail::process) == 0)

nano/core_test/thread_pool.cpp

@@ -0,0 +1,96 @@
+#include <nano/lib/thread_pool.hpp>
+#include <nano/lib/timer.hpp>
+#include <nano/test_common/testutil.hpp>
+
+#include <gtest/gtest.h>
+
+#include <future>
+
+TEST (thread_pool, thread_pool)
+{
+	std::atomic<bool> passed_sleep{ false };
+
+	auto func = [&passed_sleep] () {
+		std::this_thread::sleep_for (std::chrono::seconds (1));
+		passed_sleep = true;
+	};
+
+	nano::thread_pool workers (1u, nano::thread_role::name::unknown);
+	nano::test::start_stop_guard stop_guard{ workers };
+	workers.post (func);
+	ASSERT_FALSE (passed_sleep);
+
+	nano::timer<std::chrono::milliseconds> timer_l;
+	timer_l.start ();
+	while (!passed_sleep)
+	{
+		if (timer_l.since_start () > std::chrono::seconds (10))
+		{
+			break;
+		}
+	}
+	ASSERT_TRUE (passed_sleep);
+}
+
+TEST (thread_pool, one)
+{
+	std::atomic<bool> done (false);
+	nano::mutex mutex;
+	nano::condition_variable condition;
+	nano::thread_pool workers (1u, nano::thread_role::name::unknown);
+	nano::test::start_stop_guard stop_guard{ workers };
+	workers.post ([&] () {
+		{
+			nano::lock_guard<nano::mutex> lock{ mutex };
+			done = true;
+		}
+		condition.notify_one ();
+	});
+	nano::unique_lock<nano::mutex> unique{ mutex };
+	condition.wait (unique, [&] () { return !!done; });
+}
+
+TEST (thread_pool, many)
+{
+	std::atomic<int> count (0);
+	nano::mutex mutex;
+	nano::condition_variable condition;
+	nano::thread_pool workers (50u, nano::thread_role::name::unknown);
+	nano::test::start_stop_guard stop_guard{ workers };
+	for (auto i (0); i < 50; ++i)
+	{
+		workers.post ([&] () {
+			{
+				nano::lock_guard<nano::mutex> lock{ mutex };
+				count += 1;
+			}
+			condition.notify_one ();
+		});
+	}
+	nano::unique_lock<nano::mutex> unique{ mutex };
+	condition.wait (unique, [&] () { return count == 50; });
+}
+
+TEST (thread_pool, top_execution)
+{
+	int value1 (0);
+	int value2 (0);
+	nano::mutex mutex;
+	std::promise<bool> promise;
+	nano::thread_pool workers (1u, nano::thread_role::name::unknown);
+	nano::test::start_stop_guard stop_guard{ workers };
+	workers.post ([&] () {
+		nano::lock_guard<nano::mutex> lock{ mutex };
+		value1 = 1;
+		value2 = 1;
+	});
+	workers.post_delayed (std::chrono::milliseconds (1), [&] () {
+		nano::lock_guard<nano::mutex> lock{ mutex };
+		value2 = 2;
+		promise.set_value (false);
+	});
+	promise.get_future ().get ();
+	nano::lock_guard<nano::mutex> lock{ mutex };
+	ASSERT_EQ (1, value1);
+	ASSERT_EQ (2, value2);
+}

nano/core_test/toml.cpp

@@ -191,7 +191,6 @@ TEST (toml, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.unchecked_cutoff_time, defaults.node.unchecked_cutoff_time);
 	ASSERT_EQ (conf.node.use_memory_pools, defaults.node.use_memory_pools);
 	ASSERT_EQ (conf.node.vote_generator_delay, defaults.node.vote_generator_delay);
-	ASSERT_EQ (conf.node.vote_generator_threshold, defaults.node.vote_generator_threshold);
 	ASSERT_EQ (conf.node.vote_minimum, defaults.node.vote_minimum);
 	ASSERT_EQ (conf.node.work_peers, defaults.node.work_peers);
 	ASSERT_EQ (conf.node.work_threads, defaults.node.work_threads);
@@ -456,7 +455,6 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	unchecked_cutoff_time = 999
 	use_memory_pools = false
 	vote_generator_delay = 999
-	vote_generator_threshold = 9
 	vote_minimum = "999"
 	work_peers = ["dev.org:999"]
 	work_threads = 999
@@ -701,7 +699,6 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.unchecked_cutoff_time, defaults.node.unchecked_cutoff_time);
 	ASSERT_NE (conf.node.use_memory_pools, defaults.node.use_memory_pools);
 	ASSERT_NE (conf.node.vote_generator_delay, defaults.node.vote_generator_delay);
-	ASSERT_NE (conf.node.vote_generator_threshold, defaults.node.vote_generator_threshold);
 	ASSERT_NE (conf.node.vote_minimum, defaults.node.vote_minimum);
 	ASSERT_NE (conf.node.work_peers, defaults.node.work_peers);
 	ASSERT_NE (conf.node.work_threads, defaults.node.work_threads);

nano/core_test/utility.cpp

@@ -1,7 +1,6 @@
 #include <nano/lib/optional_ptr.hpp>
 #include <nano/lib/rate_limiting.hpp>
 #include <nano/lib/relaxed_atomic.hpp>
-#include <nano/lib/thread_pool.hpp>
 #include <nano/lib/timer.hpp>
 #include <nano/lib/utility.hpp>
 #include <nano/secure/pending_info.hpp>
@@ -147,91 +146,6 @@ TEST (optional_ptr, basic)
 	ASSERT_EQ (opt->z, 3);
 }
 
-TEST (thread, thread_pool)
-{
-	std::atomic<bool> passed_sleep{ false };
-
-	auto func = [&passed_sleep] () {
-		std::this_thread::sleep_for (std::chrono::seconds (1));
-		passed_sleep = true;
-	};
-
-	nano::thread_pool workers (1u, nano::thread_role::name::unknown);
-	workers.push_task (func);
-	ASSERT_FALSE (passed_sleep);
-
-	nano::timer<std::chrono::milliseconds> timer_l;
-	timer_l.start ();
-	while (!passed_sleep)
-	{
-		if (timer_l.since_start () > std::chrono::seconds (10))
-		{
-			break;
-		}
-	}
-	ASSERT_TRUE (passed_sleep);
-}
-
-TEST (thread_pool_alarm, one)
-{
-	std::atomic<bool> done (false);
-	nano::mutex mutex;
-	nano::condition_variable condition;
-	nano::thread_pool workers (1u, nano::thread_role::name::unknown);
-	workers.add_timed_task (std::chrono::steady_clock::now (), [&] () {
-		{
-			nano::lock_guard<nano::mutex> lock{ mutex };
-			done = true;
-		}
-		condition.notify_one ();
-	});
-	nano::unique_lock<nano::mutex> unique{ mutex };
-	condition.wait (unique, [&] () { return !!done; });
-}
-
-TEST (thread_pool_alarm, many)
-{
-	std::atomic<int> count (0);
-	nano::mutex mutex;
-	nano::condition_variable condition;
-	nano::thread_pool workers (50u, nano::thread_role::name::unknown);
-	for (auto i (0); i < 50; ++i)
-	{
-		workers.add_timed_task (std::chrono::steady_clock::now (), [&] () {
-			{
-				nano::lock_guard<nano::mutex> lock{ mutex };
-				count += 1;
-			}
-			condition.notify_one ();
-		});
-	}
-	nano::unique_lock<nano::mutex> unique{ mutex };
-	condition.wait (unique, [&] () { return count == 50; });
-}
-
-TEST (thread_pool_alarm, top_execution)
-{
-	int value1 (0);
-	int value2 (0);
-	nano::mutex mutex;
-	std::promise<bool> promise;
-	nano::thread_pool workers (1u, nano::thread_role::name::unknown);
-	workers.add_timed_task (std::chrono::steady_clock::now (), [&] () {
-		nano::lock_guard<nano::mutex> lock{ mutex };
-		value1 = 1;
-		value2 = 1;
-	});
-	workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::milliseconds (1), [&] () {
-		nano::lock_guard<nano::mutex> lock{ mutex };
-		value2 = 2;
-		promise.set_value (false);
-	});
-	promise.get_future ().get ();
-	nano::lock_guard<nano::mutex> lock{ mutex };
-	ASSERT_EQ (1, value1);
-	ASSERT_EQ (2, value2);
-}
-
 TEST (filesystem, remove_all_files)
 {
 	auto path = nano::unique_path ();

nano/core_test/wallet.cpp

@@ -104,7 +104,7 @@ TEST (wallet, empty_iteration)
 	nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis_key.pub, 1, "0");
 	ASSERT_FALSE (init);
 	auto i (wallet.begin (transaction));
-	auto j (wallet.end ());
+	auto j (wallet.end (transaction));
 	ASSERT_EQ (i, j);
 }
 
@@ -119,7 +119,7 @@ TEST (wallet, one_item_iteration)
 	ASSERT_FALSE (init);
 	nano::keypair key1;
 	wallet.insert_adhoc (transaction, key1.prv);
-	for (auto i (wallet.begin (transaction)), j (wallet.end ()); i != j; ++i)
+	for (auto i (wallet.begin (transaction)), j (wallet.end (transaction)); i != j; ++i)
 	{
 		ASSERT_EQ (key1.pub, nano::uint256_union (i->first));
 		nano::raw_key password;
@@ -147,7 +147,7 @@ TEST (wallet, two_item_iteration)
 		ASSERT_FALSE (init);
 		wallet.insert_adhoc (transaction, key1.prv);
 		wallet.insert_adhoc (transaction, key2.prv);
-		for (auto i (wallet.begin (transaction)), j (wallet.end ()); i != j; ++i)
+		for (auto i (wallet.begin (transaction)), j (wallet.end (transaction)); i != j; ++i)
 		{
 			pubs.insert (i->first);
 			nano::raw_key password;
@@ -266,7 +266,7 @@ TEST (wallet, find_none)
 	nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis_key.pub, 1, "0");
 	ASSERT_FALSE (init);
 	nano::account account (1000);
-	ASSERT_EQ (wallet.end (), wallet.find (transaction, account));
+	ASSERT_EQ (wallet.end (transaction), wallet.find (transaction, account));
 }
 
 TEST (wallet, find_existing)
@@ -283,9 +283,9 @@ TEST (wallet, find_existing)
 	wallet.insert_adhoc (transaction, key1.prv);
 	ASSERT_TRUE (wallet.exists (transaction, key1.pub));
 	auto existing (wallet.find (transaction, key1.pub));
-	ASSERT_NE (wallet.end (), existing);
+	ASSERT_NE (wallet.end (transaction), existing);
 	++existing;
-	ASSERT_EQ (wallet.end (), existing);
+	ASSERT_EQ (wallet.end (transaction), existing);
 }
 
 TEST (wallet, rekey)
@@ -487,8 +487,8 @@ TEST (wallet, serialize_json_empty)
 	ASSERT_EQ (wallet1.salt (transaction), wallet2.salt (transaction));
 	ASSERT_EQ (wallet1.check (transaction), wallet2.check (transaction));
 	ASSERT_EQ (wallet1.representative (transaction), wallet2.representative (transaction));
-	ASSERT_EQ (wallet1.end (), wallet1.begin (transaction));
-	ASSERT_EQ (wallet2.end (), wallet2.begin (transaction));
+	ASSERT_EQ (wallet1.end (transaction), wallet1.begin (transaction));
+	ASSERT_EQ (wallet2.end (transaction), wallet2.begin (transaction));
 }
 
 TEST (wallet, serialize_json_one)

nano/core_test/websocket.cpp

@@ -12,6 +12,7 @@
 
 #include <gtest/gtest.h>
 
+#include <boost/format.hpp>
 #include <boost/property_tree/json_parser.hpp>
 
 #include <chrono>
@@ -165,7 +166,7 @@ TEST (websocket, started_election)
 				 .build ();
 	nano::publish publish1{ nano::dev::network_params.network, send1 };
 	auto channel1 = std::make_shared<nano::transport::fake::channel> (*node1);
-	node1->network.inbound (publish1, channel1);
+	node1->inbound (publish1, channel1);
 	ASSERT_TIMELY (1s, node1->active.election (send1->qualified_root ()));
 	ASSERT_TIMELY_EQ (5s, future.wait_for (0s), std::future_status::ready);
 
@@ -213,7 +214,7 @@ TEST (websocket, stopped_election)
 				 .build ();
 	nano::publish publish1{ nano::dev::network_params.network, send1 };
 	auto channel1 = std::make_shared<nano::transport::fake::channel> (*node1);
-	node1->network.inbound (publish1, channel1);
+	node1->inbound (publish1, channel1);
 	ASSERT_TIMELY (5s, node1->active.election (send1->qualified_root ()));
 	node1->active.erase (*send1);
 
@@ -1005,7 +1006,7 @@ TEST (websocket, telemetry)
 
 	auto channel = node1->network.find_node_id (node2->get_node_id ());
 	ASSERT_NE (channel, nullptr);
-	ASSERT_TIMELY (5s, node1->telemetry.get_telemetry (channel->get_endpoint ()));
+	ASSERT_TIMELY (5s, node1->telemetry.get_telemetry (channel->get_remote_endpoint ()));
 
 	ASSERT_TIMELY_EQ (10s, future.wait_for (0s), std::future_status::ready);
 

nano/lib/CMakeLists.txt

@@ -94,7 +94,6 @@ add_library(
   stats_sinks.hpp
   stream.hpp
   thread_pool.hpp
-  thread_pool.cpp
   thread_roles.hpp
   thread_roles.cpp
   thread_runner.hpp

nano/lib/blocks.cpp

@@ -609,7 +609,7 @@ std::optional<nano::account> nano::send_block::destination_field () const
 	return hashables.destination;
 }
 
-nano::root const & nano::send_block::root () const
+nano::root nano::send_block::root () const
 {
 	return hashables.previous;
 }
@@ -899,7 +899,7 @@ std::optional<nano::block_hash> nano::open_block::source_field () const
 	return hashables.source;
 }
 
-nano::root const & nano::open_block::root () const
+nano::root nano::open_block::root () const
 {
 	return hashables.account;
 }
@@ -1165,7 +1165,7 @@ bool nano::change_block::valid_predecessor (nano::block const & block_a) const
 	return result;
 }
 
-nano::root const & nano::change_block::root () const
+nano::root nano::change_block::root () const
 {
 	return hashables.previous;
 }
@@ -1482,7 +1482,7 @@ bool nano::state_block::valid_predecessor (nano::block const & block_a) const
 	return true;
 }
 
-nano::root const & nano::state_block::root () const
+nano::root nano::state_block::root () const
 {
 	if (!hashables.previous.is_zero ())
 	{
@@ -1836,7 +1836,7 @@ std::optional<nano::block_hash> nano::receive_block::source_field () const
 	return hashables.source;
 }
 
-nano::root const & nano::receive_block::root () const
+nano::root nano::receive_block::root () const
 {
 	return hashables.previous;
 }

nano/lib/blocks.hpp

@@ -33,7 +33,7 @@ public:
 	virtual uint64_t block_work () const = 0;
 	virtual void block_work_set (uint64_t) = 0;
 	// Previous block or account number for open blocks
-	virtual nano::root const & root () const = 0;
+	virtual nano::root root () const = 0;
 	// Qualified root value based on previous() and root()
 	virtual nano::qualified_root qualified_root () const;
 	virtual void serialize (nano::stream &) const = 0;
@@ -123,7 +123,7 @@ public:
 	virtual ~send_block () = default;
 	uint64_t block_work () const override;
 	void block_work_set (uint64_t) override;
-	nano::root const & root () const override;
+	nano::root root () const override;
 	void serialize (nano::stream &) const override;
 	bool deserialize (nano::stream &);
 	void serialize_json (std::string &, bool = false) const override;
@@ -177,7 +177,7 @@ public:
 	virtual ~receive_block () = default;
 	uint64_t block_work () const override;
 	void block_work_set (uint64_t) override;
-	nano::root const & root () const override;
+	nano::root root () const override;
 	void serialize (nano::stream &) const override;
 	bool deserialize (nano::stream &);
 	void serialize_json (std::string &, bool = false) const override;
@@ -232,7 +232,7 @@ public:
 	virtual ~open_block () = default;
 	uint64_t block_work () const override;
 	void block_work_set (uint64_t) override;
-	nano::root const & root () const override;
+	nano::root root () const override;
 	void serialize (nano::stream &) const override;
 	bool deserialize (nano::stream &);
 	void serialize_json (std::string &, bool = false) const override;
@@ -287,7 +287,7 @@ public:
 	virtual ~change_block () = default;
 	uint64_t block_work () const override;
 	void block_work_set (uint64_t) override;
-	nano::root const & root () const override;
+	nano::root root () const override;
 	void serialize (nano::stream &) const override;
 	bool deserialize (nano::stream &);
 	void serialize_json (std::string &, bool = false) const override;
@@ -353,7 +353,7 @@ public:
 	virtual ~state_block () = default;
 	uint64_t block_work () const override;
 	void block_work_set (uint64_t) override;
-	nano::root const & root () const override;
+	nano::root root () const override;
 	void serialize (nano::stream &) const override;
 	bool deserialize (nano::stream &);
 	void serialize_json (std::string &, bool = false) const override;

nano/lib/config.cpp

@@ -6,6 +6,7 @@
 
 #include <boost/format.hpp>
 #include <boost/lexical_cast.hpp>
+#include <boost/numeric/conversion/cast.hpp>
 
 #include <valgrind/valgrind.h>
 

nano/lib/logging_enums.hpp

@@ -82,6 +82,7 @@ enum class type
 	message_processor,
 	local_block_broadcaster,
 	monitor,
+	confirming_set,
 
 	// bootstrap
 	bulk_pull_client,

nano/lib/numbers.cpp

@@ -61,11 +61,6 @@ std::string nano::public_key::to_account () const
 	return result;
 }
 
-nano::public_key::public_key () :
-	uint256_union{ 0 }
-{
-}
-
 nano::public_key const & nano::public_key::null ()
 {
 	return nano::hardened_constants::get ().not_an_account;
@@ -143,12 +138,6 @@ bool nano::public_key::decode_account (std::string const & source_a)
 	return error;
 }
 
-nano::uint256_union::uint256_union (nano::uint256_t const & number_a)
-{
-	bytes.fill (0);
-	boost::multiprecision::export_bits (number_a, bytes.rbegin (), 8, false);
-}
-
 // Construct a uint256_union = AES_ENC_CTR (cleartext, key, iv)
 void nano::uint256_union::encrypt (nano::raw_key const & cleartext, nano::raw_key const & key, uint128_union const & iv)
 {
@@ -157,11 +146,6 @@ void nano::uint256_union::encrypt (nano::raw_key const & cleartext, nano::raw_ke
 	enc.ProcessData (bytes.data (), cleartext.bytes.data (), sizeof (cleartext.bytes));
 }
 
-bool nano::uint256_union::is_zero () const
-{
-	return qwords[0] == 0 && qwords[1] == 0 && qwords[2] == 0 && qwords[3] == 0;
-}
-
 std::string nano::uint256_union::to_string () const
 {
 	std::string result;
@@ -193,22 +177,9 @@ nano::uint256_union nano::uint256_union::operator^ (nano::uint256_union const &
 nano::uint256_union::uint256_union (std::string const & hex_a)
 {
 	auto error (decode_hex (hex_a));
-
 	release_assert (!error);
 }
 
-void nano::uint256_union::clear ()
-{
-	qwords.fill (0);
-}
-
-nano::uint256_t nano::uint256_union::number () const
-{
-	nano::uint256_t result;
-	boost::multiprecision::import_bits (result, bytes.begin (), bytes.end ());
-	return result;
-}
-
 void nano::uint256_union::encode_hex (std::string & text) const
 {
 	debug_assert (text.empty ());
@@ -281,46 +252,6 @@ bool nano::uint256_union::decode_dec (std::string const & text)
 	return error;
 }
 
-nano::uint256_union::uint256_union (uint64_t value0)
-{
-	*this = nano::uint256_t (value0);
-}
-
-bool nano::uint512_union::operator== (nano::uint512_union const & other_a) const
-{
-	return bytes == other_a.bytes;
-}
-
-nano::uint512_union::uint512_union (nano::uint256_union const & upper, nano::uint256_union const & lower)
-{
-	uint256s[0] = upper;
-	uint256s[1] = lower;
-}
-
-nano::uint512_union::uint512_union (nano::uint512_t const & number_a)
-{
-	bytes.fill (0);
-	boost::multiprecision::export_bits (number_a, bytes.rbegin (), 8, false);
-}
-
-bool nano::uint512_union::is_zero () const
-{
-	return qwords[0] == 0 && qwords[1] == 0 && qwords[2] == 0 && qwords[3] == 0
-	&& qwords[4] == 0 && qwords[5] == 0 && qwords[6] == 0 && qwords[7] == 0;
-}
-
-void nano::uint512_union::clear ()
-{
-	bytes.fill (0);
-}
-
-nano::uint512_t nano::uint512_union::number () const
-{
-	nano::uint512_t result;
-	boost::multiprecision::import_bits (result, bytes.begin (), bytes.end ());
-	return result;
-}
-
 void nano::uint512_union::encode_hex (std::string & text) const
 {
 	debug_assert (text.empty ());
@@ -355,18 +286,6 @@ bool nano::uint512_union::decode_hex (std::string const & text)
 	return error;
 }
 
-bool nano::uint512_union::operator!= (nano::uint512_union const & other_a) const
-{
-	return !(*this == other_a);
-}
-
-nano::uint512_union & nano::uint512_union::operator^= (nano::uint512_union const & other_a)
-{
-	uint256s[0] ^= other_a.uint256s[0];
-	uint256s[1] ^= other_a.uint256s[1];
-	return *this;
-}
-
 std::string nano::uint512_union::to_string () const
 {
 	std::string result;
@@ -431,48 +350,9 @@ bool nano::validate_message (nano::public_key const & public_key, nano::uint256_
 nano::uint128_union::uint128_union (std::string const & string_a)
 {
 	auto error (decode_hex (string_a));
-
 	release_assert (!error);
 }
 
-nano::uint128_union::uint128_union (uint64_t value_a)
-{
-	*this = nano::uint128_t (value_a);
-}
-
-nano::uint128_union::uint128_union (nano::uint128_t const & number_a)
-{
-	bytes.fill (0);
-	boost::multiprecision::export_bits (number_a, bytes.rbegin (), 8, false);
-}
-
-bool nano::uint128_union::operator== (nano::uint128_union const & other_a) const
-{
-	return qwords[0] == other_a.qwords[0] && qwords[1] == other_a.qwords[1];
-}
-
-bool nano::uint128_union::operator!= (nano::uint128_union const & other_a) const
-{
-	return !(*this == other_a);
-}
-
-bool nano::uint128_union::operator< (nano::uint128_union const & other_a) const
-{
-	return std::memcmp (bytes.data (), other_a.bytes.data (), 16) < 0;
-}
-
-bool nano::uint128_union::operator> (nano::uint128_union const & other_a) const
-{
-	return std::memcmp (bytes.data (), other_a.bytes.data (), 16) > 0;
-}
-
-nano::uint128_t nano::uint128_union::number () const
-{
-	nano::uint128_t result;
-	boost::multiprecision::import_bits (result, bytes.begin (), bytes.end ());
-	return result;
-}
-
 void nano::uint128_union::encode_hex (std::string & text) const
 {
 	debug_assert (text.empty ());
@@ -747,16 +627,6 @@ std::string nano::uint128_union::format_balance (nano::uint128_t scale, int prec
 	return ::format_balance (number (), scale, precision, group_digits, thousands_sep, decimal_point, grouping);
 }
 
-void nano::uint128_union::clear ()
-{
-	qwords.fill (0);
-}
-
-bool nano::uint128_union::is_zero () const
-{
-	return qwords[0] == 0 && qwords[1] == 0;
-}
-
 std::string nano::uint128_union::to_string () const
 {
 	std::string result;
@@ -771,26 +641,6 @@ std::string nano::uint128_union::to_string_dec () const
 	return result;
 }
 
-nano::hash_or_account::hash_or_account () :
-	account{}
-{
-}
-
-nano::hash_or_account::hash_or_account (uint64_t value_a) :
-	raw (value_a)
-{
-}
-
-bool nano::hash_or_account::is_zero () const
-{
-	return raw.is_zero ();
-}
-
-void nano::hash_or_account::clear ()
-{
-	raw.clear ();
-}
-
 bool nano::hash_or_account::decode_hex (std::string const & text_a)
 {
 	return raw.decode_hex (text_a);
@@ -811,36 +661,6 @@ std::string nano::hash_or_account::to_account () const
 	return account.to_account ();
 }
 
-nano::block_hash const & nano::hash_or_account::as_block_hash () const
-{
-	return hash;
-}
-
-nano::account const & nano::hash_or_account::as_account () const
-{
-	return account;
-}
-
-nano::hash_or_account::operator nano::uint256_union const & () const
-{
-	return raw;
-}
-
-nano::block_hash const & nano::root::previous () const
-{
-	return hash;
-}
-
-bool nano::hash_or_account::operator== (nano::hash_or_account const & hash_or_account_a) const
-{
-	return bytes == hash_or_account_a.bytes;
-}
-
-bool nano::hash_or_account::operator!= (nano::hash_or_account const & hash_or_account_a) const
-{
-	return !(*this == hash_or_account_a);
-}
-
 std::string nano::to_string_hex (uint64_t const value_a)
 {
 	std::stringstream stream;
@@ -915,6 +735,13 @@ std::ostream & nano::operator<< (std::ostream & os, const uint512_union & val)
 	return os;
 }
 
+std::ostream & nano::operator<< (std::ostream & os, const hash_or_account & val)
+{
+	// TODO: Replace with streaming implementation
+	os << val.to_string ();
+	return os;
+}
+
 #ifdef _WIN32
 #pragma warning(push)
 #pragma warning(disable : 4146) // warning C4146: unary minus operator applied to unsigned type, result still unsigned
@@ -947,43 +774,3 @@ double nano::difficulty::to_multiplier (uint64_t const difficulty_a, uint64_t co
 #ifdef _WIN32
 #pragma warning(pop)
 #endif
-
-nano::public_key::operator nano::link const & () const
-{
-	return reinterpret_cast<nano::link const &> (*this);
-}
-
-nano::public_key::operator nano::root const & () const
-{
-	return reinterpret_cast<nano::root const &> (*this);
-}
-
-nano::public_key::operator nano::hash_or_account const & () const
-{
-	return reinterpret_cast<nano::hash_or_account const &> (*this);
-}
-
-bool nano::public_key::operator== (std::nullptr_t) const
-{
-	return bytes == null ().bytes;
-}
-
-bool nano::public_key::operator!= (std::nullptr_t) const
-{
-	return !(*this == nullptr);
-}
-
-nano::block_hash::operator nano::link const & () const
-{
-	return reinterpret_cast<nano::link const &> (*this);
-}
-
-nano::block_hash::operator nano::root const & () const
-{
-	return reinterpret_cast<nano::root const &> (*this);
-}
-
-nano::block_hash::operator nano::hash_or_account const & () const
-{
-	return reinterpret_cast<nano::hash_or_account const &> (*this);
-}

nano/lib/numbers.hpp

@@ -4,6 +4,8 @@
 #include <boost/multiprecision/cpp_int.hpp>
 
 #include <array>
+#include <compare>
+#include <limits>
 #include <ostream>
 
 #include <fmt/ostream.h>
@@ -13,6 +15,7 @@ namespace nano
 using uint128_t = boost::multiprecision::uint128_t;
 using uint256_t = boost::multiprecision::uint256_t;
 using uint512_t = boost::multiprecision::uint512_t;
+
 // SI dividers
 nano::uint128_t const Knano_ratio = nano::uint128_t ("1000000000000000000000000000000000"); // 10^33 = 1000 nano
 nano::uint128_t const nano_ratio = nano::uint128_t ("1000000000000000000000000000000"); // 10^30 = 1 nano
@@ -20,31 +23,55 @@ nano::uint128_t const raw_ratio = nano::uint128_t ("1"); // 10^0
 
 class uint128_union
 {
+public:
+	// Type that is implicitly convertible to this union
+	using underlying_type = nano::uint128_t;
+
 public:
 	uint128_union () = default;
+	uint128_union (uint64_t value) :
+		uint128_union (nano::uint128_t{ value }){};
+	uint128_union (nano::uint128_t const & value)
+	{
+		bytes.fill (0);
+		boost::multiprecision::export_bits (value, bytes.rbegin (), 8, false);
+	}
+
 	/**
 	 * Decode from hex string
 	 * @warning Aborts at runtime if the input is invalid
 	 */
-	uint128_union (std::string const &);
-	uint128_union (uint64_t);
-	uint128_union (nano::uint128_t const &);
-	bool operator== (nano::uint128_union const &) const;
-	bool operator!= (nano::uint128_union const &) const;
-	bool operator< (nano::uint128_union const &) const;
-	bool operator> (nano::uint128_union const &) const;
+	explicit uint128_union (std::string const &);
+
 	void encode_hex (std::string &) const;
 	bool decode_hex (std::string const &);
 	void encode_dec (std::string &) const;
 	bool decode_dec (std::string const &, bool = false);
 	bool decode_dec (std::string const &, nano::uint128_t);
+
 	std::string format_balance (nano::uint128_t scale, int precision, bool group_digits) const;
 	std::string format_balance (nano::uint128_t scale, int precision, bool group_digits, std::locale const & locale) const;
-	nano::uint128_t number () const;
-	void clear ();
-	bool is_zero () const;
+
+	void clear ()
+	{
+		qwords.fill (0);
+	}
+	bool is_zero () const
+	{
+		return qwords[0] == 0 && qwords[1] == 0;
+	}
+
+	nano::uint128_t number () const
+	{
+		nano::uint128_t result;
+		boost::multiprecision::import_bits (result, bytes.begin (), bytes.end ());
+		return result;
+	}
+
 	std::string to_string () const;
 	std::string to_string_dec () const;
+
+public:
 	union
 	{
 		std::array<uint8_t, 16> bytes;
@@ -52,6 +79,24 @@ public:
 		std::array<uint32_t, 4> dwords;
 		std::array<uint64_t, 2> qwords;
 	};
+
+public: // Keep operators inlined
+	std::strong_ordering operator<=> (nano::uint128_union const & other) const
+	{
+		return std::memcmp (bytes.data (), other.bytes.data (), 16) <=> 0;
+	};
+	bool operator== (nano::uint128_union const & other) const
+	{
+		return *this <=> other == 0;
+	}
+	operator nano::uint128_t () const
+	{
+		return number ();
+	}
+	uint128_union const & as_union () const
+	{
+		return *this;
+	}
 };
 static_assert (std::is_nothrow_move_constructible<uint128_union>::value, "uint128_union should be noexcept MoveConstructible");
 
@@ -61,37 +106,69 @@ class amount : public uint128_union
 public:
 	using uint128_union::uint128_union;
 
+	auto operator<=> (nano::amount const & other) const
+	{
+		return uint128_union::operator<=> (other);
+	}
 	operator nano::uint128_t () const
 	{
 		return number ();
 	}
 };
+
 class raw_key;
 
 class uint256_union
 {
+public:
+	// Type that is implicitly convertible to this union
+	using underlying_type = nano::uint256_t;
+
 public:
 	uint256_union () = default;
+	uint256_union (uint64_t value) :
+		uint256_union (nano::uint256_t{ value }){};
+	uint256_union (nano::uint256_t const & value)
+	{
+		bytes.fill (0);
+		boost::multiprecision::export_bits (value, bytes.rbegin (), 8, false);
+	}
+
 	/**
 	 * Decode from hex string
 	 * @warning Aborts at runtime if the input is invalid
 	 */
 	explicit uint256_union (std::string const &);
-	uint256_union (uint64_t);
-	uint256_union (nano::uint256_t const &);
+
 	void encrypt (nano::raw_key const &, nano::raw_key const &, uint128_union const &);
-	uint256_union & operator^= (nano::uint256_union const &);
-	uint256_union operator^ (nano::uint256_union const &) const;
+
+	uint256_union & operator^= (uint256_union const &);
+	uint256_union operator^ (uint256_union const &) const;
+
 	void encode_hex (std::string &) const;
 	bool decode_hex (std::string const &);
 	void encode_dec (std::string &) const;
 	bool decode_dec (std::string const &);
 
-	void clear ();
-	bool is_zero () const;
-	std::string to_string () const;
-	nano::uint256_t number () const;
+	void clear ()
+	{
+		qwords.fill (0);
+	}
+	bool is_zero () const
+	{
+		return owords[0].is_zero () && owords[1].is_zero ();
+	}
 
+	nano::uint256_t number () const
+	{
+		nano::uint256_t result;
+		boost::multiprecision::import_bits (result, bytes.begin (), bytes.end ());
+		return result;
+	}
+
+	std::string to_string () const;
+
+public:
 	union
 	{
 		std::array<uint8_t, 32> bytes;
@@ -100,33 +177,46 @@ public:
 		std::array<uint64_t, 4> qwords;
 		std::array<uint128_union, 2> owords;
 	};
-};
-inline bool operator== (nano::uint256_union const & lhs, nano::uint256_union const & rhs)
-{
-	return lhs.bytes == rhs.bytes;
-}
-inline bool operator!= (nano::uint256_union const & lhs, nano::uint256_union const & rhs)
-{
-	return !(lhs == rhs);
-}
-inline bool operator< (nano::uint256_union const & lhs, nano::uint256_union const & rhs)
-{
-	return std::memcmp (lhs.bytes.data (), rhs.bytes.data (), 32) < 0;
-}
-static_assert (std::is_nothrow_move_constructible<uint256_union>::value, "uint256_union should be noexcept MoveConstructible");
 
-class link;
-class root;
-class hash_or_account;
+public: // Keep operators inlined
+	std::strong_ordering operator<=> (nano::uint256_union const & other) const
+	{
+		return std::memcmp (bytes.data (), other.bytes.data (), 32) <=> 0;
+	};
+	bool operator== (nano::uint256_union const & other) const
+	{
+		return *this <=> other == 0;
+	}
+	operator nano::uint256_t () const
+	{
+		return number ();
+	}
+	uint256_union const & as_union () const
+	{
+		return *this;
+	}
+};
+static_assert (std::is_nothrow_move_constructible<uint256_union>::value, "uint256_union should be noexcept MoveConstructible");
 
 // All keys and hashes are 256 bit.
 class block_hash final : public uint256_union
 {
 public:
 	using uint256_union::uint256_union;
-	operator nano::link const & () const;
-	operator nano::root const & () const;
-	operator nano::hash_or_account const & () const;
+
+public: // Keep operators inlined
+	auto operator<=> (nano::block_hash const & other) const
+	{
+		return uint256_union::operator<=> (other);
+	}
+	bool operator== (nano::block_hash const & other) const
+	{
+		return *this <=> other == 0;
+	}
+	operator nano::uint256_t () const
+	{
+		return number ();
+	}
 };
 
 class public_key final : public uint256_union
@@ -134,21 +224,35 @@ class public_key final : public uint256_union
 public:
 	using uint256_union::uint256_union;
 
-	public_key ();
+	public_key () :
+		uint256_union{ 0 } {};
 
 	static const public_key & null ();
 
-	std::string to_node_id () const;
-	bool decode_node_id (std::string const & source_a);
+	bool decode_node_id (std::string const &);
 	void encode_account (std::string &) const;
-	std::string to_account () const;
 	bool decode_account (std::string const &);
 
-	operator nano::link const & () const;
-	operator nano::root const & () const;
-	operator nano::hash_or_account const & () const;
-	bool operator== (std::nullptr_t) const;
-	bool operator!= (std::nullptr_t) const;
+	std::string to_node_id () const;
+	std::string to_account () const;
+
+public: // Keep operators inlined
+	auto operator<=> (nano::public_key const & other) const
+	{
+		return uint256_union::operator<=> (other);
+	}
+	bool operator== (nano::public_key const & other) const
+	{
+		return *this <=> other == 0;
+	}
+	bool operator== (std::nullptr_t) const
+	{
+		return *this == null ();
+	}
+	operator nano::uint256_t () const
+	{
+		return number ();
+	}
 };
 
 class wallet_id : public uint256_union
@@ -162,24 +266,33 @@ using account = public_key;
 class hash_or_account
 {
 public:
-	hash_or_account ();
-	hash_or_account (uint64_t value_a);
+	// Type that is implicitly convertible to this union
+	using underlying_type = nano::uint256_t;
+
+public:
+	hash_or_account () :
+		account{} {};
+	hash_or_account (uint64_t value) :
+		raw{ value } {};
+	hash_or_account (uint256_union const & value) :
+		raw{ value } {};
+
+	void clear ()
+	{
+		raw.clear ();
+	}
+	bool is_zero () const
+	{
+		return raw.is_zero ();
+	}
 
-	bool is_zero () const;
-	void clear ();
-	std::string to_string () const;
 	bool decode_hex (std::string const &);
 	bool decode_account (std::string const &);
+
+	std::string to_string () const;
 	std::string to_account () const;
 
-	nano::account const & as_account () const;
-	nano::block_hash const & as_block_hash () const;
-
-	operator nano::uint256_union const & () const;
-
-	bool operator== (nano::hash_or_account const &) const;
-	bool operator!= (nano::hash_or_account const &) const;
-
+public:
 	union
 	{
 		std::array<uint8_t, 32> bytes;
@@ -187,6 +300,36 @@ public:
 		nano::account account;
 		nano::block_hash hash;
 	};
+
+public: // Keep operators inlined
+	auto operator<=> (nano::hash_or_account const & other) const
+	{
+		return raw <=> other.raw;
+	};
+	bool operator== (nano::hash_or_account const & other) const
+	{
+		return *this <=> other == 0;
+	}
+	explicit operator nano::uint256_t () const
+	{
+		return raw.number ();
+	}
+	explicit operator nano::uint256_union () const
+	{
+		return raw;
+	}
+	nano::account const & as_account () const
+	{
+		return account;
+	}
+	nano::block_hash const & as_block_hash () const
+	{
+		return hash;
+	}
+	nano::uint256_union const & as_union () const
+	{
+		return raw;
+	}
 };
 
 // A link can either be a destination account or source hash
@@ -194,6 +337,16 @@ class link final : public hash_or_account
 {
 public:
 	using hash_or_account::hash_or_account;
+
+public: // Keep operators inlined
+	auto operator<=> (nano::link const & other) const
+	{
+		return hash_or_account::operator<=> (other);
+	}
+	bool operator== (nano::link const & other) const
+	{
+		return *this <=> other == 0;
+	}
 };
 
 // A root can either be an open block hash or a previous hash
@@ -202,7 +355,20 @@ class root final : public hash_or_account
 public:
 	using hash_or_account::hash_or_account;
 
-	nano::block_hash const & previous () const;
+	nano::block_hash const & previous () const
+	{
+		return hash;
+	}
+
+public: // Keep operators inlined
+	auto operator<=> (nano::root const & other) const
+	{
+		return hash_or_account::operator<=> (other);
+	}
+	bool operator== (nano::root const & other) const
+	{
+		return *this <=> other == 0;
+	}
 };
 
 // The seed or private key
@@ -213,22 +379,52 @@ public:
 	~raw_key ();
 	void decrypt (nano::uint256_union const &, nano::raw_key const &, uint128_union const &);
 };
+
 class uint512_union
 {
+public:
+	// Type that is implicitly convertible to this union
+	using underlying_type = nano::uint512_t;
+
 public:
 	uint512_union () = default;
-	uint512_union (nano::uint256_union const &, nano::uint256_union const &);
-	uint512_union (nano::uint512_t const &);
-	bool operator== (nano::uint512_union const &) const;
-	bool operator!= (nano::uint512_union const &) const;
-	nano::uint512_union & operator^= (nano::uint512_union const &);
+	uint512_union (nano::uint512_t const & value)
+	{
+		bytes.fill (0);
+		boost::multiprecision::export_bits (value, bytes.rbegin (), 8, false);
+	}
+	uint512_union (nano::uint256_union const & upper, nano::uint256_union const & lower) :
+		uint256s{ upper, lower } {};
+
+	nano::uint512_union & operator^= (nano::uint512_union const & other)
+	{
+		uint256s[0] ^= other.uint256s[0];
+		uint256s[1] ^= other.uint256s[1];
+		return *this;
+	}
+
 	void encode_hex (std::string &) const;
 	bool decode_hex (std::string const &);
-	void clear ();
-	bool is_zero () const;
-	nano::uint512_t number () const;
+
+	void clear ()
+	{
+		bytes.fill (0);
+	}
+	bool is_zero () const
+	{
+		return uint256s[0].is_zero () && uint256s[1].is_zero ();
+	}
+
+	nano::uint512_t number () const
+	{
+		nano::uint512_t result;
+		boost::multiprecision::import_bits (result, bytes.begin (), bytes.end ());
+		return result;
+	}
+
 	std::string to_string () const;
 
+public:
 	union
 	{
 		std::array<uint8_t, 64> bytes;
@@ -236,6 +432,24 @@ public:
 		std::array<uint64_t, 8> qwords;
 		std::array<uint256_union, 2> uint256s;
 	};
+
+public: // Keep operators inlined
+	std::strong_ordering operator<=> (nano::uint512_union const & other) const
+	{
+		return std::memcmp (bytes.data (), other.bytes.data (), 64) <=> 0;
+	};
+	bool operator== (nano::uint512_union const & other) const
+	{
+		return *this <=> other == 0;
+	}
+	operator nano::uint512_t () const
+	{
+		return number ();
+	}
+	uint512_union const & as_union () const
+	{
+		return *this;
+	}
 };
 static_assert (std::is_nothrow_move_constructible<uint512_union>::value, "uint512_union should be noexcept MoveConstructible");
 
@@ -248,15 +462,19 @@ public:
 class qualified_root : public uint512_union
 {
 public:
-	using uint512_union::uint512_union;
+	qualified_root () = default;
+	qualified_root (nano::root const & root, nano::block_hash const & previous) :
+		uint512_union{ root.as_union (), previous.as_union () } {};
+	qualified_root (nano::uint512_t const & value) :
+		uint512_union{ value } {};
 
-	nano::root const & root () const
+	nano::root root () const
 	{
-		return reinterpret_cast<nano::root const &> (uint256s[0]);
+		return nano::root{ uint256s[0] };
 	}
-	nano::block_hash const & previous () const
+	nano::block_hash previous () const
 	{
-		return reinterpret_cast<nano::block_hash const &> (uint256s[1]);
+		return nano::block_hash{ uint256s[1] };
 	}
 };
 
@@ -276,6 +494,7 @@ bool from_string_hex (std::string const &, uint64_t &);
 std::ostream & operator<< (std::ostream &, const uint128_union &);
 std::ostream & operator<< (std::ostream &, const uint256_union &);
 std::ostream & operator<< (std::ostream &, const uint512_union &);
+std::ostream & operator<< (std::ostream &, const hash_or_account &);
 
 /**
  * Convert a double to string in fixed format
@@ -297,92 +516,91 @@ namespace difficulty
 namespace std
 {
 template <>
-struct hash<::nano::uint256_union>
+struct hash<::nano::uint128_union>
 {
-	size_t operator() (::nano::uint256_union const & data_a) const
+	size_t operator() (::nano::uint128_union const & value) const noexcept
 	{
-		return data_a.qwords[0] + data_a.qwords[1] + data_a.qwords[2] + data_a.qwords[3];
+		return value.qwords[0] + value.qwords[1];
 	}
 };
 template <>
-struct hash<::nano::account>
+struct hash<::nano::uint256_union>
 {
-	size_t operator() (::nano::account const & data_a) const
+	size_t operator() (::nano::uint256_union const & value) const noexcept
 	{
-		return hash<::nano::uint256_union> () (data_a);
+		return value.qwords[0] + value.qwords[1] + value.qwords[2] + value.qwords[3];
+	}
+};
+template <>
+struct hash<::nano::public_key>
+{
+	size_t operator() (::nano::public_key const & value) const noexcept
+	{
+		return hash<::nano::uint256_union>{}(value);
 	}
 };
 template <>
 struct hash<::nano::block_hash>
 {
-	size_t operator() (::nano::block_hash const & data_a) const
+	size_t operator() (::nano::block_hash const & value) const noexcept
 	{
-		return hash<::nano::uint256_union> () (data_a);
+		return hash<::nano::uint256_union>{}(value);
 	}
 };
 template <>
 struct hash<::nano::hash_or_account>
 {
-	size_t operator() (::nano::hash_or_account const & data_a) const
+	size_t operator() (::nano::hash_or_account const & value) const noexcept
 	{
-		return hash<::nano::block_hash> () (data_a.as_block_hash ());
-	}
-};
-template <>
-struct hash<::nano::raw_key>
-{
-	size_t operator() (::nano::raw_key const & data_a) const
-	{
-		return hash<::nano::uint256_union> () (data_a);
+		return hash<::nano::block_hash>{}(value.as_block_hash ());
 	}
 };
 template <>
 struct hash<::nano::root>
 {
-	size_t operator() (::nano::root const & data_a) const
+	size_t operator() (::nano::root const & value) const noexcept
 	{
-		return hash<::nano::uint256_union> () (data_a);
+		return hash<::nano::hash_or_account>{}(value);
+	}
+};
+template <>
+struct hash<::nano::link>
+{
+	size_t operator() (::nano::link const & value) const noexcept
+	{
+		return hash<::nano::hash_or_account>{}(value);
+	}
+};
+template <>
+struct hash<::nano::raw_key>
+{
+	size_t operator() (::nano::raw_key const & value) const noexcept
+	{
+		return hash<::nano::uint256_union>{}(value);
 	}
 };
 template <>
 struct hash<::nano::wallet_id>
 {
-	size_t operator() (::nano::wallet_id const & data_a) const
+	size_t operator() (::nano::wallet_id const & value) const noexcept
 	{
-		return hash<::nano::uint256_union> () (data_a);
-	}
-};
-template <>
-struct hash<::nano::uint256_t>
-{
-	size_t operator() (::nano::uint256_t const & number_a) const
-	{
-		return number_a.convert_to<size_t> ();
+		return hash<::nano::uint256_union>{}(value);
 	}
 };
 template <>
 struct hash<::nano::uint512_union>
 {
-	size_t operator() (::nano::uint512_union const & data_a) const
+	size_t operator() (::nano::uint512_union const & value) const noexcept
 	{
-		return hash<::nano::uint256_union> () (data_a.uint256s[0]) + hash<::nano::uint256_union> () (data_a.uint256s[1]);
+		return hash<::nano::uint256_union>{}(value.uint256s[0]) + hash<::nano::uint256_union> () (value.uint256s[1]);
 	}
 };
 template <>
 struct hash<::nano::qualified_root>
 {
-	size_t operator() (::nano::qualified_root const & data_a) const
+	size_t operator() (::nano::qualified_root const & value) const noexcept
 	{
-		return hash<::nano::uint512_union> () (data_a);
-	}
-};
-
-template <>
-struct equal_to<std::reference_wrapper<::nano::block_hash const>>
-{
-	bool operator() (std::reference_wrapper<::nano::block_hash const> const & lhs, std::reference_wrapper<::nano::block_hash const> const & rhs) const
-	{
-		return lhs.get () == rhs.get ();
+		return hash<::nano::uint512_union>{}(value);
 	}
 };
 }
@@ -390,20 +608,91 @@ struct equal_to<std::reference_wrapper<::nano::block_hash const>>
 namespace boost
 {
 template <>
-struct hash<std::reference_wrapper<::nano::block_hash const>>
+struct hash<::nano::uint128_union>
 {
-	size_t operator() (std::reference_wrapper<::nano::block_hash const> const & hash_a) const
+	size_t operator() (::nano::uint128_union const & value) const noexcept
 	{
-		std::hash<::nano::block_hash> hash;
-		return hash (hash_a);
+		return std::hash<::nano::uint128_union> () (value);
+	}
+};
+template <>
+struct hash<::nano::uint256_union>
+{
+	size_t operator() (::nano::uint256_union const & value) const noexcept
+	{
+		return std::hash<::nano::uint256_union> () (value);
+	}
+};
+template <>
+struct hash<::nano::public_key>
+{
+	size_t operator() (::nano::public_key const & value) const noexcept
+	{
+		return std::hash<::nano::public_key> () (value);
+	}
+};
+template <>
+struct hash<::nano::block_hash>
+{
+	size_t operator() (::nano::block_hash const & value) const noexcept
+	{
+		return std::hash<::nano::block_hash> () (value);
+	}
+};
+template <>
+struct hash<::nano::hash_or_account>
+{
+	size_t operator() (::nano::hash_or_account const & value) const noexcept
+	{
+		return std::hash<::nano::hash_or_account> () (value);
 	}
 };
 template <>
 struct hash<::nano::root>
 {
-	size_t operator() (::nano::root const & value_a) const
+	size_t operator() (::nano::root const & value) const noexcept
 	{
-		return std::hash<::nano::root> () (value_a);
+		return std::hash<::nano::root> () (value);
+	}
+};
+template <>
+struct hash<::nano::link>
+{
+	size_t operator() (::nano::link const & value) const noexcept
+	{
+		return std::hash<::nano::link> () (value);
+	}
+};
+template <>
+struct hash<::nano::raw_key>
+{
+	size_t operator() (::nano::raw_key const & value) const noexcept
+	{
+		return std::hash<::nano::raw_key> () (value);
+	}
+};
+template <>
+struct hash<::nano::wallet_id>
+{
+	size_t operator() (::nano::wallet_id const & value) const noexcept
+	{
+		return std::hash<::nano::wallet_id> () (value);
+	}
+};
+template <>
+struct hash<::nano::uint512_union>
+{
+	size_t operator() (::nano::uint512_union const & value) const noexcept
+	{
+		return std::hash<::nano::uint512_union> () (value);
+	}
+};
+template <>
+struct hash<::nano::qualified_root>
+{
+	size_t operator() (::nano::qualified_root const & value) const noexcept
+	{
+		return std::hash<::nano::qualified_root> () (value);
 	}
 };
 }
@@ -427,6 +716,11 @@ struct fmt::formatter<nano::uint512_union> : fmt::ostream_formatter
 {
 };
 
+template <>
+struct fmt::formatter<nano::hash_or_account> : fmt::ostream_formatter
+{
+};
+
 template <>
 struct fmt::formatter<nano::block_hash> : fmt::formatter<nano::uint256_union>
 {
@@ -440,4 +734,4 @@ struct fmt::formatter<nano::public_key> : fmt::formatter<nano::uint256_union>
 template <>
 struct fmt::formatter<nano::qualified_root> : fmt::formatter<nano::uint512_union>
 {
-};
+};

nano/lib/stats_enums.hpp

@@ -95,6 +95,7 @@ enum class type
 	message_processor,
 	message_processor_overfill,
 	message_processor_type,
+	process_confirmed,
 
 	_last // Must be the last enum
 };
@@ -137,6 +138,8 @@ enum class detail
 	cemented,
 	cooldown,
 	empty,
+	done,
+	retry,
 
 	// processing queue
 	queue,
@@ -252,6 +255,8 @@ enum class detail
 	generate_vote_final,
 	broadcast_block_initial,
 	broadcast_block_repeat,
+	confirm_once,
+	confirm_once_failed,
 
 	// election types
 	manual,
@@ -372,6 +377,7 @@ enum class detail
 	failed_send_telemetry_req,
 	empty_payload,
 	cleanup_outdated,
+	erase_stale,
 
 	// vote generator
 	generator_broadcasts,
@@ -409,6 +415,7 @@ enum class detail
 	// active_elections
 	started,
 	stopped,
+	confirm_dependent,
 
 	// unchecked
 	put,
@@ -455,7 +462,6 @@ enum class detail
 	dependency_update_failed,
 
 	// bootstrap_ascending_frontiers
-	done,
 	done_range,
 	done_empty,
 	next_by_requests,
@@ -521,6 +527,7 @@ enum class detail
 	already_cemented,
 	cementing,
 	cemented_hash,
+	cementing_failed,
 
 	// election_state
 	passive,

nano/lib/thread_pool.cpp

@@ -1,97 +0,0 @@
-#include <nano/lib/thread_pool.hpp>
-
-#include <boost/asio/post.hpp>
-#include <boost/asio/steady_timer.hpp>
-#include <boost/asio/thread_pool.hpp>
-
-/*
- * thread_pool
- */
-
-nano::thread_pool::thread_pool (unsigned num_threads, nano::thread_role::name thread_name) :
-	num_threads (num_threads),
-	thread_pool_m (std::make_unique<boost::asio::thread_pool> (num_threads)),
-	thread_names_latch{ num_threads }
-{
-	set_thread_names (thread_name);
-}
-
-nano::thread_pool::~thread_pool ()
-{
-	stop ();
-}
-
-void nano::thread_pool::stop ()
-{
-	nano::unique_lock<nano::mutex> lk (mutex);
-	if (!stopped)
-	{
-		stopped = true;
-#if defined(BOOST_ASIO_HAS_IOCP)
-		// A hack needed for Windows to prevent deadlock during destruction, described here: https://github.com/chriskohlhoff/asio/issues/431
-		boost::asio::use_service<boost::asio::detail::win_iocp_io_context> (*thread_pool_m).stop ();
-#endif
-		lk.unlock ();
-		thread_pool_m->stop ();
-		thread_pool_m->join ();
-		lk.lock ();
-		thread_pool_m = nullptr;
-	}
-}
-
-void nano::thread_pool::push_task (std::function<void ()> task)
-{
-	++num_tasks;
-	nano::lock_guard<nano::mutex> guard (mutex);
-	if (!stopped)
-	{
-		boost::asio::post (*thread_pool_m, [this, task] () {
-			task ();
-			--num_tasks;
-		});
-	}
-}
-
-void nano::thread_pool::add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function<void ()> task)
-{
-	nano::lock_guard<nano::mutex> guard (mutex);
-	if (!stopped && thread_pool_m)
-	{
-		auto timer = std::make_shared<boost::asio::steady_timer> (thread_pool_m->get_executor (), expiry_time);
-		timer->async_wait ([this, task, timer] (boost::system::error_code const & ec) {
-			if (!ec)
-			{
-				push_task (task);
-			}
-		});
-	}
-}
-
-unsigned nano::thread_pool::get_num_threads () const
-{
-	return num_threads;
-}
-
-uint64_t nano::thread_pool::num_queued_tasks () const
-{
-	return num_tasks;
-}
-
-void nano::thread_pool::set_thread_names (nano::thread_role::name thread_name)
-{
-	for (auto i = 0u; i < num_threads; ++i)
-	{
-		boost::asio::post (*thread_pool_m, [this, thread_name] () {
-			nano::thread_role::set (thread_name);
-			thread_names_latch.arrive_and_wait ();
-		});
-	}
-	thread_names_latch.wait ();
-}
-
-nano::container_info nano::thread_pool::container_info () const
-{
-	nano::container_info info;
-	info.put ("count", num_queued_tasks ());
-	return info;
-}

nano/lib/thread_pool.hpp

@@ -1,52 +1,154 @@
 #pragma once
 
+#include <nano/lib/relaxed_atomic.hpp>
 #include <nano/lib/thread_roles.hpp>
 #include <nano/lib/threading.hpp>
 
+#include <boost/asio/post.hpp>
+#include <boost/asio/steady_timer.hpp>
+#include <boost/asio/thread_pool.hpp>
+
 #include <atomic>
 #include <chrono>
-#include <functional>
 #include <latch>
-
-namespace boost::asio
-{
-class thread_pool;
-}
+#include <memory>
+#include <type_traits>
 
 namespace nano
 {
 class thread_pool final
 {
 public:
-	explicit thread_pool (unsigned num_threads, nano::thread_role::name);
-	~thread_pool ();
+	// TODO: Auto start should be removed once the node is refactored to start the thread pool explicitly
+	thread_pool (unsigned num_threads, nano::thread_role::name thread_name, bool auto_start = false) :
+		num_threads{ num_threads },
+		thread_name{ thread_name },
+		thread_names_latch{ num_threads }
+	{
+		if (auto_start)
+		{
+			start ();
+		}
+	}
 
-	/** This will run when there is an available thread for execution */
-	void push_task (std::function<void ()>);
+	~thread_pool ()
+	{
+		// Must be stopped before destruction to avoid running takss when node components are being destroyed
+		debug_assert (!thread_pool_impl);
+	}
 
-	/** Run a task at a certain point in time */
-	void add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function<void ()> task);
+	void start ()
+	{
+		debug_assert (!stopped);
+		debug_assert (!thread_pool_impl);
+		thread_pool_impl = std::make_unique<boost::asio::thread_pool> (num_threads);
+		set_thread_names ();
+	}
 
-	/** Stops any further pushed tasks from executing */
-	void stop ();
+	void stop ()
+	{
+		nano::unique_lock<nano::mutex> lock{ mutex };
+		if (!stopped && thread_pool_impl)
+		{
+			stopped = true;
 
-	/** Number of threads in the thread pool */
-	unsigned get_num_threads () const;
+			// TODO: Is this still needed?
+#if defined(BOOST_ASIO_HAS_IOCP)
+			// A hack needed for Windows to prevent deadlock during destruction, described here: https://github.com/chriskohlhoff/asio/issues/431
+			boost::asio::use_service<boost::asio::detail::win_iocp_io_context> (*thread_pool_impl).stop ();
+#endif
 
-	/** Returns the number of tasks which are awaiting execution by the thread pool **/
-	uint64_t num_queued_tasks () const;
+			lock.unlock ();
 
-	nano::container_info container_info () const;
+			thread_pool_impl->stop ();
+			thread_pool_impl->join ();
+
+			lock.lock ();
+			thread_pool_impl = nullptr;
+		}
+	}
+
+	template <typename F>
+	void post (F && task)
+	{
+		nano::lock_guard<nano::mutex> guard{ mutex };
+		if (!stopped)
+		{
+			++num_tasks;
+			release_assert (thread_pool_impl);
+			boost::asio::post (*thread_pool_impl, [this, t = std::forward<F> (task)] () mutable {
+				t ();
+				--num_tasks;
+			});
+		}
+	}
+
+	template <typename F>
+	void post_delayed (std::chrono::steady_clock::duration const & delay, F && task)
+	{
+		nano::lock_guard<nano::mutex> guard{ mutex };
+		if (!stopped)
+		{
+			++num_delayed;
+			release_assert (thread_pool_impl);
+			auto timer = std::make_shared<boost::asio::steady_timer> (thread_pool_impl->get_executor ());
+			timer->expires_after (delay);
+			timer->async_wait ([this, t = std::forward<F> (task), /* preserve lifetime */ timer] (boost::system::error_code const & ec) mutable {
+				if (!ec)
+				{
+					--num_delayed;
+					post (std::move (t));
+				}
+			});
+		}
+	}
+
+	bool alive () const
+	{
+		nano::lock_guard<nano::mutex> guard{ mutex };
+		return thread_pool_impl != nullptr;
+	}
+
+	uint64_t queued_tasks () const
+	{
+		return num_tasks;
+	}
+
+	uint64_t delayed_tasks () const
+	{
+		return num_delayed;
+	}
+
+	nano::container_info container_info () const
+	{
+		nano::container_info info;
+		info.put ("tasks", num_tasks);
+		info.put ("delayed", num_delayed);
+		return info;
+	}
 
 private:
-	nano::mutex mutex;
-	std::atomic<bool> stopped{ false };
-	unsigned num_threads;
-	std::unique_ptr<boost::asio::thread_pool> thread_pool_m;
-	std::atomic<uint64_t> num_tasks{ 0 };
+	void set_thread_names ()
+	{
+		for (auto i = 0u; i < num_threads; ++i)
+		{
+			boost::asio::post (*thread_pool_impl, [this] () {
+				nano::thread_role::set (thread_name);
+				thread_names_latch.arrive_and_wait ();
+			});
+		}
+		thread_names_latch.wait ();
+	}
+
+private:
+	unsigned const num_threads;
+	nano::thread_role::name const thread_name;
 
-	/** Set the names of all the threads in the thread pool for easier identification */
 	std::latch thread_names_latch;
-	void set_thread_names (nano::thread_role::name thread_name);
+	mutable nano::mutex mutex;
+	std::atomic<bool> stopped{ false };
+	std::unique_ptr<boost::asio::thread_pool> thread_pool_impl;
+	std::atomic<uint64_t> num_tasks{ 0 };
+	std::atomic<uint64_t> num_delayed{ 0 };
 };
-} // namespace nano
+}

nano/nano_node/entry.cpp

@@ -353,7 +353,7 @@ int main (int argc, char * const * argv)
 			auto inactive_node = nano::default_inactive_node (data_path, vm);
 			auto transaction = inactive_node->node->store.tx_begin_read ();
 			auto i = inactive_node->node->store.block.begin (transaction);
-			auto end = inactive_node->node->store.block.end ();
+			auto end = inactive_node->node->store.block.end (transaction);
 			for (; i != end; ++i)
 			{
 				nano::block_hash hash = i->first;
@@ -435,7 +435,7 @@ int main (int argc, char * const * argv)
 			auto current (node->online_reps.trended ());
 			std::cout << boost::str (boost::format ("Trended Weight %1%\n") % current);
 			auto transaction (node->store.tx_begin_read ());
-			for (auto i (node->store.online_weight.begin (transaction)), n (node->store.online_weight.end ()); i != n; ++i)
+			for (auto i (node->store.online_weight.begin (transaction)), n (node->store.online_weight.end (transaction)); i != n; ++i)
 			{
 				using time_point = std::chrono::system_clock::time_point;
 				time_point ts (std::chrono::duration_cast<time_point::duration> (std::chrono::nanoseconds (i->first)));
@@ -471,7 +471,7 @@ int main (int argc, char * const * argv)
 			// Cache the account heads to make searching quicker against unchecked keys.
 			auto transaction (node->store.tx_begin_read ());
 			std::unordered_set<nano::block_hash> frontier_hashes;
-			for (auto i (node->store.account.begin (transaction)), n (node->store.account.end ()); i != n; ++i)
+			for (auto i (node->store.account.begin (transaction)), n (node->store.account.end (transaction)); i != n; ++i)
 			{
 				frontier_hashes.insert (i->second.head);
 			}
@@ -1669,7 +1669,7 @@ int main (int argc, char * const * argv)
 			}
 			size_t const accounts_deque_overflow (32 * 1024);
 			auto transaction = node->ledger.tx_begin_read ();
-			for (auto i (node->store.account.begin (transaction)), n (node->store.account.end ()); i != n; ++i)
+			for (auto i (node->store.account.begin (transaction)), n (node->store.account.end (transaction)); i != n; ++i)
 			{
 				{
 					nano::unique_lock<nano::mutex> lock{ mutex };
@@ -1780,7 +1780,7 @@ int main (int argc, char * const * argv)
 			start_threads (check_pending, pending);
 
 			size_t const pending_deque_overflow (64 * 1024);
-			for (auto i (node->store.pending.begin (transaction)), n (node->store.pending.end ()); i != n; ++i)
+			for (auto i (node->store.pending.begin (transaction)), n (node->store.pending.end (transaction)); i != n; ++i)
 			{
 				{
 					nano::unique_lock<nano::mutex> lock{ mutex };
@@ -1837,7 +1837,7 @@ int main (int argc, char * const * argv)
 				auto transaction = source_node->ledger.tx_begin_read ();
 				block_count = source_node->ledger.block_count ();
 				std::cout << boost::str (boost::format ("Performing bootstrap emulation, %1% blocks in ledger...") % block_count) << std::endl;
-				for (auto i (source_node->store.account.begin (transaction)), n (source_node->store.account.end ()); i != n; ++i)
+				for (auto i (source_node->store.account.begin (transaction)), n (source_node->store.account.end (transaction)); i != n; ++i)
 				{
 					nano::account const & account (i->first);
 					nano::account_info const & info (i->second);
@@ -1942,7 +1942,7 @@ int main (int argc, char * const * argv)
 			nano::locked<std::vector<boost::unordered_set<nano::account>>> opened_account_versions_shared (epoch_count);
 			using opened_account_versions_t = decltype (opened_account_versions_shared)::value_type;
 			node->store.account.for_each_par (
-			[&opened_account_versions_shared, epoch_count] (nano::store::read_transaction const & /*unused*/, nano::store::iterator<nano::account, nano::account_info> i, nano::store::iterator<nano::account, nano::account_info> n) {
+			[&opened_account_versions_shared, epoch_count] (nano::store::read_transaction const & /*unused*/, auto i, auto n) {
 				// First cache locally
 				opened_account_versions_t opened_account_versions_l (epoch_count);
 				for (; i != n; ++i)
@@ -1979,7 +1979,7 @@ int main (int argc, char * const * argv)
 			nano::locked<boost::unordered_map<nano::account, std::underlying_type_t<nano::epoch>>> unopened_highest_pending_shared;
 			using unopened_highest_pending_t = decltype (unopened_highest_pending_shared)::value_type;
 			node->store.pending.for_each_par (
-			[&unopened_highest_pending_shared, &opened_accounts] (nano::store::read_transaction const & /*unused*/, nano::store::iterator<nano::pending_key, nano::pending_info> i, nano::store::iterator<nano::pending_key, nano::pending_info> n) {
+			[&unopened_highest_pending_shared, &opened_accounts] (nano::store::read_transaction const & /*unused*/, auto i, auto n) {
 				// First cache locally
 				unopened_highest_pending_t unopened_highest_pending_l;
 				for (; i != n; ++i)

nano/nano_wallet/entry.cpp

@@ -18,6 +18,7 @@
 #include <nano/rpc/rpc.hpp>
 #include <nano/secure/working.hpp>
 
+#include <boost/format.hpp>
 #include <boost/make_shared.hpp>
 #include <boost/program_options.hpp>
 #include <boost/property_tree/json_parser.hpp>
@@ -147,7 +148,7 @@ public:
 				{
 					auto transaction (wallet->wallets.tx_begin_write ());
 					auto existing (wallet->store.begin (transaction));
-					if (existing != wallet->store.end ())
+					if (existing != wallet->store.end (transaction))
 					{
 						wallet_config.account = existing->first;
 					}

nano/node/active_elections.cpp

@@ -25,36 +25,41 @@ nano::active_elections::active_elections (nano::node & node_a, nano::confirming_
 	confirming_set{ confirming_set_a },
 	block_processor{ block_processor_a },
 	recently_confirmed{ config.confirmation_cache },
-	recently_cemented{ config.confirmation_history_size },
-	election_time_to_live{ node_a.network_params.network.is_dev_network () ? 0s : 2s }
+	recently_cemented{ config.confirmation_history_size }
 {
 	count_by_behavior.fill (0); // Zero initialize array
 
-	confirming_set.batch_cemented.add ([this] (nano::confirming_set::cemented_notification const & notification) {
+	// Cementing blocks might implicitly confirm dependent elections
+	confirming_set.batch_cemented.add ([this] (auto const & cemented) {
+		std::deque<block_cemented_result> results;
 		{
-			auto transaction = node.ledger.tx_begin_read ();
-			for (auto const & [block, confirmation_root] : notification.cemented)
+			// Process all cemented blocks while holding the lock to avoid races where an election for a block that is already cemented is inserted
+			nano::lock_guard<nano::mutex> guard{ mutex };
+			for (auto const & [block, confirmation_root, source_election] : cemented)
 			{
-				transaction.refresh_if_needed ();
-
-				block_cemented_callback (transaction, block, confirmation_root);
+				auto result = block_cemented (block, confirmation_root, source_election);
+				results.push_back (result);
 			}
 		}
-		for (auto const & hash : notification.already_cemented)
 		{
-			block_already_cemented_callback (hash);
+			// TODO: This could be offloaded to a separate notification worker, profiling is needed
+			auto transaction = node.ledger.tx_begin_read ();
+			for (auto const & [status, votes] : results)
+			{
+				transaction.refresh_if_needed ();
+				notify_observers (transaction, status, votes);
+			}
 		}
 	});
 
 	// Notify elections about alternative (forked) blocks
-	block_processor.block_processed.add ([this] (auto const & result, auto const & context) {
-		switch (result)
+	block_processor.batch_processed.add ([this] (auto const & batch) {
+		for (auto const & [result, context] : batch)
 		{
-			case nano::block_status::fork:
+			if (result == nano::block_status::fork)
+			{
 				publish (context.block);
-				break;
-			default:
-				break;
+			}
 		}
 	});
 }
@@ -91,28 +96,32 @@ void nano::active_elections::stop ()
 	clear ();
 }
 
-void nano::active_elections::block_cemented_callback (nano::secure::transaction const & transaction, std::shared_ptr<nano::block> const & block, nano::block_hash const & confirmation_root)
+auto nano::active_elections::block_cemented (std::shared_ptr<nano::block> const & block, nano::block_hash const & confirmation_root, std::shared_ptr<nano::election> const & source_election) -> block_cemented_result
 {
+	debug_assert (!mutex.try_lock ());
 	debug_assert (node.block_confirmed (block->hash ()));
 
-	if (auto election_l = election (block->qualified_root ()))
+	// Dependent elections are implicitly confirmed when their block is cemented
+	auto dependend_election = election_impl (block->qualified_root ());
+	if (dependend_election)
 	{
-		election_l->try_confirm (block->hash ());
+		node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::confirm_dependent);
+		dependend_election->try_confirm (block->hash ()); // TODO: This should either confirm or cancel the election
 	}
-	auto election = remove_election_winner_details (block->hash ());
+
 	nano::election_status status;
 	std::vector<nano::vote_with_weight_info> votes;
 	status.winner = block;
-	if (election)
-	{
-		status = election->get_status ();
-		votes = election->votes_with_weight ();
-	}
-	if (block->hash () == confirmation_root)
+
+	// Check if the currently cemented block was part of an election that triggered the confirmation
+	if (source_election && source_election->qualified_root == block->qualified_root ())
 	{
+		status = source_election->get_status ();
+		debug_assert (status.winner->hash () == block->hash ());
+		votes = source_election->votes_with_weight ();
 		status.type = nano::election_status_type::active_confirmed_quorum;
 	}
-	else if (election)
+	else if (dependend_election)
 	{
 		status.type = nano::election_status_type::active_confirmation_height;
 	}
@@ -120,23 +129,18 @@ void nano::active_elections::block_cemented_callback (nano::secure::transaction
 	{
 		status.type = nano::election_status_type::inactive_confirmation_height;
 	}
+
 	recently_cemented.put (status);
 
 	node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::cemented);
 	node.stats.inc (nano::stat::type::active_elections_cemented, to_stat_detail (status.type));
 
-	node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_cemented, nano::log::arg{ "election", election });
+	node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_cemented,
+	nano::log::arg{ "block", block },
+	nano::log::arg{ "confirmation_root", confirmation_root },
+	nano::log::arg{ "source_election", source_election });
 
-	notify_observers (transaction, status, votes);
-
-	bool cemented_bootstrap_count_reached = node.ledger.cemented_count () >= node.ledger.bootstrap_weight_max_blocks;
-	bool was_active = status.type == nano::election_status_type::active_confirmed_quorum || status.type == nano::election_status_type::active_confirmation_height;
-
-	// Next-block activations are only done for blocks with previously active elections
-	if (cemented_bootstrap_count_reached && was_active && !node.flags.disable_activate_successors)
-	{
-		activate_successors (transaction, block);
-	}
+	return { status, votes };
 }
 
 void nano::active_elections::notify_observers (nano::secure::transaction const & transaction, nano::election_status const & status, std::vector<nano::vote_with_weight_info> const & votes) const
@@ -174,50 +178,6 @@ void nano::active_elections::notify_observers (nano::secure::transaction const &
 	}
 }
 
-void nano::active_elections::activate_successors (nano::secure::transaction const & transaction, std::shared_ptr<nano::block> const & block)
-{
-	node.scheduler.priority.activate (transaction, block->account ());
-
-	// Start or vote for the next unconfirmed block in the destination account
-	if (block->is_send () && !block->destination ().is_zero () && block->destination () != block->account ())
-	{
-		node.scheduler.priority.activate (transaction, block->destination ());
-	}
-}
-
-void nano::active_elections::add_election_winner_details (nano::block_hash const & hash_a, std::shared_ptr<nano::election> const & election_a)
-{
-	nano::lock_guard<nano::mutex> guard{ election_winner_details_mutex };
-	election_winner_details.emplace (hash_a, election_a);
-}
-
-std::shared_ptr<nano::election> nano::active_elections::remove_election_winner_details (nano::block_hash const & hash_a)
-{
-	std::shared_ptr<nano::election> result;
-	{
-		nano::lock_guard<nano::mutex> guard{ election_winner_details_mutex };
-		auto existing = election_winner_details.find (hash_a);
-		if (existing != election_winner_details.end ())
-		{
-			result = existing->second;
-			election_winner_details.erase (existing);
-		}
-	}
-
-	vacancy_update ();
-
-	return result;
-}
-
-void nano::active_elections::block_already_cemented_callback (nano::block_hash const & hash_a)
-{
-	// Depending on timing there is a situation where the election_winner_details is not reset.
-	// This can happen when a block wins an election, and the block is confirmed + observer
-	// called before the block hash gets added to election_winner_details. If the block is confirmed
-	// callbacks have already been done, so we can safely just remove it.
-	remove_election_winner_details (hash_a);
-}
-
 int64_t nano::active_elections::limit (nano::election_behavior behavior) const
 {
 	switch (behavior)
@@ -265,7 +225,7 @@ int64_t nano::active_elections::vacancy (nano::election_behavior behavior) const
 	};
 
 	auto election_winners_vacancy = [this] () -> int64_t {
-		return static_cast<int64_t> (config.max_election_winners) - static_cast<int64_t> (election_winner_details_size ());
+		return static_cast<int64_t> (config.max_election_winners) - static_cast<int64_t> (confirming_set.size ());
 	};
 
 	return std::min (election_vacancy (behavior), election_winners_vacancy ());
@@ -350,7 +310,7 @@ void nano::active_elections::cleanup_election (nano::unique_lock<nano::mutex> &
 		entry.erased_callback (election);
 	}
 
-	vacancy_update ();
+	vacancy_updated.notify ();
 
 	for (auto const & [hash, block] : blocks_l)
 	{
@@ -473,7 +433,7 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr<
 
 		node.vote_cache_processor.trigger (hash);
 		node.observers.active_started.notify (hash);
-		vacancy_update ();
+		vacancy_updated.notify ();
 	}
 
 	// Votes are generated for inserted or ongoing elections
@@ -497,11 +457,17 @@ bool nano::active_elections::active (nano::block const & block_a) const
 	return roots.get<tag_root> ().find (block_a.qualified_root ()) != roots.get<tag_root> ().end ();
 }
 
-std::shared_ptr<nano::election> nano::active_elections::election (nano::qualified_root const & root_a) const
+std::shared_ptr<nano::election> nano::active_elections::election (nano::qualified_root const & root) const
 {
-	std::shared_ptr<nano::election> result;
 	nano::lock_guard<nano::mutex> lock{ mutex };
-	auto existing = roots.get<tag_root> ().find (root_a);
+	return election_impl (root);
+}
+
+std::shared_ptr<nano::election> nano::active_elections::election_impl (nano::qualified_root const & root) const
+{
+	debug_assert (!mutex.try_lock ());
+	std::shared_ptr<nano::election> result;
+	auto existing = roots.get<tag_root> ().find (root);
 	if (existing != roots.get<tag_root> ().end ())
 	{
 		result = existing->election;
@@ -572,12 +538,6 @@ bool nano::active_elections::publish (std::shared_ptr<nano::block> const & block
 	return result;
 }
 
-std::size_t nano::active_elections::election_winner_details_size () const
-{
-	nano::lock_guard<nano::mutex> guard{ election_winner_details_mutex };
-	return election_winner_details.size ();
-}
-
 void nano::active_elections::clear ()
 {
 	// TODO: Call erased_callback for each election
@@ -585,8 +545,7 @@ void nano::active_elections::clear ()
 		nano::lock_guard<nano::mutex> guard{ mutex };
 		roots.clear ();
 	}
-
-	vacancy_update ();
+	vacancy_updated.notify ();
 }
 
 nano::container_info nano::active_elections::container_info () const
@@ -595,7 +554,6 @@ nano::container_info nano::active_elections::container_info () const
 
 	nano::container_info info;
 	info.put ("roots", roots.size ());
-	info.put ("election_winner_details", election_winner_details_size ());
 	info.put ("normal", static_cast<std::size_t> (count_by_behavior[nano::election_behavior::priority]));
 	info.put ("hinted", static_cast<std::size_t> (count_by_behavior[nano::election_behavior::hinted]));
 	info.put ("optimistic", static_cast<std::size_t> (count_by_behavior[nano::election_behavior::optimistic]));

nano/node/active_elections.hpp

@@ -2,6 +2,7 @@
 
 #include <nano/lib/enum_util.hpp>
 #include <nano/lib/numbers.hpp>
+#include <nano/lib/observer_set.hpp>
 #include <nano/node/election_behavior.hpp>
 #include <nano/node/election_insertion_result.hpp>
 #include <nano/node/election_status.hpp>
@@ -104,7 +105,7 @@ public:
 	bool active (nano::qualified_root const &) const;
 	std::shared_ptr<nano::election> election (nano::qualified_root const &) const;
 	// Returns a list of elections sorted by difficulty
-	std::vector<std::shared_ptr<nano::election>> list_active (std::size_t = std::numeric_limits<std::size_t>::max ());
+	std::vector<std::shared_ptr<nano::election>> list_active (std::size_t max_count = std::numeric_limits<std::size_t>::max ());
 	bool erase (nano::block const &);
 	bool erase (nano::qualified_root const &);
 	bool empty () const;
@@ -121,26 +122,24 @@ public:
 	 * How many election slots are available for specified election type
 	 */
 	int64_t vacancy (nano::election_behavior behavior) const;
-	std::function<void ()> vacancy_update{ [] () {} };
-
-	std::size_t election_winner_details_size () const;
-	void add_election_winner_details (nano::block_hash const &, std::shared_ptr<nano::election> const &);
-	std::shared_ptr<nano::election> remove_election_winner_details (nano::block_hash const &);
 
 	nano::container_info container_info () const;
 
+public: // Events
+	nano::observer_set<> vacancy_updated;
+
 private:
 	void request_loop ();
 	void request_confirm (nano::unique_lock<nano::mutex> &);
 	// Erase all blocks from active and, if not confirmed, clear digests from network filters
 	void cleanup_election (nano::unique_lock<nano::mutex> & lock_a, std::shared_ptr<nano::election>);
-	nano::stat::type completion_type (nano::election const & election) const;
-	// Returns a list of elections sorted by difficulty, mutex must be locked
-	std::vector<std::shared_ptr<nano::election>> list_active_impl (std::size_t) const;
-	void activate_successors (nano::secure::transaction const &, std::shared_ptr<nano::block> const & block);
+
+	using block_cemented_result = std::pair<nano::election_status, std::vector<nano::vote_with_weight_info>>;
+	block_cemented_result block_cemented (std::shared_ptr<nano::block> const & block, nano::block_hash const & confirmation_root, std::shared_ptr<nano::election> const & source_election);
 	void notify_observers (nano::secure::transaction const &, nano::election_status const & status, std::vector<nano::vote_with_weight_info> const & votes) const;
-	void block_cemented_callback (nano::secure::transaction const &, std::shared_ptr<nano::block> const & block, nano::block_hash const & confirmation_root);
-	void block_already_cemented_callback (nano::block_hash const & hash);
+
+	std::shared_ptr<nano::election> election_impl (nano::qualified_root const &) const;
+	std::vector<std::shared_ptr<nano::election>> list_active_impl (std::size_t max_count) const;
 
 private: // Dependencies
 	active_elections_config const & config;
@@ -157,12 +156,6 @@ public:
 	mutable nano::mutex mutex{ mutex_identifier (mutexes::active) };
 
 private:
-	mutable nano::mutex election_winner_details_mutex{ mutex_identifier (mutexes::election_winner_details) };
-	std::unordered_map<nano::block_hash, std::shared_ptr<nano::election>> election_winner_details;
-
-	// Maximum time an election can be kept active if it is extending the container
-	std::chrono::seconds const election_time_to_live;
-
 	/** Keeps track of number of elections by election behavior (normal, hinted, optimistic) */
 	nano::enum_array<nano::election_behavior, int64_t> count_by_behavior{};
 

nano/node/backlog_population.cpp

@@ -96,7 +96,7 @@ void nano::backlog_population::populate_backlog (nano::unique_lock<nano::mutex>
 			auto transaction = ledger.tx_begin_read ();
 
 			auto it = ledger.store.account.begin (transaction, next);
-			auto const end = ledger.store.account.end ();
+			auto const end = ledger.store.account.end (transaction);
 
 			auto should_refresh = [&transaction] () {
 				auto cutoff = std::chrono::steady_clock::now () - 100ms; // TODO: Make this configurable

nano/node/bootstrap/bootstrap.cpp

@@ -313,7 +313,7 @@ void nano::pulls_cache::add (nano::pull_info const & pull_a)
 			cache.erase (cache.begin ());
 		}
 		debug_assert (cache.size () <= cache_size_max);
-		nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original);
+		nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original);
 		auto existing (cache.get<account_head_tag> ().find (head_512));
 		if (existing == cache.get<account_head_tag> ().end ())
 		{
@@ -336,7 +336,7 @@ void nano::pulls_cache::add (nano::pull_info const & pull_a)
 void nano::pulls_cache::update_pull (nano::pull_info & pull_a)
 {
 	nano::lock_guard<nano::mutex> guard{ pulls_cache_mutex };
-	nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original);
+	nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original);
 	auto existing (cache.get<account_head_tag> ().find (head_512));
 	if (existing != cache.get<account_head_tag> ().end ())
 	{
@@ -347,7 +347,7 @@ void nano::pulls_cache::update_pull (nano::pull_info & pull_a)
 void nano::pulls_cache::remove (nano::pull_info const & pull_a)
 {
 	nano::lock_guard<nano::mutex> guard{ pulls_cache_mutex };
-	nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original);
+	nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original);
 	cache.get<account_head_tag> ().erase (head_512);
 }
 

nano/node/bootstrap/bootstrap_bulk_pull.cpp

@@ -127,7 +127,7 @@ void nano::bulk_pull_client::throttled_receive_block ()
 	else
 	{
 		auto this_l (shared_from_this ());
-		node->workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::seconds (1), [this_l] () {
+		node->workers.post_delayed (std::chrono::seconds (1), [this_l] () {
 			if (!this_l->connection->pending_stop && !this_l->attempt->stopped)
 			{
 				this_l->throttled_receive_block ();
@@ -177,7 +177,7 @@ void nano::bulk_pull_client::received_block (boost::system::error_code ec, std::
 	// Is block expected?
 	bool block_expected (false);
 	// Unconfirmed head is used only for lazy destinations if legacy bootstrap is not available, see nano::bootstrap_attempt::lazy_destinations_increment (...)
-	bool unconfirmed_account_head = node->flags.disable_legacy_bootstrap && pull_blocks == 0 && pull.retry_limit <= node->network_params.bootstrap.lazy_retry_limit && (expected == pull.account_or_head.as_block_hash ()) && (block->account_field () == pull.account_or_head.as_account ());
+	bool unconfirmed_account_head = node->flags.disable_legacy_bootstrap && pull_blocks == 0 && pull.retry_limit <= node->network_params.bootstrap.lazy_retry_limit && (expected == pull.account_or_head.as_block_hash ()) && (block->account_field ().value_or (0) == pull.account_or_head.as_account ());
 	if (hash == expected || unconfirmed_account_head)
 	{
 		expected = block->previous ();
@@ -394,7 +394,7 @@ void nano::bulk_pull_server::set_current_end ()
 			if (!request->end.is_zero ())
 			{
 				auto account (node->ledger.any.block_account (transaction, request->end));
-				if (account != request->start.as_account ())
+				if (account.value_or (0) != request->start.as_account ())
 				{
 					node->logger.debug (nano::log::type::bulk_pull_server, "Request for block that is not on account chain: {} not on {}", request->end.to_string (), request->start.to_account ());
 
@@ -530,7 +530,7 @@ void nano::bulk_pull_server::sent_action (boost::system::error_code const & ec,
 	}
 	if (!ec)
 	{
-		node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () {
+		node->bootstrap_workers.post ([this_l = shared_from_this ()] () {
 			this_l->send_next ();
 		});
 	}
@@ -816,7 +816,7 @@ void nano::bulk_pull_account_server::sent_action (boost::system::error_code cons
 	}
 	if (!ec)
 	{
-		node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () {
+		node->bootstrap_workers.post ([this_l = shared_from_this ()] () {
 			this_l->send_next_block ();
 		});
 	}

nano/node/bootstrap/bootstrap_bulk_push.cpp

@@ -144,7 +144,7 @@ void nano::bulk_push_server::throttled_receive ()
 	else
 	{
 		auto this_l (shared_from_this ());
-		node->workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::seconds (1), [this_l] () {
+		node->workers.post_delayed (std::chrono::seconds (1), [this_l] () {
 			if (!this_l->connection->stopped)
 			{
 				this_l->throttled_receive ();

nano/node/bootstrap/bootstrap_connections.cpp

@@ -102,7 +102,7 @@ void nano::bootstrap_connections::pool_connection (std::shared_ptr<nano::bootstr
 {
 	nano::unique_lock<nano::mutex> lock{ mutex };
 	auto const & socket_l = client_a->socket;
-	if (!stopped && !client_a->pending_stop && !node.network.excluded_peers.check (client_a->channel->get_tcp_endpoint ()))
+	if (!stopped && !client_a->pending_stop && !node.network.excluded_peers.check (client_a->channel->get_remote_endpoint ()))
 	{
 		socket_l->set_timeout (node.network_params.network.idle_timeout);
 		// Push into idle deque
@@ -138,7 +138,7 @@ std::shared_ptr<nano::bootstrap_client> nano::bootstrap_connections::find_connec
 	std::shared_ptr<nano::bootstrap_client> result;
 	for (auto i (idle.begin ()), end (idle.end ()); i != end && !stopped; ++i)
 	{
-		if ((*i)->channel->get_tcp_endpoint () == endpoint_a)
+		if ((*i)->channel->get_remote_endpoint () == endpoint_a)
 		{
 			result = *i;
 			idle.erase (i);
@@ -306,7 +306,7 @@ void nano::bootstrap_connections::populate_connections (bool repeat)
 	if (!stopped && repeat)
 	{
 		std::weak_ptr<nano::bootstrap_connections> this_w (shared_from_this ());
-		node.workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::seconds (1), [this_w] () {
+		node.workers.post_delayed (std::chrono::seconds (1), [this_w] () {
 			if (auto this_l = this_w.lock ())
 			{
 				this_l->populate_connections ();

nano/node/bootstrap/bootstrap_frontier.cpp

@@ -22,7 +22,7 @@ void nano::frontier_req_client::run (nano::account const & start_account_a, uint
 		return;
 	}
 	nano::frontier_req request{ node->network_params.network };
-	request.start = (start_account_a.is_zero () || start_account_a.number () == std::numeric_limits<nano::uint256_t>::max ()) ? start_account_a : start_account_a.number () + 1;
+	request.start = (start_account_a.is_zero () || start_account_a.number () == std::numeric_limits<nano::uint256_t>::max ()) ? start_account_a.number () : start_account_a.number () + 1;
 	request.age = frontiers_age_a;
 	request.count = count_a;
 	current = start_account_a;
@@ -70,7 +70,7 @@ void nano::frontier_req_client::receive_frontier ()
 		// we simply get a size of 0.
 		if (size_a == nano::frontier_req_client::size_frontier)
 		{
-			node->bootstrap_workers.push_task ([this_l, ec, size_a] () {
+			node->bootstrap_workers.post ([this_l, ec, size_a] () {
 				this_l->received_frontier (ec, size_a);
 			});
 		}
@@ -239,7 +239,7 @@ void nano::frontier_req_client::next ()
 	{
 		std::size_t max_size (128);
 		auto transaction (node->store.tx_begin_read ());
-		for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end ()); i != n && accounts.size () != max_size; ++i)
+		for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end (transaction)); i != n && accounts.size () != max_size; ++i)
 		{
 			nano::account_info const & info (i->second);
 			nano::account const & account (i->first);
@@ -355,7 +355,7 @@ void nano::frontier_req_server::sent_action (boost::system::error_code const & e
 	{
 		count++;
 
-		node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () {
+		node->bootstrap_workers.post ([this_l = shared_from_this ()] () {
 			this_l->send_next ();
 		});
 	}
@@ -381,7 +381,7 @@ void nano::frontier_req_server::next ()
 		auto transaction (node->store.tx_begin_read ());
 		if (!send_confirmed ())
 		{
-			for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end ()); i != n && accounts.size () != max_size; ++i)
+			for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end (transaction)); i != n && accounts.size () != max_size; ++i)
 			{
 				nano::account_info const & info (i->second);
 				if (disable_age_filter || (now - info.modified) <= request->age)
@@ -393,7 +393,7 @@ void nano::frontier_req_server::next ()
 		}
 		else
 		{
-			for (auto i (node->store.confirmation_height.begin (transaction, current.number () + 1)), n (node->store.confirmation_height.end ()); i != n && accounts.size () != max_size; ++i)
+			for (auto i (node->store.confirmation_height.begin (transaction, current.number () + 1)), n (node->store.confirmation_height.end (transaction)); i != n && accounts.size () != max_size; ++i)
 			{
 				nano::confirmation_height_info const & info (i->second);
 				nano::block_hash const & confirmed_frontier (info.frontier);

nano/node/bootstrap/bootstrap_lazy.cpp

@@ -293,7 +293,7 @@ bool nano::bootstrap_attempt_lazy::process_block_lazy (std::shared_ptr<nano::blo
 	if (!lazy_blocks_processed (hash))
 	{
 		// Search for new dependencies
-		if (block_a->source_field () && !node->block_or_pruned_exists (block_a->source_field ().value ()) && block_a->source_field ().value () != node->network_params.ledger.genesis->account ())
+		if (block_a->source_field () && !node->block_or_pruned_exists (block_a->source_field ().value ()) && block_a->source_field ().value () != node->network_params.ledger.genesis->account ().as_union ())
 		{
 			lazy_add (block_a->source_field ().value (), retry_limit);
 		}

nano/node/bootstrap/bootstrap_legacy.cpp

@@ -138,7 +138,7 @@ bool nano::bootstrap_attempt_legacy::request_frontier (nano::unique_lock<nano::m
 	lock_a.lock ();
 	if (connection_l && !stopped)
 	{
-		endpoint_frontier_request = connection_l->channel->get_tcp_endpoint ();
+		endpoint_frontier_request = connection_l->channel->get_remote_endpoint ();
 		std::future<bool> future;
 		{
 			auto this_l = std::dynamic_pointer_cast<nano::bootstrap_attempt_legacy> (shared_from_this ());

nano/node/bootstrap/bootstrap_server.cpp

@@ -395,7 +395,7 @@ nano::asc_pull_ack nano::bootstrap_server::process (secure::transaction const &
 	response.type = nano::asc_pull_type::frontiers;
 
 	nano::asc_pull_ack::frontiers_payload response_payload{};
-	for (auto it = store.account.begin (transaction, request.start), end = store.account.end (); it != end && response_payload.frontiers.size () < request.count; ++it)
+	for (auto it = store.account.begin (transaction, request.start), end = store.account.end (transaction); it != end && response_payload.frontiers.size () < request.count; ++it)
 	{
 		response_payload.frontiers.emplace_back (it->first, it->second.head);
 	}

nano/node/bootstrap_ascending/iterators.hpp

@@ -19,8 +19,8 @@ struct account_database_crawler
 	account_database_crawler (nano::store::component & store, nano::store::transaction const & transaction, nano::account const & start) :
 		store{ store },
 		transaction{ transaction },
-		it{ store.account.end () },
-		end{ store.account.end () }
+		it{ store.account.end (transaction) },
+		end{ store.account.end (transaction) }
 	{
 		seek (start);
 	}
@@ -97,8 +97,8 @@ struct pending_database_crawler
 	pending_database_crawler (nano::store::component & store, nano::store::transaction const & transaction, nano::account const & start) :
 		store{ store },
 		transaction{ transaction },
-		it{ store.pending.end () },
-		end{ store.pending.end () }
+		it{ store.pending.end (transaction) },
+		end{ store.pending.end (transaction) }
 	{
 		seek (start);
 	}

nano/node/bootstrap_ascending/service.cpp

@@ -63,6 +63,7 @@ nano::bootstrap_ascending::service::~service ()
 	debug_assert (!dependencies_thread.joinable ());
 	debug_assert (!frontiers_thread.joinable ());
 	debug_assert (!timeout_thread.joinable ());
+	debug_assert (!workers.alive ());
 }
 
 void nano::bootstrap_ascending::service::start ()
@@ -79,6 +80,8 @@ void nano::bootstrap_ascending::service::start ()
 		return;
 	}
 
+	workers.start ();
+
 	if (config.enable_scan)
 	{
 		priorities_thread = std::thread ([this] () {
@@ -130,6 +133,8 @@ void nano::bootstrap_ascending::service::stop ()
 	nano::join_or_pass (dependencies_thread);
 	nano::join_or_pass (frontiers_thread);
 	nano::join_or_pass (timeout_thread);
+
+	workers.stop ();
 }
 
 bool nano::bootstrap_ascending::service::send (std::shared_ptr<nano::transport::channel> const & channel, async_tag tag)
@@ -622,7 +627,7 @@ void nano::bootstrap_ascending::service::run_one_frontier ()
 		return frontiers_limiter.should_pass (1);
 	});
 	wait ([this] () {
-		return workers.num_queued_tasks () < config.frontier_scan.max_pending;
+		return workers.queued_tasks () < config.frontier_scan.max_pending;
 	});
 	wait_tags ();
 	auto channel = wait_channel ();
@@ -872,9 +877,9 @@ void nano::bootstrap_ascending::service::process (const nano::asc_pull_ack::fron
 			}
 
 			// Allow some overfill to avoid unnecessarily dropping responses
-			if (workers.num_queued_tasks () < config.frontier_scan.max_pending * 4)
+			if (workers.queued_tasks () < config.frontier_scan.max_pending * 4)
 			{
-				workers.push_task ([this, frontiers = response.frontiers] {
+				workers.post ([this, frontiers = response.frontiers] {
 					process_frontiers (frontiers);
 				});
 			}
@@ -1015,7 +1020,7 @@ nano::bootstrap_ascending::service::verify_result nano::bootstrap_ascending::ser
 		case query_type::blocks_by_account:
 		{
 			// Open & state blocks always contain account field
-			if (first->account_field () != tag.start.as_account ())
+			if (first->account_field ().value_or (0) != tag.start.as_account ())
 			{
 				// TODO: Stat & log
 				return verify_result::invalid;

nano/node/cli.cpp

@@ -1012,7 +1012,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
 						nano::raw_key seed;
 						existing->second->store.seed (seed, transaction);
 						std::cout << boost::str (boost::format ("Seed: %1%\n") % seed.to_string ());
-						for (auto i (existing->second->store.begin (transaction)), m (existing->second->store.end ()); i != m; ++i)
+						for (auto i (existing->second->store.begin (transaction)), m (existing->second->store.end (transaction)); i != m; ++i)
 						{
 							nano::account const & account (i->first);
 							nano::raw_key key;
@@ -1201,7 +1201,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
 		{
 			std::cout << boost::str (boost::format ("Wallet ID: %1%\n") % i->first.to_string ());
 			auto transaction (i->second->wallets.tx_begin_read ());
-			for (auto j (i->second->store.begin (transaction)), m (i->second->store.end ()); j != m; ++j)
+			for (auto j (i->second->store.begin (transaction)), m (i->second->store.end (transaction)); j != m; ++j)
 			{
 				std::cout << nano::account (j->first).to_account () << '\n';
 			}
@@ -1224,7 +1224,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
 					{
 						auto transaction (wallet->second->wallets.tx_begin_write ());
 						auto account (wallet->second->store.find (transaction, account_id));
-						if (account != wallet->second->store.end ())
+						if (account != wallet->second->store.end (transaction))
 						{
 							wallet->second->store.erase (transaction, account_id);
 						}

nano/node/confirming_set.cpp

@@ -1,20 +1,23 @@
+#include <nano/lib/logging.hpp>
 #include <nano/lib/thread_roles.hpp>
 #include <nano/node/confirming_set.hpp>
 #include <nano/secure/ledger.hpp>
+#include <nano/secure/ledger_set_any.hpp>
 #include <nano/secure/ledger_set_confirmed.hpp>
 #include <nano/store/component.hpp>
 #include <nano/store/write_queue.hpp>
 
-nano::confirming_set::confirming_set (confirming_set_config const & config_a, nano::ledger & ledger_a, nano::stats & stats_a) :
+nano::confirming_set::confirming_set (confirming_set_config const & config_a, nano::ledger & ledger_a, nano::stats & stats_a, nano::logger & logger_a) :
 	config{ config_a },
 	ledger{ ledger_a },
 	stats{ stats_a },
+	logger{ logger_a },
 	notification_workers{ 1, nano::thread_role::name::confirmation_height_notifications }
 {
-	batch_cemented.add ([this] (auto const & notification) {
-		for (auto const & [block, confirmation_root] : notification.cemented)
+	batch_cemented.add ([this] (auto const & cemented) {
+		for (auto const & context : cemented)
 		{
-			cemented_observers.notify (block);
+			cemented_observers.notify (context.block);
 		}
 	});
 }
@@ -24,12 +27,12 @@ nano::confirming_set::~confirming_set ()
 	debug_assert (!thread.joinable ());
 }
 
-void nano::confirming_set::add (nano::block_hash const & hash)
+void nano::confirming_set::add (nano::block_hash const & hash, std::shared_ptr<nano::election> const & election)
 {
 	bool added = false;
 	{
 		std::lock_guard lock{ mutex };
-		auto [it, inserted] = set.insert (hash);
+		auto [it, inserted] = set.push_back ({ hash, election });
 		added = inserted;
 	}
 	if (added)
@@ -47,6 +50,13 @@ void nano::confirming_set::start ()
 {
 	debug_assert (!thread.joinable ());
 
+	if (!config.enable)
+	{
+		return;
+	}
+
+	notification_workers.start ();
+
 	thread = std::thread{ [this] () {
 		nano::thread_role::set (nano::thread_role::name::confirmation_height);
 		run ();
@@ -67,10 +77,10 @@ void nano::confirming_set::stop ()
 	notification_workers.stop ();
 }
 
-bool nano::confirming_set::exists (nano::block_hash const & hash) const
+bool nano::confirming_set::contains (nano::block_hash const & hash) const
 {
 	std::lock_guard lock{ mutex };
-	return set.count (hash) != 0;
+	return set.get<tag_hash> ().contains (hash) || current.contains (hash);
 }
 
 std::size_t nano::confirming_set::size () const
@@ -99,17 +109,16 @@ void nano::confirming_set::run ()
 	}
 }
 
-std::deque<nano::block_hash> nano::confirming_set::next_batch (size_t max_count)
+auto nano::confirming_set::next_batch (size_t max_count) -> std::deque<entry>
 {
 	debug_assert (!mutex.try_lock ());
 	debug_assert (!set.empty ());
 
-	std::deque<nano::block_hash> results;
+	std::deque<entry> results;
 	while (!set.empty () && results.size () < max_count)
 	{
-		auto it = set.begin ();
-		results.push_back (*it);
-		set.erase (it);
+		results.push_back (set.front ());
+		set.pop_front ();
 	}
 	return results;
 }
@@ -120,21 +129,28 @@ void nano::confirming_set::run_batch (std::unique_lock<std::mutex> & lock)
 	debug_assert (!mutex.try_lock ());
 	debug_assert (!set.empty ());
 
-	std::deque<cemented_t> cemented;
+	std::deque<context> cemented;
 	std::deque<nano::block_hash> already;
 
-	auto batch = next_batch (256);
+	auto batch = next_batch (config.batch_size);
+
+	// Keep track of the blocks we're currently cementing, so that the .contains (...) check is accurate
+	debug_assert (current.empty ());
+	for (auto const & [hash, election] : batch)
+	{
+		current.insert (hash);
+	}
 
 	lock.unlock ();
 
-	auto notify = [this, &cemented, &already] () {
-		cemented_notification notification{};
-		notification.cemented.swap (cemented);
-		notification.already_cemented.swap (already);
+	auto notify = [this, &cemented] () {
+		std::deque<context> batch;
+		batch.swap (cemented);
 
 		std::unique_lock lock{ mutex };
 
-		while (notification_workers.num_queued_tasks () >= config.max_queued_notifications)
+		// It's possible that ledger cementing happens faster than the notifications can be processed by other components, cooldown here
+		while (notification_workers.queued_tasks () >= config.max_queued_notifications)
 		{
 			stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cooldown);
 			condition.wait_for (lock, 100ms, [this] { return stopped.load (); });
@@ -144,9 +160,9 @@ void nano::confirming_set::run_batch (std::unique_lock<std::mutex> & lock)
 			}
 		}
 
-		notification_workers.push_task ([this, notification = std::move (notification)] () {
+		notification_workers.post ([this, batch = std::move (batch)] () {
 			stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify);
-			batch_cemented.notify (notification);
+			batch_cemented.notify (batch);
 		});
 	};
 
@@ -163,8 +179,10 @@ void nano::confirming_set::run_batch (std::unique_lock<std::mutex> & lock)
 
 	{
 		auto transaction = ledger.tx_begin_write (nano::store::writer::confirmation_height);
-		for (auto const & hash : batch)
+		for (auto const & [hash, election] : batch)
 		{
+			size_t cemented_count = 0;
+			bool success = false;
 			do
 			{
 				transaction.refresh_if_needed ();
@@ -180,6 +198,13 @@ void nano::confirming_set::run_batch (std::unique_lock<std::mutex> & lock)
 
 				stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cementing);
 
+				// The block might be rolled back before it's fully cemented
+				if (!ledger.any.block_exists (transaction, hash))
+				{
+					stats.inc (nano::stat::type::confirming_set, nano::stat::detail::missing_block);
+					break;
+				}
+
 				auto added = ledger.confirm (transaction, hash, config.max_blocks);
 				if (!added.empty ())
 				{
@@ -187,8 +212,9 @@ void nano::confirming_set::run_batch (std::unique_lock<std::mutex> & lock)
 					stats.add (nano::stat::type::confirming_set, nano::stat::detail::cemented, added.size ());
 					for (auto & block : added)
 					{
-						cemented.emplace_back (block, hash);
+						cemented.push_back ({ block, hash, election });
 					}
+					cemented_count += added.size ();
 				}
 				else
 				{
@@ -196,16 +222,31 @@ void nano::confirming_set::run_batch (std::unique_lock<std::mutex> & lock)
 					already.push_back (hash);
 					debug_assert (ledger.confirmed.block_exists (transaction, hash));
 				}
-			} while (!ledger.confirmed.block_exists (transaction, hash));
 
-			stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cemented_hash);
+				success = ledger.confirmed.block_exists (transaction, hash);
+			} while (!success);
+
+			if (success)
+			{
+				stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cemented_hash);
+				logger.debug (nano::log::type::confirming_set, "Cemented block: {} (total cemented: {})", hash.to_string (), cemented_count);
+			}
+			else
+			{
+				stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cementing_failed);
+				logger.debug (nano::log::type::confirming_set, "Failed to cement block: {}", hash.to_string ());
+			}
 		}
 	}
 
 	notify ();
-
 	release_assert (cemented.empty ());
-	release_assert (already.empty ());
+
+	already_cemented.notify (already);
+
+	lock.lock ();
+	current.clear ();
+	lock.unlock ();
 }
 
 nano::container_info nano::confirming_set::container_info () const

nano/node/confirming_set.hpp

@@ -4,6 +4,13 @@
 #include <nano/lib/observer_set.hpp>
 #include <nano/lib/thread_pool.hpp>
 #include <nano/node/fwd.hpp>
+#include <nano/secure/common.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index_container.hpp>
 
 #include <condition_variable>
 #include <deque>
@@ -11,6 +18,8 @@
 #include <thread>
 #include <unordered_set>
 
+namespace mi = boost::multi_index;
+
 namespace nano
 {
 class confirming_set_config final
@@ -19,6 +28,9 @@ public:
 	// TODO: Serialization & deserialization
 
 public:
+	bool enable{ true };
+	size_t batch_size{ 256 };
+
 	/** Maximum number of dependent blocks to be stored in memory during processing */
 	size_t max_blocks{ 128 * 1024 };
 	size_t max_queued_notifications{ 8 };
@@ -33,45 +45,65 @@ class confirming_set final
 	friend class confirmation_height_pruned_source_Test;
 
 public:
-	confirming_set (confirming_set_config const &, nano::ledger &, nano::stats &);
+	confirming_set (confirming_set_config const &, nano::ledger &, nano::stats &, nano::logger &);
 	~confirming_set ();
 
 	void start ();
 	void stop ();
 
 	// Adds a block to the set of blocks to be confirmed
-	void add (nano::block_hash const & hash);
+	void add (nano::block_hash const & hash, std::shared_ptr<nano::election> const & election = nullptr);
 	// Added blocks will remain in this set until after ledger has them marked as confirmed.
-	bool exists (nano::block_hash const & hash) const;
+	bool contains (nano::block_hash const & hash) const;
 	std::size_t size () const;
 
 	nano::container_info container_info () const;
 
 public: // Events
-	// Observers will be called once ledger has blocks marked as confirmed
-	using cemented_t = std::pair<std::shared_ptr<nano::block>, nano::block_hash>; // <block, confirmation root>
-
-	struct cemented_notification
+	struct context
 	{
-		std::deque<cemented_t> cemented;
-		std::deque<nano::block_hash> already_cemented;
+		std::shared_ptr<nano::block> block;
+		nano::block_hash confirmation_root;
+		std::shared_ptr<nano::election> election;
 	};
 
-	nano::observer_set<cemented_notification const &> batch_cemented;
+	nano::observer_set<std::deque<context> const &> batch_cemented;
+	nano::observer_set<std::deque<nano::block_hash> const &> already_cemented;
+
 	nano::observer_set<std::shared_ptr<nano::block>> cemented_observers;
 
 private: // Dependencies
 	confirming_set_config const & config;
 	nano::ledger & ledger;
 	nano::stats & stats;
+	nano::logger & logger;
 
 private:
+	struct entry
+	{
+		nano::block_hash hash;
+		std::shared_ptr<nano::election> election;
+	};
+
 	void run ();
 	void run_batch (std::unique_lock<std::mutex> &);
-	std::deque<nano::block_hash> next_batch (size_t max_count);
+	std::deque<entry> next_batch (size_t max_count);
 
 private:
-	std::unordered_set<nano::block_hash> set;
+	// clang-format off
+	class tag_hash {};
+	class tag_sequenced {};
+
+	using ordered_entries = boost::multi_index_container<entry,
+	mi::indexed_by<
+		mi::sequenced<mi::tag<tag_sequenced>>,
+		mi::hashed_unique<mi::tag<tag_hash>,
+			mi::member<entry, nano::block_hash, &entry::hash>>
+	>>;
+	// clang-format on
+
+	ordered_entries set;
+	std::unordered_set<nano::block_hash> current;
 
 	nano::thread_pool notification_workers;
 

nano/node/distributed_work.cpp

@@ -4,6 +4,7 @@
 #include <nano/node/websocket.hpp>
 
 #include <boost/algorithm/string/erase.hpp>
+#include <boost/format.hpp>
 
 std::shared_ptr<request_type> nano::distributed_work::peer_request::get_prepared_json_request (std::string const & request_string_a) const
 {
@@ -400,10 +401,9 @@ void nano::distributed_work::handle_failure ()
 
 			status = work_generation_status::failure_peers;
 
-			auto now (std::chrono::steady_clock::now ());
 			std::weak_ptr<nano::node> node_weak (node.shared ());
 			auto next_backoff (std::min (backoff * 2, std::chrono::seconds (5 * 60)));
-			node.workers.add_timed_task (now + std::chrono::seconds (backoff), [node_weak, request_l = request, next_backoff] {
+			node.workers.post_delayed (std::chrono::seconds (backoff), [node_weak, request_l = request, next_backoff] {
 				bool error_l{ true };
 				if (auto node_l = node_weak.lock ())
 				{

nano/node/election.cpp

@@ -35,18 +35,16 @@ nano::election::election (nano::node & node_a, std::shared_ptr<nano::block> cons
 	last_blocks.emplace (block_a->hash (), block_a);
 }
 
-void nano::election::confirm_once (nano::unique_lock<nano::mutex> & lock_a)
+void nano::election::confirm_once (nano::unique_lock<nano::mutex> & lock)
 {
-	debug_assert (lock_a.owns_lock ());
+	debug_assert (lock.owns_lock ());
+	debug_assert (!mutex.try_lock ());
 
-	// This must be kept above the setting of election state, as dependent confirmed elections require up to date changes to election_winner_details
-	nano::unique_lock<nano::mutex> election_winners_lk{ node.active.election_winner_details_mutex };
-	auto just_confirmed = state_m != nano::election_state::confirmed;
+	bool just_confirmed = state_m != nano::election_state::confirmed;
 	state_m = nano::election_state::confirmed;
-	if (just_confirmed && (node.active.election_winner_details.count (status.winner->hash ()) == 0))
+
+	if (just_confirmed)
 	{
-		node.active.election_winner_details.emplace (status.winner->hash (), shared_from_this ());
-		election_winners_lk.unlock ();
 		status.election_end = std::chrono::duration_cast<std::chrono::milliseconds> (std::chrono::system_clock::now ().time_since_epoch ());
 		status.election_duration = std::chrono::duration_cast<std::chrono::milliseconds> (std::chrono::steady_clock::now () - election_start);
 		status.confirmation_request_count = confirmation_request_count;
@@ -56,15 +54,18 @@ void nano::election::confirm_once (nano::unique_lock<nano::mutex> & lock_a)
 
 		node.active.recently_confirmed.put (qualified_root, status_l.winner->hash ());
 
+		node.stats.inc (nano::stat::type::election, nano::stat::detail::confirm_once);
 		node.logger.trace (nano::log::type::election, nano::log::detail::election_confirmed,
 		nano::log::arg{ "id", id },
 		nano::log::arg{ "qualified_root", qualified_root },
 		nano::log::arg{ "status", current_status_locked () });
 
-		lock_a.unlock ();
+		lock.unlock ();
 
-		node.election_workers.push_task ([node_l = node.shared (), status_l, confirmation_action_l = confirmation_action] () {
-			node_l->process_confirmed (status_l);
+		node.election_workers.post ([this_l = shared_from_this (), status_l, confirmation_action_l = confirmation_action] () {
+			// This is necessary if the winner of the election is one of the forks.
+			// In that case the winning block is not yet in the ledger and cementing needs to wait for rollbacks to complete.
+			this_l->node.process_confirmed (status_l.winner->hash (), this_l);
 
 			if (confirmation_action_l)
 			{
@@ -74,7 +75,8 @@ void nano::election::confirm_once (nano::unique_lock<nano::mutex> & lock_a)
 	}
 	else
 	{
-		lock_a.unlock ();
+		node.stats.inc (nano::stat::type::election, nano::stat::detail::confirm_once_failed);
+		lock.unlock ();
 	}
 }
 
@@ -414,6 +416,7 @@ void nano::election::confirm_if_quorum (nano::unique_lock<nano::mutex> & lock_a)
 		if (final_weight >= node.online_reps.delta ())
 		{
 			confirm_once (lock_a);
+			debug_assert (!lock_a.owns_lock ());
 		}
 	}
 }
@@ -427,6 +430,7 @@ void nano::election::try_confirm (nano::block_hash const & hash)
 		if (!confirmed_locked ())
 		{
 			confirm_once (election_lock);
+			debug_assert (!election_lock.owns_lock ());
 		}
 	}
 }

nano/node/epoch_upgrader.cpp

@@ -116,7 +116,7 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc
 			{
 				auto transaction (store.tx_begin_read ());
 				// Collect accounts to upgrade
-				for (auto i (store.account.begin (transaction)), n (store.account.end ()); i != n && accounts_list.size () < count_limit; ++i)
+				for (auto i (store.account.begin (transaction)), n (store.account.end (transaction)); i != n && accounts_list.size () < count_limit; ++i)
 				{
 					nano::account const & account (i->first);
 					nano::account_info const & info (i->second);
@@ -161,7 +161,7 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc
 								upgrader_condition.wait (lock);
 							}
 						}
-						node.workers.push_task ([&upgrader_process, &upgrader_mutex, &upgrader_condition, &upgraded_accounts, &workers, epoch, difficulty, signer, root, account] () {
+						node.workers.post ([&upgrader_process, &upgrader_mutex, &upgrader_condition, &upgraded_accounts, &workers, epoch, difficulty, signer, root, account] () {
 							upgrader_process (upgraded_accounts, epoch, difficulty, signer, root, account);
 							{
 								nano::lock_guard<nano::mutex> lock{ upgrader_mutex };
@@ -241,7 +241,7 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc
 									upgrader_condition.wait (lock);
 								}
 							}
-							node.workers.push_task ([&upgrader_process, &upgrader_mutex, &upgrader_condition, &upgraded_pending, &workers, epoch, difficulty, signer, root, account] () {
+							node.workers.post ([&upgrader_process, &upgrader_mutex, &upgrader_condition, &upgraded_pending, &workers, epoch, difficulty, signer, root, account] () {
 								upgrader_process (upgraded_pending, epoch, difficulty, signer, root, account);
 								{
 									nano::lock_guard<nano::mutex> lock{ upgrader_mutex };

nano/node/fwd.hpp

@@ -9,6 +9,7 @@ namespace nano
 {
 class block;
 class container_info;
+class thread_pool;
 }
 
 namespace nano

nano/node/ipc/ipc_server.cpp

@@ -14,6 +14,7 @@
 #include <nano/node/json_handler.hpp>
 #include <nano/node/node.hpp>
 
+#include <boost/array.hpp>
 #include <boost/endian/conversion.hpp>
 #include <boost/property_tree/json_parser.hpp>
 

nano/node/json_handler.cpp

@@ -207,7 +207,7 @@ bool nano::json_handler::wallet_account_impl (store::transaction const & transac
 	bool result (false);
 	if (!ec)
 	{
-		if (wallet_a->store.find (transaction_a, account_a) != wallet_a->store.end ())
+		if (wallet_a->store.find (transaction_a, account_a) != wallet_a->store.end (transaction_a))
 		{
 			result = true;
 		}
@@ -541,7 +541,7 @@ void nano::json_handler::account_block_count ()
 
 void nano::json_handler::account_create ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		if (!rpc_l->ec)
 		{
@@ -718,7 +718,7 @@ void nano::json_handler::account_list ()
 	{
 		boost::property_tree::ptree accounts;
 		auto transaction (node.wallets.tx_begin_read ());
-		for (auto i (wallet->store.begin (transaction)), j (wallet->store.end ()); i != j; ++i)
+		for (auto i (wallet->store.begin (transaction)), j (wallet->store.end (transaction)); i != j; ++i)
 		{
 			boost::property_tree::ptree entry;
 			entry.put ("", nano::account (i->first).to_account ());
@@ -731,7 +731,7 @@ void nano::json_handler::account_list ()
 
 void nano::json_handler::account_move ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		if (!rpc_l->ec)
 		{
@@ -770,7 +770,7 @@ void nano::json_handler::account_move ()
 
 void nano::json_handler::account_remove ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		auto account (rpc_l->account_impl ());
 		if (!rpc_l->ec)
@@ -805,7 +805,7 @@ void nano::json_handler::account_representative ()
 
 void nano::json_handler::account_representative_set ()
 {
-	node.workers.push_task (create_worker_task ([work_generation_enabled = node.work_generation_enabled ()] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([work_generation_enabled = node.work_generation_enabled ()] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		auto account (rpc_l->account_impl ());
 		std::string representative_text (rpc_l->request.get<std::string> ("representative"));
@@ -948,7 +948,7 @@ void nano::json_handler::accounts_representatives ()
 
 void nano::json_handler::accounts_create ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		auto count (rpc_l->count_impl ());
 		if (!rpc_l->ec)
@@ -1030,7 +1030,7 @@ void nano::json_handler::accounts_receivable ()
 		if (!ec)
 		{
 			boost::property_tree::ptree peers_l;
-			for (auto i (node.store.pending.begin (transaction, nano::pending_key (account, 0))), n (node.store.pending.end ()); i != n && nano::pending_key (i->first).account == account && peers_l.size () < count; ++i)
+			for (auto i (node.store.pending.begin (transaction, nano::pending_key (account, 0))), n (node.store.pending.end (transaction)); i != n && nano::pending_key (i->first).account == account && peers_l.size () < count; ++i)
 			{
 				nano::pending_key const & key (i->first);
 				if (block_confirmed (node, transaction, key.hash, include_active, include_only_confirmed))
@@ -1187,7 +1187,7 @@ void nano::json_handler::block_confirm ()
 			if (!node.ledger.confirmed.block_exists_or_pruned (transaction, hash))
 			{
 				// Start new confirmation for unconfirmed (or not being confirmed) block
-				if (!node.confirming_set.exists (hash))
+				if (!node.confirming_set.contains (hash))
 				{
 					node.start_election (std::move (block_l));
 				}
@@ -2240,7 +2240,7 @@ void nano::json_handler::delegators ()
 	{
 		auto transaction (node.ledger.tx_begin_read ());
 		boost::property_tree::ptree delegators;
-		for (auto i (node.store.account.begin (transaction, start_account.number () + 1)), n (node.store.account.end ()); i != n && delegators.size () < count; ++i)
+		for (auto i (node.store.account.begin (transaction, start_account.number () + 1)), n (node.store.account.end (transaction)); i != n && delegators.size () < count; ++i)
 		{
 			nano::account_info const & info (i->second);
 			if (info.representative == representative)
@@ -2266,7 +2266,7 @@ void nano::json_handler::delegators_count ()
 	{
 		uint64_t count (0);
 		auto transaction (node.ledger.tx_begin_read ());
-		for (auto i (node.store.account.begin (transaction)), n (node.store.account.end ()); i != n; ++i)
+		for (auto i (node.store.account.begin (transaction)), n (node.store.account.end (transaction)); i != n; ++i)
 		{
 			nano::account_info const & info (i->second);
 			if (info.representative == account)
@@ -2377,7 +2377,7 @@ void nano::json_handler::frontiers ()
 	{
 		boost::property_tree::ptree frontiers;
 		auto transaction (node.ledger.tx_begin_read ());
-		for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end ()); i != n && frontiers.size () < count; ++i)
+		for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end (transaction)); i != n && frontiers.size () < count; ++i)
 		{
 			frontiers.put (i->first.to_account (), i->second.head.to_string ());
 		}
@@ -2462,7 +2462,7 @@ public:
 			// Report opens as a receive
 			tree.put ("type", "receive");
 		}
-		if (block_a.hashables.source != handler.node.ledger.constants.genesis->account ())
+		if (block_a.hashables.source != handler.node.ledger.constants.genesis->account ().as_union ())
 		{
 			bool error_or_pruned (false);
 			auto amount = handler.node.ledger.any.block_amount (transaction, hash);
@@ -2780,7 +2780,7 @@ void nano::json_handler::ledger ()
 		auto transaction = node.ledger.tx_begin_read ();
 		if (!ec && !sorting) // Simple
 		{
-			for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end ()); i != n && accounts.size () < count; ++i)
+			for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end (transaction)); i != n && accounts.size () < count; ++i)
 			{
 				nano::account_info const & info (i->second);
 				if (info.modified >= modified_since && (receivable || info.balance.number () >= threshold.number ()))
@@ -2821,7 +2821,7 @@ void nano::json_handler::ledger ()
 		else if (!ec) // Sorting
 		{
 			std::vector<std::pair<nano::uint128_union, nano::account>> ledger_l;
-			for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end ()); i != n; ++i)
+			for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end (transaction)); i != n; ++i)
 			{
 				nano::account_info const & info (i->second);
 				nano::uint128_union balance (info.balance);
@@ -2930,7 +2930,7 @@ void nano::json_handler::node_id_delete ()
 
 void nano::json_handler::password_change ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		if (!rpc_l->ec)
 		{
@@ -2953,7 +2953,7 @@ void nano::json_handler::password_change ()
 
 void nano::json_handler::password_enter ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		if (!rpc_l->ec)
 		{
@@ -2991,7 +2991,7 @@ void nano::json_handler::peers ()
 	bool const peer_details = request.get<bool> ("peer_details", false);
 	auto peers_list (node.network.list (std::numeric_limits<std::size_t>::max ()));
 	std::sort (peers_list.begin (), peers_list.end (), [] (auto const & lhs, auto const & rhs) {
-		return lhs->get_endpoint () < rhs->get_endpoint ();
+		return lhs->get_remote_endpoint () < rhs->get_remote_endpoint ();
 	});
 	for (auto i (peers_list.begin ()), n (peers_list.end ()); i != n; ++i)
 	{
@@ -3003,9 +3003,9 @@ void nano::json_handler::peers ()
 			boost::property_tree::ptree pending_tree;
 			pending_tree.put ("protocol_version", std::to_string (channel->get_network_version ()));
 			auto node_id_l (channel->get_node_id_optional ());
-			if (node_id_l.is_initialized ())
+			if (node_id_l.has_value ())
 			{
-				pending_tree.put ("node_id", node_id_l.get ().to_node_id ());
+				pending_tree.put ("node_id", node_id_l.value ().to_node_id ());
 			}
 			else
 			{
@@ -3055,7 +3055,7 @@ void nano::json_handler::receivable ()
 		// The ptree container is used if there are any children nodes (e.g source/min_version) otherwise the amount container is used.
 		std::vector<std::pair<std::string, boost::property_tree::ptree>> hash_ptree_pairs;
 		std::vector<std::pair<std::string, nano::uint128_t>> hash_amount_pairs;
-		for (auto i (node.store.pending.begin (transaction, nano::pending_key (account, 0))), n (node.store.pending.end ()); i != n && nano::pending_key (i->first).account == account && (should_sort || peers_l.size () < count); ++i)
+		for (auto i (node.store.pending.begin (transaction, nano::pending_key (account, 0))), n (node.store.pending.end (transaction)); i != n && nano::pending_key (i->first).account == account && (should_sort || peers_l.size () < count); ++i)
 		{
 			nano::pending_key const & key (i->first);
 			if (block_confirmed (node, transaction, key.hash, include_active, include_only_confirmed))
@@ -3178,7 +3178,7 @@ void nano::json_handler::receivable_exists ()
 
 void nano::json_handler::process ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		bool const is_async = rpc_l->request.get<bool> ("async", false);
 		auto block (rpc_l->block_impl (true));
 
@@ -4143,7 +4143,7 @@ void nano::json_handler::unchecked ()
 
 void nano::json_handler::unchecked_clear ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		rpc_l->node.unchecked.clear ();
 		rpc_l->response_l.put ("success", "");
 		rpc_l->response_errors ();
@@ -4240,7 +4240,7 @@ void nano::json_handler::unopened ()
 	{
 		auto transaction = node.store.tx_begin_read ();
 		auto iterator = node.store.pending.begin (transaction, nano::pending_key (start, 0));
-		auto end = node.store.pending.end ();
+		auto end = node.store.pending.end (transaction);
 		nano::account current_account = start;
 		nano::uint128_t current_account_sum{ 0 };
 		boost::property_tree::ptree accounts;
@@ -4316,7 +4316,7 @@ void nano::json_handler::validate_account_number ()
 
 void nano::json_handler::wallet_add ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		if (!rpc_l->ec)
 		{
@@ -4346,7 +4346,7 @@ void nano::json_handler::wallet_add ()
 
 void nano::json_handler::wallet_add_watch ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		if (!rpc_l->ec)
 		{
@@ -4392,7 +4392,7 @@ void nano::json_handler::wallet_info ()
 		uint64_t adhoc_count (0);
 		auto transaction (node.wallets.tx_begin_read ());
 		auto block_transaction = node.ledger.tx_begin_read ();
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 
@@ -4448,7 +4448,7 @@ void nano::json_handler::wallet_balances ()
 		boost::property_tree::ptree balances;
 		auto transaction (node.wallets.tx_begin_read ());
 		auto block_transaction = node.ledger.tx_begin_read ();
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 			nano::uint128_t balance = node.ledger.any.account_balance (block_transaction, account).value_or (0).number ();
@@ -4469,7 +4469,7 @@ void nano::json_handler::wallet_balances ()
 
 void nano::json_handler::wallet_change_seed ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		if (!rpc_l->ec)
 		{
@@ -4509,7 +4509,7 @@ void nano::json_handler::wallet_contains ()
 	if (!ec)
 	{
 		auto transaction (node.wallets.tx_begin_read ());
-		auto exists (wallet->store.find (transaction, account) != wallet->store.end ());
+		auto exists (wallet->store.find (transaction, account) != wallet->store.end (transaction));
 		response_l.put ("exists", exists ? "1" : "0");
 	}
 	response_errors ();
@@ -4517,7 +4517,7 @@ void nano::json_handler::wallet_contains ()
 
 void nano::json_handler::wallet_create ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		nano::raw_key seed;
 		auto seed_text (rpc_l->request.get_optional<std::string> ("seed"));
 		if (seed_text.is_initialized () && seed.decode_hex (seed_text.get ()))
@@ -4553,7 +4553,7 @@ void nano::json_handler::wallet_create ()
 
 void nano::json_handler::wallet_destroy ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		std::string wallet_text (rpc_l->request.get<std::string> ("wallet"));
 		nano::wallet_id wallet;
 		if (!wallet.decode_hex (wallet_text))
@@ -4599,7 +4599,7 @@ void nano::json_handler::wallet_frontiers ()
 		boost::property_tree::ptree frontiers;
 		auto transaction (node.wallets.tx_begin_read ());
 		auto block_transaction = node.ledger.tx_begin_read ();
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 			auto latest (node.ledger.any.account_head (block_transaction, account));
@@ -4630,7 +4630,7 @@ void nano::json_handler::wallet_history ()
 		std::multimap<uint64_t, boost::property_tree::ptree, std::greater<uint64_t>> entries;
 		auto transaction (node.wallets.tx_begin_read ());
 		auto block_transaction = node.ledger.tx_begin_read ();
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 			auto info = node.ledger.any.account_get (block_transaction, account);
@@ -4704,7 +4704,7 @@ void nano::json_handler::wallet_ledger ()
 		boost::property_tree::ptree accounts;
 		auto transaction (node.wallets.tx_begin_read ());
 		auto block_transaction = node.ledger.tx_begin_read ();
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 			auto info = node.ledger.any.account_get (block_transaction, account);
@@ -4780,11 +4780,11 @@ void nano::json_handler::wallet_receivable ()
 		boost::property_tree::ptree pending;
 		auto transaction (node.wallets.tx_begin_read ());
 		auto block_transaction = node.ledger.tx_begin_read ();
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 			boost::property_tree::ptree peers_l;
-			for (auto ii (node.store.pending.begin (block_transaction, nano::pending_key (account, 0))), nn (node.store.pending.end ()); ii != nn && nano::pending_key (ii->first).account == account && peers_l.size () < count; ++ii)
+			for (auto ii (node.store.pending.begin (block_transaction, nano::pending_key (account, 0))), nn (node.store.pending.end (block_transaction)); ii != nn && nano::pending_key (ii->first).account == account && peers_l.size () < count; ++ii)
 			{
 				nano::pending_key key (ii->first);
 				if (block_confirmed (node, block_transaction, key.hash, include_active, include_only_confirmed))
@@ -4845,7 +4845,7 @@ void nano::json_handler::wallet_representative ()
 
 void nano::json_handler::wallet_representative_set ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		std::string representative_text (rpc_l->request.get<std::string> ("representative"));
 		auto representative (rpc_l->account_impl (representative_text, nano::error_rpc::bad_representative_number));
@@ -4871,7 +4871,7 @@ void nano::json_handler::wallet_representative_set ()
 				{
 					auto transaction (rpc_l->node.wallets.tx_begin_read ());
 					auto block_transaction = rpc_l->node.ledger.tx_begin_read ();
-					for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+					for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 					{
 						nano::account const & account (i->first);
 						auto info = rpc_l->node.ledger.any.account_get (block_transaction, account);
@@ -4905,7 +4905,7 @@ void nano::json_handler::wallet_republish ()
 		std::deque<std::shared_ptr<nano::block>> republish_bundle;
 		auto transaction (node.wallets.tx_begin_read ());
 		auto block_transaction = node.ledger.tx_begin_read ();
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 			auto latest (node.ledger.any.account_head (block_transaction, account));
@@ -4967,7 +4967,7 @@ void nano::json_handler::wallet_work_get ()
 	{
 		boost::property_tree::ptree works;
 		auto transaction (node.wallets.tx_begin_read ());
-		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i)
+		for (auto i (wallet->store.begin (transaction)), n (wallet->store.end (transaction)); i != n; ++i)
 		{
 			nano::account const & account (i->first);
 			uint64_t work (0);
@@ -5132,7 +5132,7 @@ void nano::json_handler::work_get ()
 
 void nano::json_handler::work_set ()
 {
-	node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
+	node.workers.post (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
 		auto wallet (rpc_l->wallet_impl ());
 		auto account (rpc_l->account_impl ());
 		auto work (rpc_l->work_optional_impl ());

nano/node/message_processor.cpp

@@ -175,7 +175,7 @@ public:
 		if (peer0.address () == boost::asio::ip::address_v6{} && peer0.port () != 0)
 		{
 			// TODO: Remove this as we do not need to establish a second connection to the same peer
-			nano::endpoint new_endpoint (channel->get_tcp_endpoint ().address (), peer0.port ());
+			nano::endpoint new_endpoint (channel->get_remote_endpoint ().address (), peer0.port ());
 			node.network.merge_peer (new_endpoint);
 
 			// Remember this for future forwarding to other peers

nano/node/network.cpp

@@ -281,6 +281,15 @@ void nano::network::flood_vote (std::shared_ptr<nano::vote> const & vote, float
 	}
 }
 
+void nano::network::flood_vote_non_pr (std::shared_ptr<nano::vote> const & vote, float scale, bool rebroadcasted)
+{
+	nano::confirm_ack message{ node.network_params.network, vote, rebroadcasted };
+	for (auto & i : list_non_pr (fanout (scale)))
+	{
+		i->send (message, nullptr);
+	}
+}
+
 void nano::network::flood_vote_pr (std::shared_ptr<nano::vote> const & vote, bool rebroadcasted)
 {
 	nano::confirm_ack message{ node.network_params.network, vote, rebroadcasted };
@@ -300,7 +309,7 @@ void nano::network::flood_block_many (std::deque<std::shared_ptr<nano::block>> b
 		if (!blocks_a.empty ())
 		{
 			std::weak_ptr<nano::node> node_w (node.shared ());
-			node.workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::milliseconds (delay_a + std::rand () % delay_a), [node_w, blocks (std::move (blocks_a)), callback_a, delay_a] () {
+			node.workers.post_delayed (std::chrono::milliseconds (delay_a + std::rand () % delay_a), [node_w, blocks (std::move (blocks_a)), callback_a, delay_a] () {
 				if (auto node_l = node_w.lock ())
 				{
 					node_l->network.flood_block_many (std::move (blocks), callback_a, delay_a);
@@ -314,14 +323,6 @@ void nano::network::flood_block_many (std::deque<std::shared_ptr<nano::block>> b
 	}
 }
 
-void nano::network::inbound (const nano::message & message, const std::shared_ptr<nano::transport::channel> & channel)
-{
-	debug_assert (message.header.network == node.network_params.network.current_network);
-	debug_assert (message.header.version_using >= node.network_params.network.protocol_version_min);
-
-	node.message_processor.process (message, channel);
-}
-
 // Send keepalives to all the peers we've been notified of
 void nano::network::merge_peers (std::array<nano::endpoint, 8> const & peers_a)
 {
@@ -385,11 +386,13 @@ std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list_non_pr
 {
 	std::deque<std::shared_ptr<nano::transport::channel>> result;
 	tcp_channels.list (result);
+
+	auto partition_point = std::partition (result.begin (), result.end (),
+	[this] (std::shared_ptr<nano::transport::channel> const & channel) {
+		return !node.rep_crawler.is_pr (channel);
+	});
+	result.resize (std::distance (result.begin (), partition_point));
 	nano::random_pool_shuffle (result.begin (), result.end ());
-	result.erase (std::remove_if (result.begin (), result.end (), [this] (std::shared_ptr<nano::transport::channel> const & channel) {
-		return node.rep_crawler.is_pr (channel);
-	}),
-	result.end ());
 	if (result.size () > count_a)
 	{
 		result.resize (count_a, nullptr);
@@ -504,14 +507,14 @@ void nano::network::erase (nano::transport::channel const & channel_a)
 	auto const channel_type = channel_a.get_type ();
 	if (channel_type == nano::transport::transport_type::tcp)
 	{
-		tcp_channels.erase (channel_a.get_tcp_endpoint ());
+		tcp_channels.erase (channel_a.get_remote_endpoint ());
 	}
 }
 
 void nano::network::exclude (std::shared_ptr<nano::transport::channel> const & channel)
 {
 	// Add to peer exclusion list
-	excluded_peers.add (channel->get_tcp_endpoint ());
+	excluded_peers.add (channel->get_remote_endpoint ());
 
 	// Disconnect
 	erase (*channel);

nano/node/network.hpp

@@ -95,6 +95,7 @@ public:
 	void flood_keepalive_self (float const scale_a = 0.5f);
 	void flood_vote (std::shared_ptr<nano::vote> const &, float scale, bool rebroadcasted = false);
 	void flood_vote_pr (std::shared_ptr<nano::vote> const &, bool rebroadcasted = false);
+	void flood_vote_non_pr (std::shared_ptr<nano::vote> const &, float scale, bool rebroadcasted = false);
 	// Flood block to all PRs and a random selection of non-PRs
 	void flood_block_initial (std::shared_ptr<nano::block> const &);
 	// Flood block to a random selection of peers
@@ -127,7 +128,6 @@ public:
 	void erase (nano::transport::channel const &);
 	/** Disconnects and adds peer to exclusion list */
 	void exclude (std::shared_ptr<nano::transport::channel> const & channel);
-	void inbound (nano::message const &, std::shared_ptr<nano::transport::channel> const &);
 
 	nano::container_info container_info () const;
 

nano/node/node.cpp

@@ -1,5 +1,6 @@
 #include <nano/lib/blocks.hpp>
 #include <nano/lib/stream.hpp>
+#include <nano/lib/thread_pool.hpp>
 #include <nano/lib/thread_runner.hpp>
 #include <nano/lib/tomlconfig.hpp>
 #include <nano/lib/utility.hpp>
@@ -37,6 +38,7 @@
 #include <nano/store/component.hpp>
 #include <nano/store/rocksdb/rocksdb.hpp>
 
+#include <boost/format.hpp>
 #include <boost/property_tree/json_parser.hpp>
 
 #include <algorithm>
@@ -68,6 +70,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, uint16_t pe
 nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesystem::path const & application_path_a, nano::node_config const & config_a, nano::work_pool & work_a, nano::node_flags flags_a, unsigned seq) :
 	node_id{ load_or_create_node_id (application_path_a) },
 	config{ config_a },
+	flags{ flags_a },
 	io_ctx_shared{ std::make_shared<boost::asio::io_context> () },
 	io_ctx{ *io_ctx_shared },
 	logger{ make_logger_identifier (node_id) },
@@ -76,11 +79,14 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 	node_initialized_latch (1),
 	network_params{ config.network_params },
 	stats{ logger, config.stats_config },
-	workers{ config.background_threads, nano::thread_role::name::worker },
-	bootstrap_workers{ config.bootstrap_serving_threads, nano::thread_role::name::bootstrap_worker },
-	wallet_workers{ 1, nano::thread_role::name::wallet_worker },
-	election_workers{ 1, nano::thread_role::name::election_worker },
-	flags (flags_a),
+	workers_impl{ std::make_unique<nano::thread_pool> (config.background_threads, nano::thread_role::name::worker, /* start immediately */ true) },
+	workers{ *workers_impl },
+	bootstrap_workers_impl{ std::make_unique<nano::thread_pool> (config.bootstrap_serving_threads, nano::thread_role::name::bootstrap_worker, /* start immediately */ true) },
+	bootstrap_workers{ *bootstrap_workers_impl },
+	wallet_workers_impl{ std::make_unique<nano::thread_pool> (1, nano::thread_role::name::wallet_worker, /* start immediately */ true) },
+	wallet_workers{ *wallet_workers_impl },
+	election_workers_impl{ std::make_unique<nano::thread_pool> (1, nano::thread_role::name::election_worker, /* start immediately */ true) },
+	election_workers{ *election_workers_impl },
 	work (work_a),
 	distributed_work (*this),
 	store_impl (nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade)),
@@ -115,7 +121,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 	port_mapping_impl{ std::make_unique<nano::port_mapping> (*this) },
 	port_mapping{ *port_mapping_impl },
 	block_processor (*this),
-	confirming_set_impl{ std::make_unique<nano::confirming_set> (config.confirming_set, ledger, stats) },
+	confirming_set_impl{ std::make_unique<nano::confirming_set> (config.confirming_set, ledger, stats, logger) },
 	confirming_set{ *confirming_set_impl },
 	active_impl{ std::make_unique<nano::active_elections> (*this, confirming_set, block_processor) },
 	active{ *active_impl },
@@ -137,7 +143,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 	generator{ *generator_impl },
 	final_generator_impl{ std::make_unique<nano::vote_generator> (config, *this, ledger, wallets, vote_processor, history, network, stats, logger, /* final */ true) },
 	final_generator{ *final_generator_impl },
-	scheduler_impl{ std::make_unique<nano::scheduler::component> (*this) },
+	scheduler_impl{ std::make_unique<nano::scheduler::component> (config, *this, ledger, block_processor, active, online_reps, vote_cache, confirming_set, stats, logger) },
 	scheduler{ *scheduler_impl },
 	aggregator_impl{ std::make_unique<nano::request_aggregator> (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) },
 	aggregator{ *aggregator_impl },
@@ -187,13 +193,6 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 
 	if (!init_error ())
 	{
-		// Notify election schedulers when AEC frees election slot
-		active.vacancy_update = [this] () {
-			scheduler.priority.notify ();
-			scheduler.hinted.notify ();
-			scheduler.optimistic.notify ();
-		};
-
 		wallets.observer = [this] (bool active) {
 			observers.wallet.notify (active);
 		};
@@ -318,7 +317,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 		auto is_initialized (false);
 		{
 			auto const transaction (store.tx_begin_read ());
-			is_initialized = (store.account.begin (transaction) != store.account.end ());
+			is_initialized = (store.account.begin (transaction) != store.account.end (transaction));
 		}
 
 		if (!is_initialized && !flags.read_only)
@@ -408,7 +407,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 			// TODO: Is it neccessary to call this for all blocks?
 			if (block->is_send ())
 			{
-				wallet_workers.push_task ([this, hash = block->hash (), destination = block->destination ()] () {
+				wallet_workers.post ([this, hash = block->hash (), destination = block->destination ()] () {
 					wallets.receive_confirmed (hash, destination);
 				});
 			}
@@ -518,6 +517,16 @@ void nano::node::keepalive (std::string const & address_a, uint16_t port_a)
 	});
 }
 
+void nano::node::inbound (const nano::message & message, const std::shared_ptr<nano::transport::channel> & channel)
+{
+	debug_assert (channel->owner () == shared_from_this ()); // This node should be the channel owner
+
+	debug_assert (message.header.network == network_params.network.current_network);
+	debug_assert (message.header.version_using >= network_params.network.protocol_version_min);
+
+	message_processor.process (message, channel);
+}
+
 void nano::node::process_active (std::shared_ptr<nano::block> const & incoming)
 {
 	block_processor.add (incoming);
@@ -560,7 +569,7 @@ void nano::node::start ()
 	if (flags.enable_pruning)
 	{
 		auto this_l (shared ());
-		workers.push_task ([this_l] () {
+		workers.post ([this_l] () {
 			this_l->ongoing_ledger_pruning ();
 		});
 	}
@@ -601,7 +610,7 @@ void nano::node::start ()
 	{
 		// Delay to start wallet lazy bootstrap
 		auto this_l (shared ());
-		workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::minutes (1), [this_l] () {
+		workers.post_delayed (std::chrono::minutes (1), [this_l] () {
 			this_l->bootstrap_wallet ();
 		});
 	}
@@ -650,9 +659,7 @@ void nano::node::stop ()
 	logger.info (nano::log::type::node, "Node stopping...");
 
 	tcp_listener.stop ();
-	bootstrap_workers.stop ();
-	wallet_workers.stop ();
-	election_workers.stop ();
+
 	vote_router.stop ();
 	peer_history.stop ();
 	// Cancels ongoing work generation tasks, which may be blocking other threads
@@ -680,12 +687,16 @@ void nano::node::stop ()
 	wallets.stop ();
 	stats.stop ();
 	epoch_upgrader.stop ();
-	workers.stop ();
 	local_block_broadcaster.stop ();
 	message_processor.stop ();
 	network.stop (); // Stop network last to avoid killing in-use sockets
 	monitor.stop ();
 
+	bootstrap_workers.stop ();
+	wallet_workers.stop ();
+	election_workers.stop ();
+	workers.stop ();
+
 	// work pool is not stopped on purpose due to testing setup
 
 	// Stop the IO runner last
@@ -751,7 +762,7 @@ void nano::node::long_inactivity_cleanup ()
 	if (store.online_weight.count (transaction) > 0)
 	{
 		auto sample (store.online_weight.rbegin (transaction));
-		auto n (store.online_weight.end ());
+		auto n (store.online_weight.rend (transaction));
 		debug_assert (sample != n);
 		auto const one_week_ago = static_cast<std::size_t> ((std::chrono::system_clock::now () - std::chrono::hours (7 * 24)).time_since_epoch ().count ());
 		perform_cleanup = sample->first < one_week_ago;
@@ -800,7 +811,7 @@ void nano::node::ongoing_bootstrap ()
 			{
 				auto transaction = store.tx_begin_read ();
 				auto last_record = store.online_weight.rbegin (transaction);
-				if (last_record != store.online_weight.end ())
+				if (last_record != store.online_weight.end (transaction))
 				{
 					last_sample_time = last_record->first;
 				}
@@ -819,7 +830,7 @@ void nano::node::ongoing_bootstrap ()
 	// Bootstrap and schedule for next attempt
 	bootstrap_initiator.bootstrap (false, boost::str (boost::format ("auto_bootstrap_%1%") % previous_bootstrap_count), frontiers_age);
 	std::weak_ptr<nano::node> node_w (shared_from_this ());
-	workers.add_timed_task (std::chrono::steady_clock::now () + next_wakeup, [node_w] () {
+	workers.post_delayed (next_wakeup, [node_w] () {
 		if (auto node_l = node_w.lock ())
 		{
 			node_l->ongoing_bootstrap ();
@@ -840,7 +851,7 @@ void nano::node::backup_wallet ()
 		i->second->store.write_backup (transaction, backup_path / (i->first.to_string () + ".json"));
 	}
 	auto this_l (shared ());
-	workers.add_timed_task (std::chrono::steady_clock::now () + network_params.node.backup_interval, [this_l] () {
+	workers.post_delayed (network_params.node.backup_interval, [this_l] () {
 		this_l->backup_wallet ();
 	});
 }
@@ -852,7 +863,7 @@ void nano::node::search_receivable_all ()
 	// Search pending
 	wallets.search_receivable_all ();
 	auto this_l (shared ());
-	workers.add_timed_task (std::chrono::steady_clock::now () + network_params.node.search_pending_interval, [this_l] () {
+	workers.post_delayed (network_params.node.search_pending_interval, [this_l] () {
 		this_l->search_receivable_all ();
 	});
 }
@@ -867,7 +878,7 @@ void nano::node::bootstrap_wallet ()
 		{
 			auto & wallet (*i->second);
 			nano::lock_guard<std::recursive_mutex> wallet_lock{ wallet.store.mutex };
-			for (auto j (wallet.store.begin (transaction)), m (wallet.store.end ()); j != m && accounts.size () < 128; ++j)
+			for (auto j (wallet.store.begin (transaction)), m (wallet.store.end (transaction)); j != m && accounts.size () < 128; ++j)
 			{
 				nano::account account (j->first);
 				accounts.push_back (account);
@@ -885,7 +896,7 @@ bool nano::node::collect_ledger_pruning_targets (std::deque<nano::block_hash> &
 	uint64_t read_operations (0);
 	bool finish_transaction (false);
 	auto transaction = ledger.tx_begin_read ();
-	for (auto i (store.confirmation_height.begin (transaction, last_account_a)), n (store.confirmation_height.end ()); i != n && !finish_transaction;)
+	for (auto i (store.confirmation_height.begin (transaction, last_account_a)), n (store.confirmation_height.end (transaction)); i != n && !finish_transaction;)
 	{
 		++read_operations;
 		auto const & account (i->first);
@@ -977,8 +988,8 @@ void nano::node::ongoing_ledger_pruning ()
 	ledger_pruning (flags.block_processor_batch_size != 0 ? flags.block_processor_batch_size : 2 * 1024, bootstrap_weight_reached);
 	auto const ledger_pruning_interval (bootstrap_weight_reached ? config.max_pruning_age : std::min (config.max_pruning_age, std::chrono::seconds (15 * 60)));
 	auto this_l (shared ());
-	workers.add_timed_task (std::chrono::steady_clock::now () + ledger_pruning_interval, [this_l] () {
-		this_l->workers.push_task ([this_l] () {
+	workers.post_delayed (ledger_pruning_interval, [this_l] () {
+		this_l->workers.post ([this_l] () {
 			this_l->ongoing_ledger_pruning ();
 		});
 	});
@@ -1104,14 +1115,14 @@ void nano::node::start_election (std::shared_ptr<nano::block> const & block)
 	scheduler.manual.push (block);
 }
 
-bool nano::node::block_confirmed (nano::block_hash const & hash_a)
+bool nano::node::block_confirmed (nano::block_hash const & hash)
 {
-	return ledger.confirmed.block_exists_or_pruned (ledger.tx_begin_read (), hash_a);
+	return ledger.confirmed.block_exists_or_pruned (ledger.tx_begin_read (), hash);
 }
 
-bool nano::node::block_confirmed_or_being_confirmed (nano::secure::transaction const & transaction, nano::block_hash const & hash_a)
+bool nano::node::block_confirmed_or_being_confirmed (nano::secure::transaction const & transaction, nano::block_hash const & hash)
 {
-	return confirming_set.exists (hash_a) || ledger.confirmed.block_exists_or_pruned (transaction, hash_a);
+	return confirming_set.contains (hash) || ledger.confirmed.block_exists_or_pruned (transaction, hash);
 }
 
 bool nano::node::block_confirmed_or_being_confirmed (nano::block_hash const & hash_a)
@@ -1122,7 +1133,7 @@ bool nano::node::block_confirmed_or_being_confirmed (nano::block_hash const & ha
 void nano::node::ongoing_online_weight_calculation_queue ()
 {
 	std::weak_ptr<nano::node> node_w (shared_from_this ());
-	workers.add_timed_task (std::chrono::steady_clock::now () + (std::chrono::seconds (network_params.node.weight_period)), [node_w] () {
+	workers.post_delayed ((std::chrono::seconds (network_params.node.weight_period)), [node_w] () {
 		if (auto node_l = node_w.lock ())
 		{
 			node_l->ongoing_online_weight_calculation ();
@@ -1141,31 +1152,36 @@ void nano::node::ongoing_online_weight_calculation ()
 	ongoing_online_weight_calculation_queue ();
 }
 
-void nano::node::process_confirmed (nano::election_status const & status_a, uint64_t iteration_a)
+// TODO: Replace this with a queue of some sort. Blocks submitted here could be in a limbo for a while: neither part of an active election nor cemented
+void nano::node::process_confirmed (nano::block_hash hash, std::shared_ptr<nano::election> election, uint64_t iteration)
 {
-	auto hash (status_a.winner->hash ());
-	decltype (iteration_a) const num_iters = (config.block_processor_batch_max_time / network_params.node.process_confirmed_interval) * 4;
-	if (auto block_l = ledger.any.block_get (ledger.tx_begin_read (), hash))
-	{
-		logger.trace (nano::log::type::node, nano::log::detail::process_confirmed, nano::log::arg{ "block", block_l });
+	stats.inc (nano::stat::type::process_confirmed, nano::stat::detail::initiate);
 
-		confirming_set.add (block_l->hash ());
-	}
-	else if (iteration_a < num_iters)
+	// Limit the maximum number of iterations to avoid getting stuck
+	uint64_t const max_iterations = (config.block_processor_batch_max_time / network_params.node.process_confirmed_interval) * 4;
+
+	if (auto block = ledger.any.block_get (ledger.tx_begin_read (), hash))
 	{
-		iteration_a++;
-		std::weak_ptr<nano::node> node_w (shared ());
-		election_workers.add_timed_task (std::chrono::steady_clock::now () + network_params.node.process_confirmed_interval, [node_w, status_a, iteration_a] () {
-			if (auto node_l = node_w.lock ())
-			{
-				node_l->process_confirmed (status_a, iteration_a);
-			}
+		stats.inc (nano::stat::type::process_confirmed, nano::stat::detail::done);
+		logger.trace (nano::log::type::node, nano::log::detail::process_confirmed, nano::log::arg{ "block", block });
+
+		confirming_set.add (block->hash (), election);
+	}
+	else if (iteration < max_iterations)
+	{
+		stats.inc (nano::stat::type::process_confirmed, nano::stat::detail::retry);
+
+		// Try again later
+		election_workers.post_delayed (network_params.node.process_confirmed_interval, [this, hash, election, iteration] () {
+			process_confirmed (hash, election, iteration + 1);
 		});
 	}
 	else
 	{
+		stats.inc (nano::stat::type::process_confirmed, nano::stat::detail::timeout);
+
 		// Do some cleanup due to this block never being processed by confirmation height processor
-		active.remove_election_winner_details (hash);
+		active.recently_confirmed.erase (hash);
 	}
 }
 

nano/node/node.hpp

@@ -4,7 +4,6 @@
 #include <nano/lib/config.hpp>
 #include <nano/lib/logging.hpp>
 #include <nano/lib/stats.hpp>
-#include <nano/lib/thread_pool.hpp>
 #include <nano/lib/work.hpp>
 #include <nano/node/blockprocessor.hpp>
 #include <nano/node/bootstrap/bootstrap.hpp>
@@ -94,7 +93,8 @@ public:
 	bool copy_with_compaction (std::filesystem::path const &);
 	void keepalive (std::string const &, uint16_t);
 	int store_version ();
-	void process_confirmed (nano::election_status const &, uint64_t = 0);
+	void inbound (nano::message const &, std::shared_ptr<nano::transport::channel> const &);
+	void process_confirmed (nano::block_hash, std::shared_ptr<nano::election> = nullptr, uint64_t iteration = 0);
 	void process_active (std::shared_ptr<nano::block> const &);
 	std::optional<nano::block_status> process_local (std::shared_ptr<nano::block> const &);
 	void process_local_async (std::shared_ptr<nano::block> const &);
@@ -147,6 +147,7 @@ public:
 public:
 	const nano::keypair node_id;
 	nano::node_config config;
+	nano::node_flags flags;
 	std::shared_ptr<boost::asio::io_context> io_ctx_shared;
 	boost::asio::io_context & io_ctx;
 	nano::logger logger;
@@ -155,11 +156,14 @@ public:
 	boost::latch node_initialized_latch;
 	nano::network_params & network_params;
 	nano::stats stats;
-	nano::thread_pool workers;
-	nano::thread_pool bootstrap_workers;
-	nano::thread_pool wallet_workers;
-	nano::thread_pool election_workers;
-	nano::node_flags flags;
+	std::unique_ptr<nano::thread_pool> workers_impl;
+	nano::thread_pool & workers;
+	std::unique_ptr<nano::thread_pool> bootstrap_workers_impl;
+	nano::thread_pool & bootstrap_workers;
+	std::unique_ptr<nano::thread_pool> wallet_workers_impl;
+	nano::thread_pool & wallet_workers;
+	std::unique_ptr<nano::thread_pool> election_workers_impl;
+	nano::thread_pool & election_workers;
 	nano::work_pool & work;
 	nano::distributed_work_factory distributed_work;
 	std::unique_ptr<nano::store::component> store_impl;

nano/node/nodeconfig.cpp

@@ -120,7 +120,6 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	toml.put ("allow_local_peers", allow_local_peers, "Enable or disable local host peering.\ntype:bool");
 	toml.put ("vote_minimum", vote_minimum.to_string_dec (), "Local representatives do not vote if the delegated weight is under this threshold. Saves on system resources.\ntype:string,amount,raw");
 	toml.put ("vote_generator_delay", vote_generator_delay.count (), "Delay before votes are sent to allow for efficient bundling of hashes in votes.\ntype:milliseconds");
-	toml.put ("vote_generator_threshold", vote_generator_threshold, "Number of bundled hashes required for an additional generator delay.\ntype:uint64,[1..11]");
 	toml.put ("unchecked_cutoff_time", unchecked_cutoff_time.count (), "Number of seconds before deleting an unchecked entry.\nWarning: lower values (e.g., 3600 seconds, or 1 hour) may result in unsuccessful bootstraps, especially a bootstrap from scratch.\ntype:seconds");
 	toml.put ("tcp_io_timeout", tcp_io_timeout.count (), "Timeout for TCP connect-, read- and write operations.\nWarning: a low value (e.g., below 5 seconds) may result in TCP connections failing.\ntype:seconds");
 	toml.put ("pow_sleep_interval", pow_sleep_interval.count (), "Time to sleep between batch work generation attempts. Reduces max CPU usage at the expense of a longer generation time.\ntype:nanoseconds");
@@ -484,8 +483,6 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 		toml.get ("vote_generator_delay", delay_l);
 		vote_generator_delay = std::chrono::milliseconds (delay_l);
 
-		toml.get<unsigned> ("vote_generator_threshold", vote_generator_threshold);
-
 		auto block_processor_batch_max_time_l = block_processor_batch_max_time.count ();
 		toml.get ("block_processor_batch_max_time", block_processor_batch_max_time_l);
 		block_processor_batch_max_time = std::chrono::milliseconds (block_processor_batch_max_time_l);
@@ -600,10 +597,6 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 		{
 			toml.get_error ().set ("bandwidth_limit unbounded = 0, default = 10485760, max = 18446744073709551615");
 		}
-		if (vote_generator_threshold < 1 || vote_generator_threshold > 11)
-		{
-			toml.get_error ().set ("vote_generator_threshold must be a number between 1 and 11");
-		}
 		if (max_work_generate_multiplier < 1)
 		{
 			toml.get_error ().set ("max_work_generate_multiplier must be greater than or equal to 1");

nano/node/nodeconfig.hpp

@@ -73,7 +73,6 @@ public:
 	nano::amount vote_minimum{ nano::Knano_ratio }; // 1000 nano
 	nano::amount rep_crawler_weight_minimum{ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" };
 	std::chrono::milliseconds vote_generator_delay{ std::chrono::milliseconds (100) };
-	unsigned vote_generator_threshold{ 3 };
 	nano::amount online_weight_minimum{ 60000 * nano::Knano_ratio }; // 60 million nano
 	/*
 	 * The minimum vote weight that a representative must have for its vote to be counted.

nano/node/online_reps.cpp

@@ -45,7 +45,7 @@ void nano::online_reps::sample ()
 		while (ledger.store.online_weight.count (transaction) >= config.network_params.node.max_weight_samples)
 		{
 			auto oldest (ledger.store.online_weight.begin (transaction));
-			debug_assert (oldest != ledger.store.online_weight.end ());
+			debug_assert (oldest != ledger.store.online_weight.end (transaction));
 			ledger.store.online_weight.del (transaction, oldest->first);
 		}
 		ledger.store.online_weight.put (transaction, std::chrono::system_clock::now ().time_since_epoch ().count (), online_l);
@@ -70,7 +70,7 @@ nano::uint128_t nano::online_reps::calculate_trend (store::transaction & transac
 	std::vector<nano::uint128_t> items;
 	items.reserve (config.network_params.node.max_weight_samples + 1);
 	items.push_back (config.online_weight_minimum.number ());
-	for (auto i (ledger.store.online_weight.begin (transaction_a)), n (ledger.store.online_weight.end ()); i != n; ++i)
+	for (auto i (ledger.store.online_weight.begin (transaction_a)), n (ledger.store.online_weight.end (transaction_a)); i != n; ++i)
 	{
 		items.push_back (i->second.number ());
 	}

nano/node/peer_history.cpp

@@ -104,7 +104,7 @@ void nano::peer_history::run_one ()
 	auto const now = std::chrono::system_clock::now ();
 	auto const cutoff = now - config.erase_cutoff;
 
-	for (auto it = store.peer.begin (transaction); it != store.peer.end (); ++it)
+	for (auto it = store.peer.begin (transaction); it != store.peer.end (transaction); ++it)
 	{
 		auto const [endpoint, timestamp_millis] = *it;
 		auto timestamp = nano::from_milliseconds_since_epoch (timestamp_millis);
@@ -124,7 +124,7 @@ std::vector<nano::endpoint> nano::peer_history::peers () const
 {
 	auto transaction = store.tx_begin_read ();
 	std::vector<nano::endpoint> peers;
-	for (auto it = store.peer.begin (transaction); it != store.peer.end (); ++it)
+	for (auto it = store.peer.begin (transaction); it != store.peer.end (transaction); ++it)
 	{
 		auto const [endpoint, timestamp_millis] = *it;
 		peers.push_back (endpoint.endpoint ());

nano/node/process_live_dispatcher.cpp

@@ -43,12 +43,6 @@ void nano::process_live_dispatcher::inspect (nano::block_status const & result,
 
 void nano::process_live_dispatcher::process_live (nano::block const & block, secure::transaction const & transaction)
 {
-	// Start collecting quorum on block
-	if (ledger.dependents_confirmed (transaction, block))
-	{
-		scheduler.activate (transaction, block.account ());
-	}
-
 	if (websocket.server && websocket.server->any_subscriber (nano::websocket::topic::new_unconfirmed_block))
 	{
 		websocket.server->broadcast (nano::websocket::message_builder ().new_block_arrived (block));

nano/node/repcrawler.cpp

@@ -102,7 +102,7 @@ void nano::rep_crawler::validate_and_process (nano::unique_lock<nano::mutex> & l
 				rep.last_response = std::chrono::steady_clock::now ();
 
 				// Update if representative channel was changed
-				if (rep.channel->get_endpoint () != channel->get_endpoint ())
+				if (rep.channel->get_remote_endpoint () != channel->get_remote_endpoint ())
 				{
 					debug_assert (rep.account == vote->account);
 					updated = true;

nano/node/scheduler/component.cpp

@@ -5,16 +5,22 @@
 #include <nano/node/scheduler/optimistic.hpp>
 #include <nano/node/scheduler/priority.hpp>
 
-nano::scheduler::component::component (nano::node & node) :
-	hinted_impl{ std::make_unique<nano::scheduler::hinted> (node.config.hinted_scheduler, node, node.vote_cache, node.active, node.online_reps, node.stats) },
+nano::scheduler::component::component (nano::node_config & node_config, nano::node & node, nano::ledger & ledger, nano::block_processor & block_processor, nano::active_elections & active, nano::online_reps & online_reps, nano::vote_cache & vote_cache, nano::confirming_set & confirming_set, nano::stats & stats, nano::logger & logger) :
+	hinted_impl{ std::make_unique<nano::scheduler::hinted> (node_config.hinted_scheduler, node, vote_cache, active, online_reps, stats) },
 	manual_impl{ std::make_unique<nano::scheduler::manual> (node) },
-	optimistic_impl{ std::make_unique<nano::scheduler::optimistic> (node.config.optimistic_scheduler, node, node.ledger, node.active, node.network_params.network, node.stats) },
-	priority_impl{ std::make_unique<nano::scheduler::priority> (node, node.stats) },
+	optimistic_impl{ std::make_unique<nano::scheduler::optimistic> (node_config.optimistic_scheduler, node, ledger, active, node_config.network_params.network, stats) },
+	priority_impl{ std::make_unique<nano::scheduler::priority> (node_config, node, ledger, block_processor, active, confirming_set, stats, logger) },
 	hinted{ *hinted_impl },
 	manual{ *manual_impl },
 	optimistic{ *optimistic_impl },
 	priority{ *priority_impl }
 {
+	// Notify election schedulers when AEC frees election slot
+	active.vacancy_updated.add ([this] () {
+		priority.notify ();
+		hinted.notify ();
+		optimistic.notify ();
+	});
 }
 
 nano::scheduler::component::~component ()

nano/node/scheduler/component.hpp

@@ -10,12 +10,10 @@ namespace nano::scheduler
 class component final
 {
 public:
-	explicit component (nano::node & node);
+	component (nano::node_config &, nano::node &, nano::ledger &, nano::block_processor &, nano::active_elections &, nano::online_reps &, nano::vote_cache &, nano::confirming_set &, nano::stats &, nano::logger &);
 	~component ();
 
-	// Starts all schedulers
 	void start ();
-	// Stops all schedulers
 	void stop ();
 
 	nano::container_info container_info () const;

nano/node/scheduler/priority.cpp

@@ -7,10 +7,15 @@
 #include <nano/secure/ledger_set_any.hpp>
 #include <nano/secure/ledger_set_confirmed.hpp>
 
-nano::scheduler::priority::priority (nano::node & node_a, nano::stats & stats_a) :
-	config{ node_a.config.priority_scheduler },
+nano::scheduler::priority::priority (nano::node_config & node_config, nano::node & node_a, nano::ledger & ledger_a, nano::block_processor & block_processor_a, nano::active_elections & active_a, nano::confirming_set & confirming_set_a, nano::stats & stats_a, nano::logger & logger_a) :
+	config{ node_config.priority_scheduler },
 	node{ node_a },
-	stats{ stats_a }
+	ledger{ ledger_a },
+	block_processor{ block_processor_a },
+	active{ active_a },
+	confirming_set{ confirming_set_a },
+	stats{ stats_a },
+	logger{ logger_a }
 {
 	std::vector<nano::uint128_t> minimums;
 
@@ -34,13 +39,41 @@ nano::scheduler::priority::priority (nano::node & node_a, nano::stats & stats_a)
 	build_region (uint128_t{ 1 } << 116, uint128_t{ 1 } << 120, 2);
 	minimums.push_back (uint128_t{ 1 } << 120);
 
-	node.logger.debug (nano::log::type::election_scheduler, "Number of buckets: {}", minimums.size ());
+	logger.debug (nano::log::type::election_scheduler, "Number of buckets: {}", minimums.size ());
 
 	for (size_t i = 0u, n = minimums.size (); i < n; ++i)
 	{
-		auto bucket = std::make_unique<scheduler::bucket> (minimums[i], node.config.priority_bucket, node.active, stats);
+		auto bucket = std::make_unique<scheduler::bucket> (minimums[i], node_config.priority_bucket, active, stats);
 		buckets.emplace_back (std::move (bucket));
 	}
+
+	// Activate accounts with fresh blocks
+	block_processor.batch_processed.add ([this] (auto const & batch) {
+		auto transaction = ledger.tx_begin_read ();
+		for (auto const & [result, context] : batch)
+		{
+			if (result == nano::block_status::progress)
+			{
+				release_assert (context.block != nullptr);
+				activate (transaction, context.block->account ());
+			}
+		}
+	});
+
+	// Activate successors of cemented blocks
+	confirming_set.batch_cemented.add ([this] (auto const & batch) {
+		if (node.flags.disable_activate_successors)
+		{
+			return;
+		}
+
+		auto transaction = ledger.tx_begin_read ();
+		for (auto const & context : batch)
+		{
+			release_assert (context.block != nullptr);
+			activate_successors (transaction, *context.block);
+		}
+	});
 }
 
 nano::scheduler::priority::~priority ()
@@ -85,11 +118,10 @@ void nano::scheduler::priority::stop ()
 bool nano::scheduler::priority::activate (secure::transaction const & transaction, nano::account const & account)
 {
 	debug_assert (!account.is_zero ());
-	auto info = node.ledger.any.account_get (transaction, account);
-	if (info)
+	if (auto info = ledger.any.account_get (transaction, account))
 	{
 		nano::confirmation_height_info conf_info;
-		node.store.confirmation_height.get (transaction, account, conf_info);
+		ledger.store.confirmation_height.get (transaction, account, conf_info);
 		if (conf_info.height < info->block_count)
 		{
 			return activate (transaction, account, *info, conf_info);
@@ -103,14 +135,14 @@ bool nano::scheduler::priority::activate (secure::transaction const & transactio
 {
 	debug_assert (conf_info.frontier != account_info.head);
 
-	auto hash = conf_info.height == 0 ? account_info.open_block : node.ledger.any.block_successor (transaction, conf_info.frontier).value ();
-	auto block = node.ledger.any.block_get (transaction, hash);
+	auto hash = conf_info.height == 0 ? account_info.open_block : ledger.any.block_successor (transaction, conf_info.frontier).value ();
+	auto block = ledger.any.block_get (transaction, hash);
 	release_assert (block != nullptr);
 
-	if (node.ledger.dependents_confirmed (transaction, *block))
+	if (ledger.dependents_confirmed (transaction, *block))
 	{
 		auto const balance = block->balance ();
-		auto const previous_balance = node.ledger.any.block_balance (transaction, conf_info.frontier).value_or (0);
+		auto const previous_balance = ledger.any.block_balance (transaction, conf_info.frontier).value_or (0);
 		auto const balance_priority = std::max (balance, previous_balance);
 
 		bool added = false;
@@ -120,8 +152,8 @@ bool nano::scheduler::priority::activate (secure::transaction const & transactio
 		}
 		if (added)
 		{
-			node.stats.inc (nano::stat::type::election_scheduler, nano::stat::detail::activated);
-			node.logger.trace (nano::log::type::election_scheduler, nano::log::detail::block_activated,
+			stats.inc (nano::stat::type::election_scheduler, nano::stat::detail::activated);
+			logger.trace (nano::log::type::election_scheduler, nano::log::detail::block_activated,
 			nano::log::arg{ "account", account.to_account () }, // TODO: Convert to lazy eval
 			nano::log::arg{ "block", block },
 			nano::log::arg{ "time", account_info.modified },
@@ -131,7 +163,7 @@ bool nano::scheduler::priority::activate (secure::transaction const & transactio
 		}
 		else
 		{
-			node.stats.inc (nano::stat::type::election_scheduler, nano::stat::detail::activate_full);
+			stats.inc (nano::stat::type::election_scheduler, nano::stat::detail::activate_full);
 		}
 
 		return true; // Activated
@@ -141,6 +173,17 @@ bool nano::scheduler::priority::activate (secure::transaction const & transactio
 	return false; // Not activated
 }
 
+bool nano::scheduler::priority::activate_successors (secure::transaction const & transaction, nano::block const & block)
+{
+	bool result = activate (transaction, block.account ());
+	// Start or vote for the next unconfirmed block in the destination account
+	if (block.is_send () && !block.destination ().is_zero () && block.destination () != block.account ())
+	{
+		result |= activate (transaction, block.destination ());
+	}
+	return result;
+}
+
 void nano::scheduler::priority::notify ()
 {
 	condition.notify_all ();

nano/node/scheduler/priority.hpp

@@ -26,7 +26,7 @@ public:
 class priority final
 {
 public:
-	priority (nano::node &, nano::stats &);
+	priority (nano::node_config &, nano::node &, nano::ledger &, nano::block_processor &, nano::active_elections &, nano::confirming_set &, nano::stats &, nano::logger &);
 	~priority ();
 
 	void start ();
@@ -36,8 +36,9 @@ public:
 	 * Activates the first unconfirmed block of \p account_a
 	 * @return true if account was activated
 	 */
-	bool activate (secure::transaction const &, nano::account const &);
-	bool activate (secure::transaction const &, nano::account const &, nano::account_info const &, nano::confirmation_height_info const &);
+	bool activate (nano::secure::transaction const &, nano::account const &);
+	bool activate (nano::secure::transaction const &, nano::account const &, nano::account_info const &, nano::confirmation_height_info const &);
+	bool activate_successors (nano::secure::transaction const &, nano::block const &);
 
 	void notify ();
 	std::size_t size () const;
@@ -48,7 +49,12 @@ public:
 private: // Dependencies
 	priority_config const & config;
 	nano::node & node;
+	nano::ledger & ledger;
+	nano::block_processor & block_processor;
+	nano::active_elections & active;
+	nano::confirming_set & confirming_set;
 	nano::stats & stats;
+	nano::logger & logger;
 
 private:
 	void run ();

nano/node/telemetry.cpp

@@ -96,14 +96,11 @@ void nano::telemetry::process (const nano::telemetry_ack & telemetry, const std:
 
 	nano::unique_lock<nano::mutex> lock{ mutex };
 
-	const auto endpoint = channel->get_endpoint ();
-
-	if (auto it = telemetries.get<tag_endpoint> ().find (endpoint); it != telemetries.get<tag_endpoint> ().end ())
+	if (auto it = telemetries.get<tag_channel> ().find (channel); it != telemetries.get<tag_channel> ().end ())
 	{
 		stats.inc (nano::stat::type::telemetry, nano::stat::detail::update);
 
-		telemetries.get<tag_endpoint> ().modify (it, [&telemetry, &endpoint] (auto & entry) {
-			debug_assert (entry.endpoint == endpoint);
+		telemetries.get<tag_channel> ().modify (it, [&telemetry, &channel] (auto & entry) {
 			entry.data = telemetry.data;
 			entry.last_updated = std::chrono::steady_clock::now ();
 		});
@@ -111,7 +108,7 @@ void nano::telemetry::process (const nano::telemetry_ack & telemetry, const std:
 	else
 	{
 		stats.inc (nano::stat::type::telemetry, nano::stat::detail::insert);
-		telemetries.get<tag_endpoint> ().insert ({ endpoint, telemetry.data, std::chrono::steady_clock::now (), channel });
+		telemetries.get<tag_channel> ().insert ({ channel, telemetry.data, std::chrono::steady_clock::now () });
 
 		if (telemetries.size () > max_size)
 		{
@@ -212,7 +209,7 @@ void nano::telemetry::run_requests ()
 	}
 }
 
-void nano::telemetry::request (std::shared_ptr<nano::transport::channel> & channel)
+void nano::telemetry::request (std::shared_ptr<nano::transport::channel> const & channel)
 {
 	stats.inc (nano::stat::type::telemetry, nano::stat::detail::request);
 
@@ -231,7 +228,7 @@ void nano::telemetry::run_broadcasts ()
 	}
 }
 
-void nano::telemetry::broadcast (std::shared_ptr<nano::transport::channel> & channel, const nano::telemetry_data & telemetry)
+void nano::telemetry::broadcast (std::shared_ptr<nano::transport::channel> const & channel, const nano::telemetry_data & telemetry)
 {
 	stats.inc (nano::stat::type::telemetry, nano::stat::detail::broadcast);
 
@@ -247,10 +244,14 @@ void nano::telemetry::cleanup ()
 		// Remove if telemetry data is stale
 		if (!check_timeout (entry))
 		{
-			stats.inc (nano::stat::type::telemetry, nano::stat::detail::cleanup_outdated);
+			stats.inc (nano::stat::type::telemetry, nano::stat::detail::erase_stale);
+			return true; // Erase
+		}
+		if (!entry.channel->alive ())
+		{
+			stats.inc (nano::stat::type::telemetry, nano::stat::detail::erase_dead);
 			return true; // Erase
 		}
-
 		return false; // Do not erase
 	});
 }
@@ -283,7 +284,7 @@ std::unordered_map<nano::endpoint, nano::telemetry_data> nano::telemetry::get_al
 	{
 		if (check_timeout (entry))
 		{
-			result[entry.endpoint] = entry.data;
+			result[entry.endpoint ()] = entry.data;
 		}
 	}
 	return result;

nano/node/telemetry.hpp

@@ -86,10 +86,14 @@ private: // Dependencies
 private:
 	struct entry
 	{
-		nano::endpoint endpoint;
+		std::shared_ptr<nano::transport::channel> channel;
 		nano::telemetry_data data;
 		std::chrono::steady_clock::time_point last_updated;
-		std::shared_ptr<nano::transport::channel> channel;
+
+		nano::endpoint endpoint () const
+		{
+			return channel->get_remote_endpoint ();
+		}
 	};
 
 private:
@@ -101,8 +105,8 @@ private:
 	void run_broadcasts ();
 	void cleanup ();
 
-	void request (std::shared_ptr<nano::transport::channel> &);
-	void broadcast (std::shared_ptr<nano::transport::channel> &, nano::telemetry_data const &);
+	void request (std::shared_ptr<nano::transport::channel> const &);
+	void broadcast (std::shared_ptr<nano::transport::channel> const &, nano::telemetry_data const &);
 
 	bool verify (nano::telemetry_ack const &, std::shared_ptr<nano::transport::channel> const &) const;
 	bool check_timeout (entry const &) const;
@@ -110,13 +114,16 @@ private:
 private:
 	// clang-format off
 	class tag_sequenced {};
+	class tag_channel {};
 	class tag_endpoint {};
 
 	using ordered_telemetries = boost::multi_index_container<entry,
 	mi::indexed_by<
 		mi::sequenced<mi::tag<tag_sequenced>>,
-		mi::hashed_unique<mi::tag<tag_endpoint>,
-			mi::member<entry, nano::endpoint, &entry::endpoint>>
+		mi::ordered_unique<mi::tag<tag_channel>,
+			mi::member<entry,  std::shared_ptr<nano::transport::channel>, &entry::channel>>,
+		mi::hashed_non_unique<mi::tag<tag_endpoint>,
+			mi::const_mem_fun<entry, nano::endpoint, &entry::endpoint>>
 	>>;
 	// clang-format on
 

nano/node/transport/channel.cpp

@@ -45,27 +45,31 @@ void nano::transport::channel::send (nano::message & message_a, std::function<vo
 
 void nano::transport::channel::set_peering_endpoint (nano::endpoint endpoint)
 {
-	nano::lock_guard<nano::mutex> lock{ channel_mutex };
+	nano::lock_guard<nano::mutex> lock{ mutex };
 	peering_endpoint = endpoint;
 }
 
 nano::endpoint nano::transport::channel::get_peering_endpoint () const
 {
-	nano::unique_lock<nano::mutex> lock{ channel_mutex };
-	if (peering_endpoint)
 	{
-		return *peering_endpoint;
-	}
-	else
-	{
-		lock.unlock ();
-		return get_endpoint ();
+		nano::lock_guard<nano::mutex> lock{ mutex };
+		if (peering_endpoint)
+		{
+			return *peering_endpoint;
+		}
 	}
+	return get_remote_endpoint ();
+}
+
+std::shared_ptr<nano::node> nano::transport::channel::owner () const
+{
+	return node.shared ();
 }
 
 void nano::transport::channel::operator() (nano::object_stream & obs) const
 {
-	obs.write ("endpoint", get_endpoint ());
+	obs.write ("remote_endpoint", get_remote_endpoint ());
+	obs.write ("local_endpoint", get_local_endpoint ());
 	obs.write ("peering_endpoint", get_peering_endpoint ());
 	obs.write ("node_id", get_node_id ().to_node_id ());
 }

nano/node/transport/channel.hpp

@@ -40,10 +40,10 @@ public:
 
 	virtual void close () = 0;
 
-	virtual std::string to_string () const = 0;
-	virtual nano::endpoint get_endpoint () const = 0;
-	virtual nano::tcp_endpoint get_tcp_endpoint () const = 0;
+	virtual nano::endpoint get_remote_endpoint () const = 0;
 	virtual nano::endpoint get_local_endpoint () const = 0;
+
+	virtual std::string to_string () const = 0;
 	virtual nano::transport::transport_type get_type () const = 0;
 
 	virtual bool max (nano::transport::traffic_type = nano::transport::traffic_type::generic)
@@ -58,62 +58,55 @@ public:
 
 	std::chrono::steady_clock::time_point get_last_bootstrap_attempt () const
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		return last_bootstrap_attempt;
 	}
 
 	void set_last_bootstrap_attempt (std::chrono::steady_clock::time_point const time_a)
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		last_bootstrap_attempt = time_a;
 	}
 
 	std::chrono::steady_clock::time_point get_last_packet_received () const
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		return last_packet_received;
 	}
 
 	void set_last_packet_received (std::chrono::steady_clock::time_point const time_a)
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		last_packet_received = time_a;
 	}
 
 	std::chrono::steady_clock::time_point get_last_packet_sent () const
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		return last_packet_sent;
 	}
 
 	void set_last_packet_sent (std::chrono::steady_clock::time_point const time_a)
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		last_packet_sent = time_a;
 	}
 
-	boost::optional<nano::account> get_node_id_optional () const
+	std::optional<nano::account> get_node_id_optional () const
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		return node_id;
 	}
 
 	nano::account get_node_id () const
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
-		if (node_id.is_initialized ())
-		{
-			return node_id.get ();
-		}
-		else
-		{
-			return 0;
-		}
+		nano::lock_guard<nano::mutex> lock{ mutex };
+		return node_id.value_or (0);
 	}
 
 	void set_node_id (nano::account node_id_a)
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		node_id = node_id_a;
 	}
 
@@ -130,19 +123,20 @@ public:
 	nano::endpoint get_peering_endpoint () const;
 	void set_peering_endpoint (nano::endpoint endpoint);
 
-	mutable nano::mutex channel_mutex;
+	std::shared_ptr<nano::node> owner () const;
+
+protected:
+	nano::node & node;
+	mutable nano::mutex mutex;
 
 private:
 	std::chrono::steady_clock::time_point last_bootstrap_attempt{ std::chrono::steady_clock::time_point () };
 	std::chrono::steady_clock::time_point last_packet_received{ std::chrono::steady_clock::now () };
 	std::chrono::steady_clock::time_point last_packet_sent{ std::chrono::steady_clock::now () };
-	boost::optional<nano::account> node_id{ boost::none };
+	std::optional<nano::account> node_id{};
 	std::atomic<uint8_t> network_version{ 0 };
 	std::optional<nano::endpoint> peering_endpoint{};
 
-protected:
-	nano::node & node;
-
 public: // Logging
 	virtual void operator() (nano::object_stream &) const;
 };

nano/node/transport/fake.hpp

@@ -30,16 +30,11 @@ namespace transport
 				endpoint = endpoint_a;
 			}
 
-			nano::endpoint get_endpoint () const override
+			nano::endpoint get_remote_endpoint () const override
 			{
 				return endpoint;
 			}
 
-			nano::tcp_endpoint get_tcp_endpoint () const override
-			{
-				return nano::transport::map_endpoint_to_tcp (endpoint);
-			}
-
 			nano::endpoint get_local_endpoint () const override
 			{
 				return endpoint;

nano/node/transport/inproc.cpp

@@ -44,7 +44,7 @@ void nano::transport::inproc::channel::send_buffer (nano::shared_const_buffer co
 		// process message
 		{
 			node.stats.inc (nano::stat::type::message, to_stat_detail (message_a->type ()), nano::stat::dir::in);
-			destination.network.inbound (*message_a, remote_channel);
+			destination.inbound (*message_a, remote_channel);
 		}
 	});
 

nano/node/transport/inproc.hpp

@@ -22,16 +22,11 @@ namespace transport
 
 			std::string to_string () const override;
 
-			nano::endpoint get_endpoint () const override
+			nano::endpoint get_remote_endpoint () const override
 			{
 				return endpoint;
 			}
 
-			nano::tcp_endpoint get_tcp_endpoint () const override
-			{
-				return nano::transport::map_endpoint_to_tcp (endpoint);
-			}
-
 			nano::endpoint get_local_endpoint () const override
 			{
 				return endpoint;

nano/node/transport/tcp_channel.cpp

@@ -16,8 +16,6 @@ nano::transport::tcp_channel::tcp_channel (nano::node & node_a, std::weak_ptr<na
 
 nano::transport::tcp_channel::~tcp_channel ()
 {
-	nano::lock_guard<nano::mutex> lk{ channel_mutex };
-	// Close socket. Exception: socket is used by tcp_server
 	if (auto socket_l = socket.lock ())
 	{
 		socket_l->close ();
@@ -26,14 +24,14 @@ nano::transport::tcp_channel::~tcp_channel ()
 
 void nano::transport::tcp_channel::update_endpoints ()
 {
-	nano::lock_guard<nano::mutex> lk (channel_mutex);
+	nano::lock_guard<nano::mutex> lock{ mutex };
 
-	debug_assert (endpoint == nano::endpoint{}); // Not initialized endpoint value
+	debug_assert (remote_endpoint == nano::endpoint{}); // Not initialized endpoint value
 	debug_assert (local_endpoint == nano::endpoint{}); // Not initialized endpoint value
 
 	if (auto socket_l = socket.lock ())
 	{
-		endpoint = socket_l->remote_endpoint ();
+		remote_endpoint = socket_l->remote_endpoint ();
 		local_endpoint = socket_l->local_endpoint ();
 	}
 }
@@ -90,7 +88,7 @@ void nano::transport::tcp_channel::send_buffer (nano::shared_const_buffer const
 
 std::string nano::transport::tcp_channel::to_string () const
 {
-	return nano::util::to_str (get_tcp_endpoint ());
+	return nano::util::to_str (get_remote_endpoint ());
 }
 
 void nano::transport::tcp_channel::operator() (nano::object_stream & obs) const

nano/node/transport/tcp_channel.hpp

@@ -24,20 +24,15 @@ public:
 
 	std::string to_string () const override;
 
-	nano::endpoint get_endpoint () const override
+	nano::endpoint get_remote_endpoint () const override
 	{
-		return nano::transport::map_tcp_to_endpoint (get_tcp_endpoint ());
-	}
-
-	nano::tcp_endpoint get_tcp_endpoint () const override
-	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
-		return endpoint;
+		nano::lock_guard<nano::mutex> lock{ mutex };
+		return remote_endpoint;
 	}
 
 	nano::endpoint get_local_endpoint () const override
 	{
-		nano::lock_guard<nano::mutex> lk (channel_mutex);
+		nano::lock_guard<nano::mutex> lock{ mutex };
 		return local_endpoint;
 	}
 
@@ -77,7 +72,7 @@ public:
 	std::weak_ptr<nano::transport::tcp_socket> socket;
 
 private:
-	nano::endpoint endpoint;
+	nano::endpoint remote_endpoint;
 	nano::endpoint local_endpoint;
 
 public: // Logging

nano/node/transport/tcp_channels.cpp

@@ -196,9 +196,9 @@ void nano::transport::tcp_channels::random_fill (std::array<nano::endpoint, 8> &
 	auto j (target_a.begin ());
 	for (auto i (peers.begin ()), n (peers.end ()); i != n; ++i, ++j)
 	{
-		debug_assert ((*i)->get_endpoint ().address ().is_v6 ());
+		debug_assert ((*i)->get_remote_endpoint ().address ().is_v6 ());
 		debug_assert (j < target_a.end ());
-		*j = (*i)->get_endpoint ();
+		*j = (*i)->get_remote_endpoint ();
 	}
 }
 
@@ -417,18 +417,6 @@ void nano::transport::tcp_channels::list (std::deque<std::shared_ptr<nano::trans
 	// clang-format on
 }
 
-void nano::transport::tcp_channels::modify (std::shared_ptr<nano::transport::tcp_channel> const & channel_a, std::function<void (std::shared_ptr<nano::transport::tcp_channel> const &)> modify_callback_a)
-{
-	nano::lock_guard<nano::mutex> lock{ mutex };
-	auto existing (channels.get<endpoint_tag> ().find (channel_a->get_tcp_endpoint ()));
-	if (existing != channels.get<endpoint_tag> ().end ())
-	{
-		channels.get<endpoint_tag> ().modify (existing, [modify_callback = std::move (modify_callback_a)] (channel_entry & wrapper_a) {
-			modify_callback (wrapper_a.channel);
-		});
-	}
-}
-
 void nano::transport::tcp_channels::start_tcp (nano::endpoint const & endpoint)
 {
 	node.tcp_listener.connect (endpoint.address (), endpoint.port ());