Warn about ledger processing cooldown (#4921)

* Warn when aec notifications queue grows

* Warn about ledger processing cooldown

nano/node/active_elections.cpp

@@ -47,6 +47,11 @@ nano::active_elections::active_elections (nano::node & node_a, nano::ledger_noti
 			}
 		}
 
+		if (workers.queued_tasks () >= nano::queue_warning_threshold () && warning_interval.elapse (15s))
+		{
+			node.logger.warn (nano::log::type::active_elections, "Notification queue has {} tasks", workers.queued_tasks ());
+		}
+
 		// Notify observers about cemented blocks on a background thread
 		workers.post ([this, results = std::move (results)] () {
 			auto transaction = node.ledger.tx_begin_read ();

nano/node/active_elections.hpp

@@ -171,6 +171,7 @@ private:
 	nano::thread_pool workers;
 
 	nano::interval bootstrap_stale_interval;
+	nano::interval warning_interval;
 
 	friend class election;
 

nano/node/block_processor.cpp

@@ -277,6 +277,10 @@ void nano::block_processor::run ()
 		// It's possible that ledger processing happens faster than the notifications can be processed by other components, cooldown here
 		ledger_notifications.wait ([this] {
 			stats.inc (nano::stat::type::block_processor, nano::stat::detail::cooldown);
+			if (log_cooldown_interval.elapse (15s))
+			{
+				logger.warn (nano::log::type::block_processor, "Cooldown in block processing, waiting for remaining ledger notifications to be processed");
+			}
 		});
 
 		lock.lock ();

nano/node/block_processor.hpp

@@ -104,5 +104,6 @@ private:
 
 	nano::interval log_processing_interval;
 	nano::interval log_backlog_interval;
+	nano::interval log_cooldown_interval;
 };
 }