Graceful: Xorm, RepoIndexer, Cron and Others (#9282)

* Change graceful to use a singleton obtained through GetManager instead of a global.
* Graceful: Make TestPullRequests shutdownable
* Graceful: Make the cron tasks graceful
* Graceful: AddTestPullRequest run in graceful ctx
* Graceful: SyncMirrors shutdown
* Graceful: SetDefaultContext for Xorm to be HammerContext
* Avoid starting graceful for migrate commands and checkout
* Graceful: DeliverHooks now can be shutdown
* Fix multiple syncing errors in modules/sync/UniqueQueue &  Make UniqueQueue closable
* Begin the process of making the repo indexer shutdown gracefully

modules/graceful/manager_unix.go

@@ -19,7 +19,8 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 )
 
-type gracefulManager struct {
+// Manager manages the graceful shutdown process
+type Manager struct {
 	isChild                bool
 	forked                 bool
 	lock                   *sync.RWMutex
@@ -27,27 +28,37 @@ type gracefulManager struct {
 	shutdown               chan struct{}
 	hammer                 chan struct{}
 	terminate              chan struct{}
+	done                   chan struct{}
 	runningServerWaitGroup sync.WaitGroup
 	createServerWaitGroup  sync.WaitGroup
 	terminateWaitGroup     sync.WaitGroup
 }
 
-func newGracefulManager(ctx context.Context) *gracefulManager {
-	manager := &gracefulManager{
+func newGracefulManager(ctx context.Context) *Manager {
+	manager := &Manager{
 		isChild: len(os.Getenv(listenFDs)) > 0 && os.Getppid() > 1,
 		lock:    &sync.RWMutex{},
 	}
 	manager.createServerWaitGroup.Add(numberOfServersToCreate)
-	manager.Run(ctx)
+	manager.start(ctx)
 	return manager
 }
 
-func (g *gracefulManager) Run(ctx context.Context) {
+func (g *Manager) start(ctx context.Context) {
+	// Make channels
+	g.terminate = make(chan struct{})
+	g.shutdown = make(chan struct{})
+	g.hammer = make(chan struct{})
+	g.done = make(chan struct{})
+
+	// Set the running state & handle signals
 	g.setState(stateRunning)
 	go g.handleSignals(ctx)
-	c := make(chan struct{})
+
+	// Handle clean up of unused provided listeners	and delayed start-up
+	startupDone := make(chan struct{})
 	go func() {
-		defer close(c)
+		defer close(startupDone)
 		// Wait till we're done getting all of the listeners and then close
 		// the unused ones
 		g.createServerWaitGroup.Wait()
@@ -58,9 +69,19 @@ func (g *gracefulManager) Run(ctx context.Context) {
 	if setting.StartupTimeout > 0 {
 		go func() {
 			select {
-			case <-c:
+			case <-startupDone:
 				return
 			case <-g.IsShutdown():
+				func() {
+					// When waitgroup counter goes negative it will panic - we don't care about this so we can just ignore it.
+					defer func() {
+						_ = recover()
+					}()
+					// Ensure that the createServerWaitGroup stops waiting
+					for {
+						g.createServerWaitGroup.Done()
+					}
+				}()
 				return
 			case <-time.After(setting.StartupTimeout):
 				log.Error("Startup took too long! Shutting down")
@@ -70,7 +91,7 @@ func (g *gracefulManager) Run(ctx context.Context) {
 	}
 }
 
-func (g *gracefulManager) handleSignals(ctx context.Context) {
+func (g *Manager) handleSignals(ctx context.Context) {
 	signalChannel := make(chan os.Signal, 1)
 
 	signal.Notify(
@@ -123,7 +144,7 @@ func (g *gracefulManager) handleSignals(ctx context.Context) {
 	}
 }
 
-func (g *gracefulManager) doFork() error {
+func (g *Manager) doFork() error {
 	g.lock.Lock()
 	if g.forked {
 		g.lock.Unlock()
@@ -139,7 +160,9 @@ func (g *gracefulManager) doFork() error {
 	return err
 }
 
-func (g *gracefulManager) RegisterServer() {
+// RegisterServer registers the running of a listening server, in the case of unix this means that the parent process can now die.
+// Any call to RegisterServer must be matched by a call to ServerDone
+func (g *Manager) RegisterServer() {
 	KillParent()
 	g.runningServerWaitGroup.Add(1)
 }