Handle panics that percolate up to the graceful module (#11291)

* Handle panics in graceful goroutines

Adds a some deferred functions to handle panics in graceful goroutines

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Handle panic in webhook.Deliver

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Handle panic in mirror.syncMirror

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

1 files changed, 34 insertions, 1 deletions

diff --git a/modules/graceful/manager.go b/modules/graceful/manager.go
index eec675e297..6b134e7d0c 100644
--- a/modules/graceful/manager.go
+++ b/modules/graceful/manager.go
@@ -74,6 +74,12 @@ type RunnableWithShutdownFns func(atShutdown, atTerminate func(context.Context,
 func (g *Manager) RunWithShutdownFns(run RunnableWithShutdownFns) {
 	g.runningServerWaitGroup.Add(1)
 	defer g.runningServerWaitGroup.Done()
+	defer func() {
+		if err := recover(); err != nil {
+			log.Critical("PANIC during RunWithShutdownFns: %v\nStacktrace: %s", err, log.Stack(2))
+			g.doShutdown()
+		}
+	}()
 	run(func(ctx context.Context, atShutdown func()) {
 		go func() {
 			select {
@@ -103,6 +109,12 @@ type RunnableWithShutdownChan func(atShutdown <-chan struct{}, atTerminate Callb
 func (g *Manager) RunWithShutdownChan(run RunnableWithShutdownChan) {
 	g.runningServerWaitGroup.Add(1)
 	defer g.runningServerWaitGroup.Done()
+	defer func() {
+		if err := recover(); err != nil {
+			log.Critical("PANIC during RunWithShutdownChan: %v\nStacktrace: %s", err, log.Stack(2))
+			g.doShutdown()
+		}
+	}()
 	run(g.IsShutdown(), func(ctx context.Context, atTerminate func()) {
 		g.RunAtTerminate(ctx, atTerminate)
 	})
@@ -114,6 +126,12 @@ func (g *Manager) RunWithShutdownChan(run RunnableWithShutdownChan) {
 func (g *Manager) RunWithShutdownContext(run func(context.Context)) {
 	g.runningServerWaitGroup.Add(1)
 	defer g.runningServerWaitGroup.Done()
+	defer func() {
+		if err := recover(); err != nil {
+			log.Critical("PANIC during RunWithShutdownContext: %v\nStacktrace: %s", err, log.Stack(2))
+			g.doShutdown()
+		}
+	}()
 	run(g.ShutdownContext())
 }
 
@@ -121,18 +139,28 @@ func (g *Manager) RunWithShutdownContext(run func(context.Context)) {
 func (g *Manager) RunAtTerminate(ctx context.Context, terminate func()) {
 	g.terminateWaitGroup.Add(1)
 	go func() {
+		defer g.terminateWaitGroup.Done()
+		defer func() {
+			if err := recover(); err != nil {
+				log.Critical("PANIC during RunAtTerminate: %v\nStacktrace: %s", err, log.Stack(2))
+			}
+		}()
 		select {
 		case <-g.IsTerminate():
 			terminate()
 		case <-ctx.Done():
 		}
-		g.terminateWaitGroup.Done()
 	}()
 }
 
 // RunAtShutdown creates a go-routine to run the provided function at shutdown
 func (g *Manager) RunAtShutdown(ctx context.Context, shutdown func()) {
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Critical("PANIC during RunAtShutdown: %v\nStacktrace: %s", err, log.Stack(2))
+			}
+		}()
 		select {
 		case <-g.IsShutdown():
 			shutdown()
@@ -144,6 +172,11 @@ func (g *Manager) RunAtShutdown(ctx context.Context, shutdown func()) {
 // RunAtHammer creates a go-routine to run the provided function at shutdown
 func (g *Manager) RunAtHammer(ctx context.Context, hammer func()) {
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Critical("PANIC during RunAtHammer: %v\nStacktrace: %s", err, log.Stack(2))
+			}
+		}()
 		select {
 		case <-g.IsHammer():
 			hammer()