Pause, Resume, Release&Reopen, Add and Remove Logging from command line (#11777)

* Make LogDescriptions race safe

* Add manager commands for pausing, resuming, adding and removing loggers

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Placate lint

* Ensure that file logger is run!

* Add support for smtp and conn

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add release-and-reopen

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Lauris BH <lauris@nix.lv>

17 files changed, 924 insertions, 17 deletions

diff --git a/cmd/manager.go b/cmd/manager.go
index eed0a9e823..20c7858682 100644
--- a/cmd/manager.go
+++ b/cmd/manager.go
@@ -10,6 +10,7 @@ import (
 	"os"
 	"time"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
 
 	"github.com/urfave/cli"
@@ -25,16 +26,27 @@ var (
 			subcmdShutdown,
 			subcmdRestart,
 			subcmdFlushQueues,
+			subcmdLogging,
 		},
 	}
 	subcmdShutdown = cli.Command{
-		Name:   "shutdown",
-		Usage:  "Gracefully shutdown the running process",
+		Name:  "shutdown",
+		Usage: "Gracefully shutdown the running process",
+		Flags: []cli.Flag{
+			cli.BoolFlag{
+				Name: "debug",
+			},
+		},
 		Action: runShutdown,
 	}
 	subcmdRestart = cli.Command{
-		Name:   "restart",
-		Usage:  "Gracefully restart the running process - (not implemented for windows servers)",
+		Name:  "restart",
+		Usage: "Gracefully restart the running process - (not implemented for windows servers)",
+		Flags: []cli.Flag{
+			cli.BoolFlag{
+				Name: "debug",
+			},
+		},
 		Action: runRestart,
 	}
 	subcmdFlushQueues = cli.Command{
@@ -46,17 +58,331 @@ var (
 				Name:  "timeout",
 				Value: 60 * time.Second,
 				Usage: "Timeout for the flushing process",
-			},
-			cli.BoolFlag{
+			}, cli.BoolFlag{
 				Name:  "non-blocking",
 				Usage: "Set to true to not wait for flush to complete before returning",
 			},
+			cli.BoolFlag{
+				Name: "debug",
+			},
+		},
+	}
+	defaultLoggingFlags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "group, g",
+			Usage: "Group to add logger to - will default to \"default\"",
+		}, cli.StringFlag{
+			Name:  "name, n",
+			Usage: "Name of the new logger - will default to mode",
+		}, cli.StringFlag{
+			Name:  "level, l",
+			Usage: "Logging level for the new logger",
+		}, cli.StringFlag{
+			Name:  "stacktrace-level, L",
+			Usage: "Stacktrace logging level",
+		}, cli.StringFlag{
+			Name:  "flags, F",
+			Usage: "Flags for the logger",
+		}, cli.StringFlag{
+			Name:  "expression, e",
+			Usage: "Matching expression for the logger",
+		}, cli.StringFlag{
+			Name:  "prefix, p",
+			Usage: "Prefix for the logger",
+		}, cli.BoolFlag{
+			Name:  "color",
+			Usage: "Use color in the logs",
+		}, cli.BoolFlag{
+			Name: "debug",
+		},
+	}
+	subcmdLogging = cli.Command{
+		Name:  "logging",
+		Usage: "Adjust logging commands",
+		Subcommands: []cli.Command{
+			{
+				Name:  "pause",
+				Usage: "Pause logging (Gitea will buffer logs up to a certain point and will drop them after that point)",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					},
+				},
+				Action: runPauseLogging,
+			}, {
+				Name:  "resume",
+				Usage: "Resume logging",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					},
+				},
+				Action: runResumeLogging,
+			}, {
+				Name:  "release-and-reopen",
+				Usage: "Cause Gitea to release and re-open files used for logging",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					},
+				},
+				Action: runReleaseReopenLogging,
+			}, {
+				Name:      "remove",
+				Usage:     "Remove a logger",
+				ArgsUsage: "[name] Name of logger to remove",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					}, cli.StringFlag{
+						Name:  "group, g",
+						Usage: "Group to add logger to - will default to \"default\"",
+					},
+				},
+				Action: runRemoveLogger,
+			}, {
+				Name:  "add",
+				Usage: "Add a logger",
+				Subcommands: []cli.Command{
+					{
+						Name:  "console",
+						Usage: "Add a console logger",
+						Flags: append(defaultLoggingFlags,
+							cli.BoolFlag{
+								Name:  "stderr",
+								Usage: "Output console logs to stderr - only relevant for console",
+							}),
+						Action: runAddConsoleLogger,
+					}, {
+						Name:  "file",
+						Usage: "Add a file logger",
+						Flags: append(defaultLoggingFlags, []cli.Flag{
+							cli.StringFlag{
+								Name:  "filename, f",
+								Usage: "Filename for the logger - this must be set.",
+							}, cli.BoolTFlag{
+								Name:  "rotate, r",
+								Usage: "Rotate logs",
+							}, cli.Int64Flag{
+								Name:  "max-size, s",
+								Usage: "Maximum size in bytes before rotation",
+							}, cli.BoolTFlag{
+								Name:  "daily, d",
+								Usage: "Rotate logs daily",
+							}, cli.IntFlag{
+								Name:  "max-days, D",
+								Usage: "Maximum number of daily logs to keep",
+							}, cli.BoolTFlag{
+								Name:  "compress, z",
+								Usage: "Compress rotated logs",
+							}, cli.IntFlag{
+								Name:  "compression-level, Z",
+								Usage: "Compression level to use",
+							},
+						}...),
+						Action: runAddFileLogger,
+					}, {
+						Name:  "conn",
+						Usage: "Add a net conn logger",
+						Flags: append(defaultLoggingFlags, []cli.Flag{
+							cli.BoolFlag{
+								Name:  "reconnect-on-message, R",
+								Usage: "Reconnect to host for every message",
+							}, cli.BoolFlag{
+								Name:  "reconnect, r",
+								Usage: "Reconnect to host when connection is dropped",
+							}, cli.StringFlag{
+								Name:  "protocol, P",
+								Usage: "Set protocol to use: tcp, unix, or udp (defaults to tcp)",
+							}, cli.StringFlag{
+								Name:  "address, a",
+								Usage: "Host address and port to connect to (defaults to :7020)",
+							},
+						}...),
+						Action: runAddConnLogger,
+					}, {
+						Name:  "smtp",
+						Usage: "Add an SMTP logger",
+						Flags: append(defaultLoggingFlags, []cli.Flag{
+							cli.StringFlag{
+								Name:  "username, u",
+								Usage: "Mail server username",
+							}, cli.StringFlag{
+								Name:  "password, P",
+								Usage: "Mail server password",
+							}, cli.StringFlag{
+								Name:  "host, H",
+								Usage: "Mail server host (defaults to: 127.0.0.1:25)",
+							}, cli.StringSliceFlag{
+								Name:  "send-to, s",
+								Usage: "Email address(es) to send to",
+							}, cli.StringFlag{
+								Name:  "subject, S",
+								Usage: "Subject header of sent emails",
+							},
+						}...),
+						Action: runAddSMTPLogger,
+					},
+				},
+			},
 		},
 	}
 )
 
+func runRemoveLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	group := c.String("group")
+	if len(group) == 0 {
+		group = log.DEFAULT
+	}
+	name := c.Args().First()
+	statusCode, msg := private.RemoveLogger(group, name)
+	switch statusCode {
+	case http.StatusInternalServerError:
+		fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
+func runAddSMTPLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "smtp"
+	if c.IsSet("host") {
+		vals["host"] = c.String("host")
+	} else {
+		vals["host"] = "127.0.0.1:25"
+	}
+
+	if c.IsSet("username") {
+		vals["username"] = c.String("username")
+	}
+	if c.IsSet("password") {
+		vals["password"] = c.String("password")
+	}
+
+	if !c.IsSet("send-to") {
+		return fmt.Errorf("Some recipients must be provided")
+	}
+	vals["sendTos"] = c.StringSlice("send-to")
+
+	if c.IsSet("subject") {
+		vals["subject"] = c.String("subject")
+	} else {
+		vals["subject"] = "Diagnostic message from Gitea"
+	}
+
+	return commonAddLogger(c, mode, vals)
+}
+
+func runAddConnLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "conn"
+	vals["net"] = "tcp"
+	if c.IsSet("protocol") {
+		switch c.String("protocol") {
+		case "udp":
+			vals["net"] = "udp"
+		case "unix":
+			vals["net"] = "unix"
+		}
+	}
+	if c.IsSet("address") {
+		vals["address"] = c.String("address")
+	} else {
+		vals["address"] = ":7020"
+	}
+	if c.IsSet("reconnect") {
+		vals["reconnect"] = c.Bool("reconnect")
+	}
+	if c.IsSet("reconnect-on-message") {
+		vals["reconnectOnMsg"] = c.Bool("reconnect-on-message")
+	}
+	return commonAddLogger(c, mode, vals)
+}
+
+func runAddFileLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "file"
+	if c.IsSet("filename") {
+		vals["filename"] = c.String("filename")
+	} else {
+		return fmt.Errorf("filename must be set when creating a file logger")
+	}
+	if c.IsSet("rotate") {
+		vals["rotate"] = c.Bool("rotate")
+	}
+	if c.IsSet("max-size") {
+		vals["maxsize"] = c.Int64("max-size")
+	}
+	if c.IsSet("daily") {
+		vals["daily"] = c.Bool("daily")
+	}
+	if c.IsSet("max-days") {
+		vals["maxdays"] = c.Int("max-days")
+	}
+	if c.IsSet("compress") {
+		vals["compress"] = c.Bool("compress")
+	}
+	if c.IsSet("compression-level") {
+		vals["compressionLevel"] = c.Int("compression-level")
+	}
+	return commonAddLogger(c, mode, vals)
+}
+
+func runAddConsoleLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "console"
+	if c.IsSet("stderr") && c.Bool("stderr") {
+		vals["stderr"] = c.Bool("stderr")
+	}
+	return commonAddLogger(c, mode, vals)
+}
+
+func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) error {
+	if len(c.String("level")) > 0 {
+		vals["level"] = log.FromString(c.String("level")).String()
+	}
+	if len(c.String("stacktrace-level")) > 0 {
+		vals["stacktraceLevel"] = log.FromString(c.String("stacktrace-level")).String()
+	}
+	if len(c.String("expression")) > 0 {
+		vals["expression"] = c.String("expression")
+	}
+	if len(c.String("prefix")) > 0 {
+		vals["prefix"] = c.String("prefix")
+	}
+	if len(c.String("flags")) > 0 {
+		vals["flags"] = log.FlagsFromString(c.String("flags"))
+	}
+	if c.IsSet("color") {
+		vals["colorize"] = c.Bool("color")
+	}
+	group := "default"
+	if c.IsSet("group") {
+		group = c.String("group")
+	}
+	name := mode
+	if c.IsSet("name") {
+		name = c.String("name")
+	}
+	statusCode, msg := private.AddLogger(group, name, mode, vals)
+	switch statusCode {
+	case http.StatusInternalServerError:
+		fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
 func runShutdown(c *cli.Context) error {
-	setup("manager", false)
+	setup("manager", c.Bool("debug"))
 	statusCode, msg := private.Shutdown()
 	switch statusCode {
 	case http.StatusInternalServerError:
@@ -68,7 +394,7 @@ func runShutdown(c *cli.Context) error {
 }
 
 func runRestart(c *cli.Context) error {
-	setup("manager", false)
+	setup("manager", c.Bool("debug"))
 	statusCode, msg := private.Restart()
 	switch statusCode {
 	case http.StatusInternalServerError:
@@ -80,7 +406,7 @@ func runRestart(c *cli.Context) error {
 }
 
 func runFlushQueues(c *cli.Context) error {
-	setup("manager", false)
+	setup("manager", c.Bool("debug"))
 	statusCode, msg := private.FlushQueues(c.Duration("timeout"), c.Bool("non-blocking"))
 	switch statusCode {
 	case http.StatusInternalServerError:
@@ -90,3 +416,39 @@ func runFlushQueues(c *cli.Context) error {
 	fmt.Fprintln(os.Stdout, msg)
 	return nil
 }
+
+func runPauseLogging(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	statusCode, msg := private.PauseLogging()
+	switch statusCode {
+	case http.StatusInternalServerError:
+		fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
+func runResumeLogging(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	statusCode, msg := private.ResumeLogging()
+	switch statusCode {
+	case http.StatusInternalServerError:
+		fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
+func runReleaseReopenLogging(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	statusCode, msg := private.ReleaseReopenLogging()
+	switch statusCode {
+	case http.StatusInternalServerError:
+		fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
diff --git a/docs/content/doc/advanced/logging-documentation.en-us.md b/docs/content/doc/advanced/logging-documentation.en-us.md
index 919ccf783b..f3880be7c4 100644
--- a/docs/content/doc/advanced/logging-documentation.en-us.md
+++ b/docs/content/doc/advanced/logging-documentation.en-us.md
@@ -316,6 +316,28 @@ COLORIZE = true # Or false if your windows terminal cannot color
 
 This is equivalent to sending all logs to the console, with default go log being sent to the console log too.
 
+## Releasing-and-Reopening, Pausing and Resuming logging
+
+If you are running on Unix you may wish to release-and-reopen logs in order to use `logrotate` or other tools.
+It is possible force gitea to release and reopen it's logging files and connections by sending `SIGUSR1` to the
+running process, or running `gitea manager logging release-and-reopen`.
+
+Alternatively, you may wish to pause and resume logging - this can be accomplished through the use of the 
+`gitea manager logging pause` and `gitea manager logging resume` commands. Please note that whilst logging
+is paused log events below INFO level will not be stored and only a limited number of events will be stored.
+Logging may block, albeit temporarily, slowing gitea considerably whilst paused - therefore it is
+recommended that pausing only done for a very short period of time.
+
+## Adding and removing logging whilst Gitea is running
+
+It is possible to add and remove logging whilst Gitea is running using the `gitea manager logging add` and `remove` subcommands.
+This functionality can only adjust running log systems and cannot be used to start the access, macaron or router loggers if they
+were not already initialised. If you wish to start these systems you are advised to adjust the app.ini and (gracefully) restart
+the Gitea service.
+
+The main intention of these commands is to easily add a temporary logger to investigate problems on running systems where a restart
+may cause the issue to disappear.
+
 ## Log colorization
 
 Logs to the console will be colorized by default when not running on
diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md
index c0236f913d..3715be7cbd 100644
--- a/docs/content/doc/usage/command-line.en-us.md
+++ b/docs/content/doc/usage/command-line.en-us.md
@@ -318,3 +318,85 @@ var checklist = []check{
 ```
 
 This function will receive a command line context and return a list of details about the problems or error.
+
+#### manager
+
+Manage running server operations:
+
+- Commands:
+  - `shutdown`:      Gracefully shutdown the running process
+  - `restart`:       Gracefully restart the running process - (not implemented for windows servers)
+  - `flush-queues`:  Flush queues in the running process
+    - Options:
+      - `--timeout value`: Timeout for the flushing process (default: 1m0s)
+      - `--non-blocking`: Set to true to not wait for flush to complete before returning
+  - `logging`:       Adjust logging commands
+    - Commands:
+      - `pause`:   Pause logging
+        - Notes:
+          - The logging level will be raised to INFO temporarily if it is below this level.
+          - Gitea will buffer logs up to a certain point and will drop them after that point.
+      - `resume`:  Resume logging
+      - `release-and-reopen`: Cause Gitea to release and re-open files and connections used for logging (Equivalent to sending SIGUSR1 to Gitea.)
+      - `remove name`: Remove the named logger
+        - Options:
+          - `--group group`, `-g group`: Set the group to remove the sublogger from. (defaults to `default`)
+      - `add`:     Add a logger
+        - Commands:
+          - `console`: Add a console logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`: Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--stderr`: Output console logs to stderr - only relevant for console
+          - `file`: Add a file logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`:  Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--filename value`, `-f value`: Filename for the logger - 
+              - `--rotate`, `-r`: Rotate logs
+              - `--max-size value`, `-s value`: Maximum size in bytes before rotation
+              - `--daily`, `-d`: Rotate logs daily
+              - `--max-days value`, `-D value`: Maximum number of daily logs to keep
+              - `--compress`, `-z`: Compress rotated logs
+              - `--compression-level value`, `-Z value`: Compression level to use
+          - `conn`: Add a network connection logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`:  Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--reconnect-on-message`, `-R`: Reconnect to host for every message
+              - `--reconnect`, `-r`: Reconnect to host when connection is dropped
+              - `--protocol value`, `-P value`: Set protocol to use: tcp, unix, or udp (defaults to tcp)
+              - `--address value`, `-a value`: Host address and port to connect to (defaults to :7020)
+          - `smtp`: Add an SMTP logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`: Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--username value`, `-u value`: Mail server username
+              - `--password value`, `-P value`: Mail server password
+              - `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25)
+              - `--send-to value`, `-s value`: Email address(es) to send to
+              - `--subject value`, `-S value`: Subject header of sent emails
diff --git a/integrations/testlogger.go b/integrations/testlogger.go
index 9636c4892e..f84ed47e4f 100644
--- a/integrations/testlogger.go
+++ b/integrations/testlogger.go
@@ -170,6 +170,11 @@ func (log *TestLogger) Init(config string) error {
 func (log *TestLogger) Flush() {
 }
 
+//ReleaseReopen does nothing
+func (log *TestLogger) ReleaseReopen() error {
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *TestLogger) GetName() string {
 	return "test"
diff --git a/modules/graceful/manager_unix.go b/modules/graceful/manager_unix.go
index d56a4558b7..540974454c 100644
--- a/modules/graceful/manager_unix.go
+++ b/modules/graceful/manager_unix.go
@@ -113,7 +113,10 @@ func (g *Manager) handleSignals(ctx context.Context) {
 				log.Info("PID: %d. Received SIGHUP. Attempting GracefulRestart...", pid)
 				g.DoGracefulRestart()
 			case syscall.SIGUSR1:
-				log.Info("PID %d. Received SIGUSR1.", pid)
+				log.Warn("PID %d. Received SIGUSR1. Releasing and reopening logs", pid)
+				if err := log.ReleaseReopen(); err != nil {
+					log.Error("Error whilst releasing and reopening logs: %v", err)
+				}
 			case syscall.SIGUSR2:
 				log.Warn("PID %d. Received SIGUSR2. Hammering...", pid)
 				g.DoImmediateHammer()
diff --git a/modules/log/conn.go b/modules/log/conn.go
index 8816664526..1abe44c1d4 100644
--- a/modules/log/conn.go
+++ b/modules/log/conn.go
@@ -77,6 +77,13 @@ func (i *connWriter) connect() error {
 	return nil
 }
 
+func (i *connWriter) releaseReopen() error {
+	if i.innerWriter != nil {
+		return i.connect()
+	}
+	return nil
+}
+
 // ConnLogger implements LoggerProvider.
 // it writes messages in keep-live tcp connection.
 type ConnLogger struct {
@@ -119,6 +126,11 @@ func (log *ConnLogger) GetName() string {
 	return "conn"
 }
 
+// ReleaseReopen causes the ConnLogger to reconnect to the server
+func (log *ConnLogger) ReleaseReopen() error {
+	return log.out.(*connWriter).releaseReopen()
+}
+
 func init() {
 	Register("conn", NewConn)
 }
diff --git a/modules/log/console.go b/modules/log/console.go
index 6cfca8a733..a805021f0b 100644
--- a/modules/log/console.go
+++ b/modules/log/console.go
@@ -68,6 +68,20 @@ func (log *ConsoleLogger) Init(config string) error {
 func (log *ConsoleLogger) Flush() {
 }
 
+// ReleaseReopen causes the console logger to reconnect to os.Stdout
+func (log *ConsoleLogger) ReleaseReopen() error {
+	if log.Stderr {
+		log.NewWriterLogger(&nopWriteCloser{
+			w: os.Stderr,
+		})
+	} else {
+		log.NewWriterLogger(&nopWriteCloser{
+			w: os.Stdout,
+		})
+	}
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *ConsoleLogger) GetName() string {
 	return "console"
diff --git a/modules/log/event.go b/modules/log/event.go
index 37efa3c230..6975bf749d 100644
--- a/modules/log/event.go
+++ b/modules/log/event.go
@@ -29,6 +29,7 @@ type EventLogger interface {
 	GetLevel() Level
 	GetStacktraceLevel() Level
 	GetName() string
+	ReleaseReopen() error
 }
 
 // ChannelledLog represents a cached channel to a LoggerProvider
@@ -117,6 +118,11 @@ func (l *ChannelledLog) Flush() {
 	l.flush <- true
 }
 
+// ReleaseReopen this ChannelledLog
+func (l *ChannelledLog) ReleaseReopen() error {
+	return l.loggerProvider.ReleaseReopen()
+}
+
 // GetLevel gets the level of this ChannelledLog
 func (l *ChannelledLog) GetLevel() Level {
 	return l.loggerProvider.GetLevel()
@@ -145,6 +151,7 @@ type MultiChannelledLog struct {
 	level           Level
 	stacktraceLevel Level
 	closed          chan bool
+	paused          chan bool
 }
 
 // NewMultiChannelledLog a new logger instance with given logger provider and config.
@@ -159,6 +166,7 @@ func NewMultiChannelledLog(name string, bufferLength int64) *MultiChannelledLog
 		stacktraceLevel: NONE,
 		close:           make(chan bool),
 		closed:          make(chan bool),
+		paused:          make(chan bool),
 	}
 	return m
 }
@@ -229,6 +237,33 @@ func (m *MultiChannelledLog) closeLoggers() {
 	m.closed <- true
 }
 
+// Pause pauses this Logger
+func (m *MultiChannelledLog) Pause() {
+	m.paused <- true
+}
+
+// Resume resumes this Logger
+func (m *MultiChannelledLog) Resume() {
+	m.paused <- false
+}
+
+// ReleaseReopen causes this logger to tell its subloggers to release and reopen
+func (m *MultiChannelledLog) ReleaseReopen() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	var accumulatedErr error
+	for _, logger := range m.loggers {
+		if err := logger.ReleaseReopen(); err != nil {
+			if accumulatedErr == nil {
+				accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %v", logger.GetName(), err)
+			} else {
+				accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %v & %v", logger.GetName(), err, accumulatedErr)
+			}
+		}
+	}
+	return accumulatedErr
+}
+
 // Start processing the MultiChannelledLog
 func (m *MultiChannelledLog) Start() {
 	m.mutex.Lock()
@@ -238,8 +273,35 @@ func (m *MultiChannelledLog) Start() {
 	}
 	m.started = true
 	m.mutex.Unlock()
+	paused := false
 	for {
+		if paused {
+			select {
+			case paused = <-m.paused:
+				if !paused {
+					m.ResetLevel()
+				}
+			case _, ok := <-m.flush:
+				if !ok {
+					m.closeLoggers()
+					return
+				}
+				m.mutex.Lock()
+				for _, logger := range m.loggers {
+					logger.Flush()
+				}
+				m.mutex.Unlock()
+			case <-m.close:
+				m.closeLoggers()
+				return
+			}
+			continue
+		}
 		select {
+		case paused = <-m.paused:
+			if paused && m.level < INFO {
+				m.level = INFO
+			}
 		case event, ok := <-m.queue:
 			if !ok {
 				m.closeLoggers()
@@ -275,7 +337,7 @@ func (m *MultiChannelledLog) LogEvent(event *Event) error {
 	select {
 	case m.queue <- event:
 		return nil
-	case <-time.After(60 * time.Second):
+	case <-time.After(100 * time.Millisecond):
 		// We're blocked!
 		return ErrTimeout{
 			Name:     m.name,
diff --git a/modules/log/file.go b/modules/log/file.go
index 877820b8be..925d83f2b7 100644
--- a/modules/log/file.go
+++ b/modules/log/file.go
@@ -249,6 +249,19 @@ func (log *FileLogger) Flush() {
 	_ = log.mw.fd.Sync()
 }
 
+// ReleaseReopen releases and reopens log files
+func (log *FileLogger) ReleaseReopen() error {
+	closingErr := log.mw.fd.Close()
+	startingErr := log.StartLogger()
+	if startingErr != nil {
+		if closingErr != nil {
+			return fmt.Errorf("Error during closing: %v Error during starting: %v", closingErr, startingErr)
+		}
+		return startingErr
+	}
+	return closingErr
+}
+
 // GetName returns the default name for this implementation
 func (log *FileLogger) GetName() string {
 	return "file"
diff --git a/modules/log/log.go b/modules/log/log.go
index 71e88491f1..2a35b5752c 100644
--- a/modules/log/log.go
+++ b/modules/log/log.go
@@ -5,6 +5,7 @@
 package log
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 	"strings"
@@ -192,6 +193,42 @@ func IsFatal() bool {
 	return GetLevel() <= FATAL
 }
 
+// Pause pauses all the loggers
+func Pause() {
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		logger.Pause()
+		logger.Flush()
+		return true
+	})
+}
+
+// Resume resumes all the loggers
+func Resume() {
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		logger.Resume()
+		return true
+	})
+}
+
+// ReleaseReopen releases and reopens logging files
+func ReleaseReopen() error {
+	var accumulatedErr error
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		if err := logger.ReleaseReopen(); err != nil {
+			if accumulatedErr == nil {
+				accumulatedErr = fmt.Errorf("Error reopening %s: %v", key.(string), err)
+			} else {
+				accumulatedErr = fmt.Errorf("Error reopening %s: %v & %v", key.(string), err, accumulatedErr)
+			}
+		}
+		return true
+	})
+	return accumulatedErr
+}
+
 // Close closes all the loggers
 func Close() {
 	l, ok := NamedLoggers.Load(DEFAULT)
diff --git a/modules/log/smtp.go b/modules/log/smtp.go
index f912299a73..edf4943619 100644
--- a/modules/log/smtp.go
+++ b/modules/log/smtp.go
@@ -97,6 +97,11 @@ func (log *SMTPLogger) sendMail(p []byte) (int, error) {
 func (log *SMTPLogger) Flush() {
 }
 
+// ReleaseReopen does nothing
+func (log *SMTPLogger) ReleaseReopen() error {
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *SMTPLogger) GetName() string {
 	return "smtp"
diff --git a/modules/private/manager.go b/modules/private/manager.go
index 503acf17d6..6c9ec920bb 100644
--- a/modules/private/manager.go
+++ b/modules/private/manager.go
@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -81,3 +82,110 @@ func FlushQueues(timeout time.Duration, nonBlocking bool) (int, string) {
 
 	return http.StatusOK, "Flushed"
 }
+
+// PauseLogging pauses logging
+func PauseLogging() (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/pause-logging"
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Logging Paused"
+}
+
+// ResumeLogging resumes logging
+func ResumeLogging() (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/resume-logging"
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Logging Restarted"
+}
+
+// ReleaseReopenLogging releases and reopens logging files
+func ReleaseReopenLogging() (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/release-and-reopen-logging"
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Logging Restarted"
+}
+
+// LoggerOptions represents the options for the add logger call
+type LoggerOptions struct {
+	Group  string
+	Name   string
+	Mode   string
+	Config map[string]interface{}
+}
+
+// AddLogger adds a logger
+func AddLogger(group, name, mode string, config map[string]interface{}) (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/add-logger"
+
+	req := newInternalRequest(reqURL, "POST")
+	req = req.Header("Content-Type", "application/json")
+	jsonBytes, _ := json.Marshal(LoggerOptions{
+		Group:  group,
+		Name:   name,
+		Mode:   mode,
+		Config: config,
+	})
+	req.Body(jsonBytes)
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Added"
+
+}
+
+// RemoveLogger removes a logger
+func RemoveLogger(group, name string) (int, string) {
+	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/remove-logger/%s/%s", url.PathEscape(group), url.PathEscape(name))
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Removed"
+}
diff --git a/modules/setting/log.go b/modules/setting/log.go
index 5ffb2479dd..35bf021ac2 100644
--- a/modules/setting/log.go
+++ b/modules/setting/log.go
@@ -12,6 +12,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/modules/log"
 
@@ -20,6 +21,69 @@ import (
 
 var filenameSuffix = ""
 
+var descriptionLock = sync.RWMutex{}
+var logDescriptions = make(map[string]*LogDescription)
+
+// GetLogDescriptions returns a race safe set of descriptions
+func GetLogDescriptions() map[string]*LogDescription {
+	descriptionLock.RLock()
+	defer descriptionLock.RUnlock()
+	descs := make(map[string]*LogDescription, len(logDescriptions))
+	for k, v := range logDescriptions {
+		subLogDescriptions := make([]SubLogDescription, len(v.SubLogDescriptions))
+		for i, s := range v.SubLogDescriptions {
+			subLogDescriptions[i] = s
+		}
+		descs[k] = &LogDescription{
+			Name:               v.Name,
+			SubLogDescriptions: subLogDescriptions,
+		}
+	}
+	return descs
+}
+
+// AddLogDescription adds a set of descriptions to the complete description
+func AddLogDescription(key string, description *LogDescription) {
+	descriptionLock.Lock()
+	defer descriptionLock.Unlock()
+	logDescriptions[key] = description
+}
+
+// AddSubLogDescription adds a sub log description
+func AddSubLogDescription(key string, subLogDescription SubLogDescription) bool {
+	descriptionLock.Lock()
+	defer descriptionLock.Unlock()
+	desc, ok := logDescriptions[key]
+	if !ok {
+		return false
+	}
+	for i, sub := range desc.SubLogDescriptions {
+		if sub.Name == subLogDescription.Name {
+			desc.SubLogDescriptions[i] = subLogDescription
+			return true
+		}
+	}
+	desc.SubLogDescriptions = append(desc.SubLogDescriptions, subLogDescription)
+	return true
+}
+
+// RemoveSubLogDescription removes a sub log description
+func RemoveSubLogDescription(key string, name string) bool {
+	descriptionLock.Lock()
+	defer descriptionLock.Unlock()
+	desc, ok := logDescriptions[key]
+	if !ok {
+		return false
+	}
+	for i, sub := range desc.SubLogDescriptions {
+		if sub.Name == name {
+			desc.SubLogDescriptions = append(desc.SubLogDescriptions[:i], desc.SubLogDescriptions[i+1:]...)
+			return true
+		}
+	}
+	return false
+}
+
 type defaultLogOptions struct {
 	levelName      string // LogLevel
 	flags          string
@@ -185,7 +249,7 @@ func generateNamedLogger(key string, options defaultLogOptions) *LogDescription
 		log.Info("%s Log: %s(%s:%s)", strings.Title(key), strings.Title(name), provider, levelName)
 	}
 
-	LogDescriptions[key] = &description
+	AddLogDescription(key, &description)
 
 	return &description
 }
@@ -279,7 +343,7 @@ func newLogService() {
 		log.Info("Gitea Log Mode: %s(%s:%s)", strings.Title(name), strings.Title(provider), levelName)
 	}
 
-	LogDescriptions[log.DEFAULT] = &description
+	AddLogDescription(log.DEFAULT, &description)
 
 	// Finally redirect the default golog to here
 	golog.SetFlags(0)
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 8efc832f9d..4c2fba8048 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -289,7 +289,6 @@ var (
 	LogLevel           string
 	StacktraceLogLevel string
 	LogRootPath        string
-	LogDescriptions    = make(map[string]*LogDescription)
 	RedirectMacaronLog bool
 	DisableRouterLog   bool
 	RouterLogLevel     log.Level
diff --git a/routers/admin/admin.go b/routers/admin/admin.go
index e48e416258..f43c1e69c5 100644
--- a/routers/admin/admin.go
+++ b/routers/admin/admin.go
@@ -307,7 +307,7 @@ func Config(ctx *context.Context) {
 	}
 
 	ctx.Data["EnvVars"] = envVars
-	ctx.Data["Loggers"] = setting.LogDescriptions
+	ctx.Data["Loggers"] = setting.GetLogDescriptions()
 	ctx.Data["RedirectMacaronLog"] = setting.RedirectMacaronLog
 	ctx.Data["EnableAccessLog"] = setting.EnableAccessLog
 	ctx.Data["AccessLogTemplate"] = setting.AccessLogTemplate
diff --git a/routers/private/internal.go b/routers/private/internal.go
index 5bc01b0aea..821cf62a61 100644
--- a/routers/private/internal.go
+++ b/routers/private/internal.go
@@ -42,6 +42,10 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Post("/manager/shutdown", Shutdown)
 		m.Post("/manager/restart", Restart)
 		m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues)
-
+		m.Post("/manager/pause-logging", PauseLogging)
+		m.Post("/manager/resume-logging", ResumeLogging)
+		m.Post("/manager/release-and-reopen-logging", ReleaseReopenLogging)
+		m.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger)
+		m.Post("/manager/remove-logger/:group/:name", RemoveLogger)
 	}, CheckInternalToken)
 }
diff --git a/routers/private/manager.go b/routers/private/manager.go
index 1238ff2d28..67bd92003f 100644
--- a/routers/private/manager.go
+++ b/routers/private/manager.go
@@ -5,12 +5,15 @@
 package private
 
 import (
+	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
 	"code.gitea.io/gitea/modules/queue"
+	"code.gitea.io/gitea/modules/setting"
 
 	"gitea.com/macaron/macaron"
 )
@@ -34,8 +37,120 @@ func FlushQueues(ctx *macaron.Context, opts private.FlushOptions) {
 	err := queue.GetManager().FlushAll(ctx.Req.Request.Context(), opts.Timeout)
 	if err != nil {
 		ctx.JSON(http.StatusRequestTimeout, map[string]interface{}{
-			"err": err,
+			"err": fmt.Sprintf("%v", err),
 		})
 	}
 	ctx.PlainText(http.StatusOK, []byte("success"))
 }
+
+// PauseLogging pauses logging
+func PauseLogging(ctx *macaron.Context) {
+	log.Pause()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}
+
+// ResumeLogging resumes logging
+func ResumeLogging(ctx *macaron.Context) {
+	log.Resume()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}
+
+// ReleaseReopenLogging releases and reopens logging files
+func ReleaseReopenLogging(ctx *macaron.Context) {
+	if err := log.ReleaseReopen(); err != nil {
+		ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"err": fmt.Sprintf("Error during release and reopen: %v", err),
+		})
+		return
+	}
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}
+
+// RemoveLogger removes a logger
+func RemoveLogger(ctx *macaron.Context) {
+	group := ctx.Params("group")
+	name := ctx.Params("name")
+	ok, err := log.GetLogger(group).DelLogger(name)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"err": fmt.Sprintf("Failed to remove logger: %s %s %v", group, name, err),
+		})
+		return
+	}
+	if ok {
+		setting.RemoveSubLogDescription(group, name)
+	}
+	ctx.PlainText(http.StatusOK, []byte(fmt.Sprintf("Removed %s %s", group, name)))
+}
+
+// AddLogger adds a logger
+func AddLogger(ctx *macaron.Context, opts private.LoggerOptions) {
+	if len(opts.Group) == 0 {
+		opts.Group = log.DEFAULT
+	}
+	if _, ok := opts.Config["flags"]; !ok {
+		switch opts.Group {
+		case "access":
+			opts.Config["flags"] = log.FlagsFromString("")
+		case "router":
+			opts.Config["flags"] = log.FlagsFromString("date,time")
+		default:
+			opts.Config["flags"] = log.FlagsFromString("stdflags")
+		}
+	}
+
+	if _, ok := opts.Config["colorize"]; !ok && opts.Mode == "console" {
+		if _, ok := opts.Config["stderr"]; ok {
+			opts.Config["colorize"] = log.CanColorStderr
+		} else {
+			opts.Config["colorize"] = log.CanColorStdout
+		}
+	}
+
+	if _, ok := opts.Config["level"]; !ok {
+		opts.Config["level"] = setting.LogLevel
+	}
+
+	if _, ok := opts.Config["stacktraceLevel"]; !ok {
+		opts.Config["stacktraceLevel"] = setting.StacktraceLogLevel
+	}
+
+	if opts.Mode == "file" {
+		if _, ok := opts.Config["maxsize"]; !ok {
+			opts.Config["maxsize"] = 1 << 28
+		}
+		if _, ok := opts.Config["maxdays"]; !ok {
+			opts.Config["maxdays"] = 7
+		}
+		if _, ok := opts.Config["compressionLevel"]; !ok {
+			opts.Config["compressionLevel"] = -1
+		}
+	}
+
+	bufferLen := setting.Cfg.Section("log").Key("BUFFER_LEN").MustInt64(10000)
+	byteConfig, err := json.Marshal(opts.Config)
+	if err != nil {
+		log.Error("Failed to marshal log configuration: %v %v", opts.Config, err)
+		ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"err": fmt.Sprintf("Failed to marshal log configuration: %v %v", opts.Config, err),
+		})
+		return
+	}
+	config := string(byteConfig)
+
+	if err := log.NewNamedLogger(opts.Group, bufferLen, opts.Name, opts.Mode, config); err != nil {
+		log.Error("Failed to create new named logger: %s %v", config, err)
+		ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"err": fmt.Sprintf("Failed to create new named logger: %s %v", config, err),
+		})
+		return
+	}
+
+	setting.AddSubLogDescription(opts.Group, setting.SubLogDescription{
+		Name:     opts.Name,
+		Provider: opts.Mode,
+		Config:   config,
+	})
+
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}