Pause, Resume, Release&Reopen, Add and Remove Logging from command line (#11777)

* Make LogDescriptions race safe

* Add manager commands for pausing, resuming, adding and removing loggers

Signed-off-by: Andrew Thornton <[email protected]>

* Placate lint

* Ensure that file logger is run!

* Add support for smtp and conn

Signed-off-by: Andrew Thornton <[email protected]>

* Add release-and-reopen

Signed-off-by: Andrew Thornton <[email protected]>

Co-authored-by: techknowlogick <[email protected]>
Co-authored-by: Lauris BH <[email protected]>

Author: 3 people

cmd/manager.go

@@ -316,6 +316,28 @@ COLORIZE = true # Or false if your windows terminal cannot color
 
 This is equivalent to sending all logs to the console, with default go log being sent to the console log too.
 
+## Releasing-and-Reopening, Pausing and Resuming logging
+
+If you are running on Unix you may wish to release-and-reopen logs in order to use `logrotate` or other tools.
+It is possible force gitea to release and reopen it's logging files and connections by sending `SIGUSR1` to the
+running process, or running `gitea manager logging release-and-reopen`.
+
+Alternatively, you may wish to pause and resume logging - this can be accomplished through the use of the 
+`gitea manager logging pause` and `gitea manager logging resume` commands. Please note that whilst logging
+is paused log events below INFO level will not be stored and only a limited number of events will be stored.
+Logging may block, albeit temporarily, slowing gitea considerably whilst paused - therefore it is
+recommended that pausing only done for a very short period of time.
+
+## Adding and removing logging whilst Gitea is running
+
+It is possible to add and remove logging whilst Gitea is running using the `gitea manager logging add` and `remove` subcommands.
+This functionality can only adjust running log systems and cannot be used to start the access, macaron or router loggers if they
+were not already initialised. If you wish to start these systems you are advised to adjust the app.ini and (gracefully) restart
+the Gitea service.
+
+The main intention of these commands is to easily add a temporary logger to investigate problems on running systems where a restart
+may cause the issue to disappear.
+
 ## Log colorization
 
 Logs to the console will be colorized by default when not running on

docs/content/doc/advanced/logging-documentation.en-us.md

@@ -318,3 +318,85 @@ var checklist = []check{
 ```
 
 This function will receive a command line context and return a list of details about the problems or error.
+
+#### manager
+
+Manage running server operations:
+
+- Commands:
+  - `shutdown`:      Gracefully shutdown the running process
+  - `restart`:       Gracefully restart the running process - (not implemented for windows servers)
+  - `flush-queues`:  Flush queues in the running process
+    - Options:
+      - `--timeout value`: Timeout for the flushing process (default: 1m0s)
+      - `--non-blocking`: Set to true to not wait for flush to complete before returning
+  - `logging`:       Adjust logging commands
+    - Commands:
+      - `pause`:   Pause logging
+        - Notes:
+          - The logging level will be raised to INFO temporarily if it is below this level.
+          - Gitea will buffer logs up to a certain point and will drop them after that point.
+      - `resume`:  Resume logging
+      - `release-and-reopen`: Cause Gitea to release and re-open files and connections used for logging (Equivalent to sending SIGUSR1 to Gitea.)
+      - `remove name`: Remove the named logger
+        - Options:
+          - `--group group`, `-g group`: Set the group to remove the sublogger from. (defaults to `default`)
+      - `add`:     Add a logger
+        - Commands:
+          - `console`: Add a console logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`: Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--stderr`: Output console logs to stderr - only relevant for console
+          - `file`: Add a file logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`:  Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--filename value`, `-f value`: Filename for the logger - 
+              - `--rotate`, `-r`: Rotate logs
+              - `--max-size value`, `-s value`: Maximum size in bytes before rotation
+              - `--daily`, `-d`: Rotate logs daily
+              - `--max-days value`, `-D value`: Maximum number of daily logs to keep
+              - `--compress`, `-z`: Compress rotated logs
+              - `--compression-level value`, `-Z value`: Compression level to use
+          - `conn`: Add a network connection logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`:  Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--reconnect-on-message`, `-R`: Reconnect to host for every message
+              - `--reconnect`, `-r`: Reconnect to host when connection is dropped
+              - `--protocol value`, `-P value`: Set protocol to use: tcp, unix, or udp (defaults to tcp)
+              - `--address value`, `-a value`: Host address and port to connect to (defaults to :7020)
+          - `smtp`: Add an SMTP logger
+            - Options:
+              - `--group value`, `-g value`: Group to add logger to - will default to "default"
+              - `--name value`, `-n value`: Name of the new logger - will default to mode
+              - `--level value`, `-l value`: Logging level for the new logger
+              - `--stacktrace-level value`, `-L value`: Stacktrace logging level
+              - `--flags value`, `-F value`: Flags for the logger
+              - `--expression value`, `-e value`: Matching expression for the logger
+              - `--prefix value`, `-p value`: Prefix for the logger
+              - `--color`: Use color in the logs
+              - `--username value`, `-u value`: Mail server username
+              - `--password value`, `-P value`: Mail server password
+              - `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25)
+              - `--send-to value`, `-s value`: Email address(es) to send to
+              - `--subject value`, `-S value`: Subject header of sent emails

docs/content/doc/usage/command-line.en-us.md

@@ -170,6 +170,11 @@ func (log *TestLogger) Init(config string) error {
 func (log *TestLogger) Flush() {
 }
 
+//ReleaseReopen does nothing
+func (log *TestLogger) ReleaseReopen() error {
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *TestLogger) GetName() string {
 	return "test"

integrations/testlogger.go

@@ -113,7 +113,10 @@ func (g *Manager) handleSignals(ctx context.Context) {
 				log.Info("PID: %d. Received SIGHUP. Attempting GracefulRestart...", pid)
 				g.DoGracefulRestart()
 			case syscall.SIGUSR1:
-				log.Info("PID %d. Received SIGUSR1.", pid)
+				log.Warn("PID %d. Received SIGUSR1. Releasing and reopening logs", pid)
+				if err := log.ReleaseReopen(); err != nil {
+					log.Error("Error whilst releasing and reopening logs: %v", err)
+				}
 			case syscall.SIGUSR2:
 				log.Warn("PID %d. Received SIGUSR2. Hammering...", pid)
 				g.DoImmediateHammer()

modules/graceful/manager_unix.go

@@ -77,6 +77,13 @@ func (i *connWriter) connect() error {
 	return nil
 }
 
+func (i *connWriter) releaseReopen() error {
+	if i.innerWriter != nil {
+		return i.connect()
+	}
+	return nil
+}
+
 // ConnLogger implements LoggerProvider.
 // it writes messages in keep-live tcp connection.
 type ConnLogger struct {
@@ -119,6 +126,11 @@ func (log *ConnLogger) GetName() string {
 	return "conn"
 }
 
+// ReleaseReopen causes the ConnLogger to reconnect to the server
+func (log *ConnLogger) ReleaseReopen() error {
+	return log.out.(*connWriter).releaseReopen()
+}
+
 func init() {
 	Register("conn", NewConn)
 }

modules/log/conn.go

@@ -68,6 +68,20 @@ func (log *ConsoleLogger) Init(config string) error {
 func (log *ConsoleLogger) Flush() {
 }
 
+// ReleaseReopen causes the console logger to reconnect to os.Stdout
+func (log *ConsoleLogger) ReleaseReopen() error {
+	if log.Stderr {
+		log.NewWriterLogger(&nopWriteCloser{
+			w: os.Stderr,
+		})
+	} else {
+		log.NewWriterLogger(&nopWriteCloser{
+			w: os.Stdout,
+		})
+	}
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *ConsoleLogger) GetName() string {
 	return "console"

modules/log/console.go

@@ -29,6 +29,7 @@ type EventLogger interface {
 	GetLevel() Level
 	GetStacktraceLevel() Level
 	GetName() string
+	ReleaseReopen() error
 }
 
 // ChannelledLog represents a cached channel to a LoggerProvider
@@ -117,6 +118,11 @@ func (l *ChannelledLog) Flush() {
 	l.flush <- true
 }
 
+// ReleaseReopen this ChannelledLog
+func (l *ChannelledLog) ReleaseReopen() error {
+	return l.loggerProvider.ReleaseReopen()
+}
+
 // GetLevel gets the level of this ChannelledLog
 func (l *ChannelledLog) GetLevel() Level {
 	return l.loggerProvider.GetLevel()
@@ -145,6 +151,7 @@ type MultiChannelledLog struct {
 	level           Level
 	stacktraceLevel Level
 	closed          chan bool
+	paused          chan bool
 }
 
 // NewMultiChannelledLog a new logger instance with given logger provider and config.
@@ -159,6 +166,7 @@ func NewMultiChannelledLog(name string, bufferLength int64) *MultiChannelledLog
 		stacktraceLevel: NONE,
 		close:           make(chan bool),
 		closed:          make(chan bool),
+		paused:          make(chan bool),
 	}
 	return m
 }
@@ -229,6 +237,33 @@ func (m *MultiChannelledLog) closeLoggers() {
 	m.closed <- true
 }
 
+// Pause pauses this Logger
+func (m *MultiChannelledLog) Pause() {
+	m.paused <- true
+}
+
+// Resume resumes this Logger
+func (m *MultiChannelledLog) Resume() {
+	m.paused <- false
+}
+
+// ReleaseReopen causes this logger to tell its subloggers to release and reopen
+func (m *MultiChannelledLog) ReleaseReopen() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	var accumulatedErr error
+	for _, logger := range m.loggers {
+		if err := logger.ReleaseReopen(); err != nil {
+			if accumulatedErr == nil {
+				accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %v", logger.GetName(), err)
+			} else {
+				accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %v & %v", logger.GetName(), err, accumulatedErr)
+			}
+		}
+	}
+	return accumulatedErr
+}
+
 // Start processing the MultiChannelledLog
 func (m *MultiChannelledLog) Start() {
 	m.mutex.Lock()
@@ -238,8 +273,35 @@ func (m *MultiChannelledLog) Start() {
 	}
 	m.started = true
 	m.mutex.Unlock()
+	paused := false
 	for {
+		if paused {
+			select {
+			case paused = <-m.paused:
+				if !paused {
+					m.ResetLevel()
+				}
+			case _, ok := <-m.flush:
+				if !ok {
+					m.closeLoggers()
+					return
+				}
+				m.mutex.Lock()
+				for _, logger := range m.loggers {
+					logger.Flush()
+				}
+				m.mutex.Unlock()
+			case <-m.close:
+				m.closeLoggers()
+				return
+			}
+			continue
+		}
 		select {
+		case paused = <-m.paused:
+			if paused && m.level < INFO {
+				m.level = INFO
+			}
 		case event, ok := <-m.queue:
 			if !ok {
 				m.closeLoggers()
@@ -275,7 +337,7 @@ func (m *MultiChannelledLog) LogEvent(event *Event) error {
 	select {
 	case m.queue <- event:
 		return nil
-	case <-time.After(60 * time.Second):
+	case <-time.After(100 * time.Millisecond):
 		// We're blocked!
 		return ErrTimeout{
 			Name:     m.name,

modules/log/event.go

@@ -249,6 +249,19 @@ func (log *FileLogger) Flush() {
 	_ = log.mw.fd.Sync()
 }
 
+// ReleaseReopen releases and reopens log files
+func (log *FileLogger) ReleaseReopen() error {
+	closingErr := log.mw.fd.Close()
+	startingErr := log.StartLogger()
+	if startingErr != nil {
+		if closingErr != nil {
+			return fmt.Errorf("Error during closing: %v Error during starting: %v", closingErr, startingErr)
+		}
+		return startingErr
+	}
+	return closingErr
+}
+
 // GetName returns the default name for this implementation
 func (log *FileLogger) GetName() string {
 	return "file"

modules/log/file.go

@@ -5,6 +5,7 @@
 package log
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 	"strings"
@@ -192,6 +193,42 @@ func IsFatal() bool {
 	return GetLevel() <= FATAL
 }
 
+// Pause pauses all the loggers
+func Pause() {
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		logger.Pause()
+		logger.Flush()
+		return true
+	})
+}
+
+// Resume resumes all the loggers
+func Resume() {
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		logger.Resume()
+		return true
+	})
+}
+
+// ReleaseReopen releases and reopens logging files
+func ReleaseReopen() error {
+	var accumulatedErr error
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		if err := logger.ReleaseReopen(); err != nil {
+			if accumulatedErr == nil {
+				accumulatedErr = fmt.Errorf("Error reopening %s: %v", key.(string), err)
+			} else {
+				accumulatedErr = fmt.Errorf("Error reopening %s: %v & %v", key.(string), err, accumulatedErr)
+			}
+		}
+		return true
+	})
+	return accumulatedErr
+}
+
 // Close closes all the loggers
 func Close() {
 	l, ok := NamedLoggers.Load(DEFAULT)

modules/log/log.go

@@ -97,6 +97,11 @@ func (log *SMTPLogger) sendMail(p []byte) (int, error) {
 func (log *SMTPLogger) Flush() {
 }
 
+// ReleaseReopen does nothing
+func (log *SMTPLogger) ReleaseReopen() error {
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *SMTPLogger) GetName() string {
 	return "smtp"