How to elegantly configure a Logger with a configuration file while supporting log rotation

Question

Problem Description

• Function: Test1() is a log rotation and cutting library gopkg.in/natefinch/lumberjack.v2 recommended in the official documentation.
• Function: Test2() is a Logger that uses yaml to read configuration based on the basic configuration in the official documentation.

After executing the main function,

in the console output:

2023-05-15T08:49:16.555+0800 | INFO | logger construction succeeded:config from yaml | {"app": "jpz"}

in the log file foo.log output:

{"level":"info","ts":1684111756.5545945,"msg":"logger construction succeeded:lumberjack.Logger"}

These two logs are definitely different.

My current requirements:

1. Both support using the configuration file config_log_zap.yaml so that all configurations can take effect, and let lumberjack complete the log rotation and splitting work.

2. The output of both the console and the log file should be the same, so that I can quickly apply the desired content through the configuration file. The reason why both the console and the log file are needed is because I need to pay attention to and record past output messages during development.

   In the console output:

   2023-05-15T08:49:16.555+0800 | INFO | logger construction succeeded:config from yaml | {"app": "jpz"}

   In the log file foo.log output:

   2023-05-15T08:49:16.555+0800 | INFO | logger construction succeeded:config from yaml | {"app": "jpz"}

3. How should I merge Test1() and Test2() into one function Test0() to meet the above two requirements?

Please give me some help, I've been studying it for a long time.

main.go

package main

import (
    "gopkg.in/yaml.v3"
    "os"

    "go.uber.org/zap"
    "go.uber.org/zap/zapcore"
    "gopkg.in/natefinch/lumberjack.v2"
)

func Test1() {
    // lumberjack.Logger is already safe for concurrent use, so we don't need to
    // lock it.
    w := zapcore.AddSync(&lumberjack.Logger{
        Filename:   "./foo.log",
        MaxSize:    500, // megabytes
        MaxBackups: 3,
        MaxAge:     28, // days
    })
    core := zapcore.NewCore(
        zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()),
        w,
        zap.InfoLevel,
    )
    logger := zap.New(core)
    logger.Info("logger construction succeeded:lumberjack.Logger")
}

func Test2() {
    var cfg zap.Config
    yamlFile, _ := os.ReadFile("./config_log_zap.yaml")
    if err := yaml.Unmarshal(yamlFile, &cfg); err != nil {
        panic(err)
    }

    logger := zap.Must(cfg.Build())
    defer logger.Sync()

    logger.Info("logger construction succeeded:config from yaml")
}

func main() {
    Test1()
    Test2()
}

config_log_zap.yaml

# For the full description for the configuration, see
# https://github.com/uber-go/zap/blob/382e2511e51cda8afde24f9e6e741f934308edfa/config.go#L58-L94
level: 'debug'
development: true
disableCaller: true
disableStacktrace: false
sampling:
  initial: 100
  thereafter: 100
encoding: 'console'
encoderConfig:
  messageKey: 'msg'
  levelKey: 'level'
  timeKey: 'ts'
  nameKey: 'logger'
  callerKey: 'caller'
  functionKey: 'function'
  stacktraceKey: 'stacktrace'
  skipLineEnding: false
  lineEnding: "\n"
  levelEncoder: 'capital'
  timeEncoder: 'iso8601'
  durationEncoder: 'string'
  callerEncoder: 'full'
  nameEncoder: 'full'
  consoleSeparator: ' | '
outputPaths:
  - 'stdout'
  - './foo.log'
errorOutputPaths:
  - 'stderr'
  - './error_logs'
initialFields:
  app: 'jpz'

Answer 1

Use zap.RegisterSink to register the lumberjack logger as a new sink:

package main

import (
    "net/url"
    "os"
    "strconv"
    "strings"

    "gopkg.in/yaml.v3"

    "go.uber.org/zap"
    "gopkg.in/natefinch/lumberjack.v2"
)

type lumberjackSink struct {
    lumberjack.Logger
}

func (l *lumberjackSink) Sync() error {
    return nil
}

func parseNumber(s string, fallback int) int {
    v, err := strconv.Atoi(s)
    if err == nil {
        return v
    }
    return fallback
}

func Test0() {
    if err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) {
        // Read parameters from URL:
        // lumberjack://localhost/foo.log?maxSize=500&maxBackups=3&maxAge=28
        filename := strings.TrimLeft(u.Path, "/")
        if filename == "" {
            filename = "foo.log"
        }
        q := u.Query()
        l := &lumberjackSink{
            Logger: lumberjack.Logger{
                Filename:   filename,
                MaxSize:    parseNumber(q.Get("maxSize"), 500),
                MaxBackups: parseNumber(q.Get("maxBackups"), 3),
                MaxAge:     parseNumber(q.Get("maxAge"), 28),
            },
        }
        return l, nil
    }); err != nil {
        panic(err)
    }

    var cfg zap.Config
    yamlFile, _ := os.ReadFile("./config_log_zap.yaml")
    if err := yaml.Unmarshal(yamlFile, &cfg); err != nil {
        panic(err)
    }

    logger := zap.Must(cfg.Build())
    defer logger.Sync()

    logger.Info("logger construction succeeded:config from yaml")
}

func main() {
    Test0()
}

And modify the config file to set outputPaths like this:

outputPaths:
  - stdout
  - lumberjack://localhost/foo.log?maxSize=500&maxBackups=3&maxAge=28