« Back to Index

[Python and Go Structured Logging]

View original Gist on GitHub

Tags: #go #golang #python #logging #structured #logrus #structlog

1. go structured logging.go

package main

import (
  "os"

  log "github.com/Sirupsen/logrus"
)

func main() {
  // Standard stdout ASCII logging
  log.WithFields(log.Fields{
    "animal": "walrus",
  }).Info("A walrus appears")

  // JSON style structured logging
  log.SetFormatter(&log.JSONFormatter{})
  f, e := os.Create("logs")
  if e != nil {
    log.Fatal("Failed to create log file")
  }
  log.SetOutput(f)
  log.WithFields(log.Fields{
    "animal": "walrus",
    "size":   10,
  }).Info("A group of walrus emerges from the ocean")

  /*
  Output...

      {
        "animal": "walrus",
        "level": "info",
        "msg": "A group of walrus emerges from the ocean",
        "size": 10,
        "time": "2015-12-22T13:58:46Z"
      }
  */
}

2. python structured logging.py

import structlog


def rename_event_key(_, __, ed):
    """Datadog prefers the message field to be called 'message' not 'event'.
    
    DOCS:
    	https://www.structlog.org/en/stable/processors.html
    """

    ed["message"] = ed.pop("event")
    return ed

  
structlog.configure(
  processors=[
    rename_event_key,
    structlog.processors.JSONRenderer(sort_keys=True),
  ]
)

log = structlog.get_logger()
log.info("foo", bar=123, baz={"a": 555})

# {"bar": 123, "baz": {"a": 555}, "message": "foo"}

3. python structured logging (separate package).py

# logger.py
#
# EXAMPLE USAGE:
#
# import logger
# log = logger.log
# log.info(...)

import logging

import structlog
from bf_rig import settings


def rename_event_key(_, __, ed):
    """Datadog prefers the message field to be called 'message' not 'event'."""

    ed["message"] = ed.pop("event")
    return ed


def new_logger():
    log_format = ""
    datefmt = ""

    if settings.get("debug"):
        log_format = "[%(levelname)s %(asctime)s path:%(pathname)s lineno:%(lineno)s] %(message)s"
        datefmt = "%Y/%m/%d %I:%M:%S"

        if settings.get("cluster") == "prod":
            logging.getLogger("nsq").setLevel(logging.WARNING)
    else:
        # keep logs quiet outside of local development
        logging.getLogger("botocore").setLevel(logging.CRITICAL)
        logging.getLogger("boto3").setLevel(logging.CRITICAL)
        logging.getLogger("nsq").setLevel(logging.CRITICAL)

    level = getattr(logging, settings.get("log_level").upper())
    logging.basicConfig(level=level, format=log_format, datefmt=datefmt)

    structlog.configure(
        processors=[
            rename_event_key,
            structlog.stdlib.add_log_level,
            structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S"),
            structlog.processors.JSONRenderer(sort_keys=True),
        ],
        context_class=dict,
        cache_logger_on_first_use=True,
    )

    log = structlog.get_logger()

    return log.bind(
        rig={
            "cluster": settings.get('cluster'),
            "service": settings.get('service'),
            "version": settings.get('rig_image_version'),
        },
        meta={
            "deployment_name": settings.get("rig_deployment_name"),
            "deployment_type": settings.get("rig_deployment_type"),
        },
    )


log = new_logger()