Loki config with s3

Question

I can't get Loki to connect to AWS S3 using docker-compose. Logs are visible in Grafana but the S3 bucket remains empty.

The s3 bucket is public and I have an IAM role attached to allow s3:FullAccess.

I updated loki to v2.0.0 and changed the period to 24h but it made no difference. There are no errors in the loki logs.

Here are the selected lines from docker logs (loki):

msg="Starting Loki" version="(version=master-4e661cd, branch=master, revision=4e661cde)"
caller=server.go:225 http=[::]:3100 grpc=[::]:9095 msg="server listening on addresses"
caller=worker.go:65 msg="no address specified, not starting worker"
msg="cleaning up mapped rules directory" path=/loki/tmprules
msg=initialising module=memberlist-kv
msg=initialising module=store
msg=initialising module=server
msg=initialising module=ring
msg="value is nil" key=collectors/ring index=1
msg=initialising module=ingester
msg="not loading tokens from file, tokens file path is empty"
msg="instance not found in ring, adding with no tokens" ring=ingester
msg="auto-joining cluster after timeout" ring=ingester
msg=initialising module=table-manager
msg=initialising module=distributor
msg=initialising module=ingester-querier
msg=initialising module=ruler
msg="ruler up and running"
msg="Loki started"
msg="synching tables" expected_tables=132

Here is my loki.config:

auth_enabled: false

server:
  http_listen_port: 3100

distributor:
  ring:
    kvstore:
      store: memberlist

ingester:
  lifecycler:
    ring:
      kvstore:
        store: memberlist
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s

schema_config:
  configs:
  - from: 2020-10-27
    store: boltdb-shipper
    object_store: s3
    schema: v11
    index:
      prefix: index_
      period: 24h
  
storage_config:
  boltdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
    resync_interval: 5s
    shared_store: s3
  
  aws:
     s3: s3://AKIARE3@us-east-1/mydomain.com.docker.loki.logs
     s3forcepathstyle: true 

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h     

Here is docker-compose.yaml

version: "3.8"

networks:
  traefik:
    external: true

volumes:
  data:
   
services:
  fluentd:
    image: grafana/fluent-plugin-loki:master
    command:
      - "fluentd"
      - "-v"
      - "-p"
      - "/fluentd/plugins"
    environment:
      LOKI_URL: http://loki:3100
      LOKI_USERNAME:
      LOKI_PASSWORD:
    container_name: "fluentd"
    restart: always
    ports:
      - '24224:24224'
    networks:
      - traefik
    volumes:
      - type: bind
        source: ./config/fluent.conf
        target: /fluentd/etc/fluent.conf
    logging:
      options:
        tag: docker.monitoring
  
  loki:
    image: grafana/loki:master
    container_name: "loki"
    restart: always
    networks:
      - traefik
    volumes:
      - type: volume
        source: data
        target: /loki
    ports:
      - 3100
    volumes:
      - type: bind
        source: ./config/s3.loki.conf
        target: /loki/etc/loki.conf
    depends_on:
      - fluentd

Answer 1

I finally did work this out. It requires a compactor but gives no warning about it. Best practice is to create an AWS s3 bucket without any public access. Next create an IAM user with programmatic access only. Create an access policy which gives full access only to the bucket you created. Attach the policy to the user's permissions. You do not need to attach a policy to the bucket itself. Check if you have "/" in your URL that you escape it with %2F otherwise you will get an auth error. Note that this config is for loki v2.0.0 which was released yesterday.

Here are my complete working docker-compose and loki config files. I put them on an external network to enable prometheus monitoring.

here is my docker-compose.yaml

    version: "3.8"
    
    networks:
      appnet:
        external: true
    
    volumes:
      loki_data:
       
    services:
      fluentd:
        container_name: "fluentd"
        image: grafana/fluent-plugin-loki:master
        command:
          - "fluentd"
          - "-v"
          - "-p"
          - "/fluentd/plugins"
        environment:
          LOKI_URL: http://loki:3100
          LOKI_USERNAME:
          LOKI_PASSWORD:
        restart: always
        ports:
          - '24224:24224'
        networks:
          - appnet
        volumes:
          - type: bind
            source: ./config/fluent.conf
            target: /fluentd/etc/fluent.conf
      
      loki:
        container_name: "loki"
        image: grafana/loki:2.0.0
        restart: always
        networks:
          - appnet
        ports:
          - 3100
        volumes:
          - type: volume
            source: loki_data
            target: /data
          - type: bind
            source: ./config/s3-loki-bolt-conf.yml
            target: /etc/loki/local-config.yaml
        command: -config.file=/etc/loki/local-config.yaml        
        depends_on:
          - fluentd

Here is my loki config in prometheus/config/s3-loki-bolt-conf.yml. You can name this anything you want but keep the target file name as above as it is the loki default config file.

    auth_enabled: false
    
    ingester:
      chunk_idle_period: 3m
      chunk_block_size: 262144
      chunk_retain_period: 1m
      max_transfer_retries: 0
      lifecycler:
        ring:
          kvstore:
            store: inmemory
          replication_factor: 1
    
    limits_config:
      enforce_metric_name: false
      reject_old_samples: true
      reject_old_samples_max_age: 168h
    
    compactor:
      working_directory: /loki/boltdb-shipper-compactor
      shared_store: aws
      
    schema_config:
      configs:
      - from: 2020-07-01
        store: boltdb-shipper
        object_store: aws
        schema: v11
        index:
          prefix: loki_index_
          period: 24h
          
    server:
      http_listen_port: 3100
    
    storage_config:
      aws:
        s3: s3://ACCESS_KEY:SECRET_ACCESS_KEY@us-west-1/mydomain.com.docker.loki.logs
      boltdb_shipper:
        active_index_directory: /loki/index
        shared_store: s3
        cache_location: /loki/boltdb-cache
    
    chunk_store_config:
      max_look_back_period: 0s
    
    table_manager:
      retention_deletes_enabled: false
      retention_period: 0s

Answer 2

It looks like the loki config file was changed again, the solution that is posted here didn't work in our case. We had to specify the storage configuration under common, instead of storage_config to make it work.

This is the content of our loki-config.yaml file

    auth_enabled: false

    server:
      http_listen_port: 3100
      grpc_listen_port: 9096

    common:
      instance_addr: 127.0.0.1
      path_prefix: /loki
      storage:
        s3:
          bucketnames: xxx
          region: xxx
      replication_factor: 1
      ring:
        kvstore:
          store: inmemory

    query_range:
      results_cache:
        cache:
          embedded_cache:
            enabled: true
            max_size_mb: 100

    schema_config:
      configs:
      - from: 2023-05-15
        store: boltdb-shipper
        object_store: s3
        schema: v11
        index:
          prefix: loki_index_
          period: 24h    

    ruler:
      alertmanager_url: http://localhost:9093

Answer 3

For those who want to use boltdb-shipper and store in S3 compatible object store (in my case from Scaleway), using helm and loki 2.0.0

Here is my values.yml:

loki:
  enabled: true
  config:
    auth_enabled: false

    ingester:
      chunk_idle_period: 3m
      chunk_block_size: 262144
      chunk_retain_period: 1m
      max_transfer_retries: 0
      lifecycler:
        ring:
          kvstore:
            store: inmemory
          replication_factor: 1

    limits_config:
      enforce_metric_name: false
      reject_old_samples: true
      reject_old_samples_max_age: 168h

    compactor:
      working_directory: /data/loki/boltdb-shipper-compactor
      shared_store: aws

    schema_config:
      configs:
        - from: 2020-11-13
          store: boltdb-shipper
          object_store: aws
          schema: v11
          index:
            prefix: loki_index_
            period: 24h

    server:
      http_listen_port: 3100

    storage_config:
      aws:
        s3: s3://<key>:<secret>@s3.fr-par.scw.cloud/<bucket-name>
        region: fr-par
        s3forcepathstyle: true
      boltdb_shipper:
        active_index_directory: /data/loki/index
        shared_store: s3
        cache_location: /data/loki/boltdb-cache

    chunk_store_config:
      max_look_back_period: 0s

    table_manager:
      retention_deletes_enabled: true
      retention_period: 720h

promtail:
  enabled: true