Elasticsearch not accepting ILM-Config from Fluentd

22 views Asked by At

I have this Output-Plugin-Config for fluentd:

<match logging.**>
  @type elasticsearch_data_stream
  host <<fluent-conf-host-elasticsearch>>
  port <<fluent-conf-port-elasticsearch>>

  logstash_format true
  rollover_index true
  # logstash_prefix ""
  # logstash_prefix_separator ""
  # logstash_dateformat %Y.%m.%d

  enable_ilm true
  template_name "delete_after_30_days_template"
  template_file "/fluentd/etc/delete_after_30_days_template"
  template_overwrite true
  # customize_template

  ilm_policy_id logging_policy
  ilm_policy_overwrite true
  ilm_policy {
  "policy": {
    "phases": {
      "hot": {                                
        "actions": {
          "rollover": {
            "max_age": "1d"
          },
          "set_priority" : {
            "priority": 100
          }
        }
      },
      "warm": {                                
        "actions": {
          "set_priority" : {
            "priority": 50
          }
        },
        "transitions": [
          {
            "state_name": "delete",
            "min_age": "30d"
          }
        ]
      },
      "delete": {                    
        "actions": {
          "delete": {}                        
        }
      }
    }
  }
}

  data_stream_name logging
  data_stream_template_name delete_after_30_days_template
  data_stream_ilm_name logging_policy
  data_stream_ilm_policy_overwrite true

  user <<fluent-conf-elasticsearch-user>>
  password <<fluent-conf-elasticsearch-password>>

  scheme http
  ssl_verify false

  include_timestamp true
  time_key_format "%Y-%m-%dT%H:%M:%S.%N%z"

  <buffer>
    flush_interval 1s
  </buffer>

</match>

It does everything I want it to (Creating Datastream, with the correct index template), except for creating the right ILM-Policy.

When I send a request to inspect the ILM-Policy in elasticsearch it looks like it just took some default values:

"logging_policy" : {
    "version" : 1,
    "modified_date" : "2024-02-26T08:43:56.755Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_age" : "30d",
              "max_size" : "50gb"
            }
          }
        }
      }
    },
    "in_use_by" : {
      "indices" : [
        ".ds-logging-2024.02.26-000001"
      ],
      "data_streams" : [
        "logging"
      ],
      "composable_templates" : [
        "delete_after_30_days_template"
      ]
    }
  }

Why isn't the value of the ilm_policy inserted?

0

There are 0 answers