Merge two YAML files with ansible.builtin.include_vars and combine

Question

I am trying to merge these two YAML file together:

vars/standard_x.yaml:

---
paas_id: "aze8ea"
paas_name: "test"
components:
  - name: paas_registration
  - name: argos
    size: s
  - name: splunk
    size: s
  - name: controlm
  - name: paas
    subnets:
      cri1:
        size: 25
        zone_type: cri
        zone_name: cri1
        usage:
          - paas_ingress
          - paas_router
      anz1:
        size: 27
        zone_type: anz
        zone_name: anz1
        usage:
          - paas_ingress
          - paas_router

vars/standard_y.yaml:

---
vnet:
  size: 23
skip_components:
  - kafka
components:
  - name: e2e-tree
  - name: subscription
  - name: lz-network
  - name: paas
    delegation:
      admin_ad_group: tst-ne-01-owners
      cluster_master_node_type: Standard_E8as_v4
      cluster_worker_node_type: Standard_E8as_v4
    subnets:
      cbz1:
        size: 27
        zone_type: cbz
        zone_name: cbz1
        usage:
          - paas_ingress
      dmz1:
        size: 27
        zone_type: dmz
        zone_name: dmz1
        usage:
          - paas_ingress

This is what I got so far:

- name: Load x vars
  ansible.builtin.include_vars:
    file: vars/standard_x.yaml
    name: standard_x

- name: Load y vars
  ansible.builtin.include_vars:
    file: vars/standard_y.yaml
    name: standard_y

- name: Combine delegation into standard vars
  ansible.builtin.set_fact:
    my_vars: "{{ standard_x | combine(standard_y, recursive=True, list_merge='append_rp') }}"

But it does not merge the files correctly, for example the var "- name: paas" is repeated twice in the file, the correct output would be this one:

---
vnet:
  size: 23
skip_components:
  - kafka
paas_id: "aze8ea"
paas_name: "test"
components:
  - name: paas_registration
  - name: argos
    size: s
  - name: splunk
    size: s
  - name: controlm
  - name: e2e-tree
  - name: subscription
  - name: lz-network
  - name: paas
    delegation:
      admin_ad_group: tst-ne-01-owners
      cluster_master_node_type: Standard_E8as_v4
      cluster_worker_node_type: Standard_E8as_v4
    subnets:
      cbz1:
        size: 27
        zone_type: cbz
        zone_name: cbz1
        usage:
          - paas_ingress
      dmz1:
        size: 27
        zone_type: dmz
        zone_name: dmz1
        usage:
          - paas_ingress
      cri1:
        size: 25
        zone_type: cri
        zone_name: cri1
        usage:
          - paas_ingress
          - paas_router
      anz1:
        size: 27
        zone_type: anz
        zone_name: anz1
        usage:
          - paas_ingress
          - paas_router

I can not seem to make it work, I would appreciate some help. Let me know if anything is not clear.

Answer 1

The result you get is actually totally expected. You need a bit more than a simple combine on this one as you want to merge elements in the components key by names. Fortunately, there is the lists_mergeby filter for that.

Given your two example files in the vars subfolder above, the following merge.yml playbook:

---
- hosts: localhost
  gather_facts: false

  vars:
    merged_components:
      components: "{{ standard_x.components | community.general.lists_mergeby(
        standard_y.components,
        'name',
        recursive=True,   
      ) }}"

    my_vars: "{{ standard_x | combine(standard_y) | combine(merged_components) }}"

  tasks:
    - name: "Load vars from files"
      ansible.builtin.include_vars:
        file: "{{ item }}.yaml"
        name: "{{ item }}"
      loop:
        - standard_x
        - standard_y

    - name: Show merging result
      ansible.builtin.debug:
        var: my_vars

gives:

$ ansible-playbook merge.yml 

PLAY [localhost] **********************************************************************************************************************************************************************************************************

TASK [Load vars from files] ***********************************************************************************************************************************************************************************************
ok: [localhost] => (item=standard_x)
ok: [localhost] => (item=standard_y)

TASK [Show merging result] ************************************************************************************************************************************************************************************************
ok: [localhost] => {
    "my_vars": {
        "components": [
            {
                "name": "argos",
                "size": "s"
            },
            {
                "name": "controlm"
            },
            {
                "name": "e2e-tree"
            },
            {
                "name": "lz-network"
            },
            {
                "delegation": {
                    "admin_ad_group": "tst-ne-01-owners",
                    "cluster_master_node_type": "Standard_E8as_v4",
                    "cluster_worker_node_type": "Standard_E8as_v4"
                },
                "name": "paas",
                "subnets": {
                    "anz1": {
                        "size": 27,
                        "usage": [
                            "paas_ingress",
                            "paas_router"
                        ],
                        "zone_name": "anz1",
                        "zone_type": "anz"
                    },
                    "cbz1": {
                        "size": 27,
                        "usage": [
                            "paas_ingress"
                        ],
                        "zone_name": "cbz1",
                        "zone_type": "cbz"
                    },
                    "cri1": {
                        "size": 25,
                        "usage": [
                            "paas_ingress",
                            "paas_router"
                        ],
                        "zone_name": "cri1",
                        "zone_type": "cri"
                    },
                    "dmz1": {
                        "size": 27,
                        "usage": [
                            "paas_ingress"
                        ],
                        "zone_name": "dmz1",
                        "zone_type": "dmz"
                    }
                }
            },
            {
                "name": "paas_registration"
            },
            {
                "name": "splunk",
                "size": "s"
            },
            {
                "name": "subscription"
            }
        ],
        "paas_id": "aze8ea",
        "paas_name": "test-ngi",
        "skip_components": [
            "kafka"
        ],
        "vnet": {
            "size": 23
        }
    }
}

PLAY RECAP ****************************************************************************************************************************************************************************************************************
localhost                  : ok=2    changed=0    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0