Fix variable collisions in orchestrator provisioning

src/orchestrator/ansible.cfg

@@ -20,8 +20,8 @@ timeout = 180
 executable = /bin/bash
 interpreter_python = /usr/bin/python3
 deprecation_warnings = false
-show_task_path_on_failure = false
-stdout_callback = omnia_default
+show_task_path_on_failure = true
+stdout_callback = default
 collections_path = .:~/.ansible/collections:/usr/share/ansible/collections
 
 [persistent_connection]

src/orchestrator/roles/configure_ochami/tasks/configure_boot_svc_metadata_svc.yml

@@ -34,9 +34,9 @@
   ansible.builtin.set_fact:
     node_parsed_yaml: "{{ nodes_vars.content | b64decode | from_yaml }}"
 
-- name: Set nodes
+- name: Set boot_svc_nodes
   ansible.builtin.set_fact:
-    nodes: "{{ node_parsed_yaml.nodes }}"
+    boot_svc_nodes: "{{ node_parsed_yaml.nodes }}"
 
 - name: Create boot and metadata-service directory
   ansible.builtin.file:

src/orchestrator/roles/configure_ochami/tasks/create_groups.yml

@@ -23,7 +23,7 @@
     dest: "{{ nodes_dir }}/groups-{{ functional_group_name }}.yml"
     mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
   vars:
-    nodes: "{{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items }}"
+    smd_group_nodes: "{{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items }}"
 
 - name: Get SMD group data
   ansible.builtin.command: /usr/bin/ochami smd group get

src/orchestrator/roles/configure_ochami/tasks/create_groups_additional_fg.yml

@@ -27,7 +27,7 @@
     dest: "{{ nodes_dir }}/groups-{{ additional_metadata_svc_fg_smd_group_name }}.yml"
     mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
   vars:
-    nodes: "{{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items }}"
+    smd_group_nodes: "{{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items }}"
 
 - name: Delete SMD group - {{ additional_metadata_svc_fg_smd_group_name }}
   ansible.builtin.command: >-

src/orchestrator/roles/configure_ochami/tasks/create_groups_common.yml

@@ -23,13 +23,16 @@
     dest: "{{ nodes_dir }}/groups-common-{{ common_group_name }}.yml"
     mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
   vars:
-    _all_nodes: "{{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items }}"
-    nodes: >-
-      {{ (_all_nodes
-          | selectattr('value.FUNCTIONAL_GROUP_NAME', 'in', registered_fg_names)
-          | list)
-         if (registered_fg_names is defined)
-         else _all_nodes }}
+    # Only include nodes registered in SMD for the current target category.
+    # target_fg_names is set by the caller (register_nodes.yml).
+    # Use 'common_group_nodes' to avoid collision with the 'nodes' set_fact
+    # from configure_boot_svc.yml.
+    common_group_nodes: >-
+      {{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items
+         | selectattr('value.FUNCTIONAL_GROUP_NAME', 'in', target_fg_names | default([]))
+         | list
+         if (target_fg_names | default([]) | length > 0)
+         else hostvars['localhost']['read_mapping_file']['dict'] | dict2items }}
 
 - name: Delete the SMD common group - {{ common_group_name }}
   ansible.builtin.command: /usr/bin/ochami smd group delete --no-confirm {{ common_group_name }}

src/orchestrator/roles/configure_ochami/tasks/orchestration_mapping_nodes.yml

@@ -36,7 +36,7 @@
         dest: "{{ openchami_nodes_vars_path }}"
         mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
       vars:
-        nodes: "{{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items }}"
+        mapping_nodes: "{{ hostvars['localhost']['read_mapping_file']['dict'] | dict2items | map(attribute='value') | list }}"
 
     - name: Create orchestrator output directory on OIM
       ansible.builtin.file:

src/orchestrator/roles/configure_ochami/templates/boot_svc/boot-svc.yaml.j2

@@ -8,7 +8,7 @@ spec:
   initrd: "{{ s3_base_url }}/{{ initrd }}"
   params: "nomodeset ro root=live:{{ s3_base_url }}/{{ image }} ip=dhcp rd.live.image rd.live.ram rd.neednet=1 {% if s3_use_https %}rd.noverifyssl {% endif %}rd.driver.blacklist=ccp,edac_core,power_meter,ahci,megaraid_sas modprobe.blacklist=ccp,edac_core,power_meter,ahci,megaraid_sas libata.force=1:disable,2:disable,3:disable,4:disable rd.luks=0 rd.md=0 rd.dm=0 console=tty0 console=ttyS0,115200 selinux=0 apparmor=0 ip6=off {{ boot_svc_params_metadata_svc }}{% if boot_kernel_params | default('') | length > 0 %} {{ boot_kernel_params }}{% endif %}"
   macs:
-{% for item in nodes %}
+{% for item in boot_svc_nodes %}
 {% if item.group == functional_group_name %}
     - "{{ item.interfaces[0].mac_addr }}"
 {% endif %}

src/orchestrator/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2

@@ -17,7 +17,7 @@ modprobe ib_uverbs || true
 NETMASK_BITS="{{ hostvars['localhost']['admin_netmask_bits'] }}"
 
 # Get current node's IP from cloud-init metadata
-ADMIN_NIC_IP="{% raw %}{{"{{"}} ds.meta_data.instance_data.local_ipv4 {{"}}"}}{% endraw %}"
+ADMIN_NIC_IP="{% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %}"
 
 # Create IB mapping from all nodes (IB_NIC_NAME only)
 declare -A ADMIN_IB_NIC_NAME_MAP=(

src/orchestrator/roles/configure_ochami/templates/metadata_svc/ms-group-login_compiler_node_aarch64.yaml.j2

@@ -223,21 +223,21 @@
 {% endfor %}
 {% endif %}
 {% raw %}
-      {% if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map %}
-      {% set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) %}
-      {% if mymounts %}
-      {% for mount in mymounts.get("mounts", []) %}
-        - mkdir -pv {{"{{"}} mount[1] {{"}}"}}
-        - echo "{{"{{"}} mount | join(' ') {{"}}"}}" >> /etc/fstab
-      {% endfor %}
+      {{ "{%" }} if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map {{ "%}" }}
+      {{ "{%" }} set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) {{ "%}" }}
+      {{ "{%" }} if mymounts {{ "%}" }}
+      {{ "{%" }} for mount in mymounts.get("mounts", []) {{ "%}" }}
+        - mkdir -pv {{ "{{" }} mount[1] {{ "}}" }}
+        - echo "{{ "{{" }} mount | join(' ') {{ "}}" }}" >> /etc/fstab
+      {{ "{%" }} endfor {{ "%}" }}
         - mount -av
-      {% for decoded_cmd in mymounts.get("runcmd", []) %}
-        - {{"{{"}} decoded_cmd {{"}}"}}
-      {% endfor %}
-      {% else %}
+      {{ "{%" }} for decoded_cmd in mymounts.get("runcmd", []) {{ "%}" }}
+        - {{ "{{" }} decoded_cmd {{ "}}" }}
+      {{ "{%" }} endfor {{ "%}" }}
+      {{ "{%" }} else {{ "%}" }}
         - echo "No mount entries found for this host"
-      {% endif %}
-      {% endif %}
+      {{ "{%" }} endif {{ "%}" }}
+      {{ "{%" }} endif {{ "%}" }}
 {% endraw %}
 {% set fg_swap = metadata_svc_groups_dict.get(functional_group_name, {}).get('swap', {}) %}
 {% include 'configure_swap.yaml.j2' %}

src/orchestrator/roles/configure_ochami/templates/metadata_svc/ms-group-login_compiler_node_x86_64.yaml.j2

@@ -223,21 +223,21 @@
 {% endfor %}
 {% endif %}
 {% raw %}
-      {% if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map %}
-      {% set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) %}
-      {% if mymounts %}
-      {% for mount in mymounts.get("mounts", []) %}
-        - mkdir -pv {{"{{"}} mount[1] {{"}}"}}
-        - echo "{{"{{"}} mount | join(' ') {{"}}"}}" >> /etc/fstab
-      {% endfor %}
+      {{ "{%" }} if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map {{ "%}" }}
+      {{ "{%" }} set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) {{ "%}" }}
+      {{ "{%" }} if mymounts {{ "%}" }}
+      {{ "{%" }} for mount in mymounts.get("mounts", []) {{ "%}" }}
+        - mkdir -pv {{ "{{" }} mount[1] {{ "}}" }}
+        - echo "{{ "{{" }} mount | join(' ') {{ "}}" }}" >> /etc/fstab
+      {{ "{%" }} endfor {{ "%}" }}
         - mount -av
-      {% for decoded_cmd in mymounts.get("runcmd", []) %}
-        - {{"{{"}} decoded_cmd {{"}}"}}
-      {% endfor %}
-      {% else %}
-         - echo "No mount entries found for this host"
-      {% endif %}
-      {% endif %}
+      {{ "{%" }} for decoded_cmd in mymounts.get("runcmd", []) {{ "%}" }}
+        - {{ "{{" }} decoded_cmd {{ "}}" }}
+      {{ "{%" }} endfor {{ "%}" }}
+      {{ "{%" }} else {{ "%}" }}
+        - echo "No mount entries found for this host"
+      {{ "{%" }} endif {{ "%}" }}
+      {{ "{%" }} endif {{ "%}" }}
 {% endraw %}
 {% set fg_swap = metadata_svc_groups_dict.get(functional_group_name, {}).get('swap', {}) %}
 {% include 'configure_swap.yaml.j2' %}

src/orchestrator/roles/configure_ochami/templates/metadata_svc/ms-group-login_node_aarch64.yaml.j2

@@ -179,21 +179,21 @@
 {% endfor %}
 {% endif %}
 {% raw %}
-      {% if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map %}
-      {% set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) %}
-      {% if mymounts %}
-      {% for mount in mymounts.get("mounts", []) %}
-        - mkdir -pv {{"{{"}} mount[1] {{"}}"}}
-        - echo "{{"{{"}} mount | join(' ') {{"}}"}}" >> /etc/fstab
-      {% endfor %}
+      {{ "{%" }} if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map {{ "%}" }}
+      {{ "{%" }} set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) {{ "%}" }}
+      {{ "{%" }} if mymounts {{ "%}" }}
+      {{ "{%" }} for mount in mymounts.get("mounts", []) {{ "%}" }}
+        - mkdir -pv {{ "{{" }} mount[1] {{ "}}" }}
+        - echo "{{ "{{" }} mount | join(' ') {{ "}}" }}" >> /etc/fstab
+      {{ "{%" }} endfor {{ "%}" }}
         - mount -av
-      {% for decoded_cmd in mymounts.get("runcmd", []) %}
-        - {{"{{"}} decoded_cmd {{"}}"}}
-      {% endfor %}
-      {% else %}
-         - echo "No mount entries found for this host"
-      {% endif %}
-      {% endif %}
+      {{ "{%" }} for decoded_cmd in mymounts.get("runcmd", []) {{ "%}" }}
+        - {{ "{{" }} decoded_cmd {{ "}}" }}
+      {{ "{%" }} endfor {{ "%}" }}
+      {{ "{%" }} else {{ "%}" }}
+        - echo "No mount entries found for this host"
+      {{ "{%" }} endif {{ "%}" }}
+      {{ "{%" }} endif {{ "%}" }}
 {% endraw %}
 {% set fg_swap = metadata_svc_groups_dict.get(functional_group_name, {}).get('swap', {}) %}
 {% include 'configure_swap.yaml.j2' %}

src/orchestrator/roles/configure_ochami/templates/metadata_svc/ms-group-login_node_x86_64.yaml.j2

@@ -178,21 +178,21 @@
 {% endfor %}
 {% endif %}
 {% raw %}
-      {% if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map %}
-      {% set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) %}
-      {% if mymounts %}
-      {% for mount in mymounts.get("mounts", []) %}
-        - mkdir -pv {{"{{"}} mount[1] {{"}}"}}
-        - echo "{{"{{"}} mount | join(' ') {{"}}"}}" >> /etc/fstab
-      {% endfor %}
+      {{ "{%" }} if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map {{ "%}" }}
+      {{ "{%" }} set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) {{ "%}" }}
+      {{ "{%" }} if mymounts {{ "%}" }}
+      {{ "{%" }} for mount in mymounts.get("mounts", []) {{ "%}" }}
+        - mkdir -pv {{ "{{" }} mount[1] {{ "}}" }}
+        - echo "{{ "{{" }} mount | join(' ') {{ "}}" }}" >> /etc/fstab
+      {{ "{%" }} endfor {{ "%}" }}
         - mount -av
-      {% for decoded_cmd in mymounts.get("runcmd", []) %}
-        - {{"{{"}} decoded_cmd {{"}}"}}
-      {% endfor %}
-      {% else %}
-         - echo "No mount entries found for this host"
-      {% endif %}
-      {% endif %}
+      {{ "{%" }} for decoded_cmd in mymounts.get("runcmd", []) {{ "%}" }}
+        - {{ "{{" }} decoded_cmd {{ "}}" }}
+      {{ "{%" }} endfor {{ "%}" }}
+      {{ "{%" }} else {{ "%}" }}
+        - echo "No mount entries found for this host"
+      {{ "{%" }} endif {{ "%}" }}
+      {{ "{%" }} endif {{ "%}" }}
 {% endraw %}
 {% set fg_swap = metadata_svc_groups_dict.get(functional_group_name, {}).get('swap', {}) %}
 {% include 'configure_swap.yaml.j2' %}

src/orchestrator/roles/configure_ochami/templates/metadata_svc/ms-group-service_kube_control_plane_first_x86_64.yaml.j2

@@ -281,10 +281,10 @@
             kubeadm init --kubernetes-version={{ service_k8s_version }} \
               --pod-network-cidr={{ k8s_pod_network_cidr }} \
               --service-cidr={{ k8s_service_addresses }} \
-              --apiserver-advertise-address={% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} \
-              --node-name {% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} \
+              --apiserver-advertise-address={% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %} \
+              --node-name {% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %} \
               --cri-socket=unix:///var/run/crio/crio.sock \
-              --control-plane-endpoint={% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}:6443 \
+              --control-plane-endpoint={% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %}:6443 \
               --apiserver-cert-extra-sans {{ kube_vip }}
 
         - path: /tmp/generate-control-plane-join.sh
@@ -369,21 +369,21 @@
 {% endfor %}
 {% endif %}
 {% raw %}
-      {% if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map %}
-      {% set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) %}
-      {% if mymounts %}
-      {% for mount in mymounts.get("mounts", []) %}
-        - mkdir -pv {{ mount[1] }}
-        - echo "{{ mount | join(' ') }}" >> /etc/fstab
-      {% endfor %}
+      {{ "{%" }} if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map is mapping and ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map {{ "%}" }}
+      {{ "{%" }} set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) {{ "%}" }}
+      {{ "{%" }} if mymounts {{ "%}" }}
+      {{ "{%" }} for mount in mymounts.get("mounts", []) {{ "%}" }}
+        - mkdir -pv {{ "{{" }} mount[1] {{ "}}" }}
+        - echo "{{ "{{" }} mount | join(' ') {{ "}}" }}" >> /etc/fstab
+      {{ "{%" }} endfor {{ "%}" }}
         - mount -av
-      {% for decoded_cmd in mymounts.get("runcmd", []) %}
-        - {{ decoded_cmd }}
-      {% endfor %}
-      {% else %}
+      {{ "{%" }} for decoded_cmd in mymounts.get("runcmd", []) {{ "%}" }}
+        - {{ "{{" }} decoded_cmd {{ "}}" }}
+      {{ "{%" }} endfor {{ "%}" }}
+      {{ "{%" }} else {{ "%}" }}
         - echo "No mount entries found for this host"
-      {% endif %}
-      {% endif %}
+      {{ "{%" }} endif {{ "%}" }}
+      {{ "{%" }} endif {{ "%}" }}
 {% endraw %}
 {% for pv_entry in metadata_svc_groups_dict[functional_group_name].powervault_scripts | default([], true) %} 
         - bash /usr/local/bin/setup_iscsi_storage_{{ pv_entry.name }}.sh
@@ -393,11 +393,11 @@
 {% if etcd_on_local_disk | default(false) %}
         - /usr/local/bin/etcd-fstab-update.sh
 {% else %}
-        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/etcd      /var/lib/etcd        nfs noatime,nolock 0 0" >> /etc/fstab
+        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %}/etcd      /var/lib/etcd        nfs noatime,nolock 0 0" >> /etc/fstab
 {% endif %}
-        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet   /var/lib/kubelet     nfs noatime,nolock 0 0" >> /etc/fstab
-        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes   /etc/kubernetes      nfs noatime,nolock 0 0" >> /etc/fstab
-        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs   /var/log/pods      nfs noatime,nolock 0 0" >> /etc/fstab
+        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %}/kubelet   /var/lib/kubelet     nfs noatime,nolock 0 0" >> /etc/fstab
+        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %}/kubernetes   /etc/kubernetes      nfs noatime,nolock 0 0" >> /etc/fstab
+        - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %}/pod-logs   /var/log/pods      nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/packages   /var/lib/packages        nfs    noatime,nolock     0 0" >> /etc/fstab
         - echo "tmpfs   /tmp/crio-storage   tmpfs   size={{ k8s_crio_storage_size }},noatime,nodev,nosuid   0 0" >> /etc/fstab
         - sudo swapoff -a
@@ -519,7 +519,7 @@
             rm -rf /var/lib/etcd/*  /var/lib/kubelet/* /etc/kubernetes/*
             rm -rf /var/lib/etcd/.*  /var/lib/kubelet/.* /etc/kubernetes/.*
             #!/bin/bash
-            NODE_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}"
+            NODE_IP="{% raw %}{{ "{{" }} ip {{ "}}" }}{% endraw %}"
             # Find the interface with this IP
             VIP_IFACE=$(ip -o addr show | awk -v ip="$NODE_IP" '$4 ~ ip {print $2}')
             # Replace the vip_interface placeholder in the yaml