diff --git a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml index be8ef36..736c573 100644 --- a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml index d5f73ca..1c1744f 100644 --- a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml @@ -38,7 +38,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -61,7 +61,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -93,6 +93,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create scheduler group become: yes diff --git a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml index 89e2cb6..23e4109 100644 --- a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml index 1505647..69b51f0 100644 --- a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml @@ -6,7 +6,7 @@ remove_packages: [] packages: - - netcat + - netcat-openbsd - policycoreutils-python-utils - python3-selinux diff --git a/cloudmon/ansible/project/roles/epmon/tasks/main.yml b/cloudmon/ansible/project/roles/epmon/tasks/main.yml index ba9b0fe..4139183 100644 --- a/cloudmon/ansible/project/roles/epmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/epmon/tasks/main.yml @@ -37,7 +37,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -60,7 +60,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml index 8c46752..076d958 100644 --- a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -96,6 +96,7 @@ loop: "{{ play_hosts }}" - ansible.builtin.include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Ensure directories exist become: true diff --git a/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml b/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml index b6e6d0f..a5caca4 100644 --- a/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml @@ -1,64 +1,7 @@ -- name: Include OS-specific variables - include_vars: "{{ lookup('first_found', params) }}" - vars: - params: - files: "{{ distro_lookup_path }}" - paths: - - 'vars' - -- name: Install firewalld - ansible.builtin.package: - name: "{{ item }}" - state: "present" - loop: - - "{{ packages }}" - when: "ansible_facts.pkg_mgr != 'atomic_container'" - -- name: Enable services - ansible.posix.firewalld: - permanent: "yes" - service: "{{ item }}" - state: "enabled" - loop: "{{ firewalld_services_enable }}" - notify: - - Reload firewalld - -- name: Disable services - ansible.posix.firewalld: - permanent: "yes" - service: "{{ item }}" - state: "disabled" - loop: "{{ firewalld_services_disable }}" - notify: - - Reload firewalld - -- name: Enable ports - ansible.posix.firewalld: - permanent: "yes" - port: "{{ item }}" - state: "enabled" - loop: "{{ firewalld_ports_enable }}" - notify: - - Reload firewalld - -- name: Disable ports - ansible.posix.firewalld: - permanent: "yes" - port: "{{ item }}" - state: "disabled" - loop: "{{ firewalld_ports_disable }}" - notify: - - Reload firewalld - -- name: Disable iptables - ansible.builtin.service: - name: "iptables" - state: "stopped" - enabled: "false" - ignore_errors: "true" - -- name: Enable firewalld - ansible.builtin.service: - name: "firewalld" - state: "started" - enabled: "true" +--- +# Host firewall management is meaningless in a container (no service manager, +# no host firewall), so allow targets to opt out. Defaults to true, so +# existing VM-based deployments are unaffected. +- name: Manage host firewall + ansible.builtin.include_tasks: manage.yaml + when: cloudmon_manage_firewall | default(true) | bool diff --git a/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml b/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml new file mode 100644 index 0000000..b6e6d0f --- /dev/null +++ b/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml @@ -0,0 +1,64 @@ +- name: Include OS-specific variables + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: "{{ distro_lookup_path }}" + paths: + - 'vars' + +- name: Install firewalld + ansible.builtin.package: + name: "{{ item }}" + state: "present" + loop: + - "{{ packages }}" + when: "ansible_facts.pkg_mgr != 'atomic_container'" + +- name: Enable services + ansible.posix.firewalld: + permanent: "yes" + service: "{{ item }}" + state: "enabled" + loop: "{{ firewalld_services_enable }}" + notify: + - Reload firewalld + +- name: Disable services + ansible.posix.firewalld: + permanent: "yes" + service: "{{ item }}" + state: "disabled" + loop: "{{ firewalld_services_disable }}" + notify: + - Reload firewalld + +- name: Enable ports + ansible.posix.firewalld: + permanent: "yes" + port: "{{ item }}" + state: "enabled" + loop: "{{ firewalld_ports_enable }}" + notify: + - Reload firewalld + +- name: Disable ports + ansible.posix.firewalld: + permanent: "yes" + port: "{{ item }}" + state: "disabled" + loop: "{{ firewalld_ports_disable }}" + notify: + - Reload firewalld + +- name: Disable iptables + ansible.builtin.service: + name: "iptables" + state: "stopped" + enabled: "false" + ignore_errors: "true" + +- name: Enable firewalld + ansible.builtin.service: + name: "firewalld" + state: "started" + enabled: "true" diff --git a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml index 3061cc8..0a95f0f 100644 --- a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml @@ -42,7 +42,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -65,7 +65,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/grafana/tasks/main.yml b/cloudmon/ansible/project/roles/grafana/tasks/main.yml index 6ce8ccc..24cf8e8 100644 --- a/cloudmon/ansible/project/roles/grafana/tasks/main.yml +++ b/cloudmon/ansible/project/roles/grafana/tasks/main.yml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -91,6 +91,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create Grafana group ansible.builtin.group: diff --git a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml index 2011deb..11a44ea 100644 --- a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -91,6 +91,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create graphite sync group become: yes diff --git a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml index 001aa59..3d95ca0 100644 --- a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml @@ -6,6 +6,6 @@ remove_packages: [] packages: - - netcat + - netcat-openbsd container_command: docker diff --git a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml index 29eaa2c..ac12375 100644 --- a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -91,6 +91,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Write Postgres Systemd unit file become: true diff --git a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml index 9e3f853..3477d19 100644 --- a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -102,6 +102,7 @@ when: "item != inventory_hostname" - ansible.builtin.include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create postgres group become: yes diff --git a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml index fee2ae1..4b3e894 100644 --- a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml @@ -36,7 +36,7 @@ when: "ansible_facts.pkg_mgr != 'atomic_container'" - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml index fdaaeea..112be38 100644 --- a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml @@ -6,7 +6,7 @@ remove_packages: [] distro_packages: - - netcat + - netcat-openbsd - cron - apparmor-profiles diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index c250b68..07ffff6 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -139,47 +139,64 @@ ansible.builtin.wait_for: path: /var/run/docker.sock timeout: 60 - - # After 75s of retries the daemon still hasn't started. Capture - # stderr/stdout from the last attempt, show the process list, and - # dump the daemon log so we can diagnose why it fails. - - name: Show docker info output from last attempt - ansible.builtin.debug: - var: docker_info - when: docker_info is defined - - - name: Check if dockerd process exists - ansible.builtin.command: - cmd: ps aux | grep -v grep | grep dockerd - register: dockerd_ps - changed_when: false - failed_when: false - - - name: Show dockerd log (last 100 lines) - ansible.builtin.command: - cmd: tail -n 100 /var/log/dockerd.log - register: dockerd_tail - changed_when: false - failed_when: false - - - name: Show containerd log (last 100 lines) - ansible.builtin.command: - cmd: tail -n 100 /var/log/containerd.log - register: containerd_tail - changed_when: false - failed_when: false - - - name: Fail with diagnostic context - ansible.builtin.fail: - msg: >- - Docker daemon failed to start after 75s. - dockerd process: {{ dockerd_ps.stdout_lines | default(['no process found']) }} - Last docker info: {{ docker_info.msg | default('N/A') }} - dockerd log:\n{{ dockerd_tail.stdout | default('no log file') }} - containerd log:\n{{ containerd_tail.stdout | default('no log file') }} + register: docker_sock + ignore_errors: true + + # Only runs when the socket never appeared. wait_for above is + # non-fatal so these diagnostics can be collected before failing; + # previously it aborted the play, so they never ran on failure, and + # the fail below had no condition, so it always ran on success. + - name: Diagnose why the docker daemon did not start + when: docker_sock is failed + block: + - name: Check if dockerd process exists + ansible.builtin.command: + cmd: pgrep -a dockerd + register: dockerd_ps + changed_when: false + failed_when: false + + - name: Show dockerd log (last 100 lines) + ansible.builtin.command: + cmd: tail -n 100 /var/log/dockerd.log + register: dockerd_tail + changed_when: false + failed_when: false + + - name: Show containerd log (last 100 lines) + ansible.builtin.command: + cmd: tail -n 100 /var/log/containerd.log + register: containerd_tail + changed_when: false + failed_when: false + + - name: Fail with diagnostic context + ansible.builtin.fail: + msg: |- + Docker daemon failed to start after 60s. + dockerd process: {{ dockerd_ps.stdout_lines | default(['no process found']) }} + dockerd log: + {{ dockerd_tail.stdout | default('no log file') }} + containerd log: + {{ containerd_tail.stdout | default('no log file') }} - hosts: all become: true + pre_tasks: + # The node image ships none of these: ansible.builtin.pip imports + # `packaging` on the target, and cloudmon_venv_path makes it build a + # venv, which needs the venv/virtualenv tooling. + - name: Install Python packaging and venv tooling + ansible.builtin.apt: + name: + - python3-packaging + - python3-pip + - python3-venv + - virtualenv + state: present + register: py_prereq + retries: 5 + until: py_prereq is success roles: - install_cloudmon - deploy_cloudmon_k8s diff --git a/roles/deploy_cloudmon_k8s/tasks/main.yaml b/roles/deploy_cloudmon_k8s/tasks/main.yaml index 8dcc5c0..5e4b568 100644 --- a/roles/deploy_cloudmon_k8s/tasks/main.yaml +++ b/roles/deploy_cloudmon_k8s/tasks/main.yaml @@ -10,9 +10,37 @@ executable: "/bin/bash" ansible.builtin.shell: | source {{ cloudmon_venv_path }}/bin/activate - cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure graphite provision - cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure statsd provision - cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure postgres provision + cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure {{ item }} provision + loop: + - graphite + - statsd + - postgres + register: cloudmon_provision + failed_when: false + +# Run one component per task and report before failing. Previously all three +# ran in a single shell block with no `set -e`, so rc came from the last +# command only and the output was never surfaced -- a failure showed up as a +# bare "non-zero return code" with nothing to debug. +- name: Show cloudmon provision output + ansible.builtin.debug: + msg: |- + {{ item.item }} rc={{ item.rc }} + stdout: + {{ item.stdout | default('') }} + stderr: + {{ item.stderr | default('') }} + loop: "{{ cloudmon_provision.results }}" + loop_control: + label: "{{ item.item }}" + +- name: Fail if any cloudmon provision step failed + ansible.builtin.fail: + msg: >- + cloudmon provision failed for: + {{ cloudmon_provision.results | selectattr('rc', 'ne', 0) + | map(attribute='item') | list }} + when: cloudmon_provision.results | selectattr('rc', 'ne', 0) | list | length > 0 # cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure grafana provision # cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure apimon provision diff --git a/roles/deploy_cloudmon_k8s/templates/inventory.j2 b/roles/deploy_cloudmon_k8s/templates/inventory.j2 index e7c8bac..99387a7 100644 --- a/roles/deploy_cloudmon_k8s/templates/inventory.j2 +++ b/roles/deploy_cloudmon_k8s/templates/inventory.j2 @@ -5,29 +5,39 @@ all: # "localhost" from the pod's point of view. localhost: ansible_host: localhost + # Defining localhost explicitly loses Ansible's implicit local + # connection, and the pod runs no sshd, so this must be set. + ansible_connection: local internal_address: localhost + vars: + # The pod has no service manager and no host firewall, so firewalld and + # iptables service management cannot work here. + cloudmon_manage_firewall: false children: + # A group's `hosts` must be a mapping, not a list -- a list makes Ansible + # reject the whole inventory ("requires a dictionary"), after which every + # play matches no hosts. statsd: hosts: - - localhost + localhost: graphite: hosts: - - localhost + localhost: grafana: hosts: - - localhost + localhost: schedulers: hosts: - - localhost + localhost: executors: hosts: - - localhost + localhost: epmons: hosts: - - localhost + localhost: globalmons: hosts: - - localhost + localhost: postgres: hosts: - - localhost + localhost: