WIP: SKC baremetal environment

etc/kayobe/environments/ci-aio/controllers.yml

@@ -4,7 +4,8 @@
 
 # User with which to access the controllers via SSH during bootstrap, in order
 # to setup the Kayobe user account. Default is {{ os_distribution }}.
-controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}"
+#controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}"
+controller_bootstrap_user: 'rocky'
 
 controller_extra_network_interfaces:
   - ethernet

etc/kayobe/environments/ci-aio/stackhpc.yml

@@ -1,3 +1,3 @@
 ---
 
-stackhpc_enable_cis_benchmark_hardening_hook: true
+stackhpc_enable_cis_benchmark_hardening_hook: false

etc/kayobe/environments/stackhpc-baremetal/ansible/baremetal-0-enroll-overcloud.yml

@@ -0,0 +1,83 @@
+---
+
+- name: Register baremetal compute nodes
+  hosts: localhost
+  vars:
+    venv: "{{ virtualenv_path }}/openstack-cli"
+  tasks:
+    - name: Set up openstack cli virtualenv
+      pip:
+        virtualenv: "{{ venv }}"
+        name:
+          - python-openstackclient
+          - python-ironicclient
+        state: latest
+        virtualenv_command: "python3.{{ ansible_facts.python.version.minor }} -m venv"
+        extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
+
+- name: Ensure overcloud baremetal nodes are registered in ironic
+  hosts: baremetal-overcloud
+  gather_facts: false
+  max_fail_percentage: >-
+    {{ baremetal_compute_register_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
+  tags:
+    - baremetal
+  vars:
+    venv: "{{ virtualenv_path }}/openstack-cli"
+    controller_host: localhost
+  tasks:
+    - name: Check Ironic variables are defined
+      ansible.builtin.assert:
+        that:
+          - ironic_driver is defined
+          - ironic_driver_info is defined
+          - ironic_properties is defined
+          - ironic_resource_class is defined
+        fail_msg: One or more Ironic variables are undefined.
+
+    - block:
+      - name: Show baremetal node
+        ansible.builtin.command:
+          cmd: "{{ venv }}/bin/openstack baremetal node show {{ inventory_hostname }}"
+        register: node_show
+        failed_when:
+          - '"HTTP 404" not in node_show.stderr'
+          - node_show.rc != 0
+        changed_when: false
+
+      # NOTE: The openstack.cloud.baremetal_node module cannot be used in this
+      # script due to requiring a MAC address pre-defined, instead, this should
+      # be discovered by inpsection following this script.
+      #
+      # NOTE: IPMI address must be passed with Redfish address to ensure existing
+      # Ironic nodes match with new nodes during inspection.
+      - name: Create baremetal nodes
+        ansible.builtin.shell:
+          cmd: |
+            {{ venv }}/bin/openstack baremetal node create \
+            --name {{ inventory_hostname }} \
+            --driver {{ ironic_driver }} \
+            {% for key, value in ironic_driver_info.items() %}
+            --driver-info {{ key }}={{ value }} \
+            {% endfor %}
+            {% for key, value in ironic_properties.items() %}
+            --property {{ key }}={{ value }} \
+            {% endfor %}
+            --resource-class {{ ironic_resource_class }}
+        when:
+          - node_show.rc != 0
+
+      - name: Manage baremetal nodes
+        ansible.builtin.command:
+          cmd: "{{ venv }}/bin/openstack baremetal node manage {{ inventory_hostname }} --wait"
+        when:
+          - node_show.rc != 0
+      delegate_to: "{{ controller_host }}"
+      vars:
+        # NOTE: Without this, the controller's ansible_host variable will not
+        # be respected when using delegate_to.
+        ansible_host: "{{ hostvars[controller_host].ansible_host | default(controller_host) }}"
+      environment: "{{ openstack_auth_env }}"

etc/kayobe/environments/stackhpc-baremetal/ansible/baremetal-1-check-bmc-up.yml

@@ -0,0 +1,124 @@
+---
+- name: Check baremetal compute node bmc is up
+  hosts: baremetal
+  gather_facts: false
+  max_fail_percentage: >-
+    {{ baremetal_compute_register_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
+  tags:
+    - baremetal
+  vars:
+    venv: "{{ virtualenv_path }}/openstack-cli"
+    controller_host: localhost
+
+  tasks:
+    - name: Check Ironic variables are defined
+      ansible.builtin.assert:
+        that:
+          - ironic_driver is defined
+          - ironic_driver_info is defined
+          - ironic_properties is defined
+          - ironic_resource_class is defined
+        fail_msg: One or more Ironic variables are undefined.
+
+    - name: Show and check baremetal node
+      delegate_to: "{{ controller_host }}"
+      vars:
+        # NOTE: Without this, the controller's ansible_host variable will not
+        # be respected when using delegate_to.
+        ansible_host: "{{ hostvars[controller_host].ansible_host | default(controller_host) }}"
+      environment: "{{ openstack_auth_env }}"
+      block:
+
+        - name: Show baremetal node
+          ansible.builtin.command:
+            cmd: "{{ venv }}/bin/openstack baremetal node show {{ inventory_hostname }} -f json"
+          register: node_show
+          failed_when:
+            - node_show.rc != 0
+          changed_when: false
+
+        - name: Check if bmc is up
+          ansible.builtin.set_fact:
+            kayobe_bmc_up: "{{ (node_show.stdout | from_json)['extra'].get('kayobe_bmc_up') }}"
+            provision_state: "{{ (node_show.stdout | from_json)['provision_state'] }}"
+
+        - name: Output when bmc last up run
+          ansible.builtin.debug:
+            msg: "BMC for node {{ inventory_hostname }} was up at {{ kayobe_bmc_up }}."
+          when: kayobe_bmc_up != ""
+
+        - name: Check BMC is up
+          ansible.builtin.uri:
+            url: "{{ ironic_driver_info['redfish_address'] + '/redfish/v1' }}"
+            method: GET
+            status_code: 200
+            validate_certs: false
+            timeout: 10
+
+        - name: Get firmware inventory (to check redfish auth)
+          community.general.redfish_info:
+            category: Update
+            command: GetFirmwareInventory
+            baseuri: "{{ ironic_redfish_address }}"
+            username: "{{ ironic_redfish_username }}"
+            password: "{{ ironic_redfish_password }}"
+          register: firmware_inventory
+          failed_when: not firmware_inventory.redfish_facts.firmware.ret
+
+        # - name: Print fetched information
+        #   ansible.builtin.debug:
+        #     msg: "{{ firmware_inventory.redfish_facts.firmware | to_nice_json }}"
+
+        - name: Reboot BMC
+          community.general.redfish_command:
+            category: Manager
+            command: PowerReboot
+            resource_id: 1
+            baseuri: "{{ ironic_redfish_address }}"
+            username: "{{ ironic_redfish_username }}"
+            password: "{{ ironic_redfish_password }}"
+          when: kayobe_bmc_up == ""
+
+        - name: Wait 300 seconds for port 443 to become open
+          ansible.builtin.wait_for:
+            port: 443
+            host: "{{ ironic_redfish_address }}"
+            delay: 20
+            timeout: 300
+          when: kayobe_bmc_up == ""
+
+        - name: Check BMC back up again
+          ansible.builtin.uri:
+            url: "https://{{ ironic_driver_info['redfish_address'] }}"
+            method: GET
+            status_code: 200
+            validate_certs: false
+            timeout: 10
+          register: uri_output
+          until: uri_output.status == 200
+          delay: 5
+          retries: 24 # Retries for 24 * 5 seconds = 120 seconds = 2 minutes
+
+        - name: Note when we are able to reach the bmc, the first time
+          ansible.builtin.command:
+            cmd: |
+              {{ venv }}/bin/openstack baremetal node set {{ inventory_hostname }} --extra kayobe_bmc_up={{ now(utc=true, fmt='%Y-%m-%dT%H:%M:%SZ') }}
+          register: node_set
+          failed_when:
+            - node_set.rc != 0
+          changed_when: true
+          when: kayobe_bmc_up == ""
+
+        - name: Try move from enroll to manageable
+          ansible.builtin.command:
+            cmd: |
+              {{ venv }}/bin/openstack baremetal node manage {{ inventory_hostname }} --wait 300
+          register: node_set
+          failed_when:
+            - node_set.rc != 0
+          changed_when: true
+          when:
+            - provision_state == "enroll"

etc/kayobe/environments/stackhpc-baremetal/ansible/baremetal-2-ensure-redfish-inspect.yml

@@ -0,0 +1,86 @@
+---
+- name: Check baremetal compute node bmc is up
+  hosts: baremetal
+  gather_facts: false
+  max_fail_percentage: >-
+    {{ baremetal_compute_register_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
+  tags:
+    - baremetal
+  vars:
+    venv: "{{ virtualenv_path }}/openstack-cli"
+    controller_host: localhost
+
+  tasks:
+    - name: Show and check baremetal node
+      delegate_to: "{{ controller_host }}"
+      vars:
+        # NOTE: Without this, the controller's ansible_host variable will not
+        # be respected when using delegate_to.
+        ansible_host: "{{ hostvars[controller_host].ansible_host | default(controller_host) }}"
+        redfish_inspect_timeout: 120
+      environment: "{{ openstack_auth_env }}"
+      block:
+
+        - name: Show baremetal node
+          ansible.builtin.command:
+            cmd: "{{ venv }}/bin/openstack baremetal node show {{ inventory_hostname }} -f json"
+          register: node_show
+          failed_when:
+            - node_show.rc != 0
+          changed_when: false
+
+        - name: Check BMC is up
+          ansible.builtin.uri:
+            url: "{{ ironic_driver_info['redfish_address'] + '/redfish/v1' }}"
+            method: GET
+            status_code: 200
+            validate_certs: false
+            timeout: 10
+
+        - name: Check for redfish inspection details
+          ansible.builtin.set_fact:
+            kayobe_redfish_inspect_done: "{{ (node_show.stdout | from_json)['extra'].get('kayobe_redfish_inspect_done') }}"
+            inspect_interface: "{{ (node_show.stdout | from_json)['inspect_interface'] }}"
+            provision_state: "{{ (node_show.stdout | from_json)['provision_state'] }}"
+
+        - name: Output when redfish inspection was done
+          ansible.builtin.debug:
+            msg: "{{ inventory_hostname }} inspected at {{ kayobe_redfish_inspect_done }}."
+          when: kayobe_redfish_inspect_done != ""
+
+        - name: Fail if not redfish inspection
+          ansible.builtin.fail:
+            msg: "{{ inventory_hostname }} has the wrong inspect_interface: {{ inspect_interface }}"
+          when:
+            - inspect_interface != "redfish"
+            - kayobe_redfish_inspect_done == ""
+
+        - name: Fail if not in manageable state
+          ansible.builtin.fail:
+            msg: "{{ inventory_hostname }} has the wrong provision_state: {{ provision_state }}"
+          when:
+            - provision_state != "manageable"
+            - kayobe_redfish_inspect_done == ""
+
+        - name: Wait for inspection
+          ansible.builtin.command:
+            cmd: |
+              {{ venv }}/bin/openstack baremetal node inspect {{ inventory_hostname }} --wait {{ redfish_inspect_timeout }}
+          register: node_inspect
+          failed_when:
+            - node_inspect.rc != 0
+          changed_when: true
+          when: kayobe_redfish_inspect_done == ""
+
+        - name: Note when redfish inspection is done
+          ansible.builtin.command:
+            cmd: |
+              {{ venv }}/bin/openstack baremetal node set {{ inventory_hostname }} --extra kayobe_redfish_inspect_done={{ now(utc=true, fmt='%Y-%m-%dT%H:%M:%SZ') }}
+          register: node_set
+          failed_when:
+            - node_set.rc != 0
+          changed_when: true
+          when: kayobe_redfish_inspect_done == ""