zfs-localpv/e2e-tests/chaoslib/service_failure/service_chaos.yml

- block:

    - name: Identify the node on which application pod is scheduled
      shell: >
        kubectl get pod {{ app_pod }} -n {{ app_ns }}
        --no-headers -o custom-columns=:spec.nodeName
      args: 
        executable: /bin/bash
      register: node_name 
               
    - name: Record the node name on which application pod is scheduled
      set_fact:
        app_node: "{{ node_name.stdout }}"

    - name: Get the IP Address of the node on which application pod is scheduled
      shell: >
        kubectl get nodes {{ app_node }} --no-headers -o jsonpath='{.status.addresses[0].address}'
      args:
        executable: /bin/bash
      register: node_ip_address

    - name: Record the IP Address of the node on which application pod is scheduled
      set_fact:
        node_ip_add: "{{ node_ip_address.stdout }}"
    
    - block:
       
       - name: stop the docker service on node where application pod is scheduled
         shell: > 
           sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }}
           "echo {{ node_pwd }} | sudo -S su -c 'systemctl stop docker.socket'"
         args:
           executable: /bin/bash
         when: svc_type == "docker"

       - name: stop the container runtime (if containerd, or crio) services on the application node
         shell: > 
           sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }}
           "echo {{ node_pwd }} | sudo -S su -c 'systemctl stop {{ svc_type }}.service'"    
         args:
           executable: /bin/bash
         when: svc_type == "containerd" or svc_type == "crio"

       - name: stop the kubelet service on node where application pod is scheduled
         shell: > 
           sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }}
           "echo {{ node_pwd }} | sudo -S su -c 'systemctl stop kubelet.service'"
         args:
           executable: /bin/bash
         when: svc_type == "kubelet"

       - name: Check for the {{ svc_type }} service status
         shell: >
           sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }} 
           "echo {{ node_pwd }} | sudo -S su -c 'systemctl status {{ svc_type }}.service'" | grep 'inactive'
         args:
          executable: /bin/bash
         register: svc_status
         until: "'inactive' in svc_status.stdout"
         delay: 5
         retries: 15

       - name: Check the node {{ app_node }} status on which {{ svc_type }} failure chaos is induced
         shell:
           kubectl get nodes {{ app_node }}
         args: 
           executable: /bin/bash
         register: node_status
         until: "'NotReady' in node_status.stdout"
         delay: 10
         retries: 30

       - name: Check if the new application pod is scheduled after {{ svc_type }} failure
         shell: >
           kubectl get pods -n {{ app_ns }} -l {{ app_label }} --no-headers | wc -l
         args:
           executable: /bin/bash
         register: app_pod_count
         until: "'2' in app_pod_count.stdout"
         delay: 15
         retries: 30

       - name: Get the new application pod name
         shell: > 
           kubectl get pod -n {{ app_ns }} -l {{ app_label }} --no-headers | grep -v Terminating | awk '{print $1}'
         args:
           executable: /bin/bash
         register: new_app_pod_name

       - name: Record the new application pod name 
         set_fact:
           new_app_pod: "{{ new_app_pod_name.stdout }}"
         
       - name: Check for the newly created application pod status 
         shell: >
           kubectl get pod {{ new_app_pod }} -n {{ app_ns }} --no-headers -o custom-columns=:.status.phase
         args:
           executable: /bin/bash
         register: new_app_pod_status
         until: "'Pending' in new_app_pod_status.stdout"
         delay: 5
         retries: 20
       
      when: svc_type=="kubelet" or svc_type=="docker"

  when: action == "svc_stop"

  
- block: 

   - name: Start the docker services
     shell: > 
       sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }}
       "echo {{ node_pwd }} | sudo -S su -c 'systemctl start docker.socket'"    
     args:
       executable: /bin/bash
     when: svc_type == "docker"

   - name: Start the container runtime (if containerd, or crio) services
     shell: > 
       sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }}
       "echo {{ node_pwd }} | sudo -S su -c 'systemctl start {{ svc_type }}.service'"    
     args:
       executable: /bin/bash
     when: svc_type == "containerd" or svc_type == "crio"

   - name: Start the kubelet services
     shell: > 
       sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }}
       "echo {{ node_pwd }} | sudo -S su -c 'systemctl start kubelet.service'"    
     args:
       executable: /bin/bash
     when: svc_type == "kubelet"

   - name: Check for the {{ svc_type }} services status
     shell: >
       sshpass -p {{ node_pwd }} ssh -o StrictHostKeyChecking=no {{ user }}@{{ node_ip_add }}
       "echo {{ node_pwd }} | sudo -S su -c 'systemctl status {{ svc_type }}.service'" | grep 'active (running)'
     args: 
       executable: /bin/bash
     register: svc_status
     until: "'active (running)' in svc_status.stdout"
     delay: 5
     retries: 15

   - name: Check for the node status after starting {{ svc_type }} service
     shell: >
       kubectl get nodes {{ app_node }} --no-headers
     args: 
       executable: /bin/bash
     register: node_status
     until: "'NotReady' not in node_status.stdout"
     delay: 10
     retries: 30

   - name: Verify that previous pods are deleted successfully after restart of {{ svc_type }}
     shell: >
       kubectl get pods -n {{ app_ns }} -l {{ app_label }} --no-headers | wc -l
     args:
       executable: /bin/bash
     register: app_pod_count
     until: "'1' in app_pod_count.stdout"
     delay: 5
     retries: 60
     
   - name: Get the status of newly created application pod
     shell: >
       kubectl get pod {{ new_app_pod }} -n {{ app_ns }} --no-headers -o custom-columns=:.status.phase
     args:
       executable: /bin/bash
     register: application_pod
     until: "'Running' in application_pod.stdout"
     delay: 10
     retries: 50

  when: action == "svc_start"