feat(e2e-test): Add e2e-tests for zfs-localpv (#298)

Signed-off-by: w3aman <aman.gupta@mayadata.io>

e2e-tests/utils/applications/busybox/busybox_data_persistence.yml

@@ -0,0 +1,108 @@
+---
+- block:
+
+   - name: Create some test data in the busybox app
+     shell: >
+       kubectl exec {{ pod_name }} -n {{ ns }} 
+       -- sh -c "{{ item }}"
+     args:
+       executable: /bin/bash
+     register: result
+     failed_when: "result.rc != 0"
+     with_items:
+       - "dd if=/dev/urandom of=/busybox/{{ testfile }} bs={{ blocksize }} count={{ blockcount }}"
+       - "md5sum /busybox/{{ testfile }} > /busybox/{{ testfile }}-pre-chaos-md5"
+       - "sync;sync;sync"
+
+  when: status == "LOAD"
+
+- block:
+
+   - name: Kill the application pod
+     shell: >
+       kubectl delete pod {{ pod_name }} -n {{ ns }}
+     args:
+       executable: /bin/bash
+
+   - name: Verify if the application pod is deleted
+     shell: >
+       kubectl get pods -n {{ ns }}
+     args:
+       executable: /bin/bash
+     register: podstatus
+     until: '"{{ pod_name }}" not in podstatus.stdout'
+     retries: 2
+     delay: 150
+
+   - name: Obtain the newly created pod name for application
+     shell: >
+       kubectl get pods -n {{ ns }} -l {{ label }} -o jsonpath='{.items[].metadata.name}'
+     args:
+       executable: /bin/bash
+     register: newpod_name
+
+   - name: Checking application pod is in running state
+     shell: kubectl get pods -n {{ ns }} -o jsonpath='{.items[?(@.metadata.name=="{{ newpod_name.stdout }}")].status.phase}'
+     register: result
+     until: "((result.stdout.split()|unique)|length) == 1 and 'Running' in result.stdout"
+     delay: 2
+     retries: 150
+
+   - name: Get the container status of application.
+     shell: >
+        kubectl get pods -n {{ ns }} -o jsonpath='{.items[?(@.metadata.name=="{{ newpod_name.stdout }}")].status.containerStatuses[].state}' | grep running
+     args:
+       executable: /bin/bash
+     register: containerStatus
+     until: "'running' in containerStatus.stdout"
+     delay: 2
+     retries: 150
+
+   - name: Check the md5sum of stored data file
+     shell: >
+       kubectl exec {{ newpod_name.stdout }} -n {{ ns }}
+       -- sh -c "md5sum /busybox/{{ testfile }} > /busybox/{{ testfile }}-post-chaos-md5"
+     args:
+       executable: /bin/bash
+     register: status 
+     failed_when: "status.rc != 0"
+
+   - name: Verify whether data is consistent
+     shell: >
+           kubectl exec {{ newpod_name.stdout }} -n {{ ns }}
+           -- sh -c "diff /busybox/{{ testfile }}-pre-chaos-md5 /busybox/{{ testfile }}-post-chaos-md5"
+     args:
+       executable: /bin/bash
+     register: result 
+     failed_when: "result.rc != 0 or result.stdout != ''"   
+
+  when: status == "VERIFY"
+
+- block:
+
+   - name: Obtain the current pod name for application
+     shell: >
+       kubectl get pods -n {{ ns }} -l {{ label }} -o jsonpath='{.items[].metadata.name}'
+     args:
+       executable: /bin/bash
+     register: newpod_name
+
+   - name: Delete/drop the files 
+     shell: > 
+       kubectl exec {{ newpod_name.stdout }} -n {{ ns }}
+       -- sh -c "rm -f /busybox/{{ testfile }}*"
+     args:
+       executable: /bin/bash
+     register: status 
+
+   - name: Verify successful file delete 
+     shell: > 
+           kubectl exec {{ newpod_name.stdout }} -n {{ ns }}
+           -- ls /busybox/
+     args:
+       executable: /bin/bash
+     register: result 
+     failed_when: "testfile in result.stdout"   
+
+  when: status == "DELETE"
+

e2e-tests/utils/applications/mysql/check_db_connection.yml

@@ -0,0 +1,8 @@
+#Check if the database is ready for connection, upper bound wait time: 900s 
+- name: Check if db is ready for connections
+  shell: kubectl logs {{ pod_name.resources.0.metadata.name }} -n {{ app_ns }} | grep 'ready for connections' | wc -l
+  register: initcheck
+  until: initcheck.stdout == "2"
+  delay: 5
+  retries: 180
+

e2e-tests/utils/applications/mysql/mysql_data_persistence.yml

@@ -0,0 +1,108 @@
+---
+- block:
+
+   - name: Create some test data in the mysql database
+     shell: >
+       kubectl exec {{ pod_name }} -n {{ ns }} 
+       -- {{ item }}
+     args:
+       executable: /bin/bash
+     register: result
+     failed_when: "result.rc != 0"
+     with_items:
+       - mysql -u{{ dbuser }} -p{{ dbpassword }} -e 'create database {{ dbname }};'
+       - mysql -u{{ dbuser }} -p{{ dbpassword }} -e 'create table ttbl (Data VARCHAR(20));' {{ dbname }}
+       - mysql -u{{ dbuser }} -p{{ dbpassword }} -e 'insert into ttbl (Data) VALUES ("tdata");' {{ dbname }}
+
+  when: status == "LOAD"
+
+- block:
+
+   - name: Kill the application pod
+     shell: >
+       kubectl delete pod {{ pod_name }} -n {{ ns }}
+     args:
+       executable: /bin/bash
+
+   - name: Verify if the application pod is deleted
+     shell: >
+       kubectl get pods -n {{ ns }}
+     args:
+       executable: /bin/bash
+     register: podstatus
+     until: '"{{ pod_name }}" not in podstatus.stdout'
+     retries: 2
+     delay: 150
+
+   - name: Obtain the newly created pod name for application
+     shell: >
+       kubectl get pods -n {{ ns }} -l {{ label }} -o jsonpath='{.items[].metadata.name}'
+     args:
+       executable: /bin/bash
+     register: newpod_name
+
+   - name: Checking application pod is in running state
+     shell: kubectl get pods -n {{ ns }} -o jsonpath='{.items[?(@.metadata.name=="{{ newpod_name.stdout }}")].status.phase}'
+     register: result
+     until: "((result.stdout.split()|unique)|length) == 1 and 'Running' in result.stdout"
+     delay: 2
+     retries: 150
+
+   - name: Get the container status of application.
+     shell: >
+        kubectl get pods -n {{ ns }} -o jsonpath='{.items[?(@.metadata.name=="{{ newpod_name.stdout }}")].status.containerStatuses[].state}' | grep running
+     args:
+       executable: /bin/bash
+     register: containerStatus
+     until: "'running' in containerStatus.stdout"
+     delay: 2
+     retries: 150
+
+   - name: Check if db is ready for connections
+     shell: kubectl logs {{ newpod_name.stdout }} -n {{ ns }} | grep 'ready for connections'
+     register: initcheck
+     until: "'ready for connections' in initcheck.stdout"
+     delay: 5
+     retries: 180
+
+   - name: Checking for the Corrupted tables
+     shell: >
+       kubectl exec {{ newpod_name.stdout }} -n {{ ns }}
+       -- mysqlcheck -c {{ dbname }} -u{{ dbuser }} -p{{ dbpassword }}
+     args:
+       executable: /bin/bash
+     register: status 
+     failed_when: "'OK' not in status.stdout"
+
+   - name: Verify mysql data persistence 
+     shell: >
+           kubectl exec {{ newpod_name.stdout }} -n {{ ns }}
+           -- mysql -u{{ dbuser }} -p{{ dbpassword }} -e 'select * from ttbl' {{ dbname }};
+     args:
+       executable: /bin/bash
+     register: result 
+     failed_when: "'tdata' not in result.stdout"   
+
+  when: status == "VERIFY"
+
+- block:
+
+   - name: Delete/drop MySQL database 
+     shell: > 
+       kubectl exec {{ pod_name }} -n {{ ns }}
+       -- mysql -u{{ dbuser }} -p{{ dbpassword }} -e 'drop database {{ dbname }}'; 
+     args:
+       executable: /bin/bash
+     register: status 
+
+   - name: Verify successful db delete 
+     shell: > 
+           kubectl exec {{ pod_name }} -n {{ ns }}
+           -- mysql -u{{ dbuser }} -p{{ dbpassword }} -e 'show databases';
+     args:
+       executable: /bin/bash
+     register: result 
+     failed_when: "dbname in result.stdout"   
+
+  when: status == "DELETE"
+

e2e-tests/utils/k8s/create_ns.yml

@@ -0,0 +1,16 @@
+---
+- name: Obtain list of existing namespaces
+  shell: >
+    kubectl get ns --no-headers -o custom-columns=:metadata.name
+  args: 
+    executable: /bin/bash
+  register: ns_list
+
+- name: Create test specific namespace.
+  shell: kubectl create ns {{ app_ns }}
+  args:
+    executable: /bin/bash
+  when: app_ns != 'e2e' and app_ns not in ns_list.stdout_lines
+
+  # Check status of namespace
+- include_tasks: /e2e-tests/utils/k8s/status_testns.yml

e2e-tests/utils/k8s/deploy_single_app.yml

@@ -0,0 +1,12 @@
+---
+#Deploying application on k8's cluster and cross checking whether the
+#application is deployed successfully.
+- name: Deploying {{ application_name }}
+  k8s:
+    state: present
+    src: "{{ application }}"
+    namespace: "{{ app_ns }}"
+    merge_type: merge  
+  register: result
+
+- include_tasks: /e2e-tests/utils/k8s/status_app_pod.yml

e2e-tests/utils/k8s/deprovision_deployment.yml

@@ -0,0 +1,106 @@
+---
+# This Utility task file can delete the application and its underlying resources such as pvc and service from K8s cluster
+# This accepts application namespace, application label and application manifest file as input parameters.
+# The parameters used are
+#         - app_deployer ( Deployment spec yaml file )
+#         - app_ns       ( application namespace )
+#         - app_label    ( application label)
+#
+- block:
+
+    - name: Check if the application to be deleted is available.
+      k8s_facts:
+        kind: Pod
+        label_selectors:
+          - "{{ app_label }}"
+        namespace: "{{ app_ns }}"
+      register: po_name
+      until: "{{ po_name | json_query('resources[*].status.phase') | unique | length==1}}"
+      delay: 5
+      retries: 60
+
+    - name: Obtaining the PVC name using application label.
+      set_fact:
+        pvc_name: "{{ po_name.resources.0.spec.volumes.0.persistentVolumeClaim.claimName }}"
+        pod_name: "{{ po_name.resources.0.metadata.name }}"
+
+    - name: Obtaining the PV name from PVC name. 
+      k8s_facts:
+        kind: PersistentVolumeClaim
+        namespace: "{{ app_ns }}"
+        name: "{{ pvc_name }}"
+      register: pv_name
+
+    - set_fact:
+        pvname: "{{ pv_name | json_query('resources[0].spec.volumeName') }}"
+
+    ## Replacing the item names in the respective deployer spec file.
+    - name: Replace the PVC name in application deployer spec.
+      replace:
+        path: "{{ app_deployer }}"
+        regexp: "testclaim"
+        replace: "{{ lookup('env','APP_PVC') }}"
+      when: app_pvc is defined
+ 
+    - name: Replace the storageclass placeholder with provider
+      replace:
+        path: "{{ app_deployer }}"
+        regexp: "testclass"
+        replace: "{{ lookup('env','PROVIDER_STORAGE_CLASS') }}"
+      when: storage_class is defined
+    
+    - block:
+
+        - name: Get the application label values from env
+          set_fact:
+            app_lkey: "{{ app_label.split('=')[0] }}"
+            app_lvalue: "{{ app_label.split('=')[1] }}"
+    
+        - name: Replace the application label placeholder 
+          replace:
+            path: "{{ app_deployer }}"
+            regexp: "lkey: lvalue"
+            replace: "{{ app_lkey }}: {{ app_lvalue }}"
+      when: app_label is defined
+    
+    - name: Delete the application deployment.
+      shell: kubectl delete -f {{ app_deployer }} -n {{ app_ns }}
+      args:
+        executable: /bin/bash
+      ignore_errors: true
+    
+    - name: Check if the PVC is deleted.
+      k8s_facts:
+        kind: PersistentVolumeClaim
+        namespace: "{{ app_ns }}"
+        label_selectors:
+          - "{{ app_label }}"
+      register: resource_list
+      until: resource_list.resources | length < 1
+      delay: 5
+      retries: 120
+    
+- name: Check if the pods are deleted in the namespaces
+  shell: >
+    kubectl get pods -n {{ app_ns }} 
+  args:
+    executable: /bin/bash
+  register: result
+  until: "pod_name not in result.stdout"
+  delay: 5
+  retries: 60
+
+- name: Delete the namespace.
+  k8s:
+    state: absent
+    kind: Namespace
+    name: "{{ app_ns }}"
+
+- name: Check if the PV is deleted.
+  k8s_facts:
+    kind: PersistentVolume
+    name: "{{ pvname }}"
+    label_selectors:
+      - "{{ app_label }}"
+  register: pv_result
+  failed_when: "pv_result.resources | length > 1"

e2e-tests/utils/k8s/deprovision_statefulset.yml

@@ -0,0 +1,72 @@
+---
+- block:
+
+    - name: Check if the statefulset application exists.
+      shell: kubectl get pods -n {{ app_ns }} -l {{ app_label }}
+      register: pods
+      failed_when: "'No resources found' in pods.stdout"
+
+    - name: Obtaining PVCs related to the application.
+      shell: kubectl get pvc -n {{ app_ns }} -l {{ app_label }}  --no-headers -o custom-columns=:.metadata.name
+      register: pvc_list
+
+    - name: Obtaining the PV names.
+      shell: kubectl get pvc -l {{ app_label }} -n {{ app_ns }} --no-headers -o custom-columns=:.spec.volumeName
+      register: pv_list
+
+     ## Replacing the item names in the respective deployer spec file.
+    - name: Replace the PVC name in application deployer spec.
+      replace:
+        path: "{{ app_deployer }}"
+        regexp: "testclaim"
+        replace: "{{ lookup('env','APP_PVC') }}"
+      when: app_pvc is defined
+
+    - name: Replace the storageclass placeholder with provider
+      replace:
+        path: "{{ app_deployer }}"
+        regexp: "testclass"
+        replace: "{{ lookup('env','PROVIDER_STORAGE_CLASS') }}"
+      when: storage_class is defined
+
+    - block:
+
+        - name: Get the application label values from env
+          set_fact:
+            app_lkey: "{{ app_label.split('=')[0] }}"
+            app_lvalue: "{{ app_label.split('=')[1] }}"
+
+        - name: Replace the application label placeholder
+          replace:
+            path: "{{ app_deployer }}"
+            regexp: "lkey: lvalue"
+            replace: "{{ app_lkey }}: {{ app_lvalue }}"
+
+      when: app_label is defined
+
+    - name: Delete the application and its related service.
+      shell: kubectl delete -f {{ app_deployer }} -n {{ app_ns }}
+      register: app_status
+      until: 'app_status.rc == 0'
+      delay: 5
+      retries: 60
+
+    - name: Deleting the PVC
+      shell: kubectl delete pvc {{ item }} -n {{ app_ns }}
+      args:
+        executable: /bin/bash
+      with_items:
+        - "{{ pvc_list.stdout_lines }}"
+
+    - name: Check if the PVCs are deleted
+      shell: kubectl get pvc -n {{ app_ns }}
+      register: list_pvc
+      until: "'No resources found' in list_pvc.stderr"
+      delay: 30
+      retries: 15
+
+- name: Delete the namespace.
+  shell: kubectl delete ns {{ app_ns }}
+  args:
+    executable: /bin/bash
+

e2e-tests/utils/k8s/fetch_app_pod.yml

@@ -0,0 +1,13 @@
+---
+#Fetching the details of the application pod
+- name: Getting the {{ application_name }} POD name
+  k8s_facts:
+    kind: Pod
+    namespace: "{{ app_ns }}" 
+    label_selectors:
+      - "{{ app_label }}"
+  register: pod_name
+
+- debug:
+    msg: "{{ pod_name | json_query('resources[*].metadata.name') }}"
+

e2e-tests/utils/k8s/pre_create_app_deploy.yml

@@ -0,0 +1,40 @@
+---
+- block:
+    - name: Check whether the provider storageclass is present
+      shell: kubectl get sc "{{ lookup('env','STORAGE_CLASS') }}"
+      args:
+        executable: /bin/bash
+      register: result
+      failed_when: "result.rc != 0"
+
+    - name: Replace the storageclass placeholder with test specific value
+      replace:
+        path: "{{ application }}"
+        regexp: "testclass"
+        replace: "{{ lookup('env','STORAGE_CLASS') }}"
+      
+    - name: Replace the application pvc placeholder with test specific value
+      replace:
+        path: "{{ application }}"
+        regexp: "testclaim"
+        replace: "{{ lookup('env','APP_PVC') }}"
+
+    - name: Replace the persistent volume capcity placeholder with test specific value
+      replace:
+        path: "{{ application }}"
+        regexp: "teststorage"
+        replace: "{{ lookup('env','PV_CAPACITY') }}"
+
+    - name: Get the application label value from env
+      set_fact:
+         app_lkey: "{{ app_label.split('=')[0] }}"
+         app_lvalue: "{{ app_label.split('=')[1] }}"
+
+    - name: Replace the application label placeholder in deployment spec
+      replace:
+        path: "{{ application }}"
+        regexp: "lkey: lvalue"
+        replace: "{{ app_lkey }}: {{ app_lvalue }}"
+
+      # Create test specific namespace
+    - include_tasks: /e2e-tests/utils/k8s/create_ns.yml

e2e-tests/utils/k8s/status_app_pod.yml

@@ -0,0 +1,18 @@
+---
+- name: Checking {{ application_name }} pod is in running state
+  shell: kubectl get pods -n {{ app_ns }} -o jsonpath='{.items[?(@.metadata.labels.{{app_lkey}}=="{{app_lvalue}}")].status.phase}'
+  register: result
+  until: "((result.stdout.split()|unique)|length) == 1 and 'Running' in result.stdout"
+  delay: 3
+  retries: 60
+
+- name: Get the container status of application.
+  shell: >
+     kubectl get pod -n {{ app_ns }} -l {{app_lkey}}="{{app_lvalue}}"
+     -o custom-columns=:..containerStatuses[].state --no-headers | grep -w "running"
+  args:
+    executable: /bin/bash
+  register: containerStatus
+  until: "'running' in containerStatus.stdout"
+  delay: 3
+  retries: 60

e2e-tests/utils/k8s/status_testns.yml

@@ -0,0 +1,9 @@
+---
+- name: Checking the status  of test specific namespace.
+  k8s_facts:
+    kind: Namespace
+    name: "{{ app_ns }}"
+  register: npstatus
+  until: "'Active' in npstatus.resources.0.status.phase"
+  delay: 30
+  retries: 10