[skip ci]Fix pod failure chaos github action (#10208)

Signed-off-by: Nzhuwenxing <wenxing.zhu@zilliz.com>

.github/workflows/pod-failure-chaos-test.yaml

@@ -5,14 +5,14 @@ on:
 
 jobs:
   
-  test-pod-kill-chaos:
+  test-pod-failure-chaos:
    
     runs-on: ubuntu-latest
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        pod: [queryNode]
+        pod: [querynode]
     
     steps:
 
@@ -52,7 +52,7 @@ jobs:
           helm repo add chaos-mesh https://charts.chaos-mesh.org
           helm search repo chaos-mesh
           kubectl create ns chaos-testing
-          helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v0.5.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
+          helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.2 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
           sleep 60s
           kubectl get po -n chaos-testing
       
@@ -60,9 +60,10 @@ jobs:
         shell: bash
         working-directory: tests/python_client/chaos
         run: |
+          declare -A pod_map=( ["querynode"]="queryNode")
           helm repo add milvus https://milvus-io.github.io/milvus-helm
           helm repo update
-          helm install --wait --timeout 360s milvus-chaos milvus/milvus --set ${{ matrix.pod }}.replicas=2 -f cluster-values.yaml  -n=chaos-testing
+          helm install --wait --timeout 360s milvus-chaos milvus/milvus --set ${pod_map[${{ matrix.pod }}]}.replicas=2 -f cluster-values.yaml  -n=chaos-testing
           kubectl get pods -n chaos-testing
           sleep 20s
           kubectl get pods -n chaos-testing
@@ -81,7 +82,8 @@ jobs:
         run: |
           # replace chaos object
           sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_failure\/'/g" constants.py
-          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_podkill.yaml\'/g" constants.py
+          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_failure.yaml\'/g" constants.py
+          sed -i "s/CHAOS_DURATION =.*/CHAOS_DURATION = 80/g" constants.py
           cat constants.py
           pytest -s -v test_chaos.py --host 127.0.0.1
 

tests/python_client/chaos/chaos_objects/pod_failure/chaos_querynode_pod_failure.yaml

@@ -4,16 +4,13 @@ metadata:
   name: test-querynode-pod-failure
   namespace: chaos-testing
 spec:
-  action: pod-failure
-  mode: one
-  value: ''
-  duration: '20s'
   selector:
     namespaces:
-      - chaos-testing         # target namespace of milvus deployment
+      - chaos-testing
     labelSelectors:
       app.kubernetes.io/instance: milvus-chaos
-      app.kubernetes.io/name: milvus
       component: querynode
-  scheduler:
-    cron: '@every 30s'
+  mode: one
+  action: pod-failure
+  duration: 2m
+  gracePeriod: 0
\ No newline at end of file

tests/python_client/chaos/constants.py

@@ -27,4 +27,5 @@ CHAOS_CONFIG_ENV = 'CHAOS_CONFIG_PATH'      # env variables for chao path
 TESTS_CONFIG_LOCATION = 'chaos_objects/pod_kill/' # path to the chaos CRD
 ALL_CHAOS_YAMLS = 'chaos_datanode*.yaml'    # chaos file name(s) to be run against
 WAIT_PER_OP = 10                            # time to wait in seconds between operations
+CHAOS_DURATION = 40                         # chaos duration time in seconds
 DEFAULT_INDEX_PARAM = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": {"nlist": 64}}

tests/python_client/chaos/test_chaos.py

@@ -138,7 +138,7 @@ class TestChaos(TestChaosBase):
         cc.reset_counting(self.health_checkers)
 
         # wait 40s
-        sleep(constants.WAIT_PER_OP * 4)
+        sleep(constants.CHAOS_DURATION)
 
         for k, t in self.checker_threads.items():
             log.info(f"10s later: Thread {k} is_alive(): {t.is_alive()}")