From d13ca5441482510795a69b0d750a72dbae3aa4a9 Mon Sep 17 00:00:00 2001
From: SimFG <bang.fu@zilliz.com>
Date: Tue, 22 Aug 2023 09:04:21 +0800
Subject: [PATCH] Change the default gracefulStopTimeout and only warn when
 fail to refresh policy cache (#26450)

Signed-off-by: SimFG <bang.fu@zilliz.com>
---
 configs/milvus.yaml                    | 4 +---
 internal/rootcoord/root_coord.go       | 4 ++--
 pkg/util/paramtable/component_param.go | 4 ++--
 tests/integration/minicluster.go       | 1 +
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/configs/milvus.yaml b/configs/milvus.yaml
index 4d12048ce..88fd814b5 100644
--- a/configs/milvus.yaml
+++ b/configs/milvus.yaml
@@ -290,7 +290,6 @@ queryNode:
       enableCrossUserGrouping: false # false by default Enable Cross user grouping when using user-task-polling policy. (close it if task of any user can not merge others).
       maxPendingTaskPerUser: 1024 # 50 by default, max pending task in scheduler per user.
 
-  gracefulStopTimeout: 30
   port: 21123
   grpc:
     serverMaxSendSize: 536870912
@@ -312,7 +311,6 @@ indexNode:
     buildParallel: 1
   enableDisk: true # enable index node build disk vector index
   maxDiskUsagePercentage: 95
-  gracefulStopTimeout: 30
   port: 21121
   grpc:
     serverMaxSendSize: 536870912
@@ -467,7 +465,7 @@ common:
     LoadNumThreadRatio: 8
     BeamWidthRatio: 4
   gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency.
-  gracefulStopTimeout: 30 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
+  gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
   storageType: minio # please adjust in embedded Milvus: local
   # Default value: auto
   # Valid values: [auto, avx512, avx2, avx, sse4_2]
diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go
index 72a716879..911251968 100644
--- a/internal/rootcoord/root_coord.go
+++ b/internal/rootcoord/root_coord.go
@@ -642,12 +642,12 @@ func (c *Core) startInternal() error {
 			if err := c.proxyClientManager.RefreshPolicyInfoCache(c.ctx, &proxypb.RefreshPolicyInfoCacheRequest{
 				OpType: int32(typeutil.CacheRefresh),
 			}); err != nil {
-				log.Warn("fail to refresh policy info cache", zap.Error(err))
+				log.Info("fail to refresh policy info cache", zap.Error(err))
 				return err
 			}
 			return nil
 		}, retry.Attempts(100), retry.Sleep(time.Second)); err != nil {
-			log.Panic("fail to refresh policy info cache", zap.Error(err))
+			log.Warn("fail to refresh policy info cache", zap.Error(err))
 		}
 	}()
 
diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go
index 3f66688c0..ab9ab770e 100644
--- a/pkg/util/paramtable/component_param.go
+++ b/pkg/util/paramtable/component_param.go
@@ -35,7 +35,7 @@ const (
 	// DefaultIndexSliceSize defines the default slice size of index file when serializing.
 	DefaultIndexSliceSize                      = 16
 	DefaultGracefulTime                        = 5000 // ms
-	DefaultGracefulStopTimeout                 = 30   // s
+	DefaultGracefulStopTimeout                 = 1800 // s
 	DefaultHighPriorityThreadCoreCoefficient   = 100
 	DefaultMiddlePriorityThreadCoreCoefficient = 50
 	DefaultLowPriorityThreadCoreCoefficient    = 10
@@ -475,7 +475,7 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
 	p.GracefulStopTimeout = ParamItem{
 		Key:          "common.gracefulStopTimeout",
 		Version:      "2.2.1",
-		DefaultValue: "30",
+		DefaultValue: strconv.Itoa(DefaultGracefulStopTimeout),
 		Doc:          "seconds. it will force quit the server if the graceful stop process is not completed during this time.",
 		Export:       true,
 	}
diff --git a/tests/integration/minicluster.go b/tests/integration/minicluster.go
index 002aaab0f..a34686ed1 100644
--- a/tests/integration/minicluster.go
+++ b/tests/integration/minicluster.go
@@ -484,6 +484,7 @@ func DefaultParams() map[string]string {
 		params.LocalStorageCfg.Path.Key:               path.Join("/tmp", testPath),
 		params.CommonCfg.StorageType.Key:              "local",
 		params.DataNodeCfg.MemoryForceSyncEnable.Key:  "false", // local execution will print too many logs
+		params.CommonCfg.GracefulStopTimeout.Key:      "10",
 	}
 }
 
-- 
GitLab