Add ShowConfigurations to QueryNode (#18426)

Signed-off-by: Naoiasd <zhicheng.yue@zilliz.com>

internal/config/manager.go

@@ -18,6 +18,7 @@ package config
 import (
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 
 	"github.com/milvus-io/milvus/internal/log"
@@ -62,6 +63,26 @@ func (m *Manager) GetConfig(key string) (string, error) {
 	return m.getConfigValueBySource(realKey, sourceName)
 }
 
+//GetConfigsByPattern returns key values that matched pattern
+func (m *Manager) GetConfigsByPattern(pattern string) map[string]string {
+	m.RLock()
+	defer m.RLock()
+	matchedConfig := make(map[string]string)
+	pattern = strings.ToLower(pattern)
+	for key, value := range m.keySourceMap {
+		result := strings.HasPrefix(key, pattern)
+
+		if result {
+			sValue, err := m.getConfigValueBySource(key, value)
+			if err != nil {
+				continue
+			}
+			matchedConfig[key] = sValue
+		}
+	}
+	return matchedConfig
+}
+
 // Configs returns all the key values
 func (m *Manager) Configs() map[string]string {
 	m.RLock()

internal/distributed/querynode/client/client.go

@@ -278,6 +278,21 @@ func (c *Client) SyncReplicaSegments(ctx context.Context, req *querypb.SyncRepli
 	return ret.(*commonpb.Status), err
 }
 
+// ShowConfigurations gets specified configurations para of QueryNode
+func (c *Client) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
+	ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {
+		if !funcutil.CheckCtxValid(ctx) {
+			return nil, ctx.Err()
+		}
+		return client.(querypb.QueryNodeClient).ShowConfigurations(ctx, req)
+	})
+	if err != nil || ret == nil {
+		return nil, err
+	}
+
+	return ret.(*internalpb.ShowConfigurationsResponse), err
+}
+
 // GetMetrics gets the metrics information of QueryNode.
 func (c *Client) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) {
 	ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {

internal/distributed/querynode/client/client_test.go

@@ -104,6 +104,9 @@ func Test_NewClient(t *testing.T) {
 
 		r17, err := client.GetStatistics(ctx, nil)
 		retCheck(retNotNil, r17, err)
+
+		r18, err := client.ShowConfigurations(ctx, nil)
+		retCheck(retNotNil, r18, err)
 	}
 
 	client.grpcClient = &mock.GRPCClientBase{

internal/distributed/querynode/service.go

@@ -312,6 +312,11 @@ func (s *Server) SyncReplicaSegments(ctx context.Context, req *querypb.SyncRepli
 	return s.querynode.SyncReplicaSegments(ctx, req)
 }
 
+// ShowConfigurations gets specified configurations para of QueryNode
+func (s *Server) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
+	return s.querynode.ShowConfigurations(ctx, req)
+}
+
 // GetMetrics gets the metrics information of QueryNode.
 func (s *Server) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) {
 	return s.querynode.GetMetrics(ctx, req)

internal/distributed/querynode/service_test.go

@@ -44,6 +44,7 @@ type MockQueryNode struct {
 	strResp    *milvuspb.StringResponse
 	infoResp   *querypb.GetSegmentInfoResponse
 	metricResp *milvuspb.GetMetricsResponse
+	configResp *internalpb.ShowConfigurationsResponse
 	StatsResp  *internalpb.GetStatisticsResponse
 	searchResp *internalpb.SearchResults
 	queryResp  *internalpb.RetrieveResults
@@ -139,6 +140,10 @@ func (m *MockQueryNode) SetIndexCoord(index types.IndexCoord) error {
 	return m.err
 }
 
+func (m *MockQueryNode) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
+	return m.configResp, m.err
+}
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 type MockRootCoord struct {
 	types.RootCoord
@@ -326,6 +331,15 @@ func Test_NewServer(t *testing.T) {
 		assert.Equal(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
 	})
 
+	t.Run("ShowConfigurtaions", func(t *testing.T) {
+		req := &internalpb.ShowConfigurationsRequest{
+			Pattern: "Cache",
+		}
+		resp, err := server.ShowConfigurations(ctx, req)
+		assert.NoError(t, err)
+		assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
+	})
+
 	err = server.Stop()
 	assert.Nil(t, err)
 }

internal/proto/query_coord.proto

@@ -53,6 +53,7 @@ service QueryNode {
   rpc Search(SearchRequest) returns (internal.SearchResults) {}
   rpc Query(QueryRequest) returns (internal.RetrieveResults) {}
 
+  rpc ShowConfigurations(internal.ShowConfigurationsRequest) returns (internal.ShowConfigurationsResponse){}
   // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
   rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}
 }

internal/proxy/query_node_mock_test.go

@@ -123,3 +123,6 @@ func (m *QueryNodeMock) GetStatisticsChannel(ctx context.Context) (*milvuspb.Str
 func (m *QueryNodeMock) GetTimeTickChannel(ctx context.Context) (*milvuspb.StringResponse, error) {
 	return nil, nil
 }
+func (m *QueryNodeMock) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
+	return nil, nil
+}

internal/querycoord/mock_querynode_client_test.go

@@ -163,3 +163,7 @@ func (client *queryNodeClientMock) Query(ctx context.Context, req *querypb.Query
 func (client *queryNodeClientMock) SyncReplicaSegments(ctx context.Context, req *querypb.SyncReplicaSegmentsRequest) (*commonpb.Status, error) {
 	return client.grpcClient.SyncReplicaSegments(ctx, req)
 }
+
+func (client *queryNodeClientMock) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
+	return client.grpcClient.ShowConfigurations(ctx, req)
+}

internal/querynode/impl.go

@@ -1049,12 +1049,32 @@ func (node *QueryNode) SyncReplicaSegments(ctx context.Context, req *querypb.Syn
 	return &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, nil
 }
 
+//ShowConfigurations returns the configurations of queryNode matching req.Pattern
+func (node *QueryNode) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
+	if !node.isHealthy() {
+		log.Warn("QueryNode.ShowConfigurations failed",
+			zap.Int64("nodeId", Params.QueryNodeCfg.GetNodeID()),
+			zap.String("req", req.Pattern),
+			zap.Error(errQueryNodeIsUnhealthy(Params.QueryNodeCfg.GetNodeID())))
+
+		return &internalpb.ShowConfigurationsResponse{
+			Status: &commonpb.Status{
+				ErrorCode: commonpb.ErrorCode_UnexpectedError,
+				Reason:    msgQueryNodeIsUnhealthy(Params.QueryNodeCfg.GetNodeID()),
+			},
+			Configuations: nil,
+		}, nil
+	}
+
+	return getComponentConfigurations(ctx, req), nil
+}
+
 // GetMetrics return system infos of the query node, such as total memory, memory usage, cpu usage ...
 // TODO(dragondriver): cache the Metrics and set a retention to the cache
 func (node *QueryNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) {
 	if !node.isHealthy() {
 		log.Warn("QueryNode.GetMetrics failed",
-			zap.Int64("node_id", Params.QueryNodeCfg.GetNodeID()),
+			zap.Int64("nodeId", Params.QueryNodeCfg.GetNodeID()),
 			zap.String("req", req.Request),
 			zap.Error(errQueryNodeIsUnhealthy(Params.QueryNodeCfg.GetNodeID())))
 
@@ -1070,7 +1090,7 @@ func (node *QueryNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsR
 	metricType, err := metricsinfo.ParseMetricType(req.Request)
 	if err != nil {
 		log.Warn("QueryNode.GetMetrics failed to parse metric type",
-			zap.Int64("node_id", Params.QueryNodeCfg.GetNodeID()),
+			zap.Int64("nodeId", Params.QueryNodeCfg.GetNodeID()),
 			zap.String("req", req.Request),
 			zap.Error(err))
 
@@ -1087,9 +1107,9 @@ func (node *QueryNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsR
 		metrics, err := getSystemInfoMetrics(ctx, req, node)
 		if err != nil {
 			log.Warn("QueryNode.GetMetrics failed",
-				zap.Int64("node_id", Params.QueryNodeCfg.GetNodeID()),
+				zap.Int64("nodeId", Params.QueryNodeCfg.GetNodeID()),
 				zap.String("req", req.Request),
-				zap.String("metric_type", metricType),
+				zap.String("metricType", metricType),
 				zap.Error(err))
 		}
 
@@ -1097,9 +1117,9 @@ func (node *QueryNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsR
 	}
 
 	log.Debug("QueryNode.GetMetrics failed, request metric type is not implemented yet",
-		zap.Int64("node_id", Params.QueryNodeCfg.GetNodeID()),
+		zap.Int64("nodeId", Params.QueryNodeCfg.GetNodeID()),
 		zap.String("req", req.Request),
-		zap.String("metric_type", metricType))
+		zap.String("metricType", metricType))
 
 	return &milvuspb.GetMetricsResponse{
 		Status: &commonpb.Status{

internal/querynode/impl_test.go

@@ -320,6 +320,47 @@ func TestImpl_isHealthy(t *testing.T) {
 	assert.True(t, isHealthy)
 }
 
+func TestImpl_ShowConfigurations(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
+	assert.NoError(t, err)
+	defer etcdCli.Close()
+
+	t.Run("test ShowConfigurations", func(t *testing.T) {
+		node, err := genSimpleQueryNode(ctx)
+		assert.NoError(t, err)
+		node.session = sessionutil.NewSession(node.queryNodeLoopCtx, Params.EtcdCfg.MetaRootPath, etcdCli)
+
+		pattern := "Cache"
+		req := &internalpb.ShowConfigurationsRequest{
+			Base:    genCommonMsgBase(commonpb.MsgType_WatchQueryChannels),
+			Pattern: pattern,
+		}
+
+		reqs, err := node.ShowConfigurations(ctx, req)
+		assert.NoError(t, err)
+		assert.Equal(t, reqs.Status.ErrorCode, commonpb.ErrorCode_Success)
+	})
+
+	t.Run("test ShowConfigurations node failed", func(t *testing.T) {
+		node, err := genSimpleQueryNode(ctx)
+		assert.NoError(t, err)
+		node.session = sessionutil.NewSession(node.queryNodeLoopCtx, Params.EtcdCfg.MetaRootPath, etcdCli)
+		node.UpdateStateCode(internalpb.StateCode_Abnormal)
+
+		pattern := "Cache"
+		req := &internalpb.ShowConfigurationsRequest{
+			Base:    genCommonMsgBase(commonpb.MsgType_WatchQueryChannels),
+			Pattern: pattern,
+		}
+
+		reqs, err := node.ShowConfigurations(ctx, req)
+		assert.NoError(t, err)
+		assert.Equal(t, reqs.Status.ErrorCode, commonpb.ErrorCode_UnexpectedError)
+	})
+}
+
 func TestImpl_GetMetrics(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

internal/querynode/metrics_info.go

@@ -20,11 +20,34 @@ import (
 	"context"
 
 	"github.com/milvus-io/milvus/internal/proto/commonpb"
+	"github.com/milvus-io/milvus/internal/proto/internalpb"
 	"github.com/milvus-io/milvus/internal/proto/milvuspb"
 	"github.com/milvus-io/milvus/internal/util/metricsinfo"
 	"github.com/milvus-io/milvus/internal/util/typeutil"
 )
 
+//getComponentConfigurations returns the configurations of queryNode matching req.Pattern
+func getComponentConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) *internalpb.ShowConfigurationsResponse {
+	prefix := "querynode."
+	matchedConfig := Params.QueryNodeCfg.Base.GetByPattern(prefix + req.Pattern)
+	configList := make([]*commonpb.KeyValuePair, 0, len(matchedConfig))
+	for key, value := range matchedConfig {
+		configList = append(configList,
+			&commonpb.KeyValuePair{
+				Key:   key,
+				Value: value,
+			})
+	}
+
+	return &internalpb.ShowConfigurationsResponse{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_Success,
+			Reason:    "",
+		},
+		Configuations: configList,
+	}
+}
+
 // getSystemInfoMetrics returns metrics info of QueryNode
 func getSystemInfoMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest, node *QueryNode) (*milvuspb.GetMetricsResponse, error) {
 	usedMem := metricsinfo.GetUsedMemoryCount()

internal/querynode/metrics_info_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/milvus-io/milvus/internal/proto/commonpb"
+	"github.com/milvus-io/milvus/internal/proto/internalpb"
 	"github.com/milvus-io/milvus/internal/proto/milvuspb"
 	"github.com/milvus-io/milvus/internal/util/etcd"
 	"github.com/milvus-io/milvus/internal/util/sessionutil"
@@ -47,3 +48,24 @@ func TestGetSystemInfoMetrics(t *testing.T) {
 	assert.NoError(t, err)
 	resp.Status.ErrorCode = commonpb.ErrorCode_Success
 }
+
+func TestGetComponentConfigurationsFailed(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	node, err := genSimpleQueryNode(ctx)
+	assert.NoError(t, err)
+
+	etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
+	assert.NoError(t, err)
+	defer etcdCli.Close()
+	node.session = sessionutil.NewSession(node.queryNodeLoopCtx, Params.EtcdCfg.MetaRootPath, etcdCli)
+
+	req := &internalpb.ShowConfigurationsRequest{
+		Base:    genCommonMsgBase(commonpb.MsgType_WatchQueryChannels),
+		Pattern: "Cache",
+	}
+
+	resq := getComponentConfigurations(ctx, req)
+	assert.Equal(t, resq.Status.ErrorCode, commonpb.ErrorCode_Success)
+}

internal/types/types.go

@@ -1224,6 +1224,7 @@ type QueryNode interface {
 	Query(ctx context.Context, req *querypb.QueryRequest) (*internalpb.RetrieveResults, error)
 	SyncReplicaSegments(ctx context.Context, req *querypb.SyncReplicaSegmentsRequest) (*commonpb.Status, error)
 
+	ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error)
 	// GetMetrics gets the metrics about QueryNode.
 	GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error)
 }

internal/util/mock/grpc_querynode_client.go

@@ -44,6 +44,7 @@ func (m *GrpcQueryNodeClient) GetComponentStates(ctx context.Context, in *intern
 func (m *GrpcQueryNodeClient) GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) {
 	return &milvuspb.StringResponse{}, m.Err
 }
+
 func (m *GrpcQueryNodeClient) GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) {
 	return &milvuspb.StringResponse{}, m.Err
 }
@@ -91,3 +92,7 @@ func (m *GrpcQueryNodeClient) SyncReplicaSegments(ctx context.Context, in *query
 func (m *GrpcQueryNodeClient) GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error) {
 	return &milvuspb.GetMetricsResponse{}, m.Err
 }
+
+func (m *GrpcQueryNodeClient) ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) {
+	return &internalpb.ShowConfigurationsResponse{}, m.Err
+}

internal/util/mock/querynode_client.go

@@ -108,3 +108,7 @@ func (q QueryNodeClient) SyncReplicaSegments(ctx context.Context, req *querypb.S
 func (q QueryNodeClient) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) {
 	return q.grpcClient.GetMetrics(ctx, req)
 }
+
+func (q QueryNodeClient) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
+	return q.grpcClient.ShowConfigurations(ctx, req)
+}

internal/util/paramtable/base_table.go

@@ -211,6 +211,10 @@ func (gp *BaseTable) Get(key string) string {
 	return value
 }
 
+func (gp *BaseTable) GetByPattern(pattern string) map[string]string {
+	return gp.mgr.GetConfigsByPattern(pattern)
+}
+
 // For compatible reason, only visiable for Test
 func (gp *BaseTable) Remove(key string) error {
 	gp.mgr.DeleteConfig(key)
@@ -317,7 +321,6 @@ func (gp *BaseTable) ParseIntWithDefault(key string, defaultValue int) int {
 }
 
 // package methods
-
 // ConvertRangeToIntRange converts a range of strings to a range of ints.
 func ConvertRangeToIntRange(rangeStr, sep string) []int {
 	items := strings.Split(rangeStr, sep)

internal/util/typeutil/string_util.go

@@ -16,7 +16,9 @@
 
 package typeutil
 
-import "strings"
+import (
+	"strings"
+)
 
 // AddOne add one to last byte in string, on empty string return empty
 // it helps with key iteration upper bound