Remove useless configs in config files

Signed-off-by: NXuanYang-cn <xuan.yang@zilliz.com>

configs/advanced/channel.yaml

@@ -22,11 +22,9 @@ msgChannel:
     k2s: "k2s"
     proxyTimeTick: "proxyTimeTick"
     proxyServiceTimeTick: "proxyServiceTimeTick"
-    writeNodeTimeTick: "writeNodeTimeTick" # GOOSE TODO: remove this
     dataNodeTimeTick: "dataNodeTimeTick"
     queryTimeTick: "queryTimeTick"
     dataNodeSegStatistics: "dataNodeSegStatistics"
-    # old name: statsChannels: "statistic"
     queryNodeStats: "query-node-stats"
     # cmd for loadIndex, flush, etc...
     cmd: "cmd"
@@ -40,7 +38,6 @@ msgChannel:
     masterSubNamePrefix: "master"
     proxySubNamePrefix: "proxy"
     queryNodeSubNamePrefix: "queryNode"
-    writeNodeSubNamePrefix: "writeNode" # GOOSE TODO: remove this
     dataNodeSubNamePrefix: "dataNode"
     dataServiceSubNamePrefix: "dataService"
 

configs/advanced/common.yaml

@@ -10,6 +10,5 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 
 common:
-  defaultPartitionTag: _default #TODO, remove
   defaultPartitionName: "_default"
   defaultIndexName: "_default_idx"

configs/advanced/data_node.yaml

@@ -23,12 +23,10 @@ dataNode:
       pulsarBufSize: 64 # pulsar chan buffer size
 
     insert:
-      #streamBufSize: 1024 # msgPack chan buffer size
       recvBufSize: 1024 # msgPack chan buffer size
       pulsarBufSize: 1024 # pulsar chan buffer size
 
     delete:
-      #streamBufSize: 1024 # msgPack chan buffer size
       recvBufSize: 1024 # msgPack chan buffer size
       pulsarBufSize: 1024 # pulsar chan buffer size
 
@@ -38,5 +36,5 @@ dataNode:
 
   flush:
     # max buffer size to flush
-    insertBufSize: 32000
+    insertBufSize: 32000 # number of rows
     ddBufSize: 20

configs/advanced/data_service.yaml

@@ -3,11 +3,9 @@ dataservice:
   address: "127.0.0.1"
   port: 13333
   segment:
-    # old name: segmentThreshold: 536870912
     size: 512 # MB
     sizeFactor: 0.75
     defaultSizePerRecord: 1024
-    # old name: segmentExpireDuration: 2000
     IDAssignExpiration: 2000 # ms
   insertChannelNum: 2
   dataNodeNum: 1

configs/advanced/master.yaml

@@ -10,18 +10,6 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 
 master:
-  timeSync:
-    softTimeTickBarrierInterval: 500  # ms
-  segment:
-    # old name: segmentThreshold: 536870912
-    size: 512 # MB
-    sizeFactor: 0.75
-    defaultSizePerRecord: 1024
-    minIDAssignCnt: 1024
-    maxIDAssignCnt: 16384
-    # old name: segmentExpireDuration: 2000
-    IDAssignExpiration: 2000 # ms
-
   maxPartitionNum: 4096
   minSegmentSizeToEnableIndex: 1024
   nodeID: 100

configs/advanced/proxy_node.yaml

@@ -14,7 +14,6 @@ proxyNode:
 
   msgStream:
     insert:
-      #streamBufSize: 1024 # msgPack chan buffer size
       bufSize: 1024 # msgPack chan buffer size
 
     search:

configs/advanced/query_node.yaml

@@ -20,7 +20,6 @@ queryNode:
 
   msgStream:
     insert:
-      #streamBufSize: 1024 # msgPack chan buffer size
       recvBufSize: 1024 # msgPack chan buffer size
       pulsarBufSize: 1024 # pulsar chan buffer size
 
@@ -29,7 +28,6 @@ queryNode:
       pulsarBufSize: 64 # pulsar chan buffer size
 
     delete:
-      #streamBufSize: 1024 # msgPack chan buffer size
       recvBufSize: 1024 # msgPack chan buffer size
       pulsarBufSize: 1024 # pulsar chan buffer size
 

configs/milvus.yaml

@@ -10,8 +10,7 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 
 
-nodeID: # will be deprecated after v0.2
-  proxyIDList: [0]
+nodeID: # will be deprecated later
   queryNodeIDList: [1]
   dataNodeIDList: [3]
 
@@ -22,10 +21,7 @@ etcd:
   metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath
   kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath
   segFlushMetaSubPath: datanode/segment # Full Path = rootPath/metaSubPath/segFlushMetaSubPath
-  ddlFlushMetaSubPath: datanode/ddl     # Full Path = rootPath/metaSubPaht/ddlFlushMetaSubPath
-  writeNodeSegKvSubPath: writer/segment # GOOSE TODO: remove this
-  writeNodeDDLKvSubPath: writer/ddl # GOOSE TODO: remove this
-  segThreshold: 10000
+  ddlFlushMetaSubPath: datanode/ddl     # Full Path = rootPath/metaSubPath/ddlFlushMetaSubPath
 
 minio:
   address: localhost
@@ -38,11 +34,7 @@ minio:
 pulsar:
   address: localhost
   port: 6650
-  rest-port: 18080 # keep same with pulsar container
-  maxMessageSize: 5242880 # 5 * 1024 * 1024
-  authentication: false
-  user: user-default
-  token: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY
+  maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes
 
 master:
   address: localhost
@@ -60,7 +52,7 @@ proxyService:
   port: 21122
 
 queryNode:
-  gracefulTime: 5000 #ms
+  gracefulTime: 5000 # ms, for search
   port: 21123
 
 indexService:
@@ -78,11 +70,11 @@ dataNode:
   port: 21124
 
 log:
-  level: debug
+  level: debug # info, warn, error, panic, fatal, dpanic,
   file:
-    rootPath: ""
+    rootPath: "" # default to stdout, stderr
     maxSize: 300 # MB
     maxAge: 10 # day
     maxBackups: 20
-  dev: true
+  dev: true # false, change behaviour of dpaniclevel
   format: text # text/json

docs/developer_guides/chap05_proxy.md

@@ -403,7 +403,7 @@ type GlobalParamsTable struct {
 	MaxNameLength                      int64
 	MaxFieldNum                        int64
 	MaxDimension                       int64
-	DefaultPartitionTag                string
+	DefaultPartitionName               string
 	DefaultIndexName                   string
 }
 

internal/proxynode/impl.go

@@ -946,7 +946,7 @@ func (node *ProxyNode) Insert(ctx context.Context, request *milvuspb.InsertReque
 		rowIDAllocator: node.idAllocator,
 	}
 	if len(it.PartitionName) <= 0 {
-		it.PartitionName = Params.DefaultPartitionTag
+		it.PartitionName = Params.DefaultPartitionName
 	}
 
 	err := node.sched.DmQueue.Enqueue(it)

internal/proxynode/paramtable.go

@@ -54,7 +54,7 @@ type ParamTable struct {
 	MaxNameLength                      int64
 	MaxFieldNum                        int64
 	MaxDimension                       int64
-	DefaultPartitionTag                string
+	DefaultPartitionName               string
 	DefaultIndexName                   string
 
 	PulsarMaxMessageSize int
@@ -136,7 +136,6 @@ func (pt *ParamTable) initParams() {
 	pt.initPulsarAddress()
 	pt.initQueryNodeIDList()
 	pt.initQueryNodeNum()
-	// pt.initProxyID()
 	pt.initTimeTickInterval()
 	pt.initInsertChannelNames()
 	pt.initDeleteChannelNames()
@@ -154,7 +153,7 @@ func (pt *ParamTable) initParams() {
 	pt.initMaxNameLength()
 	pt.initMaxFieldNum()
 	pt.initMaxDimension()
-	pt.initDefaultPartitionTag()
+	pt.initDefaultPartitionName()
 	pt.initDefaultIndexName()
 
 	pt.initPulsarMaxMessageSize()
@@ -191,18 +190,6 @@ func (pt *ParamTable) initQueryNodeIDList() []UniqueID {
 	return ret
 }
 
-func (pt *ParamTable) initProxyID() {
-	proxyID, err := pt.Load("_proxyID")
-	if err != nil {
-		panic(err)
-	}
-	ID, err := strconv.Atoi(proxyID)
-	if err != nil {
-		panic(err)
-	}
-	pt.ProxyID = UniqueID(ID)
-}
-
 func (pt *ParamTable) initTimeTickInterval() {
 	intervalStr, err := pt.Load("proxyNode.timeTickInterval")
 	if err != nil {
@@ -311,11 +298,6 @@ func (pt *ParamTable) initProxySubName() {
 	if err != nil {
 		panic(err)
 	}
-	pt.ProxySubName = prefix
-	// proxyIDStr, err := pt.Load("_proxyID")
-	// if err != nil {
-	// 	panic(err)
-	// }
 	pt.ProxySubName = prefix + "-" + strconv.Itoa(int(pt.ProxyID))
 }
 
@@ -393,12 +375,12 @@ func (pt *ParamTable) initMaxDimension() {
 	pt.MaxDimension = maxDimension
 }
 
-func (pt *ParamTable) initDefaultPartitionTag() {
-	tag, err := pt.Load("common.defaultPartitionTag")
+func (pt *ParamTable) initDefaultPartitionName() {
+	name, err := pt.Load("common.defaultPartitionName")
 	if err != nil {
 		panic(err)
 	}
-	pt.DefaultPartitionTag = tag
+	pt.DefaultPartitionName = name
 }
 
 func (pt *ParamTable) initDefaultIndexName() {

internal/proxynode/task.go

@@ -159,7 +159,7 @@ func (it *InsertTask) Execute(ctx context.Context) error {
 			return err
 		}
 	} else {
-		partitionID, err = globalMetaCache.GetPartitionID(ctx, collectionName, Params.DefaultPartitionTag)
+		partitionID, err = globalMetaCache.GetPartitionID(ctx, collectionName, Params.DefaultPartitionName)
 		if err != nil {
 			return err
 		}

internal/querynode/load_service_test.go

@@ -1122,7 +1122,6 @@ func TestSegmentLoad_Search_Vector(t *testing.T) {
 
 	// mock write insert bin log
 	keyPrefix := path.Join("query-node-seg-manager-test-minio-prefix", strconv.FormatInt(collectionID, 10), strconv.FormatInt(partitionID, 10))
-	Params.WriteNodeSegKvSubPath = keyPrefix
 
 	node := newQueryNodeMock()
 	defer node.Stop()

internal/querynode/param_table.go

@@ -17,7 +17,6 @@ type ParamTable struct {
 	PulsarAddress string
 	ETCDAddress   string
 	MetaRootPath  string
-	WriteNodeSegKvSubPath string
 
 	QueryNodeIP                 string
 	QueryNodePort               int64
@@ -106,7 +105,6 @@ func (p *ParamTable) Init() {
 		p.initPulsarAddress()
 		p.initETCDAddress()
 		p.initMetaRootPath()
-		p.initWriteNodeSegKvSubPath()
 
 		p.initGracefulTime()
 		p.initMsgChannelSubName()
@@ -312,14 +310,6 @@ func (p *ParamTable) initMetaRootPath() {
 	p.MetaRootPath = rootPath + "/" + subPath
 }
 
-func (p *ParamTable) initWriteNodeSegKvSubPath() {
-	subPath, err := p.Load("etcd.writeNodeSegKvSubPath")
-	if err != nil {
-		panic(err)
-	}
-	p.WriteNodeSegKvSubPath = subPath + "/"
-}
-
 func (p *ParamTable) initGracefulTime() {
 	p.GracefulTime = p.ParseInt64("queryNode.gracefulTime")
 }

internal/util/paramtable/paramtable.go

@@ -329,32 +329,14 @@ func (gp *BaseTable) ParseInt(key string) int {
 	return value
 }
 
-// GOOSE TODO: remove writenode
-func (gp *BaseTable) WriteNodeIDList() []UniqueID {
-	proxyIDStr, err := gp.Load("nodeID.dataNodeIDList")
-	if err != nil {
-		panic(err)
-	}
-	var ret []UniqueID
-	proxyIDs := strings.Split(proxyIDStr, ",")
-	for _, i := range proxyIDs {
-		v, err := strconv.Atoi(i)
-		if err != nil {
-			log.Panicf("load write node id list error, %s", err.Error())
-		}
-		ret = append(ret, UniqueID(v))
-	}
-	return ret
-}
-
 func (gp *BaseTable) DataNodeIDList() []UniqueID {
-	proxyIDStr, err := gp.Load("nodeID.dataNodeIDList")
+	datanodeIDStr, err := gp.Load("nodeID.dataNodeIDList")
 	if err != nil {
 		panic(err)
 	}
 	var ret []UniqueID
-	proxyIDs := strings.Split(proxyIDStr, ",")
-	for _, i := range proxyIDs {
+	datanodeIDs := strings.Split(datanodeIDStr, ",")
+	for _, i := range datanodeIDs {
 		v, err := strconv.Atoi(i)
 		if err != nil {
 			log.Panicf("load write node id list error, %s", err.Error())
@@ -364,23 +346,6 @@ func (gp *BaseTable) DataNodeIDList() []UniqueID {
 	return ret
 }
 
-func (gp *BaseTable) ProxyIDList() []UniqueID {
-	proxyIDStr, err := gp.Load("nodeID.proxyIDList")
-	if err != nil {
-		panic(err)
-	}
-	var ret []UniqueID
-	proxyIDs := strings.Split(proxyIDStr, ",")
-	for _, i := range proxyIDs {
-		v, err := strconv.Atoi(i)
-		if err != nil {
-			log.Panicf("load proxy id list error, %s", err.Error())
-		}
-		ret = append(ret, UniqueID(v))
-	}
-	return ret
-}
-
 func (gp *BaseTable) QueryNodeIDList() []UniqueID {
 	queryNodeIDStr, err := gp.Load("nodeID.queryNodeIDList")
 	if err != nil {