Delete some unused code and some debug logs

Signed-off-by: Ncai.zhang <cai.zhang@zilliz.com>

.jenkins/modules/Publish/Publish.groovy

@@ -25,6 +25,14 @@ dir ('build/docker/deploy') {
             sh 'docker-compose build --force-rm proxynode'
             sh 'docker-compose push proxynode'
 
+            sh 'docker pull ${SOURCE_REPO}/indexservice:${SOURCE_TAG} || true'
+            sh 'docker-compose build --force-rm indexservice'
+            sh 'docker-compose push indexservice'
+
+            sh 'docker pull ${SOURCE_REPO}/indexnode:${SOURCE_TAG} || true'
+            sh 'docker-compose build --force-rm indexnode'
+            sh 'docker-compose push indexnode'
+
             sh 'docker pull ${SOURCE_REPO}/queryservice:${SOURCE_TAG} || true'
             sh 'docker-compose build --force-rm queryservice'
             sh 'docker-compose push queryservice'
@@ -47,25 +55,3 @@ dir ('build/docker/deploy') {
         sh 'docker-compose down --rmi all'
     }
 }
-
-dir ('build/docker/deploy/distributed') {
-	try {
-        withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
-            sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
-
-            sh 'docker pull ${SOURCE_REPO}/indexservice:${SOURCE_TAG} || true'
-            sh 'docker-compose build --force-rm indexservice'
-            sh 'docker-compose push indexservice'
-
-            sh 'docker pull ${SOURCE_REPO}/indexnode:${SOURCE_TAG} || true'
-            sh 'docker-compose build --force-rm indexnode'
-            sh 'docker-compose push indexnode'
-        }
-    } catch (exc) {
-        throw exc
-    } finally {
-        sh 'docker logout ${DOKCER_REGISTRY_URL}'
-        sh "docker rmi -f \$(docker images | grep '<none>' | awk '{print \$3}') || true"
-        sh 'docker-compose down --rmi all'
-    }
-}

.jenkins/modules/Regression/PythonRegression.groovy

@@ -8,18 +8,14 @@ try {
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d master'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxyservice'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxynode'
+        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexservice'
+        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexnode'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d queryservice'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e QUERY_NODE_ID=1 -d querynode'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e QUERY_NODE_ID=2 -d querynode'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e WRITE_NODE_ID=3 -d writenode'
     }
 
-    dir ('build/docker/deploy/distributed') {
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} pull'
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexservice'
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexnode'
-    }
-
     dir ('build/docker/test') {
         sh 'docker pull ${SOURCE_REPO}/pytest:${SOURCE_TAG} || true'
         sh 'docker-compose build --force-rm regression'

Makefile

@@ -130,9 +130,9 @@ build-go: build-cpp
 	@echo "Building singlenode ..."
 	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/singlenode $(PWD)/cmd/singlenode/main.go 1>/dev/null
 	@echo "Building distributed indexservice ..."
-	@mkdir -p $(INSTALL_PATH)/distributed && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/distributed/indexservice $(PWD)/cmd/distributed/indexservice/main.go 1>/dev/null
+	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/indexservice $(PWD)/cmd/distributed/indexservice/main.go 1>/dev/null
 	@echo "Building distributed indexnode ..."
-	@mkdir -p $(INSTALL_PATH)/distributed && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/distributed/indexnode $(PWD)/cmd/distributed/indexnode/main.go 1>/dev/null
+	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/indexnode $(PWD)/cmd/distributed/indexnode/main.go 1>/dev/null
 
 
 build-cpp:
@@ -175,6 +175,8 @@ install: all
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxyservice $(GOPATH)/bin/proxyservice
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/writenode $(GOPATH)/bin/writenode
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/singlenode $(GOPATH)/bin/singlenode
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexservice $(GOPATH)/bin/indexservice
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexnode $(GOPATH)/bin/indexnode
 	@mkdir -p $(LIBRARY_PATH) && cp -f $(PWD)/internal/core/output/lib/* $(LIBRARY_PATH)
 	@echo "Installation successful."
 
@@ -190,3 +192,5 @@ clean:
 	@rm -rf $(GOPATH)/bin/querynode
 	@rm -rf $(GOPATH)/bin/writenode
 	@rm -rf $(GOPATH)/bin/singlenode
+	@rm -rf $(GOPATH)/bin/indexservice
+	@rm -rf $(GOPATH)/bin/indexnode

build/docker/deploy/distributed/.env

-SOURCE_REPO=milvusdb
-TARGET_REPO=milvusdb
-SOURCE_TAG=latest
-TARGET_TAG=latest
-PULSAR_ADDRESS=pulsar://pulsar:6650
-ETCD_ADDRESS=etcd:2379
-MASTER_ADDRESS=master:53100
-MINIO_ADDRESS=minio:9000
-INDEX_SERVICE_ADDRESS=indexservice:31000

build/docker/deploy/distributed/docker-compose.yml

-version: '3.5'
-
-services:
-  indexservice:
-    image: ${TARGET_REPO}/indexservice:${TARGET_TAG}
-    build:
-      context: ../../../../
-      dockerfile: build/docker/deploy/distributed/indexservice/DockerFile
-      cache_from:
-        - ${SOURCE_REPO}/indexservice:${SOURCE_TAG}
-    environment:
-      MASTER_ADDRESS: ${MASTER_ADDRESS}
-      ETCD_ADDRESS: ${ETCD_ADDRESS}
-      MINIO_ADDRESS: ${MINIO_ADDRESS}
-
-    networks:
-      - milvus
-
-  indexnode:
-    image: ${TARGET_REPO}/indexnode:${TARGET_TAG}
-    build:
-      context: ../../../../
-      dockerfile: build/docker/deploy/distributed/indexnode/DockerFile
-      cache_from:
-        - ${SOURCE_REPO}/indexnode:${SOURCE_TAG}
-    environment:
-      INDEX_SERVICE_ADDRESS: ${INDEX_SERVICE_ADDRESS}
-      MINIO_ADDRESS: ${MINIO_ADDRESS}
-    depends_on:
-      - "indexservice"
-
-    networks:
-      - milvus
-
-networks:
-  milvus:

build/docker/deploy/docker-compose.yml

@@ -85,5 +85,34 @@ services:
     networks:
       - milvus
 
+  indexservice:
+    image: ${TARGET_REPO}/indexservice:${TARGET_TAG}
+    build:
+      context: ../../../
+      dockerfile: build/docker/deploy/indexservice/DockerFile
+      cache_from:
+        - ${SOURCE_REPO}/indexservice:${SOURCE_TAG}
+    environment:
+      MASTER_ADDRESS: ${MASTER_ADDRESS}
+      ETCD_ADDRESS: ${ETCD_ADDRESS}
+      MINIO_ADDRESS: ${MINIO_ADDRESS}
+    networks:
+      - milvus
+
+  indexnode:
+    image: ${TARGET_REPO}/indexnode:${TARGET_TAG}
+    build:
+      context: ../../../
+      dockerfile: build/docker/deploy/indexnode/DockerFile
+      cache_from:
+        - ${SOURCE_REPO}/indexnode:${SOURCE_TAG}
+    environment:
+      INDEX_SERVICE_ADDRESS: ${INDEX_SERVICE_ADDRESS}
+      MINIO_ADDRESS: ${MINIO_ADDRESS}
+    depends_on:
+      - "indexservice"
+    networks:
+      - milvus
+
 networks:
   milvus:

build/docker/deploy/distributed/indexnode/DockerFile → build/docker/deploy/indexnode/DockerFile

@@ -26,7 +26,7 @@ COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/
 RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \
     ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so
 
-COPY ./bin/distributed/indexnode /milvus-distributed/bin/distributed/indexnode
+COPY ./bin/indexnode /milvus-distributed/bin/indexnode
 
 COPY ./configs/ /milvus-distributed/configs/
 
@@ -36,5 +36,5 @@ ENV LD_LIBRARY_PATH=/milvus-distributed/lib:$LD_LIBRARY_PATH:/usr/lib
 
 WORKDIR /milvus-distributed/
 
-CMD ["./bin/distributed/indexnode"]
+CMD ["./bin/indexnode"]
 

build/docker/deploy/distributed/indexservice/DockerFile → build/docker/deploy/indexservice/DockerFile

@@ -26,7 +26,7 @@ COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/
 RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \
     ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so
 
-COPY ./bin/distributed/indexservice /milvus-distributed/bin/distributed/indexservice
+COPY ./bin/indexservice /milvus-distributed/bin/indexservice
 
 COPY ./configs/ /milvus-distributed/configs/
 
@@ -36,6 +36,6 @@ ENV LD_LIBRARY_PATH=/milvus-distributed/lib:$LD_LIBRARY_PATH:/usr/lib
 
 WORKDIR /milvus-distributed/
 
-CMD ["./bin/distributed/indexservice"]
+CMD ["./bin/indexservice"]
 
-EXPOSE 21118
+EXPOSE 31000

cmd/distributed/indexnode/main.go

@@ -25,7 +25,7 @@ import (
 func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	grpcindexnode.Init()
-	svr, err := grpcindexnode.CreateIndexNode(ctx)
+	svr, err := grpcindexnode.NewGrpcServer(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}

cmd/distributed/indexservice/main.go

@@ -25,7 +25,7 @@ import (
 func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	grpcindexserver.Init()
-	svr, err := grpcindexserver.CreateIndexServer(ctx)
+	svr, err := grpcindexserver.NewServer(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}

cmd/indexbuilder/indexbuilder.go

-package main
-
-//import (
-//	"context"
-//	"log"
-//	"os"
-//	"os/signal"
-//	"syscall"
-//
-//	"github.com/zilliztech/milvus-distributed/internal/indexnode"
-//	"go.uber.org/zap"
-//)
-//
-//func main() {
-//	indexnode.Init()
-//	ctx, cancel := context.WithCancel(context.Background())
-//	svr, err := indexnode.CreateIndexNode(ctx)
-//	if err != nil {
-//		log.Print("create server failed", zap.Error(err))
-//	}
-//
-//	sc := make(chan os.Signal, 1)
-//	signal.Notify(sc,
-//		syscall.SIGHUP,
-//		syscall.SIGINT,
-//		syscall.SIGTERM,
-//		syscall.SIGQUIT)
-//
-//	var sig os.Signal
-//	go func() {
-//		sig = <-sc
-//		cancel()
-//	}()
-//
-//	if err := svr.Start(); err != nil {
-//		log.Fatal("run builder server failed", zap.Error(err))
-//	}
-//
-//	<-ctx.Done()
-//	log.Print("Got signal to exit", zap.String("signal", sig.String()))
-//
-//	svr.Stop()
-//	switch sig {
-//	case syscall.SIGTERM:
-//		exit(0)
-//	default:
-//		exit(1)
-//	}
-//}
-//
-//func exit(code int) {
-//	os.Exit(code)
-//}

internal/distributed/indexnode/service.go

@@ -25,26 +25,6 @@ type Server struct {
 	loopWg       sync.WaitGroup
 }
 
-func NewGrpcServer(ctx context.Context) *Server {
-	ctx1, cancel := context.WithCancel(ctx)
-	indexServiceClient := serviceclient.NewClient(indexnode.Params.ServiceAddress)
-
-	node, err := indexnode.CreateIndexNode(ctx1)
-	if err != nil {
-		defer cancel()
-		return nil
-	}
-
-	node.SetServiceClient(indexServiceClient)
-
-	return &Server{
-		loopCtx:      ctx1,
-		loopCancel:   cancel,
-		node:         node,
-		serverClient: indexServiceClient,
-	}
-}
-
 func (s *Server) registerNode() error {
 
 	log.Printf("Registering node. IP = %s, Port = %d", indexnode.Params.NodeIP, indexnode.Params.NodePort)
@@ -132,11 +112,6 @@ func Init() error {
 	return nil
 }
 
-func CreateIndexNode(ctx context.Context) (*Server, error) {
-
-	return NewGrpcServer(ctx), nil
-}
-
 func (s *Server) Start() error {
 	return s.startIndexNode()
 }
@@ -153,6 +128,25 @@ func (s *Server) Stop() error {
 }
 
 func (s *Server) BuildIndex(ctx context.Context, req *indexpb.BuildIndexCmd) (*commonpb.Status, error) {
-	log.Println("distributed build index")
 	return s.node.BuildIndex(req)
 }
+
+func NewGrpcServer(ctx context.Context) (*Server, error) {
+	ctx1, cancel := context.WithCancel(ctx)
+	indexServiceClient := serviceclient.NewClient(indexnode.Params.ServiceAddress)
+
+	node, err := indexnode.CreateIndexNode(ctx1)
+	if err != nil {
+		defer cancel()
+		return nil, err
+	}
+
+	node.SetServiceClient(indexServiceClient)
+
+	return &Server{
+		loopCtx:      ctx1,
+		loopCancel:   cancel,
+		node:         node,
+		serverClient: indexServiceClient,
+	}, nil
+}

internal/distributed/indexservice/client/client.go

@@ -31,112 +31,6 @@ func (g *Client) Stop() error {
 	return nil
 }
 
-//func (g *Client) BuildIndex2(columnDataPaths []string, typeParams map[string]string, indexParams map[string]string) (UniqueID, error) {
-//
-//	parseMap := func(mStr string) (map[string]string, error) {
-//		buffer := make(map[string]interface{})
-//		err := json.Unmarshal([]byte(mStr), &buffer)
-//		if err != nil {
-//			return nil, errors.New("Unmarshal params failed")
-//		}
-//		ret := make(map[string]string)
-//		for key, value := range buffer {
-//			valueStr := fmt.Sprintf("%v", value)
-//			ret[key] = valueStr
-//		}
-//		return ret, nil
-//	}
-//	var typeParamsKV []*commonpb.KeyValuePair
-//	for key := range typeParams {
-//		if key == "params" {
-//			mapParams, err := parseMap(typeParams[key])
-//			if err != nil {
-//				log.Println("parse params error: ", err)
-//			}
-//			for pk, pv := range mapParams {
-//				typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
-//					Key:   pk,
-//					Value: pv,
-//				})
-//			}
-//		} else {
-//			typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
-//				Key:   key,
-//				Value: typeParams[key],
-//			})
-//		}
-//	}
-//
-//	var indexParamsKV []*commonpb.KeyValuePair
-//	for key := range indexParams {
-//		if key == "params" {
-//			mapParams, err := parseMap(indexParams[key])
-//			if err != nil {
-//				log.Println("parse params error: ", err)
-//			}
-//			for pk, pv := range mapParams {
-//				indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
-//					Key:   pk,
-//					Value: pv,
-//				})
-//			}
-//		} else {
-//			indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
-//				Key:   key,
-//				Value: indexParams[key],
-//			})
-//		}
-//	}
-//
-//	requset := &indexpb.BuildIndexRequest{
-//		DataPaths:   columnDataPaths,
-//		TypeParams:  typeParamsKV,
-//		IndexParams: indexParamsKV,
-//	}
-//	response, err := g.BuildIndex(requset)
-//	if err != nil {
-//		return 0, err
-//	}
-//
-//	indexID := response.IndexID
-//
-//	return indexID, nil
-//}
-//
-//func (g *Client) GetIndexStates2(indexIDs []UniqueID) (*indexpb.IndexStatesResponse, error) {
-//
-//	request := &indexpb.IndexStatesRequest{
-//		IndexIDs: indexIDs,
-//	}
-//
-//	response, err := g.GetIndexStates(request)
-//	return response, err
-//}
-//
-//func (g *Client) GetIndexFilePaths2(indexIDs []UniqueID) ([][]string, error) {
-//
-//	request := &indexpb.IndexFilePathsRequest{
-//		IndexIDs: indexIDs,
-//	}
-//
-//	response, err := g.GetIndexFilePaths(request)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	var filePaths [][]string
-//	for _, indexID := range indexIDs {
-//		for _, filePathInfo := range response.FilePaths {
-//			if indexID == filePathInfo.IndexID {
-//				filePaths = append(filePaths, filePathInfo.IndexFilePaths)
-//				break
-//			}
-//		}
-//	}
-//
-//	return filePaths, nil
-//}
-
 func (g *Client) GetComponentStates() (*internalpb2.ComponentStates, error) {
 	return nil, nil
 }
@@ -155,7 +49,7 @@ func (g *Client) tryConnect() error {
 	}
 	ctx1, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	log.Println("indexservice address = ", g.address)
+
 	conn, err := grpc.DialContext(ctx1, g.address, grpc.WithInsecure(), grpc.WithBlock())
 	if err != nil {
 		log.Printf("Connect to IndexService failed, error= %v", err)

internal/distributed/indexservice/service.go

@@ -62,13 +62,11 @@ func (s *Server) GetStatisticsChannel() (string, error) {
 
 func (s *Server) RegisterNode(ctx context.Context, req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error) {
 
-	log.Println("Register IndexNode starting..., node address = ", req.Address)
 	return s.server.RegisterNode(req)
 }
 
 func (s *Server) BuildIndex(ctx context.Context, req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
 
-	log.Println("Build Index ...")
 	return s.server.BuildIndex(req)
 }
 
@@ -84,22 +82,12 @@ func (s *Server) GetIndexFilePaths(ctx context.Context, req *indexpb.IndexFilePa
 
 func (s *Server) NotifyBuildIndex(ctx context.Context, nty *indexpb.BuildIndexNotification) (*commonpb.Status, error) {
 
-	log.Println("build index finished.")
 	return s.server.NotifyBuildIndex(nty)
 }
 
-func NewServer() *Server {
-
-	return &Server{
-		//server: &indexservice.IndexService{},
-		//grpcServer: indexservice.IndexService{},
-	}
-}
-
 func (s *Server) grpcLoop() {
 	defer s.loopWg.Done()
 
-	log.Println("Starting start IndexServer")
 	lis, err := net.Listen("tcp", ":"+strconv.Itoa(indexservice.Params.Port))
 	if err != nil {
 		log.Fatalf("IndexServer grpc server fatal error=%v", err)
@@ -108,7 +96,6 @@ func (s *Server) grpcLoop() {
 	s.grpcServer = grpc.NewServer()
 	indexpb.RegisterIndexServiceServer(s.grpcServer, s)
 
-	log.Println("IndexServer register finished")
 	if err = s.grpcServer.Serve(lis); err != nil {
 		log.Fatalf("IndexServer grpc server fatal error=%v", err)
 	}
@@ -122,7 +109,7 @@ func (s *Server) startIndexServer() error {
 	return s.server.Start()
 }
 
-func CreateIndexServer(ctx context.Context) (*Server, error) {
+func NewServer(ctx context.Context) (*Server, error) {
 
 	ctx1, cancel := context.WithCancel(ctx)
 	serverImp, err := indexservice.CreateIndexService(ctx)

internal/indexnode/indexnode.go

@@ -71,9 +71,7 @@ func CreateIndexNode(ctx context.Context) (*IndexNode, error) {
 		return nil, err
 	}
 
-	log.Println("loopctx = ", b.loopCtx)
 	b.sched, err = NewTaskScheduler(b.loopCtx, b.kv)
-	log.Println("err = ", err)
 	if err != nil {
 		return nil, err
 	}
@@ -132,7 +130,6 @@ func (i *IndexNode) BuildIndex(request *indexpb.BuildIndexCmd) (*commonpb.Status
 	t.cmd = request
 	t.kv = i.kv
 	t.serviceClient = i.serviceClient
-	log.Println("t.serviceClient = ", t.serviceClient)
 	t.nodeID = Params.NodeID
 	ctx, cancel := context.WithTimeout(context.Background(), reqTimeoutInterval)
 	defer cancel()

internal/indexnode/task_scheduler.go

@@ -216,10 +216,8 @@ func (sched *TaskScheduler) indexBuildLoop() {
 	for {
 		select {
 		case <-sched.ctx.Done():
-			log.Println("id Done?")
 			return
 		case <-sched.IndexBuildQueue.utChan():
-			log.Println("index build loop ...")
 			if !sched.IndexBuildQueue.utEmpty() {
 				tasks := sched.scheduleIndexBuildTask()
 				var wg sync.WaitGroup

internal/indexservice/meta_table.go

@@ -106,20 +106,6 @@ func (mt *metaTable) NotifyBuildIndex(nty *indexpb.BuildIndexNotification) error
 	return mt.saveIndexMeta(&meta)
 }
 
-func (mt *metaTable) NotifyBuildIndex2(indexID UniqueID, dataPaths []string, state commonpb.IndexState) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-
-	meta.State = state
-	meta.IndexFilePaths = dataPaths
-	return mt.saveIndexMeta(&meta)
-}
-
 func (mt *metaTable) GetIndexState(indexID UniqueID) (*indexpb.IndexInfo, error) {
 	mt.lock.Lock()
 	defer mt.lock.Unlock()