Fix LatestPosition option conflict with earliest patch (#10907)

Signed-off-by: NCongqi Xia <congqi.xia@zilliz.com>

internal/datacoord/server.go

@@ -312,24 +312,25 @@ func (s *Server) initMeta() error {
 func (s *Server) startServerLoop() {
 	s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.ctx)
 	s.serverLoopWg.Add(4)
-	go s.startStatsChannel(s.serverLoopCtx)
-	go s.startDataNodeTtLoop(s.serverLoopCtx)
-	go s.startWatchService(s.serverLoopCtx)
-	go s.startFlushLoop(s.serverLoopCtx)
+	s.startStatsChannel(s.serverLoopCtx)
+	s.startDataNodeTtLoop(s.serverLoopCtx)
+	s.startWatchService(s.serverLoopCtx)
+	s.startFlushLoop(s.serverLoopCtx)
 	go s.session.LivenessCheck(s.serverLoopCtx, func() {
 		log.Fatal("Data Coord disconnected from etcd, process will exit", zap.Int64("Server Id", s.session.ServerID))
 	})
 }
 
 func (s *Server) startStatsChannel(ctx context.Context) {
-	defer logutil.LogPanic()
-	defer s.serverLoopWg.Done()
 	statsStream, _ := s.msFactory.NewMsgStream(ctx)
 	statsStream.AsConsumer([]string{Params.StatisticsChannelName}, Params.DataCoordSubscriptionName)
 	log.Debug("dataCoord create stats channel consumer",
 		zap.String("channelName", Params.StatisticsChannelName),
 		zap.String("descriptionName", Params.DataCoordSubscriptionName))
 	statsStream.Start()
+	go func() {
+		defer logutil.LogPanic()
+		defer s.serverLoopWg.Done()
 		defer statsStream.Close()
 		for {
 			select {
@@ -355,11 +356,10 @@ func (s *Server) startStatsChannel(ctx context.Context) {
 				}
 			}
 		}
+	}()
 }
 
 func (s *Server) startDataNodeTtLoop(ctx context.Context) {
-	defer logutil.LogPanic()
-	defer s.serverLoopWg.Done()
 	ttMsgStream, err := s.msFactory.NewMsgStream(ctx)
 	if err != nil {
 		log.Error("new msg stream failed", zap.Error(err))
@@ -371,14 +371,17 @@ func (s *Server) startDataNodeTtLoop(ctx context.Context) {
 		zap.String("timeTickChannelName", Params.TimeTickChannelName),
 		zap.String("subscriptionName", Params.DataCoordSubscriptionName))
 	ttMsgStream.Start()
-	defer ttMsgStream.Close()
 
+	go func() {
 		var checker *LongTermChecker
 		if enableTtChecker {
 			checker = NewLongTermChecker(ctx, ttCheckerName, ttMaxInterval, ttCheckerWarnMsg)
 			checker.Start()
 			defer checker.Stop()
 		}
+		defer logutil.LogPanic()
+		defer s.serverLoopWg.Done()
+		defer ttMsgStream.Close()
 		for {
 			select {
 			case <-ctx.Done():
@@ -455,13 +458,16 @@ func (s *Server) startDataNodeTtLoop(ctx context.Context) {
 			}
 			s.helper.eventAfterHandleDataNodeTt()
 		}
+	}()
 }
 
-//go:norace
-// fix datarace in unittest
-// startWatchService will only be invoked at start procedure
-// otherwise, remove the annotation and add atomic protection
+// start a goroutine wto watch services
 func (s *Server) startWatchService(ctx context.Context) {
+	go s.watchService(ctx)
+}
+
+// watchService watchs services
+func (s *Server) watchService(ctx context.Context) {
 	defer logutil.LogPanic()
 	defer s.serverLoopWg.Done()
 	for {
@@ -484,6 +490,7 @@ func (s *Server) startWatchService(ctx context.Context) {
 			}
 		}
 	}
+
 }
 
 // handles session events - DataNodes Add/Del
@@ -527,6 +534,7 @@ func (s *Server) handleSessionEvent(ctx context.Context, event *sessionutil.Sess
 }
 
 func (s *Server) startFlushLoop(ctx context.Context) {
+	go func() {
 		defer logutil.LogPanic()
 		defer s.serverLoopWg.Done()
 		ctx2, cancel := context.WithCancel(ctx)
@@ -543,6 +551,7 @@ func (s *Server) startFlushLoop(ctx context.Context) {
 				_ = s.postFlush(ctx, segmentID)
 			}
 		}
+	}()
 }
 
 // post function after flush is done

internal/datacoord/server_test.go

@@ -592,7 +592,7 @@ func TestService_WatchServices(t *testing.T) {
 	signal := make(chan struct{}, 1)
 
 	go func() {
-		svr.startWatchService(context.Background())
+		svr.watchService(context.Background())
 		flag = true
 		signal <- struct{}{}
 	}()
@@ -609,7 +609,7 @@ func TestService_WatchServices(t *testing.T) {
 	svr.serverLoopWg.Add(1)
 
 	go func() {
-		svr.startWatchService(ctx)
+		svr.watchService(ctx)
 		flag = true
 		signal <- struct{}{}
 	}()

internal/util/mqclient/pulsar_client.go

@@ -67,9 +67,12 @@ func (pc *pulsarClient) Subscribe(options ConsumerOptions) (Consumer, error) {
 	if err != nil {
 		return nil, err
 	}
-	//consumer.Seek(pulsar.EarliestMessageID())
-	//consumer.SeekByTime(time.Unix(0, 0))
+
 	pConsumer := &pulsarConsumer{c: consumer, closeCh: make(chan struct{})}
+	// prevent seek to earliest patch applied when using latest position options
+	if options.SubscriptionInitialPosition == SubscriptionPositionLatest {
+		pConsumer.hasSeek = true
+	}
 
 	return pConsumer, nil
 }