Only balance segments in targets (#20636)

Signed-off-by: Nyah01 <yang.cen@zilliz.com>
Signed-off-by: Nyah01 <yang.cen@zilliz.com>

internal/querycoordv2/handlers.go

@@ -85,7 +85,11 @@ func (s *Server) balanceSegments(ctx context.Context, req *querypb.LoadBalanceRe
 	dstNodeSet.Remove(srcNode)
 
 	toBalance := typeutil.NewSet[*meta.Segment]()
+	// Only balance segments in targets
 	segments := s.dist.SegmentDistManager.GetByNode(srcNode)
+	segments = lo.Filter(segments, func(segment *meta.Segment, _ int) bool {
+		return s.targetMgr.GetSegment(segment.GetID()) != nil
+	})
 	allSegments := make(map[int64]*meta.Segment)
 	for _, segment := range segments {
 		allSegments[segment.GetID()] = segment