Fix metric QueryCoordNumCollections (#24053) (#24107)

Signed-off-by: Nlongjiquan <jiquan.long@zilliz.com>

internal/querycoordv2/job/job_load.go

@@ -204,9 +204,6 @@ func (job *LoadCollectionJob) Execute() error {
 		return utils.WrapError(msg, err)
 	}
 
-	if !colExisted {
-		metrics.QueryCoordNumCollections.WithLabelValues().Inc()
-	}
 	metrics.QueryCoordNumPartitions.WithLabelValues().Add(float64(len(partitions)))
 	return nil
 }
@@ -380,7 +377,6 @@ func (job *LoadPartitionJob) Execute() error {
 			log.Error(msg, zap.Error(err))
 			return utils.WrapError(msg, err)
 		}
-		metrics.QueryCoordNumCollections.WithLabelValues().Inc()
 	} else { // collection exists, put partitions only
 		err = job.meta.CollectionManager.PutPartition(partitions...)
 		if err != nil {

internal/querycoordv2/meta/collection_manager.go

@@ -491,6 +491,11 @@ func (m *CollectionManager) UpdateLoadPercent(partitionID int64, loadPercent int
 		saveCollection = true
 		newCollection.Status = querypb.LoadStatus_Loaded
 		elapsed := time.Since(newCollection.CreatedAt)
+
+		// TODO: what if part of the collection has been unloaded? Now we decrease the metric only after
+		// 	`ReleaseCollection` is triggered. Maybe it's hard to make this metric really accurate.
+		metrics.QueryCoordNumCollections.WithLabelValues().Inc()
+
 		metrics.QueryCoordLoadLatency.WithLabelValues().Observe(float64(elapsed.Milliseconds()))
 	}
 	return collectionPercent, m.putCollection(saveCollection, newCollection)

internal/querycoordv2/server.go

@@ -311,7 +311,10 @@ func (s *Server) initMeta() error {
 	}
 	collections := s.meta.GetAll()
 	log.Info("recovering collections...", zap.Int64s("collections", collections))
-	metrics.QueryCoordNumCollections.WithLabelValues().Set(float64(len(collections)))
+
+	// We really update the metric after observers think the collection loaded.
+	metrics.QueryCoordNumCollections.WithLabelValues().Set(0)
+
 	metrics.QueryCoordNumPartitions.WithLabelValues().Set(float64(len(s.meta.GetAllPartitions())))
 
 	err = s.meta.ReplicaManager.Recover(collections)