Make data node start only once (#24031)

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

internal/datanode/data_node.go

@@ -116,6 +116,7 @@ type DataNode struct {
 
 	//call once
 	initOnce     sync.Once
+	startOnce    sync.Once
 	sessionMu    sync.Mutex // to fix data race
 	session      *sessionutil.Session
 	watchKv      kv.MetaKv
@@ -485,9 +486,12 @@ func (node *DataNode) BackGroundGC(vChannelCh <-chan string) {
 
 // Start will update DataNode state to HEALTHY
 func (node *DataNode) Start() error {
+	var startErr error
+	node.startOnce.Do(func() {
 		if err := node.allocator.Start(); err != nil {
 			log.Error("failed to start id allocator", zap.Error(err), zap.String("role", typeutil.DataNodeRole))
-		return err
+			startErr = err
+			return
 		}
 		log.Info("start id allocator done", zap.String("role", typeutil.DataNodeRole))
 
@@ -501,7 +505,8 @@ func (node *DataNode) Start() error {
 		})
 		if err != nil || rep.Status.ErrorCode != commonpb.ErrorCode_Success {
 			log.Warn("fail to alloc timestamp", zap.Any("rep", rep), zap.Error(err))
-		return errors.New("DataNode fail to alloc timestamp")
+			startErr = errors.New("DataNode fail to alloc timestamp")
+			return
 		}
 
 		connectEtcdFn := func() error {
@@ -511,13 +516,15 @@ func (node *DataNode) Start() error {
 		}
 		err = retry.Do(node.ctx, connectEtcdFn, retry.Attempts(ConnectEtcdMaxRetryTime))
 		if err != nil {
-		return errors.New("DataNode fail to connect etcd")
+			startErr = errors.New("DataNode fail to connect etcd")
+			return
 		}
 
 		chunkManager, err := node.factory.NewPersistentStorageChunkManager(node.ctx)
 
 		if err != nil {
-		return err
+			startErr = err
+			return
 		}
 
 		node.chunkManager = chunkManager
@@ -532,7 +539,9 @@ func (node *DataNode) Start() error {
 		go node.flowgraphManager.start()
 
 		node.UpdateStateCode(commonpb.StateCode_Healthy)
-	return nil
+
+	})
+	return startErr
 }
 
 // UpdateStateCode updates datanode's state code