Fix bug and enchance system

Signed-off-by: zhenshan.cao <zhenshan.cao@zilliz.com>

.jenkins/modules/Regression/PythonRegression.groovy

@@ -54,7 +54,7 @@ timeout(time: 150, unit: 'MINUTES') {
                     echo "This is Cron Job!"
                     sh "pytest --tags=0331 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
                 } else {
-                    sh "pytest --tags=0331+l1 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
+                    sh "pytest --tags=0331+l1 -n 2 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
                 }
             }
         } catch (exc) {

cmd/singlenode/main.go

@@ -29,7 +29,12 @@ func initLogCfg() log.Config {
 	logCfg.File.MaxSize = 300
 	logCfg.File.MaxBackups = 20
 	logCfg.File.MaxDays = 10
-	logCfg.File.Filename = "/tmp/milvus/singlenode.log"
+	ciFileDir := "/milvus-distributed/logs/"
+	if _, err := os.Stat(ciFileDir); err == nil {
+		logCfg.File.Filename = ciFileDir + "singlenode.log"
+	} else {
+		logCfg.File.Filename = "/tmp/milvus/singlenode.log"
+	}
 	return logCfg
 }
 

go.mod

@@ -4,7 +4,7 @@ go 1.15
 
 require (
 	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
-	github.com/apache/pulsar-client-go v0.1.1
+	github.com/apache/pulsar-client-go v0.3.0
 	github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
@@ -14,7 +14,7 @@ require (
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/mock v1.3.1
-	github.com/golang/protobuf v1.3.2
+	github.com/golang/protobuf v1.4.2
 	github.com/google/btree v1.0.0
 	github.com/jarcoal/httpmock v1.0.8
 	github.com/klauspost/compress v1.10.11 // indirect

go.sum

@@ -10,6 +10,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4=
+github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -17,9 +19,15 @@ github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bE
 github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/apache/pulsar-client-go v0.1.1 h1:v/kU+2ZCC6yFIcbZrFtWa9/nvVzVr18L+xYJUvZSxEQ=
 github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2czVPb27D477YpyU=
+github.com/apache/pulsar-client-go v0.3.0 h1:rNhJ/ENwoEfZPHHwUHNxPBTNqNQE2LQEm7DXu043giM=
+github.com/apache/pulsar-client-go v0.3.0/go.mod h1:9eSgOadVhCfb2DfWtS1SCYaYIMk9VDOZztr4u3FO8cQ=
+github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb h1:E1P0FudxDdj2RhbveZC9i3PwukLCA/4XQSkBS/dw6/I=
+github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY=
 github.com/apache/thrift v0.14.1 h1:Yh8v0hpCj63p5edXOLaqTJW0IJ1p+eMW6+YSOqw1d6s=
 github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4 h1:orNYqmQGnSjgOauLWjHEp9/qIDT98xv/0Aa4Zet3/Y8=
 github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4/go.mod h1:V/LzksIyqd3KZuQ2SunvReTG/UkArhII1dAWY5U1sCE=
@@ -35,6 +43,8 @@ github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHL
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
@@ -42,6 +52,8 @@ github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXD
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
@@ -58,6 +70,10 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/danieljoos/wincred v1.0.2 h1:zf4bhty2iLuwgjgpraD2E9UbvO+fe54XXGJbOwe23fU=
+github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U=
+github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32 h1:QWqadCIHYA5zja4b6h9uGQn93u1vL+G/aewImumdg/M=
+github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:inRp+etsHuvVqMPNTXaFlpf/Tj7wqviBtdJoPVrPEFQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -68,6 +84,8 @@ github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4w
 github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU=
+github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
@@ -81,17 +99,22 @@ github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d8
 github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-basic/ipv4 v1.0.0 h1:gjyFAa1USC1hhXTkPOwBWDPfMcUaIM+tvo1XzV9EZxs=
 github.com/go-basic/ipv4 v1.0.0/go.mod h1:etLBnaxbidQfuqE6wgZQfs38nEWNmzALkxDZe4xY8Dg=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
+github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -119,6 +142,8 @@ github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -155,6 +180,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
+github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
+github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -176,6 +203,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
 github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
@@ -194,6 +222,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d h1:Z+RDyXzjKE0i2sTjZ/b1uxiGtPhFy34Ou/Tk0qwN0kM=
+github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -216,6 +246,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg=
+github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -247,11 +279,19 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
+github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486 h1:K35HCWaOTJIPW6cDHK4yj3QfRY/NhE0pBbfoc0M2NMQ=
 github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -276,18 +316,26 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/protocolbuffers/protobuf v3.15.3+incompatible h1:5WExaSYHEGvU73sVHvqe+3/APOOyCVg/pDCeAlfpCrw=
 github.com/protocolbuffers/protobuf v3.15.4+incompatible h1:Blv4dGFGqHXX+r5Tqoc1ziXPMDElqZ+/ryYcE4bddN4=
@@ -301,6 +349,7 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -406,6 +455,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -417,11 +467,13 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
@@ -429,13 +481,18 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -447,10 +504,17 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -535,6 +599,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -542,6 +607,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/core/src/query/SearchOnSealed.cpp

@@ -29,16 +29,24 @@ AssembleNegBitset(const BitsetSimple& bitset_simple) {
     for (auto& bitset : bitset_simple) {
         N += bitset.size();
     }
+
     aligned_vector<uint8_t> result(upper_align(upper_div(N, 8), 64));
 
-    auto acc_byte_count = 0;
+    if (bitset_simple.size() == 1) {
-    for (auto& bitset : bitset_simple) {
+        auto& bitset = bitset_simple[0];
-        auto size = bitset.size();
+        auto byte_count = upper_div(bitset.size(), 8);
-        Assert(size % 8 == 0);
-        auto byte_count = size / 8;
         auto src_ptr = boost_ext::get_data(bitset);
-        memcpy(result.data() + acc_byte_count, src_ptr, byte_count);
+        memcpy(result.data(), src_ptr, byte_count);
-        acc_byte_count += byte_count;
+    } else {
+        auto acc_byte_count = 0;
+        for (auto& bitset : bitset_simple) {
+            auto size = bitset.size();
+            Assert(size % 8 == 0);
+            auto byte_count = size / 8;
+            auto src_ptr = boost_ext::get_data(bitset);
+            memcpy(result.data() + acc_byte_count, src_ptr, byte_count);
+            acc_byte_count += byte_count;
+        }
     }
 
     // revert the bitset

internal/core/src/query/visitors/ExecExprVisitor.cpp

@@ -206,7 +206,14 @@ ExecExprVisitor::ExecRangeVisitorDispatcher(RangeExpr& expr_raw) -> RetType {
         T val1, val2;
         std::tie(op1, val1) = conditions[0];
         std::tie(op2, val2) = conditions[1];
-        Assert(val1 <= val2);
+        // TODO: disable check?
+        if (val1 > val2) {
+            // Empty
+            auto size_per_chunk = segment_.size_per_chunk();
+            auto num_chunk = upper_div(row_count_, size_per_chunk);
+            RetType ret(num_chunk, boost::dynamic_bitset<>(size_per_chunk));
+            return ret;
+        }
         auto ops = std::make_tuple(op1, op2);
         if (false) {
         } else if (ops == std::make_tuple(OpType::GreaterThan, OpType::LessThan)) {

internal/core/src/segcore/segment_c.cpp

@@ -260,6 +260,43 @@ UpdateSealedSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_inde
     }
 }
 
+CStatus
+DropFieldData(CSegmentInterface c_segment, int64_t field_id) {
+    try {
+        auto segment_interface = reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
+        auto segment = dynamic_cast<milvus::segcore::SegmentSealed*>(segment_interface);
+        AssertInfo(segment != nullptr, "segment conversion failed");
+        segment->DropFieldData(milvus::FieldId(field_id));
+        auto status = CStatus();
+        status.error_code = Success;
+        status.error_msg = "";
+        return status;
+    } catch (std::exception& e) {
+        auto status = CStatus();
+        status.error_code = UnexpectedException;
+        status.error_msg = strdup(e.what());
+        return status;
+    }
+}
+
+CStatus
+DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id) {
+    auto status = CStatus();
+    try {
+        auto segment_interface = reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
+        auto segment = dynamic_cast<milvus::segcore::SegmentSealed*>(segment_interface);
+        AssertInfo(segment != nullptr, "segment conversion failed");
+        segment->DropIndex(milvus::FieldId(field_id));
+        status.error_code = Success;
+        status.error_msg = "";
+        return status;
+    } catch (std::exception& e) {
+        status.error_code = UnexpectedException;
+        status.error_msg = strdup(e.what());
+        return status;
+    }
+}
+
 //////////////////////////////    deprecated interfaces    //////////////////////////////
 CStatus
 UpdateSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info) {

internal/core/src/segcore/segment_c.h

@@ -80,13 +80,19 @@ Delete(CSegmentInterface c_segment,
 int64_t
 PreDelete(CSegmentInterface c_segment, int64_t size);
 
-//////////////////////////////    interfaces for growing segment    //////////////////////////////
+//////////////////////////////    interfaces for sealed segment    //////////////////////////////
 CStatus
 LoadFieldData(CSegmentInterface c_segment, CLoadFieldDataInfo load_field_data_info);
 
 CStatus
 UpdateSealedSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info);
 
+CStatus
+DropFieldData(CSegmentInterface c_segment, int64_t field_id);
+
+CStatus
+DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id);
+
 //////////////////////////////    deprecated interfaces    //////////////////////////////
 CStatus
 UpdateSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info);

internal/core/unittest/test_query.cpp

@@ -223,6 +223,57 @@ TEST(Query, ExecWithPredicateLoader) {
     ASSERT_EQ(json.dump(2), ref.dump(2));
 }
 
+TEST(Query, ExecWithPredicateSmallN) {
+    using namespace milvus::query;
+    using namespace milvus::segcore;
+    auto schema = std::make_shared<Schema>();
+    schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, 7, MetricType::METRIC_L2);
+    schema->AddDebugField("age", DataType::FLOAT);
+    std::string dsl = R"({
+        "bool": {
+            "must": [
+            {
+                "range": {
+                    "age": {
+                        "GE": -1,
+                        "LT": 1
+                    }
+                }
+            },
+            {
+                "vector": {
+                    "fakevec": {
+                        "metric_type": "L2",
+                        "params": {
+                            "nprobe": 10
+                        },
+                        "query": "$0",
+                        "topk": 5
+                    }
+                }
+            }
+            ]
+        }
+    })";
+    int64_t N = 177;
+    auto dataset = DataGen(schema, N);
+    auto segment = CreateGrowingSegment(schema);
+    segment->PreInsert(N);
+    segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_);
+
+    auto plan = CreatePlan(*schema, dsl);
+    auto num_queries = 5;
+    auto ph_group_raw = CreatePlaceholderGroup(num_queries, 7, 1024);
+    auto ph_group = ParsePlaceholderGroup(plan.get(), ph_group_raw.SerializeAsString());
+    Timestamp time = 1000000;
+    std::vector<const PlaceholderGroup*> ph_group_arr = {ph_group.get()};
+    auto qr = segment->Search(plan.get(), ph_group_arr.data(), &time, 1);
+    int topk = 5;
+
+    Json json = QueryResultToJson(qr);
+    std::cout << json.dump(2);
+}
+
 TEST(Query, ExecWithPredicate) {
     using namespace milvus::query;
     using namespace milvus::segcore;

internal/datanode/collection_replica.go

@@ -46,6 +46,7 @@ type Segment struct {
 	endTime       Timestamp // not using
 	startPosition *internalpb.MsgPosition
 	endPosition   *internalpb.MsgPosition // not using
+	channelName   string
 }
 
 type CollectionSegmentReplica struct {
@@ -99,6 +100,7 @@ func (replica *CollectionSegmentReplica) addSegment(
 		createTime:    0,
 		startPosition: position,
 		endPosition:   new(internalpb.MsgPosition),
+		channelName:   channelName,
 	}
 
 	seg.isNew.Store(true)

internal/datanode/data_sync_service.go

@@ -73,7 +73,7 @@ func (dsService *dataSyncService) initNodes() {
 		}
 		return nil
 	}
-	err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
+	err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
 	if err != nil {
 		panic(err)
 	}

internal/datanode/flow_graph_insert_buffer_node.go

@@ -117,7 +117,21 @@ func (ibNode *insertBufferNode) Operate(ctx context.Context, in []Msg) ([]Msg, c
 			case iMsg.startPositions == nil || len(iMsg.startPositions) <= 0:
 				log.Error("insert Msg StartPosition empty")
 			default:
-				ibNode.replica.setStartPosition(currentSegID, iMsg.startPositions[0])
+				segment, err := ibNode.replica.getSegmentByID(currentSegID)
+				if err != nil {
+					log.Error("get segment wrong", zap.Error(err))
+				}
+				var startPosition *internalpb.MsgPosition = nil
+				for _, pos := range iMsg.startPositions {
+					if pos.ChannelName == segment.channelName {
+						startPosition = pos
+					}
+				}
+				if startPosition == nil {
+					log.Error("get position wrong", zap.Error(err))
+				} else {
+					ibNode.replica.setStartPosition(currentSegID, startPosition)
+				}
 			}
 		}
 
@@ -418,7 +432,20 @@ func (ibNode *insertBufferNode) Operate(ctx context.Context, in []Msg) ([]Msg, c
 		case iMsg.endPositions == nil || len(iMsg.endPositions) <= 0:
 			log.Error("insert Msg EndPosition empty")
 		default:
-			ibNode.replica.setEndPosition(currentSegID, iMsg.endPositions[0])
+			segment, err := ibNode.replica.getSegmentByID(currentSegID)
+			if err != nil {
+				log.Error("get segment wrong", zap.Error(err))
+			}
+			var endPosition *internalpb.MsgPosition = nil
+			for _, pos := range iMsg.endPositions {
+				if pos.ChannelName == segment.channelName {
+					endPosition = pos
+				}
+			}
+			if endPosition == nil {
+				log.Error("get position wrong", zap.Error(err))
+			}
+			ibNode.replica.setEndPosition(currentSegID, endPosition)
 		}
 
 		// 1.4 if full

internal/dataservice/meta.go

@@ -169,7 +169,6 @@ func (meta *meta) AddSegment(segmentInfo *datapb.SegmentInfo) error {
 func (meta *meta) UpdateSegment(segmentInfo *datapb.SegmentInfo) error {
 	meta.ddLock.Lock()
 	defer meta.ddLock.Unlock()
-
 	meta.segID2Info[segmentInfo.SegmentID] = segmentInfo
 	if err := meta.saveSegmentInfo(segmentInfo); err != nil {
 		_ = meta.reloadFromKV()
@@ -252,7 +251,7 @@ func (meta *meta) FlushSegment(segID UniqueID, timetick Timestamp) error {
 	}
 
 	segInfo.FlushedTime = timetick
-
+	segInfo.State = commonpb.SegmentState_Flushed
 	err := meta.saveSegmentInfo(segInfo)
 	if err != nil {
 		_ = meta.reloadFromKV()

internal/dataservice/server.go

@@ -150,7 +150,7 @@ func (s *Server) initMeta() error {
 		}
 		return nil
 	}
-	err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
+	err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
 	if err != nil {
 		return err
 	}
@@ -343,18 +343,12 @@ func (s *Server) startSegmentFlushChannel(ctx context.Context) {
 				continue
 			}
 			realMsg := msg.(*msgstream.FlushCompletedMsg)
-
+			err := s.meta.FlushSegment(realMsg.SegmentID, realMsg.BeginTimestamp)
-			segmentInfo, err := s.meta.GetSegment(realMsg.SegmentID)
+			log.Debug("dataservice flushed segment", zap.Any("segmentID", realMsg.SegmentID), zap.Error(err))
 			if err != nil {
 				log.Error("get segment from meta error", zap.Int64("segmentID", realMsg.SegmentID), zap.Error(err))
 				continue
 			}
-			segmentInfo.FlushedTime = realMsg.BeginTimestamp
-			segmentInfo.State = commonpb.SegmentState_Flushed
-			if err = s.meta.UpdateSegment(segmentInfo); err != nil {
-				log.Error("update segment error", zap.Error(err))
-				continue
-			}
 		}
 	}
 }

internal/dataservice/stats_handler.go

@@ -1,7 +1,9 @@
 package dataservice
 
 import (
+	"github.com/zilliztech/milvus-distributed/internal/log"
 	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
+	"go.uber.org/zap"
 )
 
 type statsHandler struct {
@@ -32,6 +34,6 @@ func (handler *statsHandler) HandleSegmentStat(segStats *internalpb.SegmentStati
 	segMeta.SealedTime = segStats.EndTime
 	segMeta.NumRows = segStats.NumRows
 	segMeta.MemSize = segStats.MemorySize
-
+	log.Debug("stats_handler update segment", zap.Any("segmentID", segMeta.SegmentID), zap.Any("State", segMeta.State))
 	return handler.meta.UpdateSegment(segMeta)
 }

internal/distributed/datanode/client/client.go

@@ -48,7 +48,7 @@ func (c *Client) Init() error {
 		return nil
 	}
 
-	err := retry.Retry(100, time.Millisecond*200, connectGrpcFunc)
+	err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
 	if err != nil {
 		return err
 	}

internal/distributed/datanode/service.go

@@ -182,7 +182,7 @@ func (s *Server) init() error {
 	if err = masterClient.Start(); err != nil {
 		panic(err)
 	}
-	err = funcutil.WaitForComponentHealthy(ctx, masterClient, "MasterService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentHealthy(ctx, masterClient, "MasterService", 1000000, time.Millisecond*200)
 
 	if err != nil {
 		panic(err)
@@ -202,7 +202,7 @@ func (s *Server) init() error {
 	if err = dataService.Start(); err != nil {
 		panic(err)
 	}
-	err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, time.Millisecond*200)
 	if err != nil {
 		panic(err)
 	}

internal/distributed/dataservice/client/client.go

@@ -47,7 +47,7 @@ func (c *Client) Init() error {
 		return nil
 	}
 
-	err := retry.Retry(100, time.Millisecond*200, connectGrpcFunc)
+	err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
 	if err != nil {
 		return err
 	}

internal/distributed/dataservice/service.go

@@ -104,7 +104,7 @@ func (s *Server) init() error {
 	s.dataService.UpdateStateCode(internalpb.StateCode_Initializing)
 
 	ctx := context.Background()
-	err = funcutil.WaitForComponentInitOrHealthy(ctx, client, "MasterService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentInitOrHealthy(ctx, client, "MasterService", 1000000, time.Millisecond*200)
 
 	if err != nil {
 		panic(err)

internal/distributed/indexnode/client/client.go

@@ -45,7 +45,7 @@ func (c *Client) Init() error {
 		c.grpcClient = indexpb.NewIndexNodeClient(conn)
 		return nil
 	}
-	err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
+	err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
 	if err != nil {
 		return err
 	}

internal/distributed/indexnode/paramtable.go

@@ -1,9 +1,6 @@
 package grpcindexnode
 
 import (
-	"net"
-	"os"
-	"strconv"
 	"sync"
 
 	"github.com/zilliztech/milvus-distributed/internal/util/funcutil"
@@ -35,11 +32,6 @@ func (pt *ParamTable) LoadFromArgs() {
 }
 
 func (pt *ParamTable) LoadFromEnv() {
-	indexServiceAddress := os.Getenv("INDEX_SERVICE_ADDRESS")
-	if indexServiceAddress != "" {
-		pt.IndexServerAddress = indexServiceAddress
-	}
-
 	Params.IP = funcutil.GetLocalIP()
 }
 
@@ -50,28 +42,11 @@ func (pt *ParamTable) initParams() {
 
 // todo remove and use load from env
 func (pt *ParamTable) initIndexServerAddress() {
-	addr, err := pt.Load("indexService.address")
+	ret, err := pt.Load("IndexServiceAddress")
 	if err != nil {
 		panic(err)
 	}
-
+	pt.IndexServerAddress = ret
-	hostName, _ := net.LookupHost(addr)
-	if len(hostName) <= 0 {
-		if ip := net.ParseIP(addr); ip == nil {
-			panic("invalid ip indexServer.address")
-		}
-	}
-
-	port, err := pt.Load("indexService.port")
-	if err != nil {
-		panic(err)
-	}
-	_, err = strconv.Atoi(port)
-	if err != nil {
-		panic(err)
-	}
-
-	pt.IndexServerAddress = addr + ":" + port
 }
 
 func (pt *ParamTable) initPort() {

internal/distributed/indexservice/client/client.go

@@ -50,7 +50,7 @@ func (c *Client) Init() error {
 		c.grpcClient = indexpb.NewIndexServiceClient(conn)
 		return nil
 	}
-	err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
+	err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
 	if err != nil {
 		return err
 	}

internal/distributed/indexservice/paramtable.go

@@ -1,8 +1,6 @@
 package grpcindexservice
 
 import (
-	"net"
-	"strconv"
 	"sync"
 
 	"github.com/zilliztech/milvus-distributed/internal/util/paramtable"
@@ -35,25 +33,9 @@ func (pt *ParamTable) initServicePort() {
 }
 
 func (pt *ParamTable) initServiceAddress() {
-	addr, err := pt.Load("indexService.address")
+	ret, err := pt.Load("IndexServiceAddress")
 	if err != nil {
 		panic(err)
 	}
-
+	pt.ServiceAddress = ret
-	hostName, _ := net.LookupHost(addr)
-	if len(hostName) <= 0 {
-		if ip := net.ParseIP(addr); ip == nil {
-			panic("invalid ip proxyService.address")
-		}
-	}
-
-	port, err := pt.Load("indexService.port")
-	if err != nil {
-		panic(err)
-	}
-	_, err = strconv.Atoi(port)
-	if err != nil {
-		panic(err)
-	}
-	pt.ServiceAddress = addr + ":" + port
 }

internal/distributed/masterservice/client/client.go

@@ -32,7 +32,7 @@ func NewClient(addr string, timeout time.Duration) (*GrpcClient, error) {
 		addr:        addr,
 		timeout:     timeout,
 		grpcTimeout: time.Second * 5,
-		retry:       3,
+		retry:       300,
 	}, nil
 }
 

internal/distributed/masterservice/masterservice_test.go

@@ -357,6 +357,7 @@ func TestGrpcService(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Equal(t, partMeta.PartitionName, "testPartition")
 
+		assert.Equal(t, 1, len(collectionMetaCache))
 	})
 
 	t.Run("has partition", func(t *testing.T) {
@@ -600,6 +601,7 @@ func TestGrpcService(t *testing.T) {
 		partMeta, err := core.MetaTable.GetPartitionByID(collMeta.PartitionIDs[0])
 		assert.Nil(t, err)
 		assert.Equal(t, partMeta.PartitionName, cms.Params.DefaultPartitionName)
+		assert.Equal(t, 2, len(collectionMetaCache))
 	})
 
 	t.Run("drop collection", func(t *testing.T) {
@@ -620,7 +622,7 @@ func TestGrpcService(t *testing.T) {
 		assert.Equal(t, dropCollectionArray[0].Base.MsgType, commonpb.MsgType_DropCollection)
 		assert.Equal(t, status.ErrorCode, commonpb.ErrorCode_Success)
 		assert.Equal(t, dropCollectionArray[0].CollectionName, "testColl")
-		assert.Equal(t, len(collectionMetaCache), 1)
+		assert.Equal(t, len(collectionMetaCache), 3)
 		assert.Equal(t, collectionMetaCache[0], "testColl")
 
 		req = &milvuspb.DropCollectionRequest{

internal/distributed/masterservice/server.go

@@ -123,7 +123,7 @@ func (s *Server) init() error {
 			panic(err)
 		}
 
-		err := funcutil.WaitForComponentInitOrHealthy(ctx, proxyService, "ProxyService", 100, 200*time.Millisecond)
+		err := funcutil.WaitForComponentInitOrHealthy(ctx, proxyService, "ProxyService", 1000000, 200*time.Millisecond)
 		if err != nil {
 			panic(err)
 		}
@@ -141,7 +141,7 @@ func (s *Server) init() error {
 		if err := dataService.Start(); err != nil {
 			panic(err)
 		}
-		err := funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, 200*time.Millisecond)
+		err := funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, 200*time.Millisecond)
 		if err != nil {
 			panic(err)
 		}

internal/distributed/proxynode/client/client.go

@@ -44,7 +44,7 @@ func (c *Client) Init() error {
 		c.grpcClient = proxypb.NewProxyNodeServiceClient(conn)
 		return nil
 	}
-	err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
+	err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
 	if err != nil {
 		return err
 	}

internal/distributed/proxynode/paramtable.go

@@ -1,9 +1,6 @@
 package grpcproxynode
 
 import (
-	"net"
-	"os"
-	"strconv"
 	"sync"
 
 	"github.com/zilliztech/milvus-distributed/internal/util/funcutil"
@@ -42,32 +39,6 @@ func (pt *ParamTable) LoadFromArgs() {
 }
 
 func (pt *ParamTable) LoadFromEnv() {
-
-	masterAddress := os.Getenv("MASTER_ADDRESS")
-	if masterAddress != "" {
-		pt.MasterAddress = masterAddress
-	}
-
-	proxyServiceAddress := os.Getenv("PROXY_SERVICE_ADDRESS")
-	if proxyServiceAddress != "" {
-		pt.ProxyServiceAddress = proxyServiceAddress
-	}
-
-	indexServiceAddress := os.Getenv("INDEX_SERVICE_ADDRESS")
-	if indexServiceAddress != "" {
-		pt.IndexServerAddress = indexServiceAddress
-	}
-
-	queryServiceAddress := os.Getenv("QUERY_SERVICE_ADDRESS")
-	if queryServiceAddress != "" {
-		pt.QueryServiceAddress = queryServiceAddress
-	}
-
-	dataServiceAddress := os.Getenv("DATA_SERVICE_ADDRESS")
-	if dataServiceAddress != "" {
-		pt.DataServiceAddress = dataServiceAddress
-	}
-
 	Params.IP = funcutil.GetLocalIP()
 }
 
@@ -86,96 +57,47 @@ func (pt *ParamTable) initPoxyServicePort() {
 }
 
 func (pt *ParamTable) initProxyServiceAddress() {
-	addr, err := pt.Load("proxyService.address")
+	ret, err := pt.Load("_PROXY_SERVICE_ADDRESS")
 	if err != nil {
 		panic(err)
 	}
-
+	pt.ProxyServiceAddress = ret
-	hostName, _ := net.LookupHost(addr)
-	if len(hostName) <= 0 {
-		if ip := net.ParseIP(addr); ip == nil {
-			panic("invalid ip proxyService.address")
-		}
-	}
-
-	port, err := pt.Load("proxyService.port")
-	if err != nil {
-		panic(err)
-	}
-	_, err = strconv.Atoi(port)
-	if err != nil {
-		panic(err)
-	}
-	pt.ProxyServiceAddress = addr + ":" + port
 }
 
 // todo remove and use load from env
 func (pt *ParamTable) initIndexServerAddress() {
-	addr, err := pt.Load("indexService.address")
+	ret, err := pt.Load("IndexServiceAddress")
 	if err != nil {
 		panic(err)
 	}
-
+	pt.IndexServerAddress = ret
-	hostName, _ := net.LookupHost(addr)
-	if len(hostName) <= 0 {
-		if ip := net.ParseIP(addr); ip == nil {
-			panic("invalid ip indexService.address")
-		}
-	}
-
-	port, err := pt.Load("indexService.port")
-	if err != nil {
-		panic(err)
-	}
-	_, err = strconv.Atoi(port)
-	if err != nil {
-		panic(err)
-	}
-
-	pt.IndexServerAddress = addr + ":" + port
 }
 
 // todo remove and use load from env
 func (pt *ParamTable) initMasterAddress() {
-
+	ret, err := pt.Load("_MasterAddress")
-	masterHost, err := pt.Load("master.address")
 	if err != nil {
 		panic(err)
 	}
-	port, err := pt.Load("master.port")
+	pt.MasterAddress = ret
-	if err != nil {
-		panic(err)
-	}
-	pt.MasterAddress = masterHost + ":" + port
-
 }
 
 // todo remove and use load from env
 func (pt *ParamTable) initDataServiceAddress() {
-	addr, err := pt.Load("dataService.address")
+	ret, err := pt.Load("_DataServiceAddress")
 	if err != nil {
 		panic(err)
 	}
-
+	pt.DataServiceAddress = ret
-	port, err := pt.Load("dataService.port")
-	if err != nil {
-		panic(err)
-	}
-	pt.DataServiceAddress = addr + ":" + port
 }
 
 // todo remove and use load from env
 func (pt *ParamTable) initQueryServiceAddress() {
-	addr, err := pt.Load("queryService.address")
+	ret, err := pt.Load("_QueryServiceAddress")
 	if err != nil {
 		panic(err)
 	}
-
+	pt.QueryServiceAddress = ret
-	port, err := pt.Load("queryService.port")
-	if err != nil {
-		panic(err)
-	}
-	pt.QueryServiceAddress = addr + ":" + port
 }
 
 func (pt *ParamTable) initPort() {

internal/distributed/proxynode/service.go

@@ -185,7 +185,7 @@ func (s *Server) init() error {
 	if err != nil {
 		return err
 	}
-	err = funcutil.WaitForComponentHealthy(ctx, s.masterServiceClient, "MasterService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentHealthy(ctx, s.masterServiceClient, "MasterService", 1000000, time.Millisecond*200)
 
 	if err != nil {
 		panic(err)

internal/distributed/proxyservice/client/client.go

@@ -44,7 +44,7 @@ func (c *Client) Init() error {
 		c.proxyServiceClient = proxypb.NewProxyServiceClient(conn)
 		return nil
 	}
-	err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
+	err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
 	if err != nil {
 		return err
 	}

internal/distributed/proxyservice/paramtable.go

@@ -1,8 +1,6 @@
 package grpcproxyservice
 
 import (
-	"net"
-	"strconv"
 	"sync"
 
 	"github.com/zilliztech/milvus-distributed/internal/util/paramtable"
@@ -35,25 +33,9 @@ func (pt *ParamTable) initServicePort() {
 }
 
 func (pt *ParamTable) initServiceAddress() {
-	addr, err := pt.Load("proxyService.address")
+	ret, err := pt.Load("_PROXY_SERVICE_ADDRESS")
 	if err != nil {
 		panic(err)
 	}
-
+	pt.ServiceAddress = ret
-	hostName, _ := net.LookupHost(addr)
-	if len(hostName) <= 0 {
-		if ip := net.ParseIP(addr); ip == nil {
-			panic("invalid ip proxyService.address")
-		}
-	}
-
-	port, err := pt.Load("proxyService.port")
-	if err != nil {
-		panic(err)
-	}
-	_, err = strconv.Atoi(port)
-	if err != nil {
-		panic(err)
-	}
-	pt.ServiceAddress = addr + ":" + port
 }

internal/distributed/querynode/client/client.go

@@ -37,7 +37,7 @@ func (c *Client) Init() error {
 	ctx, cancel := context.WithTimeout(context.Background(), RPCConnectionTimeout)
 	defer cancel()
 	var err error
-	for i := 0; i < Retry; i++ {
+	for i := 0; i < Retry*100; i++ {
 		if c.conn, err = grpc.DialContext(ctx, c.addr, grpc.WithInsecure(), grpc.WithBlock(),
 			grpc.WithUnaryInterceptor(
 				otgrpc.OpenTracingClientInterceptor(tracer)),

internal/distributed/querynode/service.go

@@ -111,7 +111,7 @@ func (s *Server) init() error {
 		panic(err)
 	}
 
-	err = funcutil.WaitForComponentInitOrHealthy(ctx, queryService, "QueryService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentInitOrHealthy(ctx, queryService, "QueryService", 1000000, time.Millisecond*200)
 	if err != nil {
 		panic(err)
 	}
@@ -139,7 +139,7 @@ func (s *Server) init() error {
 		panic(err)
 	}
 
-	err = funcutil.WaitForComponentHealthy(ctx, masterService, "MasterService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentHealthy(ctx, masterService, "MasterService", 1000000, time.Millisecond*200)
 	if err != nil {
 		panic(err)
 	}
@@ -160,7 +160,7 @@ func (s *Server) init() error {
 		panic(err)
 	}
 	// wait indexservice healthy
-	err = funcutil.WaitForComponentHealthy(ctx, indexService, "IndexService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentHealthy(ctx, indexService, "IndexService", 1000000, time.Millisecond*200)
 	if err != nil {
 		panic(err)
 	}
@@ -180,7 +180,7 @@ func (s *Server) init() error {
 	if err = dataService.Start(); err != nil {
 		panic(err)
 	}
-	err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, time.Millisecond*200)
 	if err != nil {
 		panic(err)
 	}

internal/distributed/queryservice/client/client.go

@@ -32,7 +32,7 @@ func NewClient(address string, timeout time.Duration) (*Client, error) {
 		conn:       nil,
 		addr:       address,
 		timeout:    timeout,
-		retry:      3,
+		retry:      300,
 	}, nil
 }
 

internal/distributed/queryservice/service.go

@@ -101,7 +101,7 @@ func (s *Server) init() error {
 		panic(err)
 	}
 	// wait for master init or healthy
-	err = funcutil.WaitForComponentInitOrHealthy(ctx, masterService, "MasterService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentInitOrHealthy(ctx, masterService, "MasterService", 1000000, time.Millisecond*200)
 	if err != nil {
 		panic(err)
 	}
@@ -121,7 +121,7 @@ func (s *Server) init() error {
 	if err = dataService.Start(); err != nil {
 		panic(err)
 	}
-	err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, time.Millisecond*200)
+	err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, time.Millisecond*200)
 	if err != nil {
 		panic(err)
 	}

internal/indexnode/indexnode.go

@@ -67,7 +67,7 @@ func NewIndexNode(ctx context.Context) (*IndexNode, error) {
 
 func (i *IndexNode) Init() error {
 	ctx := context.Background()
-	err := funcutil.WaitForComponentHealthy(ctx, i.serviceClient, "IndexService", 100, time.Millisecond*200)
+	err := funcutil.WaitForComponentHealthy(ctx, i.serviceClient, "IndexService", 1000000, time.Millisecond*200)
 
 	if err != nil {
 		return err

internal/indexservice/indexservice.go

@@ -86,7 +86,7 @@ func (i *IndexService) Init() error {
 		i.metaTable = metakv
 		return nil
 	}
-	err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
+	err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
 	if err != nil {
 		return err
 	}

internal/indexservice/paramtable.go

@@ -1,7 +1,6 @@
 package indexservice
 
 import (
-	"net"
 	"path"
 	"strconv"
 	"sync"
@@ -38,8 +37,6 @@ func (pt *ParamTable) Init() {
 	once.Do(func() {
 		pt.BaseTable.Init()
 		pt.initLogCfg()
-		pt.initAddress()
-		pt.initPort()
 		pt.initEtcdAddress()
 		pt.initMasterAddress()
 		pt.initMetaRootPath()
@@ -52,35 +49,6 @@ func (pt *ParamTable) Init() {
 	})
 }
 
-func (pt *ParamTable) initAddress() {
-	addr, err := pt.Load("indexService.address")
-	if err != nil {
-		panic(err)
-	}
-
-	hostName, _ := net.LookupHost(addr)
-	if len(hostName) <= 0 {
-		if ip := net.ParseIP(addr); ip == nil {
-			panic("invalid ip indexServer.address")
-		}
-	}
-
-	port, err := pt.Load("indexService.port")
-	if err != nil {
-		panic(err)
-	}
-	_, err = strconv.Atoi(port)
-	if err != nil {
-		panic(err)
-	}
-
-	pt.Address = addr + ":" + port
-}
-
-func (pt *ParamTable) initPort() {
-	pt.Port = pt.ParseInt("indexService.port")
-}
-
 func (pt *ParamTable) initEtcdAddress() {
 	addr, err := pt.Load("_EtcdAddress")
 	if err != nil {

internal/kv/minio/minio_kv.go

@@ -43,7 +43,7 @@ func NewMinIOKV(ctx context.Context, option *Option) (*MinIOKV, error) {
 		return nil
 	}
 
-	err := retry.Retry(200, time.Millisecond*200, connectMinIOFn)
+	err := retry.Retry(100000, time.Millisecond*200, connectMinIOFn)
 	if err != nil {
 		return nil, err
 	}

internal/masterservice/master_service.go

@@ -765,7 +765,7 @@ func (c *Core) Init() error {
 			c.kvBase = etcdkv.NewEtcdKV(c.etcdCli, Params.KvRootPath)
 			return nil
 		}
-		err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
+		err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
 		if err != nil {
 			return
 		}
@@ -1001,7 +1001,7 @@ func (c *Core) DescribeCollection(ctx context.Context, in *milvuspb.DescribeColl
 	c.ddReqQueue <- t
 	err := t.WaitToFinish()
 	if err != nil {
-		log.Debug("DescribeCollection Failed", zap.String("name", in.CollectionName))
+		log.Debug("DescribeCollection Failed", zap.String("name", in.CollectionName), zap.Error(err))
 		return &milvuspb.DescribeCollectionResponse{
 			Status: &commonpb.Status{
 				ErrorCode: commonpb.ErrorCode_UnexpectedError,

internal/masterservice/master_service_test.go

@@ -508,6 +508,9 @@ func TestMasterService(t *testing.T) {
 		assert.True(t, ok)
 		assert.Equal(t, partMsg.CollectionID, collMeta.ID)
 		assert.Equal(t, partMsg.PartitionID, partMeta.PartitionID)
+
+		assert.Equal(t, 1, len(pm.GetCollArray()))
+		assert.Equal(t, "testColl", pm.GetCollArray()[0])
 	})
 
 	t.Run("has partition", func(t *testing.T) {
@@ -893,6 +896,9 @@ func TestMasterService(t *testing.T) {
 		assert.True(t, ok)
 		assert.Equal(t, dmsg.CollectionID, collMeta.ID)
 		assert.Equal(t, dmsg.PartitionID, dropPartID)
+
+		assert.Equal(t, 2, len(pm.GetCollArray()))
+		assert.Equal(t, "testColl", pm.GetCollArray()[1])
 	})
 
 	t.Run("drop collection", func(t *testing.T) {
@@ -919,8 +925,8 @@ func TestMasterService(t *testing.T) {
 		assert.True(t, ok)
 		assert.Equal(t, dmsg.CollectionID, collMeta.ID)
 		collArray := pm.GetCollArray()
-		assert.Equal(t, len(collArray), 1)
+		assert.Equal(t, len(collArray), 3)
-		assert.Equal(t, collArray[0], "testColl")
+		assert.Equal(t, collArray[2], "testColl")
 
 		time.Sleep(time.Millisecond * 100)
 		qm.mutex.Lock()
@@ -944,8 +950,8 @@ func TestMasterService(t *testing.T) {
 		time.Sleep(time.Second)
 		assert.Zero(t, len(ddStream.Chan()))
 		collArray = pm.GetCollArray()
-		assert.Equal(t, len(collArray), 1)
+		assert.Equal(t, len(collArray), 3)
-		assert.Equal(t, collArray[0], "testColl")
+		assert.Equal(t, collArray[2], "testColl")
 	})
 
 	err = core.Stop()

internal/masterservice/task.go

@@ -419,6 +419,9 @@ func (t *CreatePartitionReqTask) Execute(ctx context.Context) error {
 		return err
 	}
 
+	// error doesn't matter here
+	_ = t.core.InvalidateCollectionMetaCache(ctx, t.Req.Base.Timestamp, t.Req.DbName, t.Req.CollectionName)
+
 	return nil
 }
 
@@ -467,6 +470,9 @@ func (t *DropPartitionReqTask) Execute(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+
+	// error doesn't matter here
+	_ = t.core.InvalidateCollectionMetaCache(ctx, t.Req.Base.Timestamp, t.Req.DbName, t.Req.CollectionName)
 	return nil
 }
 

internal/msgstream/pulsarms/pulsar_msgstream.go

@@ -2,17 +2,18 @@ package pulsarms
 
 import (
 	"context"
+	"errors"
 	"path/filepath"
 	"reflect"
 	"strconv"
 	"sync"
 	"time"
 
-	"errors"
-
 	"github.com/apache/pulsar-client-go/pulsar"
 	"github.com/golang/protobuf/proto"
 	"github.com/opentracing/opentracing-go"
+	"go.uber.org/zap"
+
 	"github.com/zilliztech/milvus-distributed/internal/log"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream/util"
@@ -20,7 +21,6 @@ import (
 	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
 	"github.com/zilliztech/milvus-distributed/internal/util/trace"
 	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-	"go.uber.org/zap"
 )
 
 type TsMsg = msgstream.TsMsg
@@ -41,8 +41,9 @@ type UnmarshalDispatcher = msgstream.UnmarshalDispatcher
 type PulsarMsgStream struct {
 	ctx              context.Context
 	client           pulsar.Client
-	producers        []Producer
+	producers        map[string]Producer
-	consumers        []Consumer
+	producerChannels []string
+	consumers        map[string]Consumer
 	consumerChannels []string
 	repackFunc       RepackFunc
 	unmarshal        UnmarshalDispatcher
@@ -50,6 +51,7 @@ type PulsarMsgStream struct {
 	wait             *sync.WaitGroup
 	streamCancel     func()
 	pulsarBufSize    int64
+	producerLock     *sync.Mutex
 	consumerLock     *sync.Mutex
 	consumerReflects []reflect.SelectCase
 
@@ -63,8 +65,9 @@ func newPulsarMsgStream(ctx context.Context,
 	unmarshal UnmarshalDispatcher) (*PulsarMsgStream, error) {
 
 	streamCtx, streamCancel := context.WithCancel(ctx)
-	producers := make([]Producer, 0)
+	producers := make(map[string]Producer)
-	consumers := make([]Consumer, 0)
+	consumers := make(map[string]Consumer)
+	producerChannels := make([]string, 0)
 	consumerChannels := make([]string, 0)
 	consumerReflects := make([]reflect.SelectCase, 0)
 	receiveBuf := make(chan *MsgPack, receiveBufSize)
@@ -85,6 +88,7 @@ func newPulsarMsgStream(ctx context.Context,
 		ctx:              streamCtx,
 		client:           client,
 		producers:        producers,
+		producerChannels: producerChannels,
 		consumers:        consumers,
 		consumerChannels: consumerChannels,
 		unmarshal:        unmarshal,
@@ -92,6 +96,7 @@ func newPulsarMsgStream(ctx context.Context,
 		receiveBuf:       receiveBuf,
 		streamCancel:     streamCancel,
 		consumerReflects: consumerReflects,
+		producerLock:     &sync.Mutex{},
 		consumerLock:     &sync.Mutex{},
 		wait:             &sync.WaitGroup{},
 		scMap:            &sync.Map{},
@@ -101,22 +106,24 @@ func newPulsarMsgStream(ctx context.Context,
 }
 
 func (ms *PulsarMsgStream) AsProducer(channels []string) {
-	for i := 0; i < len(channels); i++ {
+	for _, channel := range channels {
 		fn := func() error {
-			pp, err := ms.client.CreateProducer(pulsar.ProducerOptions{Topic: channels[i]})
+			pp, err := ms.client.CreateProducer(pulsar.ProducerOptions{Topic: channel})
 			if err != nil {
 				return err
 			}
 			if pp == nil {
 				return errors.New("pulsar is not ready, producer is nil")
 			}
-
+			ms.producerLock.Lock()
-			ms.producers = append(ms.producers, pp)
+			ms.producers[channel] = pp
+			ms.producerChannels = append(ms.producerChannels, channel)
+			ms.producerLock.Unlock()
 			return nil
 		}
 		err := util.Retry(20, time.Millisecond*200, fn)
 		if err != nil {
-			errMsg := "Failed to create producer " + channels[i] + ", error = " + err.Error()
+			errMsg := "Failed to create producer " + channel + ", error = " + err.Error()
 			panic(errMsg)
 		}
 	}
@@ -124,14 +131,17 @@ func (ms *PulsarMsgStream) AsProducer(channels []string) {
 
 func (ms *PulsarMsgStream) AsConsumer(channels []string,
 	subName string) {
-	for i := 0; i < len(channels); i++ {
+	for _, channel := range channels {
+		if _, ok := ms.consumers[channel]; ok {
+			continue
+		}
 		fn := func() error {
 			receiveChannel := make(chan pulsar.ConsumerMessage, ms.pulsarBufSize)
 			pc, err := ms.client.Subscribe(pulsar.ConsumerOptions{
-				Topic:                       channels[i],
+				Topic:                       channel,
 				SubscriptionName:            subName,
 				Type:                        pulsar.KeyShared,
-				SubscriptionInitialPosition: pulsar.SubscriptionPositionLatest,
+				SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
 				MessageChannel:              receiveChannel,
 			})
 			if err != nil {
@@ -141,8 +151,8 @@ func (ms *PulsarMsgStream) AsConsumer(channels []string,
 				return errors.New("pulsar is not ready, consumer is nil")
 			}
 
-			ms.consumers = append(ms.consumers, pc)
+			ms.consumers[channel] = pc
-			ms.consumerChannels = append(ms.consumerChannels, channels[i])
+			ms.consumerChannels = append(ms.consumerChannels, channel)
 			ms.consumerReflects = append(ms.consumerReflects, reflect.SelectCase{
 				Dir:  reflect.SelectRecv,
 				Chan: reflect.ValueOf(pc.Chan()),
@@ -153,7 +163,7 @@ func (ms *PulsarMsgStream) AsConsumer(channels []string,
 		}
 		err := util.Retry(20, time.Millisecond*200, fn)
 		if err != nil {
-			errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
+			errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
 			panic(errMsg)
 		}
 	}
@@ -233,6 +243,7 @@ func (ms *PulsarMsgStream) Produce(ctx context.Context, msgPack *MsgPack) error
 		return err
 	}
 	for k, v := range result {
+		channel := ms.producerChannels[k]
 		for i := 0; i < len(v.Msgs); i++ {
 			mb, err := v.Msgs[i].Marshal(v.Msgs[i])
 			if err != nil {
@@ -249,7 +260,7 @@ func (ms *PulsarMsgStream) Produce(ctx context.Context, msgPack *MsgPack) error
 			sp, spanCtx := trace.MsgSpanFromCtx(ctx, v.Msgs[i])
 			trace.InjectContextToPulsarMsgProperties(sp.Context(), msg.Properties)
 
-			if _, err := ms.producers[k].Send(
+			if _, err := ms.producers[channel].Send(
 				spanCtx,
 				msg,
 			); err != nil {
@@ -264,7 +275,6 @@ func (ms *PulsarMsgStream) Produce(ctx context.Context, msgPack *MsgPack) error
 }
 
 func (ms *PulsarMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) error {
-	producerLen := len(ms.producers)
 	for _, v := range msgPack.Msgs {
 		mb, err := v.Marshal(v)
 		if err != nil {
@@ -281,8 +291,9 @@ func (ms *PulsarMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) erro
 		sp, spanCtx := trace.MsgSpanFromCtx(ctx, v)
 		trace.InjectContextToPulsarMsgProperties(sp.Context(), msg.Properties)
 
-		for i := 0; i < producerLen; i++ {
+		ms.producerLock.Lock()
-			if _, err := ms.producers[i].Send(
+		for _, producer := range ms.producers {
+			if _, err := producer.Send(
 				spanCtx,
 				msg,
 			); err != nil {
@@ -291,6 +302,7 @@ func (ms *PulsarMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) erro
 				return err
 			}
 		}
+		ms.producerLock.Unlock()
 		sp.Finish()
 	}
 	return nil
@@ -319,7 +331,7 @@ func (ms *PulsarMsgStream) Consume() (*MsgPack, context.Context) {
 			sp.Finish()
 			return cm, ctx
 		case <-ms.ctx.Done():
-			log.Debug("context closed")
+			//log.Debug("context closed")
 			return nil, nil
 		}
 	}
@@ -469,18 +481,17 @@ func (ms *PulsarMsgStream) Chan() <-chan *MsgPack {
 }
 
 func (ms *PulsarMsgStream) Seek(mp *internalpb.MsgPosition) error {
-	for index, channel := range ms.consumerChannels {
+	if _, ok := ms.consumers[mp.ChannelName]; ok {
-		if channel == mp.ChannelName {
+		consumer := ms.consumers[mp.ChannelName]
-			messageID, err := typeutil.StringToPulsarMsgID(mp.MsgID)
+		messageID, err := typeutil.StringToPulsarMsgID(mp.MsgID)
-			if err != nil {
+		if err != nil {
-				return err
+			return err
-			}
-			err = ms.consumers[index].Seek(messageID)
-			if err != nil {
-				return err
-			}
-			return nil
 		}
+		err = consumer.Seek(messageID)
+		if err != nil {
+			return err
+		}
+		return nil
 	}
 
 	return errors.New("msgStream seek fail")
@@ -488,11 +499,12 @@ func (ms *PulsarMsgStream) Seek(mp *internalpb.MsgPosition) error {
 
 type PulsarTtMsgStream struct {
 	PulsarMsgStream
-	unsolvedBuf   map[Consumer][]TsMsg
+	unsolvedBuf     map[Consumer][]TsMsg
-	msgPositions  map[Consumer]*internalpb.MsgPosition
+	msgPositions    map[Consumer]*internalpb.MsgPosition
-	unsolvedMutex *sync.Mutex
+	unsolvedMutex   *sync.Mutex
-	lastTimeStamp Timestamp
+	lastTimeStamp   Timestamp
-	syncConsumer  chan int
+	syncConsumer    chan int
+	stopConsumeChan map[Consumer]chan bool
 }
 
 func newPulsarTtMsgStream(ctx context.Context,
@@ -505,6 +517,7 @@ func newPulsarTtMsgStream(ctx context.Context,
 		return nil, err
 	}
 	unsolvedBuf := make(map[Consumer][]TsMsg)
+	stopChannel := make(map[Consumer]chan bool)
 	msgPositions := make(map[Consumer]*internalpb.MsgPosition)
 	syncConsumer := make(chan int, 1)
 
@@ -514,19 +527,39 @@ func newPulsarTtMsgStream(ctx context.Context,
 		msgPositions:    msgPositions,
 		unsolvedMutex:   &sync.Mutex{},
 		syncConsumer:    syncConsumer,
+		stopConsumeChan: stopChannel,
 	}, nil
 }
 
+func (ms *PulsarTtMsgStream) addConsumer(consumer Consumer, channel string) {
+	if len(ms.consumers) == 0 {
+		ms.syncConsumer <- 1
+	}
+	ms.consumers[channel] = consumer
+	ms.unsolvedBuf[consumer] = make([]TsMsg, 0)
+	ms.consumerChannels = append(ms.consumerChannels, channel)
+	ms.msgPositions[consumer] = &internalpb.MsgPosition{
+		ChannelName: channel,
+		MsgID:       "",
+		Timestamp:   ms.lastTimeStamp,
+	}
+	stopConsumeChan := make(chan bool)
+	ms.stopConsumeChan[consumer] = stopConsumeChan
+}
+
 func (ms *PulsarTtMsgStream) AsConsumer(channels []string,
 	subName string) {
-	for i := 0; i < len(channels); i++ {
+	for _, channel := range channels {
+		if _, ok := ms.consumers[channel]; ok {
+			continue
+		}
 		fn := func() error {
 			receiveChannel := make(chan pulsar.ConsumerMessage, ms.pulsarBufSize)
 			pc, err := ms.client.Subscribe(pulsar.ConsumerOptions{
-				Topic:                       channels[i],
+				Topic:                       channel,
 				SubscriptionName:            subName,
 				Type:                        pulsar.KeyShared,
-				SubscriptionInitialPosition: pulsar.SubscriptionPositionLatest,
+				SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
 				MessageChannel:              receiveChannel,
 			})
 			if err != nil {
@@ -537,23 +570,13 @@ func (ms *PulsarTtMsgStream) AsConsumer(channels []string,
 			}
 
 			ms.consumerLock.Lock()
-			if len(ms.consumers) == 0 {
+			ms.addConsumer(pc, channel)
-				ms.syncConsumer <- 1
-			}
-			ms.consumers = append(ms.consumers, pc)
-			ms.unsolvedBuf[pc] = make([]TsMsg, 0)
-			ms.msgPositions[pc] = &internalpb.MsgPosition{
-				ChannelName: channels[i],
-				MsgID:       "",
-				Timestamp:   ms.lastTimeStamp,
-			}
-			ms.consumerChannels = append(ms.consumerChannels, channels[i])
 			ms.consumerLock.Unlock()
 			return nil
 		}
 		err := util.Retry(10, time.Millisecond*200, fn)
 		if err != nil {
-			errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
+			errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
 			panic(errMsg)
 		}
 	}
@@ -728,79 +751,87 @@ func (ms *PulsarTtMsgStream) findTimeTick(consumer Consumer,
 				return
 			}
 			sp.Finish()
+		case <-ms.stopConsumeChan[consumer]:
+			return
 		}
 	}
 }
 
 func (ms *PulsarTtMsgStream) Seek(mp *internalpb.MsgPosition) error {
+	if len(mp.MsgID) == 0 {
+		return errors.New("when msgID's length equal to 0, please use AsConsumer interface")
+	}
 	var consumer Consumer
-	var messageID MessageID
+	var err error
-	for index, channel := range ms.consumerChannels {
+	var hasWatched bool
-		if filepath.Base(channel) == filepath.Base(mp.ChannelName) {
+	seekChannel := mp.ChannelName
-			consumer = ms.consumers[index]
+	subName := mp.MsgGroup
-			if len(mp.MsgID) == 0 {
+	ms.consumerLock.Lock()
-				// TODO:: collection should has separate channels; otherwise will consume redundant msg
+	defer ms.consumerLock.Unlock()
-				messageID = pulsar.EarliestMessageID()
+	consumer, hasWatched = ms.consumers[seekChannel]
-				break
+
-			}
+	if hasWatched {
-			seekMsgID, err := typeutil.StringToPulsarMsgID(mp.MsgID)
+		return errors.New("the channel should has been subscribed")
-			if err != nil {
-				return err
-			}
-			messageID = seekMsgID
-			break
-		}
 	}
 
-	if consumer != nil {
+	receiveChannel := make(chan pulsar.ConsumerMessage, ms.pulsarBufSize)
-		err := (consumer).Seek(messageID)
+	consumer, err = ms.client.Subscribe(pulsar.ConsumerOptions{
-		if err != nil {
+		Topic:                       seekChannel,
-			return err
+		SubscriptionName:            subName,
-		}
+		Type:                        pulsar.KeyShared,
-		if messageID == nil {
+		SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
+		MessageChannel:              receiveChannel,
+	})
+	if err != nil {
+		return err
+	}
+	if consumer == nil {
+		return errors.New("pulsar is not ready, consumer is nil")
+	}
+	seekMsgID, err := typeutil.StringToPulsarMsgID(mp.MsgID)
+	if err != nil {
+		return err
+	}
+	consumer.Seek(seekMsgID)
+	ms.addConsumer(consumer, seekChannel)
+
+	if len(consumer.Chan()) == 0 {
+		return nil
+	}
+	for {
+		select {
+		case <-ms.ctx.Done():
 			return nil
-		}
+		case pulsarMsg, ok := <-consumer.Chan():
+			if !ok {
+				return errors.New("consumer closed")
+			}
+			consumer.Ack(pulsarMsg)
 
-		ms.unsolvedMutex.Lock()
+			headerMsg := commonpb.MsgHeader{}
-		ms.unsolvedBuf[consumer] = make([]TsMsg, 0)
+			err := proto.Unmarshal(pulsarMsg.Payload(), &headerMsg)
-		for {
+			if err != nil {
-			select {
+				log.Error("Failed to unmarshal message header", zap.Error(err))
-			case <-ms.ctx.Done():
+			}
-				return nil
+			tsMsg, err := ms.unmarshal.Unmarshal(pulsarMsg.Payload(), headerMsg.Base.MsgType)
-			case pulsarMsg, ok := <-consumer.Chan():
+			if err != nil {
-				if !ok {
+				log.Error("Failed to unmarshal tsMsg", zap.Error(err))
-					return errors.New("consumer closed")
+			}
-				}
+			if tsMsg.Type() == commonpb.MsgType_TimeTick {
-				consumer.Ack(pulsarMsg)
+				if tsMsg.BeginTs() >= mp.Timestamp {
-
+					return nil
-				headerMsg := commonpb.MsgHeader{}
-				err := proto.Unmarshal(pulsarMsg.Payload(), &headerMsg)
-				if err != nil {
-					log.Error("Failed to unmarshal message header", zap.Error(err))
-				}
-				tsMsg, err := ms.unmarshal.Unmarshal(pulsarMsg.Payload(), headerMsg.Base.MsgType)
-				if err != nil {
-					log.Error("Failed to unmarshal tsMsg", zap.Error(err))
-				}
-				if tsMsg.Type() == commonpb.MsgType_TimeTick {
-					if tsMsg.BeginTs() >= mp.Timestamp {
-						ms.unsolvedMutex.Unlock()
-						return nil
-					}
-					continue
-				}
-				if tsMsg.BeginTs() > mp.Timestamp {
-					tsMsg.SetPosition(&msgstream.MsgPosition{
-						ChannelName: filepath.Base(pulsarMsg.Topic()),
-						MsgID:       typeutil.PulsarMsgIDToString(pulsarMsg.ID()),
-					})
-					ms.unsolvedBuf[consumer] = append(ms.unsolvedBuf[consumer], tsMsg)
 				}
+				continue
+			}
+			if tsMsg.BeginTs() > mp.Timestamp {
+				tsMsg.SetPosition(&msgstream.MsgPosition{
+					ChannelName: filepath.Base(pulsarMsg.Topic()),
+					MsgID:       typeutil.PulsarMsgIDToString(pulsarMsg.ID()),
+				})
+				ms.unsolvedBuf[consumer] = append(ms.unsolvedBuf[consumer], tsMsg)
 			}
 		}
 	}
-
-	return errors.New("msgStream seek fail")
 }
 
 func checkTimeTickMsg(msg map[Consumer]Timestamp,
@@ -839,10 +870,8 @@ func checkTimeTickMsg(msg map[Consumer]Timestamp,
 type InMemMsgStream struct {
 	buffer chan *MsgPack
 }
-
 func (ms *InMemMsgStream) Start() {}
 func (ms *InMemMsgStream) Close() {}
-
 func (ms *InMemMsgStream) ProduceOne(msg TsMsg) error {
 	msgPack := MsgPack{}
 	msgPack.BeginTs = msg.BeginTs()
@@ -851,23 +880,19 @@ func (ms *InMemMsgStream) ProduceOne(msg TsMsg) error {
 	buffer <- &msgPack
 	return nil
 }
-
 func (ms *InMemMsgStream) Produce(msgPack *MsgPack) error {
 	buffer <- msgPack
 	return nil
 }
-
 func (ms *InMemMsgStream) Broadcast(msgPack *MsgPack) error {
 	return ms.Produce(msgPack)
 }
-
 func (ms *InMemMsgStream) Consume() *MsgPack {
 	select {
 	case msgPack := <-ms.buffer:
 		return msgPack
 	}
 }
-
 func (ms *InMemMsgStream) Chan() <- chan *MsgPack {
 	return buffer
 }

internal/msgstream/rmqms/rmq_msgstream.go

@@ -9,14 +9,15 @@ import (
 	"sync"
 	"time"
 
+	"go.uber.org/zap"
+
 	"github.com/gogo/protobuf/proto"
 	"github.com/zilliztech/milvus-distributed/internal/log"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream/util"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
 	client "github.com/zilliztech/milvus-distributed/internal/util/rocksmq/client/rocksmq"
-
-	"go.uber.org/zap"
 )
 
 type TsMsg = msgstream.TsMsg
@@ -35,8 +36,9 @@ type Consumer = client.Consumer
 type RmqMsgStream struct {
 	ctx              context.Context
 	client           client.Client
-	producers        []Producer
+	producers        map[string]Producer
-	consumers        []Consumer
+	producerChannels []string
+	consumers        map[string]Consumer
 	consumerChannels []string
 	unmarshal        msgstream.UnmarshalDispatcher
 	repackFunc       msgstream.RepackFunc
@@ -45,6 +47,7 @@ type RmqMsgStream struct {
 	wait             *sync.WaitGroup
 	streamCancel     func()
 	rmqBufSize       int64
+	producerLock     *sync.Mutex
 	consumerLock     *sync.Mutex
 	consumerReflects []reflect.SelectCase
 
@@ -55,10 +58,11 @@ func newRmqMsgStream(ctx context.Context, receiveBufSize int64, rmqBufSize int64
 	unmarshal msgstream.UnmarshalDispatcher) (*RmqMsgStream, error) {
 
 	streamCtx, streamCancel := context.WithCancel(ctx)
-	producers := make([]Producer, 0)
+	producers := make(map[string]Producer)
-	consumers := make([]Consumer, 0)
+	producerChannels := make([]string, 0)
-	consumerChannels := make([]string, 0)
 	consumerReflects := make([]reflect.SelectCase, 0)
+	consumers := make(map[string]Consumer)
+	consumerChannels := make([]string, 0)
 	receiveBuf := make(chan *MsgPack, receiveBufSize)
 
 	var clientOpts client.ClientOptions
@@ -73,12 +77,14 @@ func newRmqMsgStream(ctx context.Context, receiveBufSize int64, rmqBufSize int64
 		ctx:              streamCtx,
 		client:           client,
 		producers:        producers,
+		producerChannels: producerChannels,
 		consumers:        consumers,
 		consumerChannels: consumerChannels,
 		unmarshal:        unmarshal,
 		receiveBuf:       receiveBuf,
 		streamCancel:     streamCancel,
 		consumerReflects: consumerReflects,
+		producerLock:     &sync.Mutex{},
 		consumerLock:     &sync.Mutex{},
 		wait:             &sync.WaitGroup{},
 		scMap:            &sync.Map{},
@@ -92,6 +98,8 @@ func (rms *RmqMsgStream) Start() {
 
 func (rms *RmqMsgStream) Close() {
 	rms.streamCancel()
+	rms.wait.Wait()
+
 	if rms.client != nil {
 		rms.client.Close()
 	}
@@ -105,7 +113,10 @@ func (rms *RmqMsgStream) AsProducer(channels []string) {
 	for _, channel := range channels {
 		pp, err := rms.client.CreateProducer(client.ProducerOptions{Topic: channel})
 		if err == nil {
-			rms.producers = append(rms.producers, pp)
+			rms.producerLock.Lock()
+			rms.producers[channel] = pp
+			rms.producerChannels = append(rms.producerChannels, channel)
+			rms.producerLock.Unlock()
 		} else {
 			errMsg := "Failed to create producer " + channel + ", error = " + err.Error()
 			panic(errMsg)
@@ -114,11 +125,14 @@ func (rms *RmqMsgStream) AsProducer(channels []string) {
 }
 
 func (rms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
-	for i := 0; i < len(channels); i++ {
+	for _, channel := range channels {
+		if _, ok := rms.consumers[channel]; ok {
+			continue
+		}
 		fn := func() error {
 			receiveChannel := make(chan client.ConsumerMessage, rms.rmqBufSize)
 			pc, err := rms.client.Subscribe(client.ConsumerOptions{
-				Topic:            channels[i],
+				Topic:            channel,
 				SubscriptionName: groupName,
 				MessageChannel:   receiveChannel,
 			})
@@ -129,8 +143,8 @@ func (rms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
 				return errors.New("RocksMQ is not ready, consumer is nil")
 			}
 
-			rms.consumers = append(rms.consumers, pc)
+			rms.consumers[channel] = pc
-			rms.consumerChannels = append(rms.consumerChannels, channels[i])
+			rms.consumerChannels = append(rms.consumerChannels, channel)
 			rms.consumerReflects = append(rms.consumerReflects, reflect.SelectCase{
 				Dir:  reflect.SelectRecv,
 				Chan: reflect.ValueOf(pc.Chan()),
@@ -141,7 +155,7 @@ func (rms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
 		}
 		err := util.Retry(20, time.Millisecond*200, fn)
 		if err != nil {
-			errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
+			errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
 			panic(errMsg)
 		}
 	}
@@ -194,6 +208,7 @@ func (rms *RmqMsgStream) Produce(ctx context.Context, pack *msgstream.MsgPack) e
 		return err
 	}
 	for k, v := range result {
+		channel := rms.producerChannels[k]
 		for i := 0; i < len(v.Msgs); i++ {
 			mb, err := v.Msgs[i].Marshal(v.Msgs[i])
 			if err != nil {
@@ -205,7 +220,7 @@ func (rms *RmqMsgStream) Produce(ctx context.Context, pack *msgstream.MsgPack) e
 				return err
 			}
 			msg := &client.ProducerMessage{Payload: m}
-			if err := rms.producers[k].Send(msg); err != nil {
+			if err := rms.producers[channel].Send(msg); err != nil {
 				return err
 			}
 		}
@@ -214,7 +229,6 @@ func (rms *RmqMsgStream) Produce(ctx context.Context, pack *msgstream.MsgPack) e
 }
 
 func (rms *RmqMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) error {
-	producerLen := len(rms.producers)
 	for _, v := range msgPack.Msgs {
 		mb, err := v.Marshal(v)
 		if err != nil {
@@ -228,13 +242,15 @@ func (rms *RmqMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) error
 
 		msg := &client.ProducerMessage{Payload: m}
 
-		for i := 0; i < producerLen; i++ {
+		rms.producerLock.Lock()
-			if err := rms.producers[i].Send(
+		for _, producer := range rms.producers {
+			if err := producer.Send(
 				msg,
 			); err != nil {
 				return err
 			}
 		}
+		rms.producerLock.Unlock()
 	}
 	return nil
 }
@@ -249,7 +265,7 @@ func (rms *RmqMsgStream) Consume() (*msgstream.MsgPack, context.Context) {
 			}
 			return cm, nil
 		case <-rms.ctx.Done():
-			log.Debug("context closed")
+			//log.Debug("context closed")
 			return nil, nil
 		}
 	}
@@ -298,19 +314,17 @@ func (rms *RmqMsgStream) Chan() <-chan *msgstream.MsgPack {
 }
 
 func (rms *RmqMsgStream) Seek(mp *msgstream.MsgPosition) error {
-	for index, channel := range rms.consumerChannels {
+	if _, ok := rms.consumers[mp.ChannelName]; ok {
-		if channel == mp.ChannelName {
+		consumer := rms.consumers[mp.ChannelName]
-			msgID, err := strconv.ParseInt(mp.MsgID, 10, 64)
+		msgID, err := strconv.ParseInt(mp.MsgID, 10, 64)
-			if err != nil {
+		if err != nil {
-				return err
+			return err
-			}
-			messageID := UniqueID(msgID)
-			err = rms.consumers[index].Seek(messageID)
-			if err != nil {
-				return err
-			}
-			return nil
 		}
+		err = consumer.Seek(msgID)
+		if err != nil {
+			return err
+		}
+		return nil
 	}
 
 	return errors.New("msgStream seek fail")
@@ -319,6 +333,7 @@ func (rms *RmqMsgStream) Seek(mp *msgstream.MsgPosition) error {
 type RmqTtMsgStream struct {
 	RmqMsgStream
 	unsolvedBuf   map[Consumer][]TsMsg
+	msgPositions  map[Consumer]*internalpb.MsgPosition
 	unsolvedMutex *sync.Mutex
 	lastTimeStamp Timestamp
 	syncConsumer  chan int
@@ -330,24 +345,44 @@ func newRmqTtMsgStream(ctx context.Context, receiveBufSize int64, rmqBufSize int
 	if err != nil {
 		return nil, err
 	}
+
 	unsolvedBuf := make(map[Consumer][]TsMsg)
 	syncConsumer := make(chan int, 1)
+	msgPositions := make(map[Consumer]*internalpb.MsgPosition)
 
 	return &RmqTtMsgStream{
 		RmqMsgStream:  *rmqMsgStream,
 		unsolvedBuf:   unsolvedBuf,
+		msgPositions:  msgPositions,
 		unsolvedMutex: &sync.Mutex{},
 		syncConsumer:  syncConsumer,
 	}, nil
 }
 
+func (rtms *RmqTtMsgStream) addConsumer(consumer Consumer, channel string) {
+	if len(rtms.consumers) == 0 {
+		rtms.syncConsumer <- 1
+	}
+	rtms.consumers[channel] = consumer
+	rtms.unsolvedBuf[consumer] = make([]TsMsg, 0)
+	rtms.msgPositions[consumer] = &internalpb.MsgPosition{
+		ChannelName: channel,
+		MsgID:       "",
+		Timestamp:   rtms.lastTimeStamp,
+	}
+	rtms.consumerChannels = append(rtms.consumerChannels, channel)
+}
+
 func (rtms *RmqTtMsgStream) AsConsumer(channels []string,
 	groupName string) {
-	for i := 0; i < len(channels); i++ {
+	for _, channel := range channels {
+		if _, ok := rtms.consumers[channel]; ok {
+			continue
+		}
 		fn := func() error {
 			receiveChannel := make(chan client.ConsumerMessage, rtms.rmqBufSize)
 			pc, err := rtms.client.Subscribe(client.ConsumerOptions{
-				Topic:            channels[i],
+				Topic:            channel,
 				SubscriptionName: groupName,
 				MessageChannel:   receiveChannel,
 			})
@@ -355,22 +390,17 @@ func (rtms *RmqTtMsgStream) AsConsumer(channels []string,
 				return err
 			}
 			if pc == nil {
-				return errors.New("pulsar is not ready, consumer is nil")
+				return errors.New("RocksMQ is not ready, consumer is nil")
 			}
 
 			rtms.consumerLock.Lock()
-			if len(rtms.consumers) == 0 {
+			rtms.addConsumer(pc, channel)
-				rtms.syncConsumer <- 1
-			}
-			rtms.consumers = append(rtms.consumers, pc)
-			rtms.unsolvedBuf[pc] = make([]TsMsg, 0)
-			rtms.consumerChannels = append(rtms.consumerChannels, channels[i])
 			rtms.consumerLock.Unlock()
 			return nil
 		}
 		err := util.Retry(10, time.Millisecond*200, fn)
 		if err != nil {
-			errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
+			errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
 			panic(errMsg)
 		}
 	}
@@ -427,7 +457,8 @@ func (rtms *RmqTtMsgStream) bufMsgPackToChannel() {
 				continue
 			}
 			timeTickBuf := make([]TsMsg, 0)
-			msgPositions := make([]*msgstream.MsgPosition, 0)
+			startMsgPosition := make([]*internalpb.MsgPosition, 0)
+			endMsgPositions := make([]*internalpb.MsgPosition, 0)
 			rtms.unsolvedMutex.Lock()
 			for consumer, msgs := range rtms.unsolvedBuf {
 				if len(msgs) == 0 {
@@ -448,19 +479,24 @@ func (rtms *RmqTtMsgStream) bufMsgPackToChannel() {
 				}
 				rtms.unsolvedBuf[consumer] = tempBuffer
 
+				startMsgPosition = append(startMsgPosition, rtms.msgPositions[consumer])
+				var newPos *internalpb.MsgPosition
 				if len(tempBuffer) > 0 {
-					msgPositions = append(msgPositions, &msgstream.MsgPosition{
+					newPos = &internalpb.MsgPosition{
 						ChannelName: tempBuffer[0].Position().ChannelName,
 						MsgID:       tempBuffer[0].Position().MsgID,
 						Timestamp:   timeStamp,
-					})
+					}
+					endMsgPositions = append(endMsgPositions, newPos)
 				} else {
-					msgPositions = append(msgPositions, &msgstream.MsgPosition{
+					newPos = &internalpb.MsgPosition{
 						ChannelName: timeTickMsg.Position().ChannelName,
 						MsgID:       timeTickMsg.Position().MsgID,
 						Timestamp:   timeStamp,
-					})
+					}
+					endMsgPositions = append(endMsgPositions, newPos)
 				}
+				rtms.msgPositions[consumer] = newPos
 			}
 			rtms.unsolvedMutex.Unlock()
 
@@ -468,7 +504,8 @@ func (rtms *RmqTtMsgStream) bufMsgPackToChannel() {
 				BeginTs:        rtms.lastTimeStamp,
 				EndTs:          timeStamp,
 				Msgs:           timeTickBuf,
-				StartPositions: msgPositions,
+				StartPositions: startMsgPosition,
+				EndPositions:   endMsgPositions,
 			}
 
 			rtms.receiveBuf <- &msgPack
@@ -524,73 +561,78 @@ func (rtms *RmqTtMsgStream) findTimeTick(consumer Consumer,
 }
 
 func (rtms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
+	if len(mp.MsgID) == 0 {
+		return errors.New("when msgID's length equal to 0, please use AsConsumer interface")
+	}
 	var consumer Consumer
-	var messageID UniqueID
+	var err error
-	for index, channel := range rtms.consumerChannels {
+	var hasWatched bool
-		if filepath.Base(channel) == filepath.Base(mp.ChannelName) {
+	seekChannel := mp.ChannelName
-			consumer = rtms.consumers[index]
+	subName := mp.MsgGroup
-			if len(mp.MsgID) == 0 {
+	rtms.consumerLock.Lock()
-				messageID = -1
+	defer rtms.consumerLock.Unlock()
-				break
+	consumer, hasWatched = rtms.consumers[seekChannel]
-			}
+
-			seekMsgID, err := strconv.ParseInt(mp.MsgID, 10, 64)
+	if hasWatched {
-			if err != nil {
+		return errors.New("the channel should has been subscribed")
-				return err
-			}
-			messageID = seekMsgID
-			break
-		}
 	}
 
-	if consumer != nil {
+	receiveChannel := make(chan client.ConsumerMessage, rtms.rmqBufSize)
-		err := (consumer).Seek(messageID)
+	consumer, err = rtms.client.Subscribe(client.ConsumerOptions{
-		if err != nil {
+		Topic:            seekChannel,
-			return err
+		SubscriptionName: subName,
-		}
+		MessageChannel:   receiveChannel,
-		//TODO: Is this right?
+	})
-		if messageID == 0 {
+	if err != nil {
+		return err
+	}
+	if consumer == nil {
+		return errors.New("RocksMQ is not ready, consumer is nil")
+	}
+	seekMsgID, err := strconv.ParseInt(mp.MsgID, 10, 64)
+	if err != nil {
+		return err
+	}
+	consumer.Seek(seekMsgID)
+	rtms.addConsumer(consumer, seekChannel)
+
+	if len(consumer.Chan()) == 0 {
+		return nil
+	}
+
+	for {
+		select {
+		case <-rtms.ctx.Done():
 			return nil
-		}
+		case rmqMsg, ok := <-consumer.Chan():
+			if !ok {
+				return errors.New("consumer closed")
+			}
 
-		rtms.unsolvedMutex.Lock()
+			headerMsg := commonpb.MsgHeader{}
-		rtms.unsolvedBuf[consumer] = make([]TsMsg, 0)
+			err := proto.Unmarshal(rmqMsg.Payload, &headerMsg)
-		for {
+			if err != nil {
-			select {
+				log.Error("Failed to unmarshal message header", zap.Error(err))
-			case <-rtms.ctx.Done():
+			}
-				return nil
+			tsMsg, err := rtms.unmarshal.Unmarshal(rmqMsg.Payload, headerMsg.Base.MsgType)
-			case rmqMsg, ok := <-consumer.Chan():
+			if err != nil {
-				if !ok {
+				log.Error("Failed to unmarshal tsMsg", zap.Error(err))
-					return errors.New("consumer closed")
+			}
-				}
+			if tsMsg.Type() == commonpb.MsgType_TimeTick {
-
+				if tsMsg.BeginTs() >= mp.Timestamp {
-				headerMsg := commonpb.MsgHeader{}
+					return nil
-				err := proto.Unmarshal(rmqMsg.Payload, &headerMsg)
-				if err != nil {
-					log.Error("Failed to unmarshal message header", zap.Error(err))
-				}
-				tsMsg, err := rtms.unmarshal.Unmarshal(rmqMsg.Payload, headerMsg.Base.MsgType)
-				if err != nil {
-					log.Error("Failed to unmarshal tsMsg", zap.Error(err))
-				}
-				if tsMsg.Type() == commonpb.MsgType_TimeTick {
-					if tsMsg.BeginTs() >= mp.Timestamp {
-						rtms.unsolvedMutex.Unlock()
-						return nil
-					}
-					continue
-				}
-				if tsMsg.BeginTs() > mp.Timestamp {
-					tsMsg.SetPosition(&msgstream.MsgPosition{
-						ChannelName: filepath.Base(consumer.Topic()),
-						MsgID:       strconv.Itoa(int(rmqMsg.MsgID)),
-					})
-					rtms.unsolvedBuf[consumer] = append(rtms.unsolvedBuf[consumer], tsMsg)
 				}
+				continue
+			}
+			if tsMsg.BeginTs() > mp.Timestamp {
+				tsMsg.SetPosition(&msgstream.MsgPosition{
+					ChannelName: filepath.Base(consumer.Topic()),
+					MsgID:       strconv.Itoa(int(rmqMsg.MsgID)),
+				})
+				rtms.unsolvedBuf[consumer] = append(rtms.unsolvedBuf[consumer], tsMsg)
 			}
 		}
 	}
-
-	return errors.New("msgStream seek fail")
 }
 
 func checkTimeTickMsg(msg map[Consumer]Timestamp,

internal/proto/internal.proto

@@ -209,5 +209,6 @@ message QueryNodeStats {
 message MsgPosition {
   string channel_name = 1;
   string msgID = 2;
-  uint64 timestamp = 3;
+  string msgGroup = 3;
+  uint64 timestamp = 4;
 }

internal/proto/internalpb/internal.pb.go

@@ -1802,7 +1802,8 @@ func (m *QueryNodeStats) GetFieldStats() []*FieldStats {
 type MsgPosition struct {
 	ChannelName          string   `protobuf:"bytes,1,opt,name=channel_name,json=channelName,proto3" json:"channel_name,omitempty"`
 	MsgID                string   `protobuf:"bytes,2,opt,name=msgID,proto3" json:"msgID,omitempty"`
-	Timestamp            uint64   `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	MsgGroup             string   `protobuf:"bytes,3,opt,name=msgGroup,proto3" json:"msgGroup,omitempty"`
+	Timestamp            uint64   `protobuf:"varint,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -1847,6 +1848,13 @@ func (m *MsgPosition) GetMsgID() string {
 	return ""
 }
 
+func (m *MsgPosition) GetMsgGroup() string {
+	if m != nil {
+		return m.MsgGroup
+	}
+	return ""
+}
+
 func (m *MsgPosition) GetTimestamp() uint64 {
 	if m != nil {
 		return m.Timestamp
@@ -1890,101 +1898,102 @@ func init() {
 func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) }
 
 var fileDescriptor_41f4a519b878ee3b = []byte{
-	// 1524 bytes of a gzipped FileDescriptorProto
+	// 1539 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x6f, 0x1b, 0x45,
-	0x14, 0x67, 0x6d, 0x27, 0xb6, 0xdf, 0x3a, 0xa9, 0xbb, 0xfd, 0xda, 0xd0, 0x94, 0xba, 0xcb, 0x57,
+	0x1b, 0x7f, 0xd7, 0x76, 0x62, 0xfb, 0x59, 0x27, 0x75, 0xb7, 0x5f, 0x9b, 0x36, 0x7d, 0xeb, 0xee,
-	0xa0, 0x22, 0xa9, 0x52, 0x84, 0x10, 0x97, 0xb6, 0x89, 0x69, 0xb0, 0xda, 0x44, 0x61, 0x9d, 0x56,
+	0xfb, 0x02, 0x81, 0x8a, 0xa4, 0x4a, 0x11, 0x42, 0x5c, 0xda, 0x26, 0xa6, 0xc1, 0x6a, 0x13, 0x85,
-	0x82, 0xcb, 0x6a, 0xbc, 0x3b, 0xb1, 0xa7, 0xdd, 0x0f, 0x77, 0x66, 0xb6, 0xa9, 0x73, 0xe6, 0x86,
+	0x75, 0x5a, 0x09, 0x2e, 0xab, 0xf1, 0xee, 0xc4, 0x9e, 0x76, 0x3f, 0xdc, 0x99, 0xd9, 0xa6, 0xce,
-	0xe0, 0x80, 0xc4, 0x3f, 0xc0, 0x1f, 0xc0, 0x99, 0x13, 0x20, 0x4e, 0x48, 0xdc, 0x91, 0x90, 0xf8,
+	0x89, 0x03, 0x37, 0x04, 0x07, 0x24, 0xfe, 0x01, 0xfe, 0x00, 0xce, 0x9c, 0x00, 0x71, 0x42, 0xe2,
-	0x4b, 0x38, 0xa1, 0xf9, 0xd8, 0xf5, 0x47, 0x9d, 0x34, 0x35, 0x54, 0x08, 0xc1, 0xcd, 0xf3, 0x9b,
+	0x8e, 0x84, 0xc4, 0x5f, 0xc2, 0x09, 0xcd, 0xc7, 0xae, 0x3f, 0xea, 0xa4, 0xa9, 0xa1, 0x42, 0x08,
-	0xb7, 0x6f, 0xe6, 0xf7, 0x7b, 0xef, 0xcd, 0x9b, 0x31, 0x2c, 0x92, 0x98, 0x63, 0x1a, 0xa3, 0x70,
+	0x6e, 0x9e, 0xdf, 0xf3, 0xec, 0x33, 0xf3, 0xfc, 0x9e, 0xaf, 0x19, 0xc3, 0x22, 0x89, 0x39, 0xa6,
-	0xb5, 0x4f, 0x13, 0x9e, 0x58, 0xe7, 0x22, 0x12, 0x3e, 0x4e, 0x99, 0x1a, 0xad, 0x66, 0x93, 0x2f,
+	0x31, 0x0a, 0x57, 0xfb, 0x34, 0xe1, 0x89, 0x75, 0x2e, 0x22, 0xe1, 0x93, 0x94, 0xa9, 0xd5, 0x6a,
-	0xd7, 0xfc, 0x24, 0x8a, 0x92, 0x58, 0xc1, 0xce, 0xf7, 0x06, 0x2c, 0x6c, 0x26, 0x51, 0x3f, 0x89,
+	0x26, 0xbc, 0x58, 0xf3, 0x93, 0x28, 0x4a, 0x62, 0x05, 0x3b, 0xdf, 0x1a, 0xb0, 0xb0, 0x99, 0x44,
-	0x71, 0xcc, 0x5b, 0xf1, 0x7e, 0x62, 0x9d, 0x87, 0xf9, 0x38, 0x09, 0x70, 0xab, 0x69, 0x1b, 0x0d,
+	0xfd, 0x24, 0xc6, 0x31, 0x6f, 0xc5, 0xfb, 0x89, 0x75, 0x1e, 0xe6, 0xe3, 0x24, 0xc0, 0xad, 0xa6,
-	0x63, 0xa5, 0xe8, 0xea, 0x91, 0x65, 0x41, 0x89, 0x26, 0x21, 0xb6, 0x0b, 0x0d, 0x63, 0xa5, 0xea,
+	0x6d, 0x34, 0x8c, 0x95, 0xa2, 0xab, 0x57, 0x96, 0x05, 0x25, 0x9a, 0x84, 0xd8, 0x2e, 0x34, 0x8c,
-	0xca, 0xdf, 0xd6, 0x0d, 0x00, 0xc6, 0x11, 0xc7, 0x9e, 0x9f, 0x04, 0xd8, 0x2e, 0x36, 0x8c, 0x95,
+	0x95, 0xaa, 0x2b, 0x7f, 0x5b, 0x37, 0x01, 0x18, 0x47, 0x1c, 0x7b, 0x7e, 0x12, 0x60, 0xbb, 0xd8,
-	0xc5, 0xf5, 0xc6, 0xea, 0xd4, 0x75, 0x57, 0xdb, 0xc2, 0x70, 0x33, 0x09, 0xb0, 0x5b, 0x65, 0xd9,
+	0x30, 0x56, 0x16, 0xd7, 0x1b, 0xab, 0x53, 0xf7, 0x5d, 0x6d, 0x0b, 0xc5, 0xcd, 0x24, 0xc0, 0x6e,
-	0x4f, 0xeb, 0x26, 0x00, 0x7e, 0xc2, 0x29, 0xf2, 0x48, 0xbc, 0x9f, 0xd8, 0xa5, 0x46, 0x71, 0xc5,
+	0x95, 0x65, 0x3f, 0xad, 0x5b, 0x00, 0xf8, 0x29, 0xa7, 0xc8, 0x23, 0xf1, 0x7e, 0x62, 0x97, 0x1a,
-	0x5c, 0xbf, 0x32, 0xee, 0x40, 0x6f, 0xf7, 0x0e, 0x1e, 0xdc, 0x47, 0x61, 0x8a, 0x77, 0x11, 0xa1,
+	0xc5, 0x15, 0x73, 0xfd, 0xea, 0xb8, 0x01, 0x7d, 0xdc, 0xbb, 0x78, 0xf0, 0x00, 0x85, 0x29, 0xde,
-	0x6e, 0x55, 0x7e, 0x24, 0xb6, 0xeb, 0xfc, 0x66, 0xc0, 0xa9, 0x9c, 0x80, 0x5c, 0x83, 0x59, 0x1f,
+	0x45, 0x84, 0xba, 0x55, 0xf9, 0x91, 0x38, 0xae, 0xf3, 0x8b, 0x01, 0xa7, 0x72, 0x07, 0xe4, 0x1e,
-	0xc0, 0x9c, 0x5c, 0x42, 0x32, 0x30, 0xd7, 0x5f, 0x3b, 0x62, 0x47, 0x63, 0xbc, 0x5d, 0xf5, 0x89,
+	0xcc, 0x7a, 0x17, 0xe6, 0xe4, 0x16, 0xd2, 0x03, 0x73, 0xfd, 0xff, 0x47, 0x9c, 0x68, 0xcc, 0x6f,
-	0x75, 0x0f, 0xce, 0xb0, 0xb4, 0xe3, 0x67, 0x53, 0x9e, 0x44, 0x99, 0x5d, 0x90, 0x5b, 0x3b, 0x99,
+	0x57, 0x7d, 0x62, 0xdd, 0x87, 0x33, 0x2c, 0xed, 0xf8, 0x99, 0xc8, 0x93, 0x28, 0xb3, 0x0b, 0xf2,
-	0x27, 0x6b, 0xd4, 0x81, 0xde, 0xd2, 0x75, 0x98, 0x17, 0x9e, 0x52, 0x26, 0x55, 0x32, 0xd7, 0x2f,
+	0x68, 0x27, 0xb3, 0x64, 0x8d, 0x1a, 0xd0, 0x47, 0xba, 0x01, 0xf3, 0xc2, 0x52, 0xca, 0x24, 0x4b,
-	0x4e, 0x25, 0xd9, 0x96, 0x26, 0xae, 0x36, 0x75, 0x2e, 0xc2, 0xd2, 0x16, 0xe6, 0x13, 0xec, 0x5c,
+	0xe6, 0xfa, 0xa5, 0xa9, 0x4e, 0xb6, 0xa5, 0x8a, 0xab, 0x55, 0x9d, 0x4b, 0xb0, 0xb4, 0x85, 0xf9,
-	0xfc, 0x28, 0xc5, 0x8c, 0xeb, 0xc9, 0x3d, 0x12, 0xe1, 0x3d, 0xe2, 0x3f, 0xdc, 0xec, 0xa1, 0x38,
+	0x84, 0x77, 0x2e, 0x7e, 0x9c, 0x62, 0xc6, 0xb5, 0x70, 0x8f, 0x44, 0x78, 0x8f, 0xf8, 0x8f, 0x36,
-	0xc6, 0x61, 0x36, 0x79, 0x09, 0x2e, 0x6e, 0x61, 0xf9, 0x01, 0x61, 0x9c, 0xf8, 0x6c, 0x62, 0xfa,
+	0x7b, 0x28, 0x8e, 0x71, 0x98, 0x09, 0x2f, 0xc3, 0xa5, 0x2d, 0x2c, 0x3f, 0x20, 0x8c, 0x13, 0x9f,
-	0x1c, 0x9c, 0xd9, 0xc2, 0xbc, 0x19, 0x4c, 0xc0, 0xf7, 0xa1, 0xb2, 0x23, 0x82, 0x2d, 0xd2, 0xe0,
+	0x4d, 0x88, 0xcf, 0xc1, 0x99, 0x2d, 0xcc, 0x9b, 0xc1, 0x04, 0xfc, 0x00, 0x2a, 0x3b, 0x22, 0xd8,
-	0x3d, 0x28, 0xa3, 0x20, 0xa0, 0x98, 0x31, 0xad, 0xe2, 0xf2, 0xd4, 0x1d, 0xdf, 0x52, 0x36, 0x6e,
+	0x22, 0x0d, 0xde, 0x86, 0x32, 0x0a, 0x02, 0x8a, 0x19, 0xd3, 0x2c, 0x2e, 0x4f, 0x3d, 0xf1, 0x6d,
-	0x66, 0x3c, 0x2d, 0x4d, 0x9c, 0x07, 0x00, 0xad, 0x98, 0xf0, 0x5d, 0x44, 0x51, 0xc4, 0x8e, 0x4c,
+	0xa5, 0xe3, 0x66, 0xca, 0xd3, 0xd2, 0xc4, 0x79, 0x08, 0xd0, 0x8a, 0x09, 0xdf, 0x45, 0x14, 0x45,
-	0xb0, 0x26, 0xd4, 0x18, 0x47, 0x94, 0x7b, 0x7d, 0x69, 0xa7, 0x25, 0x3f, 0x41, 0x36, 0x98, 0xf2,
+	0xec, 0xc8, 0x04, 0x6b, 0x42, 0x8d, 0x71, 0x44, 0xb9, 0xd7, 0x97, 0x7a, 0x9a, 0xf2, 0x13, 0x64,
-	0x33, 0xe5, 0xdd, 0xf9, 0x04, 0xa0, 0xcd, 0x29, 0x89, 0xbb, 0x77, 0x09, 0xe3, 0x62, 0xad, 0xc7,
+	0x83, 0x29, 0x3f, 0x53, 0xd6, 0x9d, 0x0f, 0x01, 0xda, 0x9c, 0x92, 0xb8, 0x7b, 0x8f, 0x30, 0x2e,
-	0xc2, 0x4e, 0x90, 0x28, 0xae, 0x54, 0x5d, 0x3d, 0x1a, 0x09, 0x47, 0xe1, 0xe4, 0xe1, 0xb8, 0x01,
+	0xf6, 0x7a, 0x22, 0xf4, 0x84, 0x13, 0xc5, 0x95, 0xaa, 0xab, 0x57, 0x23, 0xe1, 0x28, 0x9c, 0x3c,
-	0x66, 0x26, 0xf7, 0x36, 0xeb, 0x5a, 0xd7, 0xa0, 0xd4, 0x41, 0x0c, 0x1f, 0x2b, 0xcf, 0x36, 0xeb,
+	0x1c, 0x37, 0xc1, 0xcc, 0xe8, 0xde, 0x66, 0x5d, 0xeb, 0x3a, 0x94, 0x3a, 0x88, 0xe1, 0x63, 0xe9,
-	0x6e, 0x20, 0x86, 0x5d, 0x69, 0xe9, 0xfc, 0x6e, 0xc0, 0x85, 0x4d, 0x8a, 0x65, 0xf2, 0x87, 0x21,
+	0xd9, 0x66, 0xdd, 0x0d, 0xc4, 0xb0, 0x2b, 0x35, 0x9d, 0x5f, 0x0d, 0xb8, 0xb0, 0x49, 0xb1, 0x4c,
-	0xf6, 0x39, 0x49, 0x62, 0xad, 0xfd, 0xf3, 0x7b, 0xb3, 0x2e, 0x40, 0x39, 0xe8, 0x78, 0x31, 0x8a,
+	0xfe, 0x30, 0xc4, 0x3e, 0x27, 0x49, 0xac, 0xb9, 0x7f, 0x71, 0x6b, 0xd6, 0x05, 0x28, 0x07, 0x1d,
-	0x32, 0xb1, 0xe7, 0x83, 0xce, 0x0e, 0x8a, 0xb0, 0xf5, 0x06, 0x2c, 0xfa, 0xb9, 0x7f, 0x81, 0xc8,
+	0x2f, 0x46, 0x51, 0x46, 0xf6, 0x7c, 0xd0, 0xd9, 0x41, 0x11, 0xb6, 0x5e, 0x85, 0x45, 0x3f, 0xb7,
-	0x9c, 0xab, 0xba, 0x13, 0xa8, 0x08, 0x55, 0xd0, 0x69, 0x35, 0xed, 0x92, 0x0c, 0x83, 0xfc, 0x6d,
+	0x2f, 0x10, 0x99, 0x73, 0x55, 0x77, 0x02, 0x15, 0xa1, 0x0a, 0x3a, 0xad, 0xa6, 0x5d, 0x92, 0x61,
-	0x39, 0x50, 0x1b, 0x5a, 0xb5, 0x9a, 0xf6, 0x9c, 0x9c, 0x1b, 0xc3, 0x84, 0xa8, 0xcc, 0xef, 0xe1,
+	0x90, 0xbf, 0x2d, 0x07, 0x6a, 0x43, 0xad, 0x56, 0xd3, 0x9e, 0x93, 0xb2, 0x31, 0x4c, 0x90, 0xca,
-	0x08, 0xd9, 0xf3, 0x0d, 0x63, 0xa5, 0xe6, 0xea, 0x91, 0xf3, 0x93, 0x01, 0xe7, 0x9a, 0x34, 0xe9,
+	0xfc, 0x1e, 0x8e, 0x90, 0x3d, 0xdf, 0x30, 0x56, 0x6a, 0xae, 0x5e, 0x39, 0x3f, 0x18, 0x70, 0xae,
-	0xff, 0x9b, 0xc9, 0x39, 0x5f, 0x14, 0xe0, 0xbc, 0x8a, 0xd1, 0x2e, 0xa2, 0x9c, 0xbc, 0x20, 0x16,
+	0x49, 0x93, 0xfe, 0xdf, 0xd9, 0x39, 0xe7, 0xb3, 0x02, 0x9c, 0x57, 0x31, 0xda, 0x45, 0x94, 0x93,
-	0x6f, 0xc2, 0xa9, 0xe1, 0xaa, 0xca, 0x60, 0x3a, 0x8d, 0xd7, 0x61, 0xb1, 0x9f, 0xed, 0x43, 0xd9,
+	0x97, 0xe4, 0xc5, 0x6b, 0x70, 0x6a, 0xb8, 0xab, 0x52, 0x98, 0xee, 0xc6, 0x2b, 0xb0, 0xd8, 0xcf,
-	0x95, 0xa4, 0xdd, 0x42, 0x8e, 0x8e, 0xb1, 0x9d, 0x3b, 0x86, 0xed, 0xfc, 0x94, 0x50, 0x36, 0xc0,
+	0xce, 0xa1, 0xf4, 0x4a, 0x52, 0x6f, 0x21, 0x47, 0xc7, 0xbc, 0x9d, 0x3b, 0xc6, 0xdb, 0xf9, 0x29,
-	0xcc, 0x1d, 0xb5, 0x9a, 0x76, 0x59, 0x9a, 0x8c, 0x42, 0xce, 0xe7, 0x05, 0x38, 0x2b, 0x82, 0xfa,
+	0xa1, 0x6c, 0x80, 0x99, 0x1b, 0x6a, 0x35, 0xed, 0xb2, 0x54, 0x19, 0x85, 0x9c, 0x4f, 0x0b, 0x70,
-	0xbf, 0x1a, 0x42, 0x8d, 0x1f, 0x0a, 0x60, 0xa9, 0xec, 0x68, 0xc5, 0x01, 0x7e, 0xf2, 0x4f, 0x6a,
+	0x56, 0x04, 0xf5, 0x5f, 0x36, 0x04, 0x1b, 0xdf, 0x15, 0xc0, 0x52, 0xd9, 0xd1, 0x8a, 0x03, 0xfc,
-	0x71, 0x09, 0x60, 0x9f, 0xe0, 0x30, 0x18, 0xd5, 0xa1, 0x2a, 0x91, 0xbf, 0xa4, 0x81, 0x0d, 0x65,
+	0xf4, 0xaf, 0xe4, 0xe2, 0x32, 0xc0, 0x3e, 0xc1, 0x61, 0x30, 0xca, 0x43, 0x55, 0x22, 0x7f, 0x88,
-	0xe9, 0x24, 0xe7, 0x9f, 0x0d, 0xc5, 0xf9, 0xac, 0x7a, 0xb5, 0x3e, 0x9f, 0x2b, 0x27, 0x3e, 0x9f,
+	0x03, 0x1b, 0xca, 0xd2, 0x48, 0xee, 0x7f, 0xb6, 0x14, 0xfd, 0x59, 0xcd, 0x6a, 0xdd, 0x9f, 0x2b,
-	0xe5, 0x67, 0xfa, 0x7c, 0xfe, 0xb6, 0x08, 0x0b, 0xad, 0x98, 0x61, 0xca, 0xff, 0xcb, 0x89, 0x64,
+	0x27, 0xee, 0xcf, 0xf2, 0x33, 0xdd, 0x9f, 0xbf, 0x2e, 0xc2, 0x42, 0x2b, 0x66, 0x98, 0xf2, 0x7f,
-	0x2d, 0x43, 0x95, 0xe1, 0x6e, 0x24, 0xae, 0x0c, 0x4d, 0xbb, 0x22, 0xe7, 0x87, 0x80, 0x98, 0xf5,
+	0x72, 0x22, 0x59, 0xcb, 0x50, 0x65, 0xb8, 0x1b, 0x89, 0x2b, 0x43, 0xd3, 0xae, 0x48, 0xf9, 0x10,
-	0x55, 0x6b, 0x6e, 0x35, 0xed, 0xaa, 0x0a, 0x6d, 0x0e, 0x58, 0xaf, 0x00, 0x70, 0x12, 0x61, 0xc6,
+	0x10, 0x52, 0x5f, 0x8d, 0xe6, 0x56, 0xd3, 0xae, 0xaa, 0xd0, 0xe6, 0x80, 0xf5, 0x5f, 0x00, 0x4e,
-	0x51, 0xd4, 0x67, 0x36, 0x34, 0x8a, 0x2b, 0x25, 0x77, 0x04, 0x11, 0xe7, 0x33, 0x4d, 0x0e, 0x5a,
+	0x22, 0xcc, 0x38, 0x8a, 0xfa, 0xcc, 0x86, 0x46, 0x71, 0xa5, 0xe4, 0x8e, 0x20, 0xa2, 0x3f, 0xd3,
-	0x4d, 0x66, 0x9b, 0x8d, 0xa2, 0x68, 0xb0, 0x6a, 0x64, 0xbd, 0x0b, 0x15, 0x9a, 0x1c, 0x78, 0x01,
+	0xe4, 0xa0, 0xd5, 0x64, 0xb6, 0xd9, 0x28, 0x8a, 0x01, 0xab, 0x56, 0xd6, 0x5b, 0x50, 0xa1, 0xc9,
-	0xe2, 0xc8, 0xae, 0xc9, 0xe0, 0x2d, 0x4d, 0x15, 0x7b, 0x23, 0x4c, 0x3a, 0x6e, 0x99, 0x26, 0x07,
+	0x81, 0x17, 0x20, 0x8e, 0xec, 0x9a, 0x0c, 0xde, 0xd2, 0x54, 0xb2, 0x37, 0xc2, 0xa4, 0xe3, 0x96,
-	0x4d, 0xc4, 0x91, 0xf3, 0x5d, 0x01, 0x16, 0xda, 0x18, 0x51, 0xbf, 0x37, 0x7b, 0xc0, 0xde, 0x82,
+	0x69, 0x72, 0xd0, 0x44, 0x1c, 0x39, 0xdf, 0x14, 0x60, 0xa1, 0x8d, 0x11, 0xf5, 0x7b, 0xb3, 0x07,
-	0x3a, 0xc5, 0x2c, 0x0d, 0xb9, 0x37, 0xa4, 0xa5, 0x22, 0x77, 0x4a, 0xe1, 0x9b, 0x39, 0xb9, 0x4c,
+	0xec, 0x75, 0xa8, 0x53, 0xcc, 0xd2, 0x90, 0x7b, 0x43, 0xb7, 0x54, 0xe4, 0x4e, 0x29, 0x7c, 0x33,
-	0xf2, 0xe2, 0x31, 0x92, 0x97, 0xa6, 0x48, 0xee, 0x40, 0x6d, 0x44, 0x5f, 0x66, 0xcf, 0x49, 0xea,
+	0x77, 0x2e, 0xa3, 0xbc, 0x78, 0x0c, 0xe5, 0xa5, 0x29, 0x94, 0x3b, 0x50, 0x1b, 0xe1, 0x97, 0xd9,
-	0x63, 0x98, 0x55, 0x87, 0x62, 0xc0, 0x42, 0x19, 0xb1, 0xaa, 0x2b, 0x7e, 0x5a, 0x57, 0xe1, 0x74,
+	0x73, 0xd2, 0xf5, 0x31, 0xcc, 0xaa, 0x43, 0x31, 0x60, 0xa1, 0x8c, 0x58, 0xd5, 0x15, 0x3f, 0xad,
-	0x3f, 0x44, 0x3e, 0xee, 0x25, 0x61, 0x80, 0xa9, 0xd7, 0xa5, 0x49, 0xda, 0x97, 0xe1, 0xaa, 0xb9,
+	0x6b, 0x70, 0xba, 0x1f, 0x22, 0x1f, 0xf7, 0x92, 0x30, 0xc0, 0xd4, 0xeb, 0xd2, 0x24, 0xed, 0xcb,
-	0xf5, 0x91, 0x89, 0x2d, 0x81, 0x5b, 0x6b, 0x30, 0xf7, 0x28, 0xc5, 0x74, 0x20, 0xe3, 0x75, 0xac,
+	0x70, 0xd5, 0xdc, 0xfa, 0x88, 0x60, 0x4b, 0xe0, 0xd6, 0x1a, 0xcc, 0x3d, 0x4e, 0x31, 0x1d, 0xc8,
-	0x78, 0xca, 0xce, 0xf9, 0xd5, 0x18, 0x4a, 0x27, 0x58, 0xb2, 0x19, 0xa4, 0x9b, 0xe5, 0xa6, 0x32,
+	0x78, 0x1d, 0x4b, 0x9e, 0xd2, 0x73, 0x7e, 0x36, 0x86, 0xd4, 0x09, 0x2f, 0xd9, 0x0c, 0xd4, 0xcd,
-	0x55, 0xef, 0xe2, 0x74, 0xbd, 0x2f, 0x83, 0x19, 0x61, 0x4e, 0x89, 0xef, 0xf1, 0x41, 0x3f, 0x2b,
+	0x72, 0x53, 0x99, 0xca, 0x77, 0x71, 0x3a, 0xdf, 0x57, 0xc0, 0x8c, 0x30, 0xa7, 0xc4, 0xf7, 0xf8,
-	0x03, 0x50, 0xd0, 0xde, 0xa0, 0x2f, 0x6b, 0xa0, 0x47, 0xb8, 0x12, 0xb4, 0xe6, 0xca, 0xdf, 0xce,
+	0xa0, 0x9f, 0x95, 0x01, 0x28, 0x68, 0x6f, 0xd0, 0x97, 0x35, 0xd0, 0x23, 0x5c, 0x11, 0x5a, 0x73,
-	0x2f, 0x06, 0x2c, 0x34, 0x71, 0x88, 0x39, 0x9e, 0x3d, 0x27, 0xa6, 0xd4, 0x6a, 0x61, 0x6a, 0xad,
+	0xe5, 0x6f, 0xe7, 0x27, 0x03, 0x16, 0x9a, 0x38, 0xc4, 0x1c, 0xcf, 0x9e, 0x13, 0x53, 0x6a, 0xb5,
-	0x8e, 0x15, 0x43, 0xf1, 0xf8, 0x62, 0x28, 0x3d, 0x55, 0x0c, 0x57, 0xa0, 0xd6, 0xa7, 0x24, 0x42,
+	0x30, 0xb5, 0x56, 0xc7, 0x8a, 0xa1, 0x78, 0x7c, 0x31, 0x94, 0x9e, 0x29, 0x86, 0xab, 0x50, 0xeb,
-	0x74, 0xe0, 0x3d, 0xc4, 0x83, 0x2c, 0x2f, 0x4c, 0x8d, 0xdd, 0xc1, 0x03, 0xe6, 0x7c, 0x63, 0x40,
+	0x53, 0x12, 0x21, 0x3a, 0xf0, 0x1e, 0xe1, 0x41, 0x96, 0x17, 0xa6, 0xc6, 0xee, 0xe2, 0x01, 0x73,
-	0xe5, 0x76, 0x98, 0xb2, 0xde, 0x4c, 0xb7, 0xba, 0xf1, 0x52, 0x2e, 0x4c, 0x96, 0xf2, 0x64, 0xee,
+	0xbe, 0x32, 0xa0, 0x72, 0x27, 0x4c, 0x59, 0x6f, 0xa6, 0x5b, 0xdd, 0x78, 0x29, 0x17, 0x26, 0x4b,
-	0x16, 0x9f, 0x91, 0xbb, 0x7b, 0xa8, 0xab, 0x83, 0x30, 0x86, 0x39, 0x7f, 0x18, 0x50, 0xbd, 0x9b,
+	0x79, 0x32, 0x77, 0x8b, 0xcf, 0xc9, 0xdd, 0x3d, 0xd4, 0xd5, 0x41, 0x18, 0xc3, 0x9c, 0xdf, 0x0c,
-	0xa0, 0x40, 0xf6, 0x9d, 0xbf, 0x7d, 0x97, 0xcb, 0x30, 0x6c, 0x1d, 0x99, 0xc6, 0xc3, 0x5e, 0x32,
+	0xa8, 0xde, 0x4b, 0x50, 0x20, 0xe7, 0xce, 0x9f, 0x7e, 0xca, 0x65, 0x18, 0x8e, 0x8e, 0x8c, 0xe3,
-	0xd2, 0x13, 0x4a, 0xe3, 0x3d, 0xe1, 0x32, 0x98, 0x44, 0x6c, 0xc8, 0xeb, 0x23, 0xde, 0x53, 0xe2,
+	0xe1, 0x2c, 0x19, 0x99, 0x09, 0xa5, 0xf1, 0x99, 0x70, 0x05, 0x4c, 0x22, 0x0e, 0xe4, 0xf5, 0x11,
-	0x56, 0x5d, 0x90, 0xd0, 0xae, 0x40, 0x44, 0xd3, 0xc8, 0x0c, 0x64, 0xd3, 0x98, 0x3f, 0x71, 0xd3,
+	0xef, 0x29, 0x72, 0xab, 0x2e, 0x48, 0x68, 0x57, 0x20, 0x62, 0x68, 0x64, 0x0a, 0x72, 0x68, 0xcc,
-	0xd0, 0x4e, 0x64, 0xd3, 0xf8, 0xb1, 0x00, 0x76, 0x5b, 0x6d, 0x76, 0xf8, 0xa6, 0xb9, 0xd7, 0x0f,
+	0x9f, 0x78, 0x68, 0x68, 0x23, 0x72, 0x68, 0x7c, 0x5f, 0x00, 0xbb, 0xad, 0x0e, 0x3b, 0x7c, 0xd3,
-	0xe4, 0xd3, 0x6a, 0x19, 0xaa, 0xed, 0x9c, 0x99, 0x7a, 0x52, 0x0c, 0x01, 0x91, 0x1f, 0xdb, 0x38,
+	0xdc, 0xef, 0x07, 0xf2, 0x69, 0xb5, 0x0c, 0xd5, 0x76, 0xee, 0x99, 0x7a, 0x52, 0x0c, 0x01, 0x91,
-	0x4a, 0xe8, 0xa0, 0x4d, 0x0e, 0xb1, 0x26, 0x3e, 0x82, 0x08, 0x6e, 0x3b, 0x69, 0xe4, 0x26, 0x07,
+	0x1f, 0xdb, 0x38, 0x4a, 0xe8, 0xa0, 0x4d, 0x0e, 0xb1, 0x76, 0x7c, 0x04, 0x11, 0xbe, 0xed, 0xa4,
-	0x4c, 0x87, 0x26, 0x1b, 0x0a, 0x6e, 0xbe, 0x6c, 0xf5, 0x9e, 0x48, 0x27, 0xc9, 0xbc, 0xe4, 0x82,
+	0x91, 0x9b, 0x1c, 0x30, 0x1d, 0x9a, 0x6c, 0x29, 0x7c, 0xf3, 0xe5, 0xa8, 0xf7, 0x44, 0x3a, 0x49,
-	0x82, 0xc4, 0x3b, 0xc0, 0x5a, 0x82, 0x0a, 0x8e, 0x03, 0x35, 0x3b, 0x27, 0x67, 0xcb, 0x38, 0x0e,
+	0xcf, 0x4b, 0x2e, 0x28, 0x48, 0xbc, 0x03, 0xac, 0x25, 0xa8, 0xe0, 0x38, 0x50, 0xd2, 0x39, 0x29,
-	0xe4, 0x54, 0x0b, 0x16, 0xf5, 0x5b, 0x26, 0x61, 0x32, 0x84, 0xf2, 0xd0, 0x31, 0xd7, 0x9d, 0x23,
+	0x2d, 0xe3, 0x38, 0x90, 0xa2, 0x16, 0x2c, 0xea, 0xb7, 0x4c, 0xc2, 0x64, 0x08, 0x65, 0xd3, 0x31,
-	0x1e, 0x90, 0xdb, 0xac, 0xbb, 0xab, 0x2d, 0xdd, 0x05, 0xf5, 0x9c, 0xd1, 0x43, 0xeb, 0x43, 0xa8,
+	0xd7, 0x9d, 0x23, 0x1e, 0x90, 0xdb, 0xac, 0xbb, 0xab, 0x35, 0xdd, 0x05, 0xf5, 0x9c, 0xd1, 0x4b,
-	0x89, 0x55, 0x72, 0x47, 0xe5, 0x13, 0x3b, 0x32, 0x71, 0x1c, 0x64, 0x03, 0xe7, 0x2b, 0x03, 0x4e,
+	0xeb, 0x3d, 0xa8, 0x89, 0x5d, 0x72, 0x43, 0xe5, 0x13, 0x1b, 0x32, 0x71, 0x1c, 0x64, 0x0b, 0xe7,
-	0x3f, 0x25, 0xe1, 0x0c, 0x79, 0x74, 0x07, 0x2a, 0x6d, 0xdc, 0x15, 0x2e, 0xb2, 0x17, 0xda, 0xda,
+	0x0b, 0x03, 0x4e, 0x3f, 0x43, 0xe1, 0x0c, 0x79, 0x74, 0x17, 0x2a, 0x6d, 0xdc, 0x15, 0x26, 0xb2,
-	0x51, 0x0f, 0xfe, 0x23, 0x02, 0xe6, 0xe6, 0x0e, 0x9c, 0x07, 0x79, 0x58, 0x65, 0xfd, 0x89, 0x97,
+	0x17, 0xda, 0xda, 0x51, 0x0f, 0xfe, 0x23, 0x02, 0xe6, 0xe6, 0x06, 0x9c, 0x87, 0x79, 0x58, 0x65,
-	0xae, 0x38, 0x54, 0x82, 0x17, 0x50, 0x88, 0xce, 0x67, 0x86, 0x78, 0x85, 0x06, 0xf8, 0x89, 0x5c,
+	0xfd, 0x89, 0x97, 0xae, 0x68, 0x2a, 0xc1, 0x4b, 0x28, 0x44, 0xe7, 0x13, 0x43, 0xbc, 0x42, 0x03,
-	0xfa, 0xa9, 0xc4, 0x34, 0x66, 0x49, 0x4c, 0xeb, 0x1a, 0x9c, 0x8d, 0xd3, 0xc8, 0xa3, 0x38, 0x44,
+	0xfc, 0x54, 0x6e, 0xfd, 0x4c, 0x62, 0x1a, 0xb3, 0x24, 0xa6, 0x75, 0x1d, 0xce, 0xc6, 0x69, 0xe4,
-	0x1c, 0x07, 0x9e, 0x5e, 0x8d, 0xe9, 0xd5, 0xad, 0x38, 0x8d, 0x5c, 0x35, 0xa5, 0x69, 0x32, 0xe7,
+	0x51, 0x1c, 0x22, 0x8e, 0x03, 0x4f, 0xef, 0xc6, 0xf4, 0xee, 0x56, 0x9c, 0x46, 0xae, 0x12, 0x69,
-	0x4b, 0x03, 0xe0, 0xb6, 0xa8, 0x1e, 0xb5, 0x8d, 0xc9, 0xe3, 0xc1, 0x38, 0xfe, 0x4a, 0x56, 0x18,
+	0x37, 0x99, 0xf3, 0xb9, 0x01, 0x70, 0x47, 0x54, 0x8f, 0x3a, 0xc6, 0x64, 0x7b, 0x30, 0x8e, 0xbf,
-	0x2f, 0xbf, 0x8d, 0xac, 0xfc, 0x98, 0x8c, 0x47, 0x71, 0x1a, 0x87, 0x3c, 0x1e, 0x43, 0xf2, 0xba,
+	0x92, 0x15, 0xc6, 0xcb, 0x6f, 0x23, 0x2b, 0x3f, 0x26, 0xe3, 0x51, 0x9c, 0xe6, 0x43, 0x1e, 0x8f,
-	0x42, 0x55, 0x0c, 0xbe, 0x36, 0xa0, 0x36, 0x12, 0x2a, 0x36, 0x2e, 0xa3, 0x31, 0x79, 0x52, 0xc8,
+	0xa1, 0xf3, 0xba, 0x42, 0x55, 0x0c, 0xbe, 0x34, 0xa0, 0x36, 0x12, 0x2a, 0x36, 0x4e, 0xa3, 0x31,
-	0x7e, 0x21, 0xaa, 0xc7, 0x63, 0x23, 0x05, 0x15, 0x0d, 0x0b, 0x6a, 0x09, 0x2a, 0x52, 0x92, 0x91,
+	0xd9, 0x29, 0xe4, 0xbc, 0x10, 0xd5, 0xe3, 0xb1, 0x91, 0x82, 0x8a, 0x86, 0x05, 0xb5, 0x04, 0x15,
-	0x8a, 0x8a, 0x75, 0x45, 0x5d, 0x85, 0xd3, 0x14, 0xfb, 0x38, 0xe6, 0xe1, 0xc0, 0x8b, 0x92, 0x80,
+	0x49, 0xc9, 0x48, 0x45, 0xc5, 0xba, 0xa2, 0xae, 0xc1, 0x69, 0x8a, 0x7d, 0x1c, 0xf3, 0x70, 0xe0,
-	0xec, 0x13, 0x1c, 0xc8, 0xba, 0xaa, 0xb8, 0xf5, 0x6c, 0x62, 0x5b, 0xe3, 0xce, 0xcf, 0x06, 0x2c,
+	0x45, 0x49, 0x40, 0xf6, 0x09, 0x0e, 0x64, 0x5d, 0x55, 0xdc, 0x7a, 0x26, 0xd8, 0xd6, 0xb8, 0xf3,
-	0x7e, 0x2c, 0xda, 0xe8, 0x4e, 0x12, 0x60, 0xb5, 0xb3, 0xe7, 0x4f, 0x89, 0x9b, 0x92, 0x8b, 0x96,
+	0xa3, 0x01, 0x8b, 0x1f, 0x88, 0x31, 0xba, 0x93, 0x04, 0x58, 0x9d, 0xec, 0xc5, 0x53, 0xe2, 0x96,
-	0x47, 0xa5, 0xeb, 0xab, 0xcf, 0x4e, 0x57, 0xe6, 0x56, 0x98, 0x4e, 0x51, 0x21, 0xb1, 0xba, 0x66,
+	0xf4, 0x45, 0xd3, 0xa3, 0xd2, 0xf5, 0x7f, 0xcf, 0x4f, 0x57, 0xe6, 0x56, 0x98, 0x4e, 0x51, 0x41,
-	0x9f, 0x44, 0xe2, 0x61, 0x60, 0x5d, 0x75, 0x39, 0x57, 0x12, 0x07, 0x60, 0x8e, 0xd4, 0xa5, 0x68,
+	0xb1, 0xba, 0x66, 0x9f, 0x84, 0xe2, 0x61, 0x60, 0x5d, 0x75, 0x39, 0x57, 0x14, 0x7f, 0x6c, 0x80,
-	0x49, 0xba, 0x7f, 0xa9, 0xb6, 0x67, 0xc8, 0xf3, 0xd6, 0xd4, 0x98, 0x3c, 0x71, 0xcf, 0xc2, 0x5c,
+	0x39, 0x52, 0x98, 0x62, 0x26, 0xe9, 0x01, 0xa6, 0xe6, 0x9e, 0x21, 0x1b, 0xae, 0xa9, 0x31, 0xd9,
-	0xc4, 0xba, 0xf9, 0x2d, 0x49, 0x0d, 0x44, 0x64, 0xf2, 0xce, 0x26, 0xb5, 0x2d, 0xb9, 0x43, 0xe0,
+	0x72, 0xcf, 0xc2, 0x5c, 0xc4, 0xba, 0xf9, 0x35, 0x49, 0x2d, 0xac, 0x8b, 0x50, 0x89, 0x58, 0x57,
-	0xed, 0xf7, 0xa1, 0x9a, 0xff, 0xc7, 0x66, 0xd5, 0xa1, 0xd6, 0x8a, 0x09, 0x27, 0x28, 0x24, 0x87,
+	0xde, 0x46, 0x74, 0x97, 0xce, 0xd7, 0x22, 0x6c, 0xf9, 0xd8, 0xd3, 0xcd, 0x6a, 0x08, 0xbc, 0xf1,
-	0x24, 0xee, 0xd6, 0x5f, 0xb2, 0x4c, 0x28, 0x7f, 0x84, 0x51, 0xc8, 0x7b, 0x83, 0xba, 0x61, 0xd5,
+	0x0e, 0x54, 0xf3, 0x3f, 0xe0, 0xac, 0x3a, 0xd4, 0x5a, 0x31, 0xe1, 0x04, 0x85, 0xe4, 0x90, 0xc4,
-	0xa0, 0x72, 0xab, 0x13, 0x27, 0x34, 0x42, 0x61, 0xbd, 0xb0, 0xd1, 0xfc, 0x74, 0xa3, 0x4b, 0x78,
+	0xdd, 0xfa, 0x7f, 0x2c, 0x13, 0xca, 0xef, 0x63, 0x14, 0xf2, 0xde, 0xa0, 0x6e, 0x58, 0x35, 0xa8,
-	0x2f, 0xed, 0x08, 0x11, 0xd7, 0x0e, 0x49, 0x18, 0x92, 0x43, 0x8e, 0xfd, 0xde, 0x9a, 0x62, 0xf9,
+	0xdc, 0xee, 0xc4, 0x09, 0x8d, 0x50, 0x58, 0x2f, 0x6c, 0x34, 0x3f, 0xda, 0xe8, 0x12, 0xde, 0x4b,
-	0x4e, 0x40, 0x18, 0xa7, 0xa4, 0x93, 0x72, 0x1c, 0xac, 0x65, 0x5c, 0xd7, 0x24, 0xf5, 0x7c, 0xd8,
+	0x3b, 0x82, 0xe1, 0xb5, 0x43, 0x12, 0x86, 0xe4, 0x90, 0x63, 0xbf, 0xb7, 0xa6, 0x28, 0x78, 0x33,
-	0xef, 0x74, 0xe6, 0x25, 0x72, 0xfd, 0xcf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x4c, 0x6c, 0x9f,
+	0x20, 0x8c, 0x53, 0xd2, 0x49, 0x39, 0x0e, 0xd6, 0x32, 0x22, 0xd6, 0x24, 0x2f, 0xf9, 0xb2, 0xdf,
-	0x88, 0x14, 0x00, 0x00,
+	0xe9, 0xcc, 0x4b, 0xe4, 0xc6, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0x54, 0x28, 0x8f, 0xa8, 0xa5,
+	0x14, 0x00, 0x00,
 }

internal/proxynode/meta_cache.go

@@ -6,6 +6,9 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/zilliztech/milvus-distributed/internal/log"
+	"go.uber.org/zap"
+
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
@@ -16,6 +19,7 @@ import (
 type Cache interface {
 	GetCollectionID(ctx context.Context, collectionName string) (typeutil.UniqueID, error)
 	GetPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error)
+	GetPartitions(ctx context.Context, collectionName string) (map[string]typeutil.UniqueID, error)
 	GetCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error)
 	RemoveCollection(ctx context.Context, collectionName string)
 	RemovePartition(ctx context.Context, collectionName string, partitionName string)
@@ -52,83 +56,143 @@ func NewMetaCache(client types.MasterService) (*MetaCache, error) {
 	}, nil
 }
 
-func (m *MetaCache) readCollectionID(ctx context.Context, collectionName string) (typeutil.UniqueID, error) {
+func (m *MetaCache) GetCollectionID(ctx context.Context, collectionName string) (typeutil.UniqueID, error) {
 	m.mu.RLock()
+	collInfo, ok := m.collInfo[collectionName]
+
+	if !ok {
+		m.mu.RUnlock()
+		coll, err := m.describeCollection(ctx, collectionName)
+		if err != nil {
+			return 0, err
+		}
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		m.updateCollection(coll, collectionName)
+		collInfo = m.collInfo[collectionName]
+		return collInfo.collID, nil
+	}
 	defer m.mu.RUnlock()
 
-	collInfo, ok := m.collInfo[collectionName]
-	if !ok {
-		return 0, fmt.Errorf("can't find collection name:%s", collectionName)
-	}
 	return collInfo.collID, nil
 }
 
-func (m *MetaCache) readCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error) {
+func (m *MetaCache) GetCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error) {
 	m.mu.RLock()
+	collInfo, ok := m.collInfo[collectionName]
+
+	if !ok {
+		m.mu.RUnlock()
+		coll, err := m.describeCollection(ctx, collectionName)
+		if err != nil {
+			return nil, err
+		}
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		m.updateCollection(coll, collectionName)
+		collInfo = m.collInfo[collectionName]
+		return collInfo.schema, nil
+	}
 	defer m.mu.RUnlock()
 
-	collInfo, ok := m.collInfo[collectionName]
-	if !ok {
-		return nil, fmt.Errorf("can't find collection name:%s", collectionName)
-	}
 	return collInfo.schema, nil
 }
 
-func (m *MetaCache) readPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error) {
+func (m *MetaCache) updateCollection(coll *milvuspb.DescribeCollectionResponse, collectionName string) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	collInfo, ok := m.collInfo[collectionName]
-	if !ok {
-		return 0, fmt.Errorf("can't find collection name:%s", collectionName)
-	}
-
-	partitionID, ok := collInfo.partInfo[partitionName]
-	if !ok {
-		return 0, fmt.Errorf("can't find partition name:%s", partitionName)
-	}
-	return partitionID, nil
-}
-
-func (m *MetaCache) GetCollectionID(ctx context.Context, collectionName string) (typeutil.UniqueID, error) {
-	collID, err := m.readCollectionID(ctx, collectionName)
-	if err == nil {
-		return collID, nil
-	}
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	req := &milvuspb.DescribeCollectionRequest{
-		Base: &commonpb.MsgBase{
-			MsgType: commonpb.MsgType_DescribeCollection,
-		},
-		CollectionName: collectionName,
-	}
-	coll, err := m.client.DescribeCollection(ctx, req)
-	if err != nil {
-		return 0, err
-	}
-	if coll.Status.ErrorCode != commonpb.ErrorCode_Success {
-		return 0, errors.New(coll.Status.Reason)
-	}
-
 	_, ok := m.collInfo[collectionName]
 	if !ok {
 		m.collInfo[collectionName] = &collectionInfo{}
 	}
 	m.collInfo[collectionName].schema = coll.Schema
 	m.collInfo[collectionName].collID = coll.CollectionID
-
-	return m.collInfo[collectionName].collID, nil
 }
-func (m *MetaCache) GetCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error) {
-	collSchema, err := m.readCollectionSchema(ctx, collectionName)
-	if err == nil {
-		return collSchema, nil
-	}
-	m.mu.Lock()
-	defer m.mu.Unlock()
 
+func (m *MetaCache) GetPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error) {
+	_, err := m.GetCollectionID(ctx, collectionName)
+	if err != nil {
+		return 0, err
+	}
+
+	m.mu.RLock()
+
+	collInfo, ok := m.collInfo[collectionName]
+	if !ok {
+		m.mu.RUnlock()
+		return 0, fmt.Errorf("can't find collection name:%s", collectionName)
+	}
+
+	partitionID, ok := collInfo.partInfo[partitionName]
+	m.mu.RUnlock()
+
+	if !ok {
+		partitions, err := m.showPartitions(ctx, collectionName)
+		if err != nil {
+			return 0, err
+		}
+
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		log.Debug("proxynode", zap.Any("GetPartitionID:partitions before update", partitions), zap.Any("collectionName", collectionName))
+		m.updatePartitions(partitions, collectionName)
+		log.Debug("proxynode", zap.Any("GetPartitionID:partitions after update", partitions), zap.Any("collectionName", collectionName))
+
+		partInfo := m.collInfo[collectionName].partInfo
+		_, ok := partInfo[partitionName]
+		if !ok {
+			return 0, fmt.Errorf("partitionID of partitionName:%s can not be find", partitionName)
+		}
+		return partInfo[partitionName], nil
+	}
+	return partitionID, nil
+}
+
+func (m *MetaCache) GetPartitions(ctx context.Context, collectionName string) (map[string]typeutil.UniqueID, error) {
+	_, err := m.GetCollectionID(ctx, collectionName)
+	if err != nil {
+		return nil, err
+	}
+
+	m.mu.RLock()
+
+	collInfo, ok := m.collInfo[collectionName]
+	if !ok {
+		m.mu.RUnlock()
+		return nil, fmt.Errorf("can't find collection name:%s", collectionName)
+	}
+
+	if collInfo.partInfo == nil || len(collInfo.partInfo) == 0 {
+		m.mu.RUnlock()
+
+		partitions, err := m.showPartitions(ctx, collectionName)
+		if err != nil {
+			return nil, err
+		}
+
+		m.mu.Lock()
+		defer m.mu.Unlock()
+
+		m.updatePartitions(partitions, collectionName)
+
+		ret := make(map[string]typeutil.UniqueID)
+		partInfo := m.collInfo[collectionName].partInfo
+		for k, v := range partInfo {
+			ret[k] = v
+		}
+		return ret, nil
+
+	}
+	defer m.mu.RUnlock()
+
+	ret := make(map[string]typeutil.UniqueID)
+	partInfo := m.collInfo[collectionName].partInfo
+	for k, v := range partInfo {
+		ret[k] = v
+	}
+
+	return ret, nil
+}
+
+func (m *MetaCache) describeCollection(ctx context.Context, collectionName string) (*milvuspb.DescribeCollectionResponse, error) {
 	req := &milvuspb.DescribeCollectionRequest{
 		Base: &commonpb.MsgBase{
 			MsgType: commonpb.MsgType_DescribeCollection,
@@ -142,45 +206,34 @@ func (m *MetaCache) GetCollectionSchema(ctx context.Context, collectionName stri
 	if coll.Status.ErrorCode != commonpb.ErrorCode_Success {
 		return nil, errors.New(coll.Status.Reason)
 	}
-
+	return coll, nil
-	_, ok := m.collInfo[collectionName]
-	if !ok {
-		m.collInfo[collectionName] = &collectionInfo{}
-	}
-	m.collInfo[collectionName].schema = coll.Schema
-	m.collInfo[collectionName].collID = coll.CollectionID
-
-	return m.collInfo[collectionName].schema, nil
 }
 
-func (m *MetaCache) GetPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error) {
+func (m *MetaCache) showPartitions(ctx context.Context, collectionName string) (*milvuspb.ShowPartitionsResponse, error) {
-	partitionID, err := m.readPartitionID(ctx, collectionName, partitionName)
-	if err == nil {
-		return partitionID, nil
-	}
-
 	req := &milvuspb.ShowPartitionsRequest{
 		Base: &commonpb.MsgBase{
 			MsgType: commonpb.MsgType_ShowPartitions,
 		},
 		CollectionName: collectionName,
 	}
+
 	partitions, err := m.client.ShowPartitions(ctx, req)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
 	if partitions.Status.ErrorCode != commonpb.ErrorCode_Success {
-		return 0, fmt.Errorf("%s", partitions.Status.Reason)
+		return nil, fmt.Errorf("%s", partitions.Status.Reason)
 	}
 
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
 	if len(partitions.PartitionIDs) != len(partitions.PartitionNames) {
-		return 0, fmt.Errorf("partition ids len: %d doesn't equal Partition name len %d",
+		return nil, fmt.Errorf("partition ids len: %d doesn't equal Partition name len %d",
 			len(partitions.PartitionIDs), len(partitions.PartitionNames))
 	}
 
+	return partitions, nil
+}
+
+func (m *MetaCache) updatePartitions(partitions *milvuspb.ShowPartitionsResponse, collectionName string) {
 	_, ok := m.collInfo[collectionName]
 	if !ok {
 		m.collInfo[collectionName] = &collectionInfo{
@@ -198,12 +251,7 @@ func (m *MetaCache) GetPartitionID(ctx context.Context, collectionName string, p
 			partInfo[partitions.PartitionNames[i]] = partitions.PartitionIDs[i]
 		}
 	}
-	_, ok = partInfo[partitionName]
+	m.collInfo[collectionName].partInfo = partInfo
-	if !ok {
-		return 0, fmt.Errorf("partitionID of partitionName:%s can not be find", partitionName)
-	}
-
-	return partInfo[partitionName], nil
 }
 
 func (m *MetaCache) RemoveCollection(ctx context.Context, collectionName string) {

internal/proxynode/proxy_node.go

@@ -74,7 +74,7 @@ func (node *ProxyNode) Init() error {
 	// todo wait for proxyservice state changed to Healthy
 	ctx := context.Background()
 
-	err := funcutil.WaitForComponentHealthy(ctx, node.proxyService, "ProxyService", 100, time.Millisecond*200)
+	err := funcutil.WaitForComponentHealthy(ctx, node.proxyService, "ProxyService", 1000000, time.Millisecond*200)
 	if err != nil {
 		return err
 	}
@@ -102,7 +102,7 @@ func (node *ProxyNode) Init() error {
 
 	// wait for dataservice state changed to Healthy
 	if node.dataService != nil {
-		err := funcutil.WaitForComponentHealthy(ctx, node.dataService, "DataService", 100, time.Millisecond*200)
+		err := funcutil.WaitForComponentHealthy(ctx, node.dataService, "DataService", 1000000, time.Millisecond*200)
 		if err != nil {
 			return err
 		}
@@ -110,7 +110,7 @@ func (node *ProxyNode) Init() error {
 
 	// wait for queryService state changed to Healthy
 	if node.queryService != nil {
-		err := funcutil.WaitForComponentHealthy(ctx, node.queryService, "QueryService", 100, time.Millisecond*200)
+		err := funcutil.WaitForComponentHealthy(ctx, node.queryService, "QueryService", 1000000, time.Millisecond*200)
 		if err != nil {
 			return err
 		}
@@ -118,7 +118,7 @@ func (node *ProxyNode) Init() error {
 
 	// wait for indexservice state changed to Healthy
 	if node.indexService != nil {
-		err := funcutil.WaitForComponentHealthy(ctx, node.indexService, "IndexService", 100, time.Millisecond*200)
+		err := funcutil.WaitForComponentHealthy(ctx, node.indexService, "IndexService", 1000000, time.Millisecond*200)
 		if err != nil {
 			return err
 		}

internal/proxynode/task.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"regexp"
 	"strconv"
 
 	"go.uber.org/zap"
@@ -552,13 +553,35 @@ func (st *SearchTask) PreExecute(ctx context.Context) error {
 	}
 	st.CollectionID = collectionID
 	st.PartitionIDs = make([]UniqueID, 0)
-	for _, partitionName := range st.query.PartitionNames {
+
-		partitionID, err := globalMetaCache.GetPartitionID(ctx, collectionName, partitionName)
+	partitionsMap, err := globalMetaCache.GetPartitions(ctx, collectionName)
-		if err != nil {
+	if err != nil {
-			continue
+		return err
-		}
-		st.PartitionIDs = append(st.PartitionIDs, partitionID)
 	}
+
+	partitionsRecord := make(map[UniqueID]bool)
+	for _, partitionName := range st.query.PartitionNames {
+		pattern := fmt.Sprintf("^%s$", partitionName)
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return errors.New("invalid partition names")
+		}
+		found := false
+		for name, pID := range partitionsMap {
+			if re.MatchString(name) {
+				if _, exist := partitionsRecord[pID]; !exist {
+					st.PartitionIDs = append(st.PartitionIDs, pID)
+					partitionsRecord[pID] = true
+				}
+				found = true
+			}
+		}
+		if !found {
+			errMsg := fmt.Sprintf("PartitonName: %s not found", partitionName)
+			return errors.New(errMsg)
+		}
+	}
+
 	st.Dsl = st.query.Dsl
 	st.PlaceholderGroup = st.query.PlaceholderGroup
 

internal/querynode/collection_replica.go

@@ -68,21 +68,31 @@ type ReplicaInterface interface {
 	getSegmentsToLoadBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID)
 	getSegmentStatistics() []*internalpb.SegmentStats
 
+	// excluded segments
+	initExcludedSegments(collectionID UniqueID)
+	removeExcludedSegments(collectionID UniqueID)
+	addExcludedSegments(collectionID UniqueID, segmentIDs []UniqueID) error
+	getExcludedSegments(collectionID UniqueID) ([]UniqueID, error)
+
 	getEnabledSegmentsBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID)
 	getSegmentsBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID)
 	replaceGrowingSegmentBySealedSegment(segment *Segment) error
 
-	getTSafe() tSafer
+	getTSafe(collectionID UniqueID) tSafer
+	addTSafe(collectionID UniqueID)
+	removeTSafe(collectionID UniqueID)
 	freeAll()
 }
 
 type collectionReplica struct {
-	tSafe tSafer
+	tSafes map[UniqueID]tSafer // map[collectionID]tSafer
 
 	mu          sync.RWMutex // guards all
 	collections map[UniqueID]*Collection
 	partitions  map[UniqueID]*Partition
 	segments    map[UniqueID]*Segment
+
+	excludedSegments map[UniqueID][]UniqueID // map[collectionID]segmentIDs
 }
 
 //----------------------------------------------------------------------------------------------------- collection
@@ -101,7 +111,7 @@ func (colReplica *collectionReplica) addCollection(collectionID UniqueID, schema
 	defer colReplica.mu.Unlock()
 
 	if ok := colReplica.hasCollectionPrivate(collectionID); ok {
-		return fmt.Errorf("collection has been existed, id %d", collectionID)
+		return errors.New("collection has been loaded, id %d" + strconv.FormatInt(collectionID, 10))
 	}
 
 	var newCollection = newCollection(collectionID, schema)
@@ -143,7 +153,7 @@ func (colReplica *collectionReplica) getCollectionByID(collectionID UniqueID) (*
 func (colReplica *collectionReplica) getCollectionByIDPrivate(collectionID UniqueID) (*Collection, error) {
 	collection, ok := colReplica.collections[collectionID]
 	if !ok {
-		return nil, fmt.Errorf("cannot find collection, id = %d", collectionID)
+		return nil, errors.New("collection hasn't been loaded or has been released, collection id = %d" + strconv.FormatInt(collectionID, 10))
 	}
 
 	return collection, nil
@@ -195,7 +205,7 @@ func (colReplica *collectionReplica) getVecFieldIDsByCollectionID(collectionID U
 	}
 
 	if len(vecFields) <= 0 {
-		return nil, fmt.Errorf("no vector field in collection %d", collectionID)
+		return nil, errors.New("no vector field in collection %d" + strconv.FormatInt(collectionID, 10))
 	}
 
 	return vecFields, nil
@@ -228,7 +238,7 @@ func (colReplica *collectionReplica) getFieldsByCollectionIDPrivate(collectionID
 	}
 
 	if len(collection.Schema().Fields) <= 0 {
-		return nil, fmt.Errorf("no field in collection %d", collectionID)
+		return nil, errors.New("no field in collection %d" + strconv.FormatInt(collectionID, 10))
 	}
 
 	return collection.Schema().Fields, nil
@@ -291,7 +301,7 @@ func (colReplica *collectionReplica) getPartitionByID(partitionID UniqueID) (*Pa
 func (colReplica *collectionReplica) getPartitionByIDPrivate(partitionID UniqueID) (*Partition, error) {
 	partition, ok := colReplica.partitions[partitionID]
 	if !ok {
-		return nil, fmt.Errorf("cannot find partition, id = %d", partitionID)
+		return nil, errors.New("partition hasn't been loaded or has been released, partition id = %d" + strconv.FormatInt(partitionID, 10))
 	}
 
 	return partition, nil
@@ -426,7 +436,7 @@ func (colReplica *collectionReplica) getSegmentByID(segmentID UniqueID) (*Segmen
 func (colReplica *collectionReplica) getSegmentByIDPrivate(segmentID UniqueID) (*Segment, error) {
 	segment, ok := colReplica.segments[segmentID]
 	if !ok {
-		return nil, errors.New("cannot find segment, id = " + strconv.FormatInt(segmentID, 10))
+		return nil, errors.New("cannot find segment in query node, id = " + strconv.FormatInt(segmentID, 10))
 	}
 
 	return segment, nil
@@ -529,7 +539,7 @@ func (colReplica *collectionReplica) getSegmentsBySegmentType(segType segmentTyp
 func (colReplica *collectionReplica) replaceGrowingSegmentBySealedSegment(segment *Segment) error {
 	colReplica.mu.Lock()
 	defer colReplica.mu.Unlock()
-	if segment.segmentType != segmentTypeSealed && segment.segmentType != segTypeIndexing {
+	if segment.segmentType != segmentTypeSealed && segment.segmentType != segmentTypeIndexing {
 		return errors.New("unexpected segment type")
 	}
 	targetSegment, err := colReplica.getSegmentByIDPrivate(segment.ID())
@@ -573,9 +583,66 @@ func (colReplica *collectionReplica) setSegmentEnableLoadBinLog(segmentID Unique
 	return nil
 }
 
+func (colReplica *collectionReplica) initExcludedSegments(collectionID UniqueID) {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	colReplica.excludedSegments[collectionID] = make([]UniqueID, 0)
+}
+
+func (colReplica *collectionReplica) removeExcludedSegments(collectionID UniqueID) {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	delete(colReplica.excludedSegments, collectionID)
+}
+
+func (colReplica *collectionReplica) addExcludedSegments(collectionID UniqueID, segmentIDs []UniqueID) error {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	if _, ok := colReplica.excludedSegments[collectionID]; !ok {
+		return errors.New("addExcludedSegments failed, cannot found collection, id =" + fmt.Sprintln(collectionID))
+	}
+
+	colReplica.excludedSegments[collectionID] = append(colReplica.excludedSegments[collectionID], segmentIDs...)
+	return nil
+}
+
+func (colReplica *collectionReplica) getExcludedSegments(collectionID UniqueID) ([]UniqueID, error) {
+	colReplica.mu.RLock()
+	defer colReplica.mu.RUnlock()
+
+	if _, ok := colReplica.excludedSegments[collectionID]; !ok {
+		return nil, errors.New("getExcludedSegments failed, cannot found collection, id =" + fmt.Sprintln(collectionID))
+	}
+
+	return colReplica.excludedSegments[collectionID], nil
+}
+
 //-----------------------------------------------------------------------------------------------------
-func (colReplica *collectionReplica) getTSafe() tSafer {
+func (colReplica *collectionReplica) getTSafe(collectionID UniqueID) tSafer {
-	return colReplica.tSafe
+	colReplica.mu.RLock()
+	defer colReplica.mu.RUnlock()
+	return colReplica.getTSafePrivate(collectionID)
+}
+
+func (colReplica *collectionReplica) getTSafePrivate(collectionID UniqueID) tSafer {
+	return colReplica.tSafes[collectionID]
+}
+
+func (colReplica *collectionReplica) addTSafe(collectionID UniqueID) {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+	colReplica.tSafes[collectionID] = newTSafe()
+}
+
+func (colReplica *collectionReplica) removeTSafe(collectionID UniqueID) {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+	ts := colReplica.getTSafePrivate(collectionID)
+	ts.close()
+	delete(colReplica.tSafes, collectionID)
 }
 
 func (colReplica *collectionReplica) freeAll() {
@@ -591,24 +658,6 @@ func (colReplica *collectionReplica) freeAll() {
 	colReplica.segments = make(map[UniqueID]*Segment)
 }
 
-func newCollectionReplica() ReplicaInterface {
-	collections := make(map[int64]*Collection)
-	partitions := make(map[int64]*Partition)
-	segments := make(map[int64]*Segment)
-
-	tSafe := newTSafe()
-
-	var replica ReplicaInterface = &collectionReplica{
-		collections: collections,
-		partitions:  partitions,
-		segments:    segments,
-
-		tSafe: tSafe,
-	}
-
-	return replica
-}
-
 func (colReplica *collectionReplica) getSegmentsToLoadBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID) {
 	colReplica.mu.RLock()
 	defer colReplica.mu.RUnlock()
@@ -634,3 +683,22 @@ func (colReplica *collectionReplica) getSegmentsToLoadBySegmentType(segType segm
 
 	return targetCollectionIDs, targetPartitionIDs, targetSegmentIDs
 }
+
+func newCollectionReplica() ReplicaInterface {
+	collections := make(map[UniqueID]*Collection)
+	partitions := make(map[UniqueID]*Partition)
+	segments := make(map[UniqueID]*Segment)
+	excludedSegments := make(map[UniqueID][]UniqueID)
+
+	var replica ReplicaInterface = &collectionReplica{
+		collections: collections,
+		partitions:  partitions,
+		segments:    segments,
+
+		excludedSegments: excludedSegments,
+
+		tSafes: make(map[UniqueID]tSafer),
+	}
+
+	return replica
+}

internal/querynode/data_sync_service.go

@@ -12,8 +12,11 @@ import (
 )
 
 type dataSyncService struct {
-	ctx context.Context
+	ctx    context.Context
-	fg  *flowgraph.TimeTickedFlowGraph
+	cancel context.CancelFunc
+
+	collectionID UniqueID
+	fg           *flowgraph.TimeTickedFlowGraph
 
 	dmStream  msgstream.MsgStream
 	msFactory msgstream.Factory
@@ -21,12 +24,16 @@ type dataSyncService struct {
 	replica ReplicaInterface
 }
 
-func newDataSyncService(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory) *dataSyncService {
+func newDataSyncService(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory, collectionID UniqueID) *dataSyncService {
+	ctx1, cancel := context.WithCancel(ctx)
+
 	service := &dataSyncService{
-		ctx:       ctx,
+		ctx:          ctx1,
-		fg:        nil,
+		cancel:       cancel,
-		replica:   replica,
+		collectionID: collectionID,
-		msFactory: factory,
+		fg:           nil,
+		replica:      replica,
+		msFactory:    factory,
 	}
 
 	service.initNodes()
@@ -38,6 +45,7 @@ func (dsService *dataSyncService) start() {
 }
 
 func (dsService *dataSyncService) close() {
+	dsService.cancel()
 	if dsService.fg != nil {
 		dsService.fg.Close()
 	}
@@ -50,10 +58,10 @@ func (dsService *dataSyncService) initNodes() {
 
 	var dmStreamNode node = dsService.newDmInputNode(dsService.ctx)
 
-	var filterDmNode node = newFilteredDmNode(dsService.replica)
+	var filterDmNode node = newFilteredDmNode(dsService.replica, dsService.collectionID)
 
-	var insertNode node = newInsertNode(dsService.replica)
+	var insertNode node = newInsertNode(dsService.replica, dsService.collectionID)
-	var serviceTimeNode node = newServiceTimeNode(dsService.ctx, dsService.replica, dsService.msFactory)
+	var serviceTimeNode node = newServiceTimeNode(dsService.ctx, dsService.replica, dsService.msFactory, dsService.collectionID)
 
 	dsService.fg.AddNode(dmStreamNode)
 

internal/querynode/data_sync_service_test.go

@@ -18,6 +18,8 @@ import (
 func TestDataSyncService_Start(t *testing.T) {
 	ctx := context.Background()
 
+	collectionID := UniqueID(0)
+
 	node := newQueryNodeMock()
 	initTestMeta(t, node, 0, 0)
 	// test data generate
@@ -64,7 +66,7 @@ func TestDataSyncService_Start(t *testing.T) {
 					Timestamp: uint64(i + 1000),
 					SourceID:  0,
 				},
-				CollectionID: UniqueID(0),
+				CollectionID: collectionID,
 				PartitionID:  defaultPartitionID,
 				SegmentID:    int64(0),
 				ChannelID:    "0",
@@ -132,8 +134,8 @@ func TestDataSyncService_Start(t *testing.T) {
 	assert.NoError(t, err)
 
 	// dataSync
-	node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory)
+	node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory, collectionID)
-	go node.dataSyncService.start()
+	go node.dataSyncServices[collectionID].start()
 
 	<-node.queryNodeLoopCtx.Done()
 	node.Stop()

internal/querynode/flow_graph_filter_dm_node.go

@@ -2,6 +2,7 @@ package querynode
 
 import (
 	"context"
+	"fmt"
 
 	"go.uber.org/zap"
 
@@ -12,7 +13,8 @@ import (
 
 type filterDmNode struct {
 	baseNode
-	replica ReplicaInterface
+	collectionID UniqueID
+	replica      ReplicaInterface
 }
 
 func (fdmNode *filterDmNode) Name() string {
@@ -33,6 +35,10 @@ func (fdmNode *filterDmNode) Operate(ctx context.Context, in []Msg) ([]Msg, cont
 		// TODO: add error handling
 	}
 
+	if msgStreamMsg == nil {
+		return []Msg{}, ctx
+	}
+
 	var iMsg = insertMsg{
 		insertMessages: make([]*msgstream.InsertMsg, 0),
 		timeRange: TimeRange{
@@ -60,14 +66,31 @@ func (fdmNode *filterDmNode) Operate(ctx context.Context, in []Msg) ([]Msg, cont
 }
 
 func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg) *msgstream.InsertMsg {
-	// TODO: open this check
+	// check if collection and partition exist
-	// check if partition dm enable
+	collection := fdmNode.replica.hasCollection(msg.CollectionID)
-	enableCollection := fdmNode.replica.hasCollection(msg.CollectionID)
+	partition := fdmNode.replica.hasPartition(msg.PartitionID)
-	enablePartition := fdmNode.replica.hasPartition(msg.PartitionID)
+	if !collection || !partition {
-	if !enableCollection || !enablePartition {
 		return nil
 	}
 
+	// check if the collection from message is target collection
+	if msg.CollectionID != fdmNode.collectionID {
+		return nil
+	}
+
+	// check if the segment is in excluded segments
+	excludedSegments, err := fdmNode.replica.getExcludedSegments(fdmNode.collectionID)
+	log.Debug("excluded segments", zap.String("segmentIDs", fmt.Sprintln(excludedSegments)))
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+	for _, id := range excludedSegments {
+		if msg.SegmentID == id {
+			return nil
+		}
+	}
+
 	// TODO: If the last record is drop type, all insert requests are invalid.
 	//if !records[len(records)-1].createOrDrop {
 	//	return nil
@@ -80,27 +103,14 @@ func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg
 		return nil
 	}
 
-	tmpTimestamps := make([]Timestamp, 0)
+	if len(msg.Timestamps) <= 0 {
-	tmpRowIDs := make([]int64, 0)
-	tmpRowData := make([]*commonpb.Blob, 0)
-
-	for i, t := range msg.Timestamps {
-		tmpTimestamps = append(tmpTimestamps, t)
-		tmpRowIDs = append(tmpRowIDs, msg.RowIDs[i])
-		tmpRowData = append(tmpRowData, msg.RowData[i])
-	}
-
-	if len(tmpRowIDs) <= 0 {
 		return nil
 	}
 
-	msg.Timestamps = tmpTimestamps
-	msg.RowIDs = tmpRowIDs
-	msg.RowData = tmpRowData
 	return msg
 }
 
-func newFilteredDmNode(replica ReplicaInterface) *filterDmNode {
+func newFilteredDmNode(replica ReplicaInterface, collectionID UniqueID) *filterDmNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
@@ -109,7 +119,8 @@ func newFilteredDmNode(replica ReplicaInterface) *filterDmNode {
 	baseNode.SetMaxParallelism(maxParallelism)
 
 	return &filterDmNode{
-		baseNode: baseNode,
+		baseNode:     baseNode,
-		replica:  replica,
+		collectionID: collectionID,
+		replica:      replica,
 	}
 }

internal/querynode/flow_graph_insert_node.go

@@ -12,7 +12,8 @@ import (
 
 type insertNode struct {
 	baseNode
-	replica ReplicaInterface
+	collectionID UniqueID
+	replica      ReplicaInterface
 }
 
 type InsertData struct {
@@ -48,6 +49,10 @@ func (iNode *insertNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.
 		insertOffset:     make(map[int64]int64),
 	}
 
+	if iMsg == nil {
+		return []Msg{}, ctx
+	}
+
 	// 1. hash insertMessages to insertData
 	for _, task := range iMsg.insertMessages {
 		// check if segment exists, if not, create this segment
@@ -119,6 +124,11 @@ func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *syn
 		return
 	}
 
+	if targetSegment.segmentType != segmentTypeGrowing {
+		wg.Done()
+		return
+	}
+
 	ids := insertData.insertIDs[segmentID]
 	timestamps := insertData.insertTimestamps[segmentID]
 	records := insertData.insertRecords[segmentID]
@@ -132,11 +142,13 @@ func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *syn
 		return
 	}
 
-	log.Debug("Do insert done", zap.Int("len", len(insertData.insertIDs[segmentID])))
+	log.Debug("Do insert done", zap.Int("len", len(insertData.insertIDs[segmentID])),
+		zap.Int64("segmentID", segmentID),
+		zap.Int64("collectionID", iNode.collectionID))
 	wg.Done()
 }
 
-func newInsertNode(replica ReplicaInterface) *insertNode {
+func newInsertNode(replica ReplicaInterface, collectionID UniqueID) *insertNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
@@ -145,7 +157,8 @@ func newInsertNode(replica ReplicaInterface) *insertNode {
 	baseNode.SetMaxParallelism(maxParallelism)
 
 	return &insertNode{
-		baseNode: baseNode,
+		baseNode:     baseNode,
-		replica:  replica,
+		collectionID: collectionID,
+		replica:      replica,
 	}
 }

internal/querynode/flow_graph_service_time_node.go

@@ -13,6 +13,7 @@ import (
 
 type serviceTimeNode struct {
 	baseNode
+	collectionID      UniqueID
 	replica           ReplicaInterface
 	timeTickMsgStream msgstream.MsgStream
 }
@@ -35,9 +36,18 @@ func (stNode *serviceTimeNode) Operate(ctx context.Context, in []Msg) ([]Msg, co
 		// TODO: add error handling
 	}
 
+	if serviceTimeMsg == nil {
+		return []Msg{}, ctx
+	}
+
 	// update service time
-	stNode.replica.getTSafe().set(serviceTimeMsg.timeRange.timestampMax)
+	ts := stNode.replica.getTSafe(stNode.collectionID)
-	//log.Debug("update tSafe to:", getPhysicalTime(serviceTimeMsg.timeRange.timestampMax))
+	if ts != nil {
+		ts.set(serviceTimeMsg.timeRange.timestampMax)
+		log.Debug("update tSafe:",
+			zap.Int64("tSafe", int64(serviceTimeMsg.timeRange.timestampMax)),
+			zap.Int64("collectionID", stNode.collectionID))
+	}
 
 	if err := stNode.sendTimeTick(serviceTimeMsg.timeRange.timestampMax); err != nil {
 		log.Error("Error: send time tick into pulsar channel failed", zap.Error(err))
@@ -71,7 +81,7 @@ func (stNode *serviceTimeNode) sendTimeTick(ts Timestamp) error {
 	return stNode.timeTickMsgStream.Produce(context.TODO(), &msgPack)
 }
 
-func newServiceTimeNode(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory) *serviceTimeNode {
+func newServiceTimeNode(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory, collectionID UniqueID) *serviceTimeNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
@@ -85,6 +95,7 @@ func newServiceTimeNode(ctx context.Context, replica ReplicaInterface, factory m
 
 	return &serviceTimeNode{
 		baseNode:          baseNode,
+		collectionID:      collectionID,
 		replica:           replica,
 		timeTickMsgStream: timeTimeMsgStream,
 	}

internal/querynode/index_loader.go

@@ -111,7 +111,12 @@ func (loader *indexLoader) execute(l *loadIndex) error {
 	if err != nil {
 		return err
 	}
-	// 3. update segment index stats
+	// 3. drop vector field data if index loaded successfully
+	err = loader.dropVectorFieldData(l.segmentID, l.fieldID)
+	if err != nil {
+		return err
+	}
+	// 4. update segment index stats
 	err = loader.updateSegmentIndexStats(indexParams, indexName, indexID, l)
 	if err != nil {
 		return err
@@ -277,6 +282,14 @@ func (loader *indexLoader) updateSegmentIndex(indexParams indexParam, bytesIndex
 	return segment.updateSegmentIndex(loadIndexInfo)
 }
 
+func (loader *indexLoader) dropVectorFieldData(segmentID UniqueID, vecFieldID int64) error {
+	segment, err := loader.replica.getSegmentByID(segmentID)
+	if err != nil {
+		return err
+	}
+	return segment.dropFieldData(vecFieldID)
+}
+
 func (loader *indexLoader) sendQueryNodeStats() error {
 	resultFieldsStats := make([]*internalpb.FieldStats, 0)
 	for fieldStatsKey, indexStats := range loader.fieldIndexes {

internal/querynode/load_service.go

@@ -2,19 +2,16 @@ package querynode
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
 
-	"github.com/zilliztech/milvus-distributed/internal/types"
-
-	"errors"
-
 	"go.uber.org/zap"
 
 	"github.com/zilliztech/milvus-distributed/internal/log"
-	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/types"
 )
 
 const loadingCheckInterval = 3
@@ -76,7 +73,7 @@ func (s *loadService) loadSegmentActively(wg *sync.WaitGroup) {
 }
 
 // load segment passively
-func (s *loadService) loadSegment(collectionID UniqueID, partitionID UniqueID, segmentIDs []UniqueID, fieldIDs []int64) error {
+func (s *loadService) loadSegmentPassively(collectionID UniqueID, partitionID UniqueID, segmentIDs []UniqueID, fieldIDs []int64) error {
 	// TODO: interim solution
 	if len(fieldIDs) == 0 {
 		var err error
@@ -168,10 +165,10 @@ func (s *loadService) loadSegmentInternal(collectionID UniqueID, partitionID Uni
 	return nil
 }
 
-func newLoadService(ctx context.Context, masterService types.MasterService, dataService types.DataService, indexService types.IndexService, replica ReplicaInterface, dmStream msgstream.MsgStream) *loadService {
+func newLoadService(ctx context.Context, masterService types.MasterService, dataService types.DataService, indexService types.IndexService, replica ReplicaInterface) *loadService {
 	ctx1, cancel := context.WithCancel(ctx)
 
-	segLoader := newSegmentLoader(ctx1, masterService, indexService, dataService, replica, dmStream)
+	segLoader := newSegmentLoader(ctx1, masterService, indexService, dataService, replica)
 
 	return &loadService{
 		ctx:    ctx1,

internal/querynode/load_service_test.go

@@ -1129,7 +1129,7 @@ func TestSegmentLoad_Search_Vector(t *testing.T) {
 	defer node.Stop()
 
 	ctx := node.queryNodeLoopCtx
-	node.loadService = newLoadService(ctx, nil, nil, nil, node.replica, nil)
+	node.loadService = newLoadService(ctx, nil, nil, nil, node.replica)
 
 	initTestMeta(t, node, collectionID, 0)
 

internal/querynode/meta_service.go

@@ -46,7 +46,7 @@ func newMetaService(ctx context.Context, replica ReplicaInterface) *metaService
 		}
 		return nil
 	}
-	err = retry.Retry(200, time.Millisecond*200, connectEtcdFn)
+	err = retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
 	if err != nil {
 		panic(err)
 	}

internal/querynode/query_node.go

@@ -14,17 +14,14 @@ import "C"
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
 	"math/rand"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
 
-	"github.com/zilliztech/milvus-distributed/internal/types"
-
-	"errors"
-
 	"go.uber.org/zap"
 
 	"github.com/zilliztech/milvus-distributed/internal/log"
@@ -33,7 +30,9 @@ import (
 	"github.com/zilliztech/milvus-distributed/internal/msgstream/rmqms"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
 	queryPb "github.com/zilliztech/milvus-distributed/internal/proto/querypb"
+	"github.com/zilliztech/milvus-distributed/internal/types"
 	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
 )
 
@@ -47,11 +46,11 @@ type QueryNode struct {
 	replica ReplicaInterface
 
 	// internal services
-	dataSyncService *dataSyncService
+	dataSyncServices map[UniqueID]*dataSyncService
-	metaService     *metaService
+	metaService      *metaService
-	searchService   *searchService
+	searchService    *searchService
-	loadService     *loadService
+	loadService      *loadService
-	statsService    *statsService
+	statsService     *statsService
 
 	// clients
 	masterService types.MasterService
@@ -70,10 +69,10 @@ func NewQueryNode(ctx context.Context, queryNodeID UniqueID, factory msgstream.F
 		queryNodeLoopCancel: cancel,
 		QueryNodeID:         queryNodeID,
 
-		dataSyncService: nil,
+		dataSyncServices: make(map[UniqueID]*dataSyncService),
-		metaService:     nil,
+		metaService:      nil,
-		searchService:   nil,
+		searchService:    nil,
-		statsService:    nil,
+		statsService:     nil,
 
 		msFactory: factory,
 	}
@@ -89,10 +88,10 @@ func NewQueryNodeWithoutID(ctx context.Context, factory msgstream.Factory) *Quer
 		queryNodeLoopCtx:    ctx1,
 		queryNodeLoopCancel: cancel,
 
-		dataSyncService: nil,
+		dataSyncServices: make(map[UniqueID]*dataSyncService),
-		metaService:     nil,
+		metaService:      nil,
-		searchService:   nil,
+		searchService:    nil,
-		statsService:    nil,
+		statsService:     nil,
 
 		msFactory: factory,
 	}
@@ -167,15 +166,13 @@ func (node *QueryNode) Start() error {
 	}
 
 	// init services and manager
-	node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, node.msFactory)
 	node.searchService = newSearchService(node.queryNodeLoopCtx, node.replica, node.msFactory)
 	//node.metaService = newMetaService(node.queryNodeLoopCtx, node.replica)
 
-	node.loadService = newLoadService(node.queryNodeLoopCtx, node.masterService, node.dataService, node.indexService, node.replica, node.dataSyncService.dmStream)
+	node.loadService = newLoadService(node.queryNodeLoopCtx, node.masterService, node.dataService, node.indexService, node.replica)
 	node.statsService = newStatsService(node.queryNodeLoopCtx, node.replica, node.loadService.segLoader.indexLoader.fieldStatsChan, node.msFactory)
 
 	// start services
-	go node.dataSyncService.start()
 	go node.searchService.start()
 	//go node.metaService.start()
 	go node.loadService.start()
@@ -192,8 +189,10 @@ func (node *QueryNode) Stop() error {
 	node.replica.freeAll()
 
 	// close services
-	if node.dataSyncService != nil {
+	for _, dsService := range node.dataSyncServices {
-		node.dataSyncService.close()
+		if dsService != nil {
+			dsService.close()
+		}
 	}
 	if node.searchService != nil {
 		node.searchService.close()
@@ -366,17 +365,20 @@ func (node *QueryNode) RemoveQueryChannel(ctx context.Context, in *queryPb.Remov
 }
 
 func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmChannelsRequest) (*commonpb.Status, error) {
-	if node.dataSyncService == nil || node.dataSyncService.dmStream == nil {
+	log.Debug("starting WatchDmChannels ...", zap.String("ChannelIDs", fmt.Sprintln(in.ChannelIDs)))
-		errMsg := "null data sync service or null data manipulation stream"
+	collectionID := in.CollectionID
+	service, ok := node.dataSyncServices[collectionID]
+	if !ok || service.dmStream == nil {
+		errMsg := "null data sync service or null data manipulation stream, collectionID = " + fmt.Sprintln(collectionID)
 		status := &commonpb.Status{
 			ErrorCode: commonpb.ErrorCode_UnexpectedError,
 			Reason:    errMsg,
 		}
-
+		log.Error(errMsg)
 		return status, errors.New(errMsg)
 	}
 
-	switch t := node.dataSyncService.dmStream.(type) {
+	switch t := service.dmStream.(type) {
 	case *pulsarms.PulsarTtMsgStream:
 	case *rmqms.RmqTtMsgStream:
 	default:
@@ -386,19 +388,61 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC
 			ErrorCode: commonpb.ErrorCode_UnexpectedError,
 			Reason:    errMsg,
 		}
-
+		log.Error(errMsg)
 		return status, errors.New(errMsg)
 	}
 
+	getUniqueSubName := func() string {
+		prefixName := Params.MsgChannelSubName
+		return prefixName + "-" + strconv.FormatInt(collectionID, 10)
+	}
+
 	// add request channel
 	consumeChannels := in.ChannelIDs
-	consumeSubName := Params.MsgChannelSubName
+	toSeekInfo := make([]*internalpb.MsgPosition, 0)
-	node.dataSyncService.dmStream.AsConsumer(consumeChannels, consumeSubName)
+	toDirSubChannels := make([]string, 0)
+
+	consumeSubName := getUniqueSubName()
+
+	for _, info := range in.Infos {
+		if len(info.Pos.MsgID) == 0 {
+			toDirSubChannels = append(toDirSubChannels, info.ChannelID)
+			continue
+		}
+		info.Pos.MsgGroup = consumeSubName
+		toSeekInfo = append(toSeekInfo, info.Pos)
+
+		log.Debug("prevent inserting segments", zap.String("segmentIDs", fmt.Sprintln(info.ExcludedSegments)))
+		err := node.replica.addExcludedSegments(collectionID, info.ExcludedSegments)
+		if err != nil {
+			status := &commonpb.Status{
+				ErrorCode: commonpb.ErrorCode_UnexpectedError,
+				Reason:    err.Error(),
+			}
+			log.Error(err.Error())
+			return status, err
+		}
+	}
+
+	service.dmStream.AsConsumer(toDirSubChannels, consumeSubName)
+	for _, pos := range toSeekInfo {
+		err := service.dmStream.Seek(pos)
+		if err != nil {
+			errMsg := "msgStream seek error :" + err.Error()
+			status := &commonpb.Status{
+				ErrorCode: commonpb.ErrorCode_UnexpectedError,
+				Reason:    errMsg,
+			}
+			log.Error(errMsg)
+			return status, errors.New(errMsg)
+		}
+	}
 	log.Debug("querynode AsConsumer: " + strings.Join(consumeChannels, ", ") + " : " + consumeSubName)
 
 	status := &commonpb.Status{
 		ErrorCode: commonpb.ErrorCode_Success,
 	}
+	log.Debug("WatchDmChannels done", zap.String("ChannelIDs", fmt.Sprintln(in.ChannelIDs)))
 	return status, nil
 }
 
@@ -418,12 +462,18 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment
 	hasCollection := node.replica.hasCollection(collectionID)
 	hasPartition := node.replica.hasPartition(partitionID)
 	if !hasCollection {
+		// loading init
 		err := node.replica.addCollection(collectionID, schema)
 		if err != nil {
 			status.ErrorCode = commonpb.ErrorCode_UnexpectedError
 			status.Reason = err.Error()
 			return status, err
 		}
+		node.replica.initExcludedSegments(collectionID)
+		node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, node.msFactory, collectionID)
+		go node.dataSyncServices[collectionID].start()
+		node.replica.addTSafe(collectionID)
+		node.searchService.register(collectionID)
 	}
 	if !hasPartition {
 		err := node.replica.addPartition(collectionID, partitionID)
@@ -444,48 +494,28 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment
 		return status, nil
 	}
 
-	if len(in.SegmentIDs) != len(in.SegmentStates) {
+	err = node.loadService.loadSegmentPassively(collectionID, partitionID, segmentIDs, fieldIDs)
-		err := errors.New("len(segmentIDs) should equal to len(segmentStates)")
-		status.ErrorCode = commonpb.ErrorCode_UnexpectedError
-		status.Reason = err.Error()
-		return status, err
-	}
-
-	// segments are ordered before LoadSegments calling
-	//var position *internalpb.MsgPosition = nil
-	for i, state := range in.SegmentStates {
-		//thisPosition := state.StartPosition
-		if state.State <= commonpb.SegmentState_Growing {
-			//if position == nil {
-			//	position = &internalpb2.MsgPosition{
-			//		ChannelName: thisPosition.ChannelName,
-			//	}
-			//}
-			segmentIDs = segmentIDs[:i]
-			break
-		}
-		//position = state.StartPosition
-	}
-
-	//err = node.dataSyncService.seekSegment(position)
-	//if err != nil {
-	//	status := &commonpb.Status{
-	//		ErrorCode: commonpb.ErrorCode_UnexpectedError,
-	//		Reason:    err.Error(),
-	//	}
-	//	return status, err
-	//}
-
-	err = node.loadService.loadSegment(collectionID, partitionID, segmentIDs, fieldIDs)
 	if err != nil {
 		status.ErrorCode = commonpb.ErrorCode_UnexpectedError
 		status.Reason = err.Error()
 		return status, err
 	}
+
+	log.Debug("LoadSegments done", zap.String("segmentIDs", fmt.Sprintln(in.SegmentIDs)))
 	return status, nil
 }
 
 func (node *QueryNode) ReleaseCollection(ctx context.Context, in *queryPb.ReleaseCollectionRequest) (*commonpb.Status, error) {
+	if _, ok := node.dataSyncServices[in.CollectionID]; ok {
+		node.dataSyncServices[in.CollectionID].close()
+		delete(node.dataSyncServices, in.CollectionID)
+		node.searchService.tSafeMutex.Lock()
+		delete(node.searchService.tSafeWatcher, in.CollectionID)
+		node.searchService.tSafeMutex.Unlock()
+		node.replica.removeTSafe(in.CollectionID)
+		node.replica.removeExcludedSegments(in.CollectionID)
+	}
+
 	err := node.replica.removeCollection(in.CollectionID)
 	if err != nil {
 		status := &commonpb.Status{
@@ -495,6 +525,7 @@ func (node *QueryNode) ReleaseCollection(ctx context.Context, in *queryPb.Releas
 		return status, err
 	}
 
+	log.Debug("ReleaseCollection done", zap.Int64("collectionID", in.CollectionID))
 	return &commonpb.Status{
 		ErrorCode: commonpb.ErrorCode_Success,
 	}, nil

internal/querynode/search_service.go

@@ -4,12 +4,12 @@ import "C"
 import (
 	"context"
 	"errors"
+	"fmt"
+	"go.uber.org/zap"
 	"strconv"
 	"strings"
 	"sync"
 
-	"go.uber.org/zap"
-
 	"github.com/golang/protobuf/proto"
 	"github.com/zilliztech/milvus-distributed/internal/log"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
@@ -24,13 +24,14 @@ type searchService struct {
 	cancel context.CancelFunc
 
 	replica      ReplicaInterface
-	tSafeWatcher *tSafeWatcher
+	tSafeMutex   *sync.Mutex
+	tSafeWatcher map[UniqueID]*tSafeWatcher
 
 	serviceableTimeMutex sync.Mutex // guards serviceableTime
-	serviceableTime      Timestamp
+	serviceableTime      map[UniqueID]Timestamp
 
-	msgBuffer             chan msgstream.TsMsg
+	msgBuffer             chan *msgstream.SearchMsg
-	unsolvedMsg           []msgstream.TsMsg
+	unsolvedMsg           []*msgstream.SearchMsg
 	searchMsgStream       msgstream.MsgStream
 	searchResultMsgStream msgstream.MsgStream
 	queryNodeID           UniqueID
@@ -54,17 +55,18 @@ func newSearchService(ctx context.Context, replica ReplicaInterface, factory msg
 	log.Debug("querynode AsProducer: " + strings.Join(producerChannels, ", "))
 
 	searchServiceCtx, searchServiceCancel := context.WithCancel(ctx)
-	msgBuffer := make(chan msgstream.TsMsg, receiveBufSize)
+	msgBuffer := make(chan *msgstream.SearchMsg, receiveBufSize)
-	unsolvedMsg := make([]msgstream.TsMsg, 0)
+	unsolvedMsg := make([]*msgstream.SearchMsg, 0)
 	return &searchService{
 		ctx:             searchServiceCtx,
 		cancel:          searchServiceCancel,
-		serviceableTime: Timestamp(0),
+		serviceableTime: make(map[UniqueID]Timestamp),
 		msgBuffer:       msgBuffer,
 		unsolvedMsg:     unsolvedMsg,
 
 		replica:      replica,
-		tSafeWatcher: newTSafeWatcher(),
+		tSafeMutex:   &sync.Mutex{},
+		tSafeWatcher: make(map[UniqueID]*tSafeWatcher),
 
 		searchMsgStream:       searchStream,
 		searchResultMsgStream: searchResultStream,
@@ -75,7 +77,6 @@ func newSearchService(ctx context.Context, replica ReplicaInterface, factory msg
 func (ss *searchService) start() {
 	ss.searchMsgStream.Start()
 	ss.searchResultMsgStream.Start()
-	ss.register()
 	ss.wait.Add(2)
 	go ss.receiveSearchMsg()
 	go ss.doUnsolvedMsgSearch()
@@ -92,32 +93,63 @@ func (ss *searchService) close() {
 	ss.cancel()
 }
 
-func (ss *searchService) register() {
+func (ss *searchService) register(collectionID UniqueID) {
-	tSafe := ss.replica.getTSafe()
+	tSafe := ss.replica.getTSafe(collectionID)
-	tSafe.registerTSafeWatcher(ss.tSafeWatcher)
+	ss.tSafeMutex.Lock()
+	ss.tSafeWatcher[collectionID] = newTSafeWatcher()
+	ss.tSafeMutex.Unlock()
+	tSafe.registerTSafeWatcher(ss.tSafeWatcher[collectionID])
 }
 
-func (ss *searchService) waitNewTSafe() Timestamp {
+func (ss *searchService) waitNewTSafe(collectionID UniqueID) (Timestamp, error) {
 	// block until dataSyncService updating tSafe
-	ss.tSafeWatcher.hasUpdate()
+	ss.tSafeWatcher[collectionID].hasUpdate()
-	timestamp := ss.replica.getTSafe().get()
+	ts := ss.replica.getTSafe(collectionID)
-	return timestamp
+	if ts != nil {
+		return ts.get(), nil
+	}
+	return 0, errors.New("tSafe closed, collectionID =" + fmt.Sprintln(collectionID))
 }
 
-func (ss *searchService) getServiceableTime() Timestamp {
+func (ss *searchService) getServiceableTime(collectionID UniqueID) Timestamp {
 	ss.serviceableTimeMutex.Lock()
 	defer ss.serviceableTimeMutex.Unlock()
-	return ss.serviceableTime
+	//t, ok := ss.serviceableTime[collectionID]
+	//if !ok {
+	//	return 0, errors.New("cannot found")
+	//}
+	return ss.serviceableTime[collectionID]
 }
 
-func (ss *searchService) setServiceableTime(t Timestamp) {
+func (ss *searchService) setServiceableTime(collectionID UniqueID, t Timestamp) {
 	ss.serviceableTimeMutex.Lock()
 	// hard code gracefultime to 1 second
 	// TODO: use config to set gracefultime
-	ss.serviceableTime = t + 1000*1000*1000
+	ss.serviceableTime[collectionID] = t + 1000*1000*1000
 	ss.serviceableTimeMutex.Unlock()
 }
 
+func (ss *searchService) collectionCheck(collectionID UniqueID) error {
+	// check if collection exists
+	if _, ok := ss.tSafeWatcher[collectionID]; !ok {
+		err := errors.New("no collection found, collectionID = " + strconv.FormatInt(collectionID, 10))
+		log.Error(err.Error())
+		return err
+	}
+	return nil
+}
+
+func (ss *searchService) emptySearch(searchMsg *msgstream.SearchMsg) {
+	err := ss.search(searchMsg)
+	if err != nil {
+		log.Error(err.Error())
+		err2 := ss.publishFailedSearchResult(searchMsg, err.Error())
+		if err2 != nil {
+			log.Error("publish FailedSearchResult failed", zap.Error(err2))
+		}
+	}
+}
+
 func (ss *searchService) receiveSearchMsg() {
 	defer ss.wait.Done()
 	for {
@@ -129,26 +161,34 @@ func (ss *searchService) receiveSearchMsg() {
 			if msgPack == nil || len(msgPack.Msgs) <= 0 {
 				continue
 			}
-			searchMsg := make([]msgstream.TsMsg, 0)
+			searchNum := 0
-			serverTime := ss.getServiceableTime()
+			for _, msg := range msgPack.Msgs {
-			for i, msg := range msgPack.Msgs {
+				sm, ok := msg.(*msgstream.SearchMsg)
-				if msg.BeginTs() > serverTime {
+				if !ok {
-					ss.msgBuffer <- msg
 					continue
 				}
-				searchMsg = append(searchMsg, msgPack.Msgs[i])
+				err := ss.collectionCheck(sm.CollectionID)
-			}
+				if err != nil {
-			for _, msg := range searchMsg {
+					ss.emptySearch(sm)
-				err := ss.search(msg)
+					searchNum++
+					continue
+				}
+				serviceTime := ss.getServiceableTime(sm.CollectionID)
+				if msg.BeginTs() > serviceTime {
+					ss.msgBuffer <- sm
+					continue
+				}
+				err = ss.search(sm)
 				if err != nil {
 					log.Error(err.Error())
-					err2 := ss.publishFailedSearchResult(msg, err.Error())
+					err2 := ss.publishFailedSearchResult(sm, err.Error())
 					if err2 != nil {
 						log.Error("publish FailedSearchResult failed", zap.Error(err2))
 					}
 				}
+				searchNum++
 			}
-			log.Debug("ReceiveSearchMsg, do search done", zap.Int("num of searchMsg", len(searchMsg)))
+			log.Debug("ReceiveSearchMsg, do search done", zap.Int("num of searchMsg", searchNum))
 		}
 	}
 }
@@ -160,18 +200,36 @@ func (ss *searchService) doUnsolvedMsgSearch() {
 		case <-ss.ctx.Done():
 			return
 		default:
-			serviceTime := ss.waitNewTSafe()
+			searchMsg := make([]*msgstream.SearchMsg, 0)
-			ss.setServiceableTime(serviceTime)
+			tempMsg := make([]*msgstream.SearchMsg, 0)
-			searchMsg := make([]msgstream.TsMsg, 0)
-			tempMsg := make([]msgstream.TsMsg, 0)
 			tempMsg = append(tempMsg, ss.unsolvedMsg...)
 			ss.unsolvedMsg = ss.unsolvedMsg[:0]
-			for _, msg := range tempMsg {
+
-				if msg.EndTs() <= serviceTime {
+			serviceTimeTmpTable := make(map[UniqueID]Timestamp)
-					searchMsg = append(searchMsg, msg)
+
+			searchNum := 0
+			for _, sm := range tempMsg {
+				err := ss.collectionCheck(sm.CollectionID)
+				if err != nil {
+					ss.emptySearch(sm)
+					searchNum++
 					continue
 				}
-				ss.unsolvedMsg = append(ss.unsolvedMsg, msg)
+				_, ok := serviceTimeTmpTable[sm.CollectionID]
+				if !ok {
+					serviceTime, err := ss.waitNewTSafe(sm.CollectionID)
+					if err != nil {
+						// TODO: emptySearch or continue, note: collection has been released
+						continue
+					}
+					ss.setServiceableTime(sm.CollectionID, serviceTime)
+					serviceTimeTmpTable[sm.CollectionID] = serviceTime
+				}
+				if sm.EndTs() <= serviceTimeTmpTable[sm.CollectionID] {
+					searchMsg = append(searchMsg, sm)
+					continue
+				}
+				ss.unsolvedMsg = append(ss.unsolvedMsg, sm)
 			}
 
 			for {
@@ -179,40 +237,52 @@ func (ss *searchService) doUnsolvedMsgSearch() {
 				if msgBufferLength <= 0 {
 					break
 				}
-				msg := <-ss.msgBuffer
+				sm := <-ss.msgBuffer
-				if msg.EndTs() <= serviceTime {
+				err := ss.collectionCheck(sm.CollectionID)
-					searchMsg = append(searchMsg, msg)
+				if err != nil {
+					ss.emptySearch(sm)
+					searchNum++
 					continue
 				}
-				ss.unsolvedMsg = append(ss.unsolvedMsg, msg)
+				_, ok := serviceTimeTmpTable[sm.CollectionID]
+				if !ok {
+					serviceTime, err := ss.waitNewTSafe(sm.CollectionID)
+					if err != nil {
+						// TODO: emptySearch or continue, note: collection has been released
+						continue
+					}
+					ss.setServiceableTime(sm.CollectionID, serviceTime)
+					serviceTimeTmpTable[sm.CollectionID] = serviceTime
+				}
+				if sm.EndTs() <= serviceTimeTmpTable[sm.CollectionID] {
+					searchMsg = append(searchMsg, sm)
+					continue
+				}
+				ss.unsolvedMsg = append(ss.unsolvedMsg, sm)
 			}
 
 			if len(searchMsg) <= 0 {
 				continue
 			}
-			for _, msg := range searchMsg {
+			for _, sm := range searchMsg {
-				err := ss.search(msg)
+				err := ss.search(sm)
 				if err != nil {
 					log.Error(err.Error())
-					err2 := ss.publishFailedSearchResult(msg, err.Error())
+					err2 := ss.publishFailedSearchResult(sm, err.Error())
 					if err2 != nil {
 						log.Error("publish FailedSearchResult failed", zap.Error(err2))
 					}
 				}
+				searchNum++
 			}
-			log.Debug("doUnsolvedMsgSearch, do search done", zap.Int("num of searchMsg", len(searchMsg)))
+			log.Debug("doUnsolvedMsgSearch, do search done", zap.Int("num of searchMsg", searchNum))
 		}
 	}
 }
 
 // TODO:: cache map[dsl]plan
 // TODO: reBatched search requests
-func (ss *searchService) search(msg msgstream.TsMsg) error {
+func (ss *searchService) search(searchMsg *msgstream.SearchMsg) error {
-	searchMsg, ok := msg.(*msgstream.SearchMsg)
-	if !ok {
-		return errors.New("invalid request type = " + string(msg.Type()))
-	}
-
 	searchTimestamp := searchMsg.Base.Timestamp
 	var queryBlob = searchMsg.Query.Value
 	query := milvuspb.SearchRequest{}
@@ -250,21 +320,17 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
 	partitionIDsInQuery := searchMsg.PartitionIDs
 	if len(partitionIDsInQuery) == 0 {
 		if len(partitionIDsInCol) == 0 {
-			return errors.New("can't find any partition in this collection on query node")
+			return errors.New("none of this collection's partition has been loaded")
 		}
 		searchPartitionIDs = partitionIDsInCol
 	} else {
-		findPartition := false
 		for _, id := range partitionIDsInQuery {
-			_, err := ss.replica.getPartitionByID(id)
+			_, err2 := ss.replica.getPartitionByID(id)
-			if err == nil {
+			if err2 != nil {
-				searchPartitionIDs = append(searchPartitionIDs, id)
+				return err2
-				findPartition = true
 			}
 		}
-		if !findPartition {
+		searchPartitionIDs = partitionIDsInQuery
-			return errors.New("partition to be searched not exist in query node")
-		}
 	}
 
 	for _, partitionID := range searchPartitionIDs {
@@ -380,14 +446,15 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
 		}
 
 		// For debugging, please don't delete.
+		//fmt.Println("==================== search result ======================")
 		//for i := 0; i < len(hits); i++ {
 		//	testHits := milvuspb.Hits{}
 		//	err := proto.Unmarshal(hits[i], &testHits)
 		//	if err != nil {
 		//		panic(err)
 		//	}
-		//	log.Debug(testHits.IDs)
+		//	fmt.Println(testHits.IDs)
-		//	log.Debug(testHits.Scores)
+		//	fmt.Println(testHits.Scores)
 		//}
 		err = ss.publishSearchResult(searchResultMsg)
 		if err != nil {
@@ -412,16 +479,12 @@ func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error {
 	return err
 }
 
-func (ss *searchService) publishFailedSearchResult(msg msgstream.TsMsg, errMsg string) error {
+func (ss *searchService) publishFailedSearchResult(searchMsg *msgstream.SearchMsg, errMsg string) error {
 	// span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "receive search msg")
 	// defer span.Finish()
 	// msg.SetMsgContext(ctx)
 	//log.Debug("Public fail SearchResult!")
 	msgPack := msgstream.MsgPack{}
-	searchMsg, ok := msg.(*msgstream.SearchMsg)
-	if !ok {
-		return errors.New("invalid request type = " + string(msg.Type()))
-	}
 
 	resultChannelInt, _ := strconv.ParseInt(searchMsg.ResultChannelID, 10, 64)
 	searchResultMsg := &msgstream.SearchResultMsg{

internal/querynode/search_service_test.go

@@ -21,6 +21,8 @@ import (
 func TestSearch_Search(t *testing.T) {
 	ctx := context.Background()
 
+	collectionID := UniqueID(0)
+
 	node := newQueryNodeMock()
 	initTestMeta(t, node, 0, 0)
 
@@ -111,6 +113,8 @@ func TestSearch_Search(t *testing.T) {
 
 	node.searchService = newSearchService(node.queryNodeLoopCtx, node.replica, msFactory)
 	go node.searchService.start()
+	node.replica.addTSafe(collectionID)
+	node.searchService.register(collectionID)
 
 	// start insert
 	timeRange := TimeRange{
@@ -143,7 +147,7 @@ func TestSearch_Search(t *testing.T) {
 					Timestamp: uint64(10 + 1000),
 					SourceID:  0,
 				},
-				CollectionID: UniqueID(0),
+				CollectionID: collectionID,
 				PartitionID:  defaultPartitionID,
 				SegmentID:    int64(0),
 				ChannelID:    "0",
@@ -209,8 +213,8 @@ func TestSearch_Search(t *testing.T) {
 	assert.NoError(t, err)
 
 	// dataSync
-	node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory)
+	node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory, collectionID)
-	go node.dataSyncService.start()
+	go node.dataSyncServices[collectionID].start()
 
 	time.Sleep(1 * time.Second)
 
@@ -219,6 +223,9 @@ func TestSearch_Search(t *testing.T) {
 
 func TestSearch_SearchMultiSegments(t *testing.T) {
 	ctx := context.Background()
+
+	collectionID := UniqueID(0)
+
 	pulsarURL := Params.PulsarAddress
 	const receiveBufSize = 1024
 
@@ -309,6 +316,8 @@ func TestSearch_SearchMultiSegments(t *testing.T) {
 
 	node.searchService = newSearchService(node.queryNodeLoopCtx, node.replica, msFactory)
 	go node.searchService.start()
+	node.replica.addTSafe(collectionID)
+	node.searchService.register(collectionID)
 
 	// start insert
 	timeRange := TimeRange{
@@ -345,7 +354,7 @@ func TestSearch_SearchMultiSegments(t *testing.T) {
 					Timestamp: uint64(i + 1000),
 					SourceID:  0,
 				},
-				CollectionID: UniqueID(0),
+				CollectionID: collectionID,
 				PartitionID:  defaultPartitionID,
 				SegmentID:    int64(segmentID),
 				ChannelID:    "0",
@@ -411,8 +420,8 @@ func TestSearch_SearchMultiSegments(t *testing.T) {
 	assert.NoError(t, err)
 
 	// dataSync
-	node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory)
+	node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory, collectionID)
-	go node.dataSyncService.start()
+	go node.dataSyncServices[collectionID].start()
 
 	time.Sleep(1 * time.Second)
 

internal/querynode/segment.go

@@ -12,6 +12,7 @@ package querynode
 */
 import "C"
 import (
+	"fmt"
 	"strconv"
 	"sync"
 	"unsafe"
@@ -31,7 +32,7 @@ const (
 	segmentTypeInvalid segmentType = iota
 	segmentTypeGrowing
 	segmentTypeSealed
-	segTypeIndexing
+	segmentTypeIndexing
 )
 
 type indexParam = map[string]string
@@ -268,34 +269,6 @@ func (s *Segment) fillTargetEntry(plan *Plan,
 	return nil
 }
 
-// segment, err := loadService.replica.getSegmentByID(segmentID)
-func (s *Segment) updateSegmentIndex(loadIndexInfo *LoadIndexInfo) error {
-	if s.segmentPtr == nil {
-		return errors.New("null seg core pointer")
-	}
-	var status C.CStatus
-
-	if s.segmentType == segmentTypeGrowing {
-		status = C.UpdateSegmentIndex(s.segmentPtr, loadIndexInfo.cLoadIndexInfo)
-	} else if s.segmentType == segmentTypeSealed {
-		status = C.UpdateSealedSegmentIndex(s.segmentPtr, loadIndexInfo.cLoadIndexInfo)
-	} else {
-		return errors.New("illegal segment type")
-	}
-
-	errorCode := status.error_code
-
-	if errorCode != 0 {
-		errorMsg := C.GoString(status.error_msg)
-		defer C.free(unsafe.Pointer(status.error_msg))
-		return errors.New("updateSegmentIndex failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
-	}
-
-	s.setType(segTypeIndexing)
-
-	return nil
-}
-
 func (s *Segment) setIndexParam(fieldID int64, indexParamKv []*commonpb.KeyValuePair) error {
 	s.paramMutex.Lock()
 	defer s.paramMutex.Unlock()
@@ -461,7 +434,8 @@ func (s *Segment) segmentLoadFieldData(fieldID int64, rowCount int, data interfa
 		return errors.New("null seg core pointer")
 	}
 	if s.segmentType != segmentTypeSealed {
-		return errors.New("illegal segment type when loading field data")
+		errMsg := fmt.Sprintln("segmentLoadFieldData failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
+		return errors.New(errMsg)
 	}
 
 	// data interface check
@@ -536,7 +510,86 @@ func (s *Segment) segmentLoadFieldData(fieldID int64, rowCount int, data interfa
 		return errors.New("LoadFieldData failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
 	}
 
-	log.Debug("load field done", zap.Int64("fieldID", fieldID), zap.Int("row count", rowCount))
+	log.Debug("load field done",
+		zap.Int64("fieldID", fieldID),
+		zap.Int("row count", rowCount),
+		zap.Int64("segmentID", s.ID()))
+
+	return nil
+}
+
+func (s *Segment) dropFieldData(fieldID int64) error {
+	/*
+		CStatus
+		DropFieldData(CSegmentInterface c_segment, int64_t field_id);
+	*/
+	if s.segmentPtr == nil {
+		return errors.New("null seg core pointer")
+	}
+	if s.segmentType != segmentTypeIndexing {
+		errMsg := fmt.Sprintln("dropFieldData failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
+		return errors.New(errMsg)
+	}
+
+	var status = C.DropFieldData(s.segmentPtr, C.long(fieldID))
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return errors.New("dropFieldData failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
+
+	log.Debug("dropFieldData done", zap.Int64("fieldID", fieldID), zap.Int64("segmentID", s.ID()))
+
+	return nil
+}
+
+func (s *Segment) updateSegmentIndex(loadIndexInfo *LoadIndexInfo) error {
+	if s.segmentPtr == nil {
+		return errors.New("null seg core pointer")
+	}
+
+	if s.segmentType != segmentTypeSealed {
+		errMsg := fmt.Sprintln("updateSegmentIndex failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
+		return errors.New(errMsg)
+	}
+
+	status := C.UpdateSealedSegmentIndex(s.segmentPtr, loadIndexInfo.cLoadIndexInfo)
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return errors.New("updateSegmentIndex failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
+
+	s.setType(segmentTypeIndexing)
+	log.Debug("updateSegmentIndex done", zap.Int64("segmentID", s.ID()))
+
+	return nil
+}
+
+func (s *Segment) dropSegmentIndex(fieldID int64) error {
+	/*
+		CStatus
+		DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id);
+	*/
+	if s.segmentPtr == nil {
+		return errors.New("null seg core pointer")
+	}
+	if s.segmentType != segmentTypeIndexing {
+		errMsg := fmt.Sprintln("dropFieldData failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
+		return errors.New(errMsg)
+	}
+
+	var status = C.DropSealedSegmentIndex(s.segmentPtr, C.long(fieldID))
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return errors.New("dropSegmentIndex failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
+
+	log.Debug("dropSegmentIndex done", zap.Int64("fieldID", fieldID), zap.Int64("segmentID", s.ID()))
 
 	return nil
 }

internal/querynode/segment_loader.go

@@ -2,31 +2,25 @@ package querynode
 
 import (
 	"context"
-	"strconv"
-
-	"github.com/zilliztech/milvus-distributed/internal/types"
-
 	"errors"
+	"strconv"
 
 	"github.com/zilliztech/milvus-distributed/internal/kv"
 	minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio"
-	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/datapb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
 	"github.com/zilliztech/milvus-distributed/internal/storage"
+	"github.com/zilliztech/milvus-distributed/internal/types"
 )
 
 // segmentLoader is only responsible for loading the field data from binlog
 type segmentLoader struct {
 	replica ReplicaInterface
 
-	dmStream msgstream.MsgStream
-
 	dataService types.DataService
 
-	kv     kv.Base // minio kv
+	kv kv.Base // minio kv
-	iCodec *storage.InsertCodec
 
 	indexLoader *indexLoader
 }
@@ -117,6 +111,8 @@ func (loader *segmentLoader) checkTargetFields(paths []*internalpb.StringList, s
 }
 
 func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetFields map[int64]*internalpb.StringList) error {
+	iCodec := storage.InsertCodec{}
+	defer iCodec.Close()
 	for id, p := range targetFields {
 		if id == timestampFieldID {
 			// seg core doesn't need timestamp field
@@ -136,7 +132,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetField
 				Value: []byte(binLog),
 			})
 		}
-		_, _, insertData, err := loader.iCodec.Deserialize(blobs)
+		_, _, insertData, err := iCodec.Deserialize(blobs)
 		if err != nil {
 			// TODO: return or continue
 			return err
@@ -193,7 +189,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetField
 	return nil
 }
 
-func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface, dmStream msgstream.MsgStream) *segmentLoader {
+func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface) *segmentLoader {
 	option := &minioKV.Option{
 		Address:           Params.MinioEndPoint,
 		AccessKeyID:       Params.MinioAccessKeyID,
@@ -212,12 +208,9 @@ func newSegmentLoader(ctx context.Context, masterService types.MasterService, in
 	return &segmentLoader{
 		replica: replica,
 
-		dmStream: dmStream,
-
 		dataService: dataService,
 
-		kv:     client,
+		kv: client,
-		iCodec: &storage.InsertCodec{},
 
 		indexLoader: iLoader,
 	}

internal/querynode/tsafe.go

@@ -28,6 +28,7 @@ type tSafer interface {
 	get() Timestamp
 	set(t Timestamp)
 	registerTSafeWatcher(t *tSafeWatcher)
+	close()
 }
 
 type tSafe struct {
@@ -64,3 +65,12 @@ func (ts *tSafe) set(t Timestamp) {
 		watcher.notify()
 	}
 }
+
+func (ts *tSafe) close() {
+	ts.tSafeMu.Lock()
+	defer ts.tSafeMu.Unlock()
+
+	for _, watcher := range ts.watcherList {
+		close(watcher.notifyChan)
+	}
+}

internal/queryservice/meta_replica.go

@@ -1,10 +1,9 @@
 package queryservice
 
 import (
-	"strconv"
-
 	"errors"
 
+	internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/querypb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
 )
@@ -21,8 +20,9 @@ type Replica interface {
 	getPartitionStates(dbID UniqueID, collectionID UniqueID, partitionIDs []UniqueID) ([]*querypb.PartitionStates, error)
 	releaseCollection(dbID UniqueID, collectionID UniqueID) error
 	releasePartition(dbID UniqueID, collectionID UniqueID, partitionID UniqueID) error
-	addDmChannels(dbID UniqueID, collectionID UniqueID, channels2NodeID map[string]int64) error
+	addDmChannel(dbID UniqueID, collectionID UniqueID, channel string, watchedStartPos *internalPb.MsgPosition) error
-	getAssignedNodeIDByChannelName(dbID UniqueID, collectionID UniqueID, channel string) (int64, error)
+	addExcludeSegmentIDs(dbID UniqueID, collectionID UniqueID, excludeSegments []UniqueID) error
+	//getAssignedNodeIDByChannelName(dbID UniqueID, collectionID UniqueID, channel string) (int64, error)
 }
 
 type segment struct {
@@ -36,10 +36,12 @@ type partition struct {
 }
 
 type collection struct {
-	id              UniqueID
+	id                UniqueID
-	partitions      map[UniqueID]*partition
+	partitions        map[UniqueID]*partition
-	dmChannels2Node map[string]int64
+	dmChannels        []string
-	schema          *schemapb.CollectionSchema
+	dmChannels2Pos    map[string]*internalPb.MsgPosition
+	excludeSegmentIds []UniqueID
+	schema            *schemapb.CollectionSchema
 }
 
 type metaReplica struct {
@@ -62,12 +64,16 @@ func (mp *metaReplica) addCollection(dbID UniqueID, collectionID UniqueID, schem
 	//TODO:: assert dbID = 0 exist
 	if _, ok := mp.db2collections[dbID]; ok {
 		partitions := make(map[UniqueID]*partition)
-		channels := make(map[string]int64)
+		channels := make([]string, 0)
+		startPos := make(map[string]*internalPb.MsgPosition)
+		excludeSegmentIDs := make([]UniqueID, 0)
 		newCollection := &collection{
-			id:              collectionID,
+			id:                collectionID,
-			partitions:      partitions,
+			partitions:        partitions,
-			schema:          schema,
+			schema:            schema,
-			dmChannels2Node: channels,
+			dmChannels:        channels,
+			dmChannels2Pos:    startPos,
+			excludeSegmentIds: excludeSegmentIDs,
 		}
 		mp.db2collections[dbID] = append(mp.db2collections[dbID], newCollection)
 		return nil
@@ -216,8 +222,7 @@ func (mp *metaReplica) releaseCollection(dbID UniqueID, collectionID UniqueID) e
 		}
 	}
 
-	errorStr := "releaseCollection: can't find dbID or collectionID " + strconv.FormatInt(collectionID, 10)
+	return nil
-	return errors.New(errorStr)
 }
 
 func (mp *metaReplica) releasePartition(dbID UniqueID, collectionID UniqueID, partitionID UniqueID) error {
@@ -232,17 +237,15 @@ func (mp *metaReplica) releasePartition(dbID UniqueID, collectionID UniqueID, pa
 		}
 	}
 
-	errorStr := "releasePartition: can't find dbID or collectionID or partitionID " + strconv.FormatInt(partitionID, 10)
+	return nil
-	return errors.New(errorStr)
 }
 
-func (mp *metaReplica) addDmChannels(dbID UniqueID, collectionID UniqueID, channels2NodeID map[string]int64) error {
+func (mp *metaReplica) addDmChannel(dbID UniqueID, collectionID UniqueID, channel string, watchedStartPos *internalPb.MsgPosition) error {
 	if collections, ok := mp.db2collections[dbID]; ok {
 		for _, collection := range collections {
 			if collectionID == collection.id {
-				for channel, id := range channels2NodeID {
+				collection.dmChannels = append(collection.dmChannels, channel)
-					collection.dmChannels2Node[channel] = id
+				collection.dmChannels2Pos[channel] = watchedStartPos
-				}
 				return nil
 			}
 		}
@@ -250,16 +253,14 @@ func (mp *metaReplica) addDmChannels(dbID UniqueID, collectionID UniqueID, chann
 	return errors.New("addDmChannels: can't find dbID or collectionID")
 }
 
-func (mp *metaReplica) getAssignedNodeIDByChannelName(dbID UniqueID, collectionID UniqueID, channel string) (int64, error) {
+func (mp *metaReplica) addExcludeSegmentIDs(dbID UniqueID, collectionID UniqueID, excludeSegments []UniqueID) error {
 	if collections, ok := mp.db2collections[dbID]; ok {
 		for _, collection := range collections {
 			if collectionID == collection.id {
-				if id, ok := collection.dmChannels2Node[channel]; ok {
+				collection.excludeSegmentIds = append(collection.excludeSegmentIds, excludeSegments...)
-					return id, nil
+				return nil
-				}
 			}
 		}
 	}
-
+	return errors.New("addExcludeSegmentIDs: can't find dbID or collectionID")
-	return 0, errors.New("getAssignedNodeIDByChannelName: can't find dbID or collectionID")
 }

internal/queryservice/querynode.go

@@ -10,9 +10,9 @@ import (
 )
 
 type queryNodeInfo struct {
-	client         types.QueryNode
+	client       types.QueryNode
-	segments       []UniqueID
+	segments     map[UniqueID][]UniqueID
-	dmChannelNames []string
+	channels2Col map[UniqueID][]string
 }
 
 func (qn *queryNodeInfo) GetComponentStates(ctx context.Context) (*internalpb.ComponentStates, error) {
@@ -31,8 +31,20 @@ func (qn *queryNodeInfo) WatchDmChannels(ctx context.Context, in *querypb.WatchD
 	return qn.client.WatchDmChannels(ctx, in)
 }
 
-func (qn *queryNodeInfo) AddDmChannels(channels []string) {
+func (qn *queryNodeInfo) AddDmChannels(channels []string, collectionID UniqueID) {
-	qn.dmChannelNames = append(qn.dmChannelNames, channels...)
+	if _, ok := qn.channels2Col[collectionID]; !ok {
+		chs := make([]string, 0)
+		qn.channels2Col[collectionID] = chs
+	}
+	qn.channels2Col[collectionID] = append(qn.channels2Col[collectionID], channels...)
+}
+
+func (qn *queryNodeInfo) AddSegments(segmentIDs []UniqueID, collectionID UniqueID) {
+	if _, ok := qn.segments[collectionID]; !ok {
+		seg := make([]UniqueID, 0)
+		qn.segments[collectionID] = seg
+	}
+	qn.segments[collectionID] = append(qn.segments[collectionID], segmentIDs...)
 }
 
 func (qn *queryNodeInfo) AddQueryChannel(ctx context.Context, in *querypb.AddQueryChannelRequest) (*commonpb.Status, error) {
@@ -40,7 +52,13 @@ func (qn *queryNodeInfo) AddQueryChannel(ctx context.Context, in *querypb.AddQue
 }
 
 func (qn *queryNodeInfo) ReleaseCollection(ctx context.Context, in *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) {
-	return qn.client.ReleaseCollection(ctx, in)
+	status, err := qn.client.ReleaseCollection(ctx, in)
+	if err != nil {
+		return status, err
+	}
+	delete(qn.segments, in.CollectionID)
+	delete(qn.channels2Col, in.CollectionID)
+	return status, nil
 }
 
 func (qn *queryNodeInfo) ReleasePartitions(ctx context.Context, in *querypb.ReleasePartitionsRequest) (*commonpb.Status, error) {
@@ -48,11 +66,11 @@ func (qn *queryNodeInfo) ReleasePartitions(ctx context.Context, in *querypb.Rele
 }
 
 func newQueryNodeInfo(client types.QueryNode) *queryNodeInfo {
-	segments := make([]UniqueID, 0)
+	segments := make(map[UniqueID][]UniqueID)
-	dmChannelNames := make([]string, 0)
+	channels := make(map[UniqueID][]string)
 	return &queryNodeInfo{
-		client:         client,
+		client:       client,
-		segments:       segments,
+		segments:     segments,
-		dmChannelNames: dmChannelNames,
+		channels2Col: channels,
 	}
 }

internal/queryservice/queryservice.go

@@ -14,17 +14,17 @@ import (
 
 	"github.com/opentracing/opentracing-go"
 	"github.com/uber/jaeger-client-go/config"
-	nodeclient "github.com/zilliztech/milvus-distributed/internal/distributed/querynode/client"
-	"github.com/zilliztech/milvus-distributed/internal/log"
-	"github.com/zilliztech/milvus-distributed/internal/types"
 	"go.uber.org/zap"
 
+	nodeclient "github.com/zilliztech/milvus-distributed/internal/distributed/querynode/client"
+	"github.com/zilliztech/milvus-distributed/internal/log"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/datapb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/querypb"
+	"github.com/zilliztech/milvus-distributed/internal/types"
 	"github.com/zilliztech/milvus-distributed/internal/util/retry"
 )
 
@@ -232,11 +232,6 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol
 		if err != nil {
 			return fn(err), err
 		}
-
-		err = qs.watchDmChannels(dbID, collectionID)
-		if err != nil {
-			return fn(err), err
-		}
 	}
 
 	// get partitionIDs
@@ -297,9 +292,15 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol
 		log.Error("LoadCollectionRequest failed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Error(err))
 		return status, fmt.Errorf("load partitions: %s", err)
 	}
+
+	err = qs.watchDmChannels(dbID, collectionID)
+	if err != nil {
+		log.Error("LoadCollectionRequest failed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Error(err))
+		return fn(err), err
+	}
+
 	log.Debug("LoadCollectionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID))
 	return status, nil
-
 }
 
 func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) {
@@ -330,7 +331,7 @@ func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.Rele
 		}, err
 	}
 
-	log.Debug("release collection end")
+	log.Debug("release collection end", zap.Int64("collectionID", collectionID))
 	//TODO:: queryNode cancel subscribe dmChannels
 	return &commonpb.Status{
 		ErrorCode: commonpb.ErrorCode_Success,
@@ -388,16 +389,14 @@ func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPar
 		return fn(err), err
 	}
 
+	watchNeeded := false
 	_, err := qs.replica.getCollectionByID(dbID, collectionID)
 	if err != nil {
 		err = qs.replica.addCollection(dbID, collectionID, schema)
 		if err != nil {
 			return fn(err), err
 		}
-		err = qs.watchDmChannels(dbID, collectionID)
+		watchNeeded = true
-		if err != nil {
-			return fn(err), err
-		}
 	}
 
 	for _, partitionID := range partitionIDs {
@@ -446,7 +445,7 @@ func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPar
 			return fn(err), err
 		}
 		for _, state := range resp.States {
-			log.Error("segment ", zap.String("state.SegmentID", fmt.Sprintln(state.SegmentID)), zap.String("state", fmt.Sprintln(state.StartPosition)))
+			log.Debug("segment ", zap.String("state.SegmentID", fmt.Sprintln(state.SegmentID)), zap.String("state", fmt.Sprintln(state.StartPosition)))
 			segmentID := state.SegmentID
 			segmentStates[segmentID] = state
 			channelName := state.StartPosition.ChannelName
@@ -459,38 +458,79 @@ func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPar
 			}
 		}
 
+		excludeSegment := make([]UniqueID, 0)
+		for id, state := range segmentStates {
+			if state.State > commonpb.SegmentState_Growing {
+				excludeSegment = append(excludeSegment, id)
+			}
+		}
 		for channel, segmentIDs := range channel2segs {
 			sort.Slice(segmentIDs, func(i, j int) bool {
 				return segmentStates[segmentIDs[i]].StartPosition.Timestamp < segmentStates[segmentIDs[j]].StartPosition.Timestamp
 			})
+			toLoadSegmentIDs := make([]UniqueID, 0)
+			var watchedStartPos *internalpb.MsgPosition = nil
+			var startPosition *internalpb.MsgPosition = nil
+			for index, id := range segmentIDs {
+				if segmentStates[id].State <= commonpb.SegmentState_Growing {
+					if index > 0 {
+						pos := segmentStates[id].StartPosition
+						if len(pos.MsgID) == 0 {
+							watchedStartPos = startPosition
+							break
+						}
+					}
+					watchedStartPos = segmentStates[id].StartPosition
+					break
+				}
+				toLoadSegmentIDs = append(toLoadSegmentIDs, id)
+				watchedStartPos = segmentStates[id].EndPosition
+				startPosition = segmentStates[id].StartPosition
+			}
+			if watchedStartPos == nil {
+				watchedStartPos = &internalpb.MsgPosition{
+					ChannelName: channel,
+				}
+			}
 
-			states := make([]*datapb.SegmentStateInfo, 0)
+			err = qs.replica.addDmChannel(dbID, collectionID, channel, watchedStartPos)
-			for _, id := range segmentIDs {
-				states = append(states, segmentStates[id])
-			}
-			loadSegmentRequest := &querypb.LoadSegmentsRequest{
-				CollectionID:  collectionID,
-				PartitionID:   partitionID,
-				SegmentIDs:    segmentIDs,
-				SegmentStates: states,
-				Schema:        schema,
-			}
-			nodeID, err := qs.replica.getAssignedNodeIDByChannelName(dbID, collectionID, channel)
 			if err != nil {
 				return fn(err), err
 			}
-			queryNode := qs.queryNodes[nodeID]
+			err = qs.replica.addExcludeSegmentIDs(dbID, collectionID, toLoadSegmentIDs)
-			//TODO:: seek when loadSegment may cause more msgs consumed
-			//TODO:: all query node should load partition's msg
-			status, err := queryNode.LoadSegments(ctx, loadSegmentRequest)
 			if err != nil {
-				return status, err
+				return fn(err), err
+			}
+
+			segment2Node := qs.shuffleSegmentsToQueryNode(toLoadSegmentIDs)
+			for nodeID, assignedSegmentIDs := range segment2Node {
+				loadSegmentRequest := &querypb.LoadSegmentsRequest{
+					CollectionID: collectionID,
+					PartitionID:  partitionID,
+					SegmentIDs:   assignedSegmentIDs,
+					Schema:       schema,
+				}
+
+				queryNode := qs.queryNodes[nodeID]
+				status, err := queryNode.LoadSegments(ctx, loadSegmentRequest)
+				if err != nil {
+					return status, err
+				}
+				queryNode.AddSegments(assignedSegmentIDs, collectionID)
 			}
 		}
 
 		qs.replica.updatePartitionState(dbID, collectionID, partitionID, querypb.PartitionState_InMemory)
 	}
 
+	if watchNeeded {
+		err = qs.watchDmChannels(dbID, collectionID)
+		if err != nil {
+			log.Debug("LoadPartitionRequest completed", zap.Int64("msgID", req.Base.MsgID), zap.Int64s("partitionIDs", partitionIDs), zap.Error(err))
+			return fn(err), err
+		}
+	}
+
 	log.Debug("LoadPartitionRequest completed", zap.Int64("msgID", req.Base.MsgID), zap.Int64s("partitionIDs", partitionIDs))
 	return &commonpb.Status{
 		ErrorCode: commonpb.ErrorCode_Success,
@@ -529,7 +569,7 @@ func (qs *QueryService) ReleasePartitions(ctx context.Context, req *querypb.Rele
 		}
 	}
 
-	log.Debug("start release partitions end")
+	log.Debug("start release partitions end", zap.String("partitionIDs", fmt.Sprintln(partitionIDs)))
 	//TODO:: queryNode cancel subscribe dmChannels
 	return &commonpb.Status{
 		ErrorCode: commonpb.ErrorCode_Success,
@@ -683,93 +723,159 @@ func (qs *QueryService) watchDmChannels(dbID UniqueID, collectionID UniqueID) er
 	}
 
 	dmChannels := resp.Values
-	watchedChannels2NodeID := make(map[string]int64)
+	channelsWithoutPos := make([]string, 0)
-	unWatchedChannels := make([]string, 0)
 	for _, channel := range dmChannels {
 		findChannel := false
-		for nodeID, node := range qs.queryNodes {
+		ChannelsWithPos := collection.dmChannels
-			watchedChannels := node.dmChannelNames
+		for _, ch := range ChannelsWithPos {
-			for _, watchedChannel := range watchedChannels {
+			if channel == ch {
-				if channel == watchedChannel {
+				findChannel = true
-					findChannel = true
+				break
-					watchedChannels2NodeID[channel] = nodeID
-					break
-				}
 			}
 		}
 		if !findChannel {
-			unWatchedChannels = append(unWatchedChannels, channel)
+			channelsWithoutPos = append(channelsWithoutPos, channel)
 		}
 	}
-	channels2NodeID := qs.shuffleChannelsToQueryNode(unWatchedChannels)
+	for _, ch := range channelsWithoutPos {
-	err = qs.replica.addDmChannels(dbID, collection.id, channels2NodeID)
+		pos := &internalpb.MsgPosition{
-	if err != nil {
+			ChannelName: ch,
-		return err
+		}
-	}
+		err = qs.replica.addDmChannel(dbID, collectionID, ch, pos)
-	err = qs.replica.addDmChannels(dbID, collection.id, watchedChannels2NodeID)
+		if err != nil {
-	if err != nil {
+			return err
-		return err
-	}
-	node2channels := make(map[int64][]string)
-	for channel, nodeID := range channels2NodeID {
-		if _, ok := node2channels[nodeID]; ok {
-			node2channels[nodeID] = append(node2channels[nodeID], channel)
-		} else {
-			channels := make([]string, 0)
-			channels = append(channels, channel)
-			node2channels[nodeID] = channels
 		}
 	}
 
-	for nodeID, channels := range node2channels {
+	channels2NodeID := qs.shuffleChannelsToQueryNode(dmChannels)
+	for nodeID, channels := range channels2NodeID {
 		node := qs.queryNodes[nodeID]
+		watchDmChannelsInfo := make([]*querypb.WatchDmChannelInfo, 0)
+		for _, ch := range channels {
+			info := &querypb.WatchDmChannelInfo{
+				ChannelID:        ch,
+				Pos:              collection.dmChannels2Pos[ch],
+				ExcludedSegments: collection.excludeSegmentIds,
+			}
+			watchDmChannelsInfo = append(watchDmChannelsInfo, info)
+		}
 		request := &querypb.WatchDmChannelsRequest{
-			ChannelIDs: channels,
+			CollectionID: collectionID,
+			ChannelIDs:   channels,
+			Infos:        watchDmChannelsInfo,
 		}
 		_, err := node.WatchDmChannels(ctx, request)
 		if err != nil {
 			return err
 		}
+		node.AddDmChannels(channels, collectionID)
 		log.Debug("query node ", zap.String("nodeID", strconv.FormatInt(nodeID, 10)), zap.String("watch channels", fmt.Sprintln(channels)))
-		node.AddDmChannels(channels)
 	}
 
 	return nil
 }
 
-func (qs *QueryService) shuffleChannelsToQueryNode(dmChannels []string) map[string]int64 {
+func (qs *QueryService) shuffleChannelsToQueryNode(dmChannels []string) map[int64][]string {
-	maxNumDMChannel := 0
+	maxNumChannels := 0
-	res := make(map[string]int64)
+	for _, node := range qs.queryNodes {
-	if len(dmChannels) == 0 {
+		numChannels := 0
-		return res
+		for _, chs := range node.channels2Col {
-	}
+			numChannels += len(chs)
-	node2lens := make(map[int64]int)
+		}
-	for id, node := range qs.queryNodes {
+		if numChannels > maxNumChannels {
-		node2lens[id] = len(node.dmChannelNames)
+			maxNumChannels = numChannels
+		}
 	}
+	res := make(map[int64][]string)
 	offset := 0
+	loopAll := false
 	for {
 		lastOffset := offset
-		for id, len := range node2lens {
+		if !loopAll {
-			if len >= maxNumDMChannel {
+			for id, node := range qs.queryNodes {
-				maxNumDMChannel = len
+				if len(node.segments) >= maxNumChannels {
-			} else {
+					continue
-				res[dmChannels[offset]] = id
+				}
-				node2lens[id]++
+				if _, ok := res[id]; !ok {
+					res[id] = make([]string, 0)
+				}
+				res[id] = append(res[id], dmChannels[offset])
 				offset++
+				if offset == len(dmChannels) {
+					return res
+				}
+			}
+		} else {
+			for id := range qs.queryNodes {
+				if _, ok := res[id]; !ok {
+					res[id] = make([]string, 0)
+				}
+				res[id] = append(res[id], dmChannels[offset])
+				offset++
+				if offset == len(dmChannels) {
+					return res
+				}
 			}
 		}
 		if lastOffset == offset {
-			for id := range node2lens {
+			loopAll = true
-				res[dmChannels[offset]] = id
+		}
-				node2lens[id]++
+	}
-				offset++
+}
-				break
+
-			}
+func (qs *QueryService) shuffleSegmentsToQueryNode(segmentIDs []UniqueID) map[int64][]UniqueID {
-		}
+	maxNumSegments := 0
-		if offset == len(dmChannels) {
+	for _, node := range qs.queryNodes {
-			break
+		numSegments := 0
+		for _, ids := range node.segments {
+			numSegments += len(ids)
+		}
+		if numSegments > maxNumSegments {
+			maxNumSegments = numSegments
+		}
+	}
+	res := make(map[int64][]UniqueID)
+	for nodeID := range qs.queryNodes {
+		segments := make([]UniqueID, 0)
+		res[nodeID] = segments
+	}
+
+	if len(segmentIDs) == 0 {
+		return res
+	}
+
+	offset := 0
+	loopAll := false
+	for {
+		lastOffset := offset
+		if !loopAll {
+			for id, node := range qs.queryNodes {
+				if len(node.segments) >= maxNumSegments {
+					continue
+				}
+				if _, ok := res[id]; !ok {
+					res[id] = make([]UniqueID, 0)
+				}
+				res[id] = append(res[id], segmentIDs[offset])
+				offset++
+				if offset == len(segmentIDs) {
+					return res
+				}
+			}
+		} else {
+			for id := range qs.queryNodes {
+				if _, ok := res[id]; !ok {
+					res[id] = make([]UniqueID, 0)
+				}
+				res[id] = append(res[id], segmentIDs[offset])
+				offset++
+				if offset == len(segmentIDs) {
+					return res
+				}
+			}
+		}
+		if lastOffset == offset {
+			loopAll = true
 		}
 	}
-	return res
 }

internal/util/flowgraph/flow_graph.go

@@ -77,6 +77,7 @@ func (fg *TimeTickedFlowGraph) Close() {
 			}
 			(*inStream.inStream).Close()
 		}
+		// v.Close()
 	}
 }
 

internal/util/flowgraph/input_node.go

@@ -2,11 +2,9 @@ package flowgraph
 
 import (
 	"context"
-	"log"
-
-	"errors"
 
 	"github.com/opentracing/opentracing-go"
+
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/util/trace"
 )
@@ -40,8 +38,6 @@ func (inNode *InputNode) Operate(ctx context.Context, msgs []Msg) ([]Msg, contex
 
 	// TODO: add status
 	if msgPack == nil {
-		log.Println("null msg pack")
-		trace.LogError(sp, errors.New("null msg pack"))
 		return nil, ctx
 	}
 

internal/util/flowgraph/node.go

@@ -77,7 +77,7 @@ func (nodeCtx *nodeCtx) Start(ctx context.Context, wg *sync.WaitGroup) {
 				log.Println("nodeCtx.downstream length = ", len(nodeCtx.downstream))
 			}
 			if len(res) < downstreamLength {
-				log.Println("node result length = ", len(res))
+				// log.Println("node result length = ", len(res))
 				break
 			}
 

tests/python_test/collection/test_describe_collection.py

@@ -68,10 +68,11 @@ class TestDescribeCollection:
     @pytest.mark.tags("0331", "l1")
     def test_describe_collection_after_index_created(self, connect, collection, get_simple_index):
         connect.create_index(collection, default_float_vec_field_name, get_simple_index)
-        index = connect.describe_index(collection, default_float_vec_field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index["index_type"] == get_simple_index["index_type"]
+            index = connect.describe_index(collection, default_float_vec_field_name)
-        assert index["metric_type"] == get_simple_index["metric_type"]
+            assert index["index_type"] == get_simple_index["index_type"]
-        assert index["params"] == get_simple_index["params"]
+            assert index["metric_type"] == get_simple_index["metric_type"]
+            assert index["params"] == get_simple_index["params"]
 
     @pytest.mark.level(2)
     @pytest.mark.tags("0331")

tests/python_test/entity/test_insert.py

@@ -133,8 +133,9 @@ class TestInsertBase:
         assert len(ids) == default_nb
         connect.flush([collection])
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.timeout(ADD_TIMEOUT)
     @pytest.mark.tags("0331", "l1")
@@ -147,8 +148,9 @@ class TestInsertBase:
         connect.create_index(collection, field_name, get_simple_index)
         ids = connect.insert(collection, default_entities)
         assert len(ids) == default_nb
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.timeout(ADD_TIMEOUT)
     @pytest.mark.tags("0331", "l1")
@@ -507,9 +509,10 @@ class TestInsertBase:
         with pytest.raises(Exception):
             connect.insert(collection, tmp_entity)
 
-    @pytest.mark.level(2)
+# todo fix timeout
-    @pytest.mark.timeout(30)
+#     @pytest.mark.level(2)
-    @pytest.mark.tags("0331")
+#     @pytest.mark.timeout(30)
+#     @pytest.mark.tags("0331")
     def test_collection_insert_rows_count_multi_threading(self, args, collection):
         '''
         target: test collection rows_count is correct or not with multi threading
@@ -839,8 +842,9 @@ class TestInsertMultiCollections:
         connect.create_index(collection, field_name, get_simple_index)
         ids = connect.insert(collection_name, default_entity)
         assert len(ids) == 1
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
         connect.drop_collection(collection_name)
 
     @pytest.mark.timeout(ADD_TIMEOUT)
@@ -856,8 +860,9 @@ class TestInsertMultiCollections:
         ids = connect.insert(collection, default_entity)
         connect.flush([collection])
         connect.create_index(collection_name, field_name, get_simple_index)
-        index = connect.describe_index(collection_name, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection_name, field_name)
+            assert index == get_simple_index
         stats = connect.get_collection_stats(collection)
         assert stats[row_count] == 1
 

tests/python_test/entity/test_search.py

@@ -25,12 +25,12 @@ default_binary_query, default_binary_query_vecs = gen_query_vectors(binary_field
                                                                     nq)
 
 
-def init_data(connect, collection, nb=1200, partition_tags=None, auto_id=True):
+def init_data(connect, collection, nb=3000, partition_tags=None, auto_id=True):
     '''
     Generate entities and add it in collection
     '''
     global entities
-    if nb == 1200:
+    if nb == 3000:
         insert_entities = entities
     else:
         insert_entities = gen_entities(nb, is_normal=True)
@@ -48,14 +48,14 @@ def init_data(connect, collection, nb=1200, partition_tags=None, auto_id=True):
     return insert_entities, ids
 
 
-def init_binary_data(connect, collection, nb=1200, insert=True, partition_tags=None):
+def init_binary_data(connect, collection, nb=3000, insert=True, partition_tags=None):
     '''
     Generate entities and add it in collection
     '''
     ids = []
     global binary_entities
     global raw_vectors
-    if nb == 1200:
+    if nb == 3000:
         insert_entities = binary_entities
         insert_raw_vectors = raw_vectors
     else:
@@ -92,7 +92,7 @@ class TestSearchBase:
         # if str(connect._cmd("mode")) == "CPU":
         #     if request.param["index_type"] in index_cpu_not_support():
         #         pytest.skip("sq8h not support in CPU mode")
-        return request.param
+        return copy.deepcopy(request.param)
 
     @pytest.fixture(
         scope="function",
@@ -257,7 +257,6 @@ class TestSearchBase:
             assert res2[0][0].id == res[0][1].id
             assert res2[0][0].entity.get("int64") == res[0][1].entity.get("int64")
 
-    # TODO:
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_search_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq):
@@ -287,7 +286,7 @@ class TestSearchBase:
             assert res[0]._distances[0] < epsilon
             assert check_id_result(res[0], ids[0])
 
-    # @pytest.mark.tags("0331")
+    @pytest.mark.tags("0331")
     def test_search_after_index_different_metric_type(self, connect, collection, get_simple_index):
         '''
         target: test search with different metric_type
@@ -311,12 +310,7 @@ class TestSearchBase:
             with pytest.raises(Exception) as e:
                 res = connect.search(collection, query)
 
-    # TODO: need to enable 
+    @pytest.mark.tags("0331")
-    # description: create/load/search
-    # @pytest.mark.tags("0331")
-    def _test_search_after_index_different_metric_type_2(self, connect, collection, get_simple_index):
-        pass
-
     @pytest.mark.level(2)
     def test_search_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
         '''
@@ -350,7 +344,9 @@ class TestSearchBase:
             res = connect.search(collection, query, partition_tags=[default_tag])
             assert len(res[0]) == 0
 
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
+    @pytest.mark.timeout(600)
     def test_search_index_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
         '''
         target: test basic search function, all the search params is correct, test all index params, and build
@@ -368,18 +364,19 @@ class TestSearchBase:
         connect.create_index(collection, field_name, get_simple_index)
         search_param = get_search_param(index_type)
         query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param)
-        for tags in [[default_tag], [default_tag, "new_tag"]]:
+        if top_k > max_top_k:
-            if top_k > max_top_k:
+            with pytest.raises(Exception) as e:
-                with pytest.raises(Exception) as e:
+                res = connect.search(collection, query, partition_tags=[default_tag])
-                    res = connect.search(collection, query, partition_tags=tags)
+        else:
-            else:
+            connect.load_partitions(collection, [default_tag])
-                connect.load_partitions(collection, tags)
+            res = connect.search(collection, query, partition_tags=[default_tag])
-                res = connect.search(collection, query, partition_tags=tags)
+            assert len(res) == nq
-                assert len(res) == nq
+            assert len(res[0]) == top_k
-                assert len(res[0]) >= top_k
+            assert res[0]._distances[0] < epsilon
-                assert res[0]._distances[0] < epsilon
+            assert check_id_result(res[0], ids[0])
-                assert check_id_result(res[0], ids[0])
 
+
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_search_index_partition_not_existed(self, connect, collection, get_top_k, get_nq, get_simple_index):
         '''
@@ -396,10 +393,9 @@ class TestSearchBase:
             with pytest.raises(Exception) as e:
                 res = connect.search(collection, query, partition_tags=["new_tag"])
         else:
-            connect.load_partitions(collection, ["new_tag"])
+            connect.load_collection(collection)
-            res = connect.search(collection, query, partition_tags=["new_tag"])
+            with pytest.raises(Exception) as e:
-            assert len(res) == nq
+                connect.search(collection, query, partition_tags=["new_tag"])
-            assert len(res[0]) == 0
 
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
@@ -473,10 +469,7 @@ class TestSearchBase:
             assert res[1]._distances[0] < epsilon
             connect.release_collection(collection)
 
-    #
+    @pytest.mark.tags("0331")
-    # test for ip metric
-    #
-    # @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_search_ip_flat(self, connect, collection, get_simple_index, get_top_k, get_nq):
         '''
@@ -520,7 +513,7 @@ class TestSearchBase:
         assert check_id_result(res[0], ids[0])
         assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0])
 
-    # @pytest.mark.tags("0331")
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_search_ip_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
         '''
@@ -630,8 +623,7 @@ class TestSearchBase:
         res = connect.search(collection, query)
         assert abs(np.sqrt(res[0]._distances[0]) - min(distance_0, distance_1)) <= gen_inaccuracy(res[0]._distances[0])
 
-    # TODO
+    @pytest.mark.tags("0331")
-    # @pytest.mark.tags("0331")
     def test_search_distance_l2_after_index(self, connect, id_collection, get_simple_index):
         '''
         target: search collection, and check the result: distance
@@ -662,7 +654,7 @@ class TestSearchBase:
         # TODO:
         # assert abs(np.sqrt(res[0]._distances[0]) - min_distance) <= tmp_epsilon
 
-    # @pytest.mark.tags("0331", "l1")
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_search_distance_ip(self, connect, collection):
         '''
@@ -685,7 +677,7 @@ class TestSearchBase:
         res = connect.search(collection, query)
         assert abs(res[0]._distances[0] - max(distance_0, distance_1)) <= epsilon
 
-    # @pytest.mark.tags("0331")
+    @pytest.mark.tags("0331")
     def test_search_distance_ip_after_index(self, connect, id_collection, get_simple_index):
         '''
         target: search collection, and check the result: distance
@@ -769,7 +761,6 @@ class TestSearchBase:
         res = connect.search(binary_collection, query)
         assert abs(res[0][0].distance - min(distance_0, distance_1).astype(float)) <= epsilon
 
-    # TODO
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_search_distance_substructure_flat_index(self, connect, binary_collection):
@@ -850,7 +841,7 @@ class TestSearchBase:
         assert res[1][0].id in ids
         assert res[1][0].distance <= epsilon
 
-    # @pytest.mark.tags("0331")
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_search_distance_tanimoto_flat_index(self, connect, binary_collection):
         '''
@@ -870,7 +861,7 @@ class TestSearchBase:
 
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
-    @pytest.mark.timeout(30)
+    @pytest.mark.timeout(300)
     def test_search_concurrent_multithreads(self, connect, args):
         '''
         target: test concurrent search with multiprocessess
@@ -906,7 +897,7 @@ class TestSearchBase:
 
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
-    @pytest.mark.timeout(30)
+    @pytest.mark.timeout(300)
     def test_search_concurrent_multithreads_single_connection(self, connect, args):
         '''
         target: test concurrent search with multiprocessess
@@ -1071,6 +1062,7 @@ class TestSearchDSL(object):
     ******************************************************************
     """
 
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_term_value_not_in(self, connect, collection):
         '''
@@ -1087,7 +1079,7 @@ class TestSearchDSL(object):
         assert len(res[0]) == 0
         # TODO:
 
-    # TODO:
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_term_value_all_in(self, connect, collection):
         '''
@@ -1103,7 +1095,7 @@ class TestSearchDSL(object):
         assert len(res[0]) == 1
         # TODO:
 
-    # TODO:
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_term_values_not_in(self, connect, collection):
         '''
@@ -1120,6 +1112,7 @@ class TestSearchDSL(object):
         assert len(res[0]) == 0
         # TODO:
 
+    @pytest.mark.tags("0331")
     def test_query_term_values_all_in(self, connect, collection):
         '''
         method: build query with vector and term expr, with all term can be filtered
@@ -1139,6 +1132,7 @@ class TestSearchDSL(object):
                 assert result.id in ids[:limit]
         # TODO:
 
+    @pytest.mark.tags("0331")
     def test_query_term_values_parts_in(self, connect, collection):
         '''
         method: build query with vector and term expr, with parts of term can be filtered
@@ -1155,7 +1149,7 @@ class TestSearchDSL(object):
         assert len(res[0]) == default_top_k
         # TODO:
 
-    # TODO:
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_term_values_repeat(self, connect, collection):
         '''
@@ -1209,7 +1203,6 @@ class TestSearchDSL(object):
     ******************************************************************
     """
 
-    # TODO
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_term_key_error(self, connect, collection):
@@ -1244,7 +1237,7 @@ class TestSearchDSL(object):
         with pytest.raises(Exception) as e:
             res = connect.search(collection, query)
 
-    # TODO
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_term_field_named_term(self, connect, collection):
         '''
@@ -1266,7 +1259,7 @@ class TestSearchDSL(object):
         expr = {"must": [gen_default_vector_expr(default_query),
                          term_param]}
         query = update_query_expr(default_query, expr=expr)
-        connect.load_collection(collection)
+        connect.load_collection(collection_term)
         res = connect.search(collection_term, query)
         assert len(res) == nq
         assert len(res[0]) == default_top_k
@@ -1293,7 +1286,6 @@ class TestSearchDSL(object):
     ******************************************************************
     """
 
-    # TODO
     @pytest.mark.tags("0331", "l1")
     def test_query_range_key_error(self, connect, collection):
         '''
@@ -1313,7 +1305,6 @@ class TestSearchDSL(object):
     def get_invalid_range(self, request):
         return request.param
 
-    # TODO
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_range_wrong_format(self, connect, collection, get_invalid_range):
@@ -1343,6 +1334,7 @@ class TestSearchDSL(object):
         with pytest.raises(Exception) as e:
             res = connect.search(collection, query)
 
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_range_invalid_ranges(self, connect, collection):
         '''
@@ -1365,6 +1357,7 @@ class TestSearchDSL(object):
     def get_valid_ranges(self, request):
         return request.param
 
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_range_valid_ranges(self, connect, collection, get_valid_ranges):
         '''
@@ -1401,7 +1394,7 @@ class TestSearchDSL(object):
     ************************************************************************
     """
 
-    # TODO
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_multi_term_has_common(self, connect, collection):
         '''
@@ -1418,7 +1411,7 @@ class TestSearchDSL(object):
         assert len(res) == nq
         assert len(res[0]) == default_top_k
 
-    # TODO
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_multi_term_no_common(self, connect, collection):
         '''
@@ -1435,7 +1428,7 @@ class TestSearchDSL(object):
         assert len(res) == nq
         assert len(res[0]) == 0
 
-    # TODO
+    @pytest.mark.tags("0331")
     def test_query_multi_term_different_fields(self, connect, collection):
         '''
          method: build query with multi range with same field, and ranges no common
@@ -1452,7 +1445,6 @@ class TestSearchDSL(object):
         assert len(res) == nq
         assert len(res[0]) == 0
 
-    # TODO
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_single_term_multi_fields(self, connect, collection):
@@ -1469,7 +1461,7 @@ class TestSearchDSL(object):
         with pytest.raises(Exception) as e:
             res = connect.search(collection, query)
 
-    # TODO
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_multi_range_has_common(self, connect, collection):
         '''
@@ -1486,7 +1478,7 @@ class TestSearchDSL(object):
         assert len(res) == nq
         assert len(res[0]) == default_top_k
 
-    # TODO
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_multi_range_no_common(self, connect, collection):
         '''
@@ -1503,7 +1495,7 @@ class TestSearchDSL(object):
         assert len(res) == nq
         assert len(res[0]) == 0
 
-    # TODO
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_multi_range_different_fields(self, connect, collection):
         '''
@@ -1520,7 +1512,6 @@ class TestSearchDSL(object):
         assert len(res) == nq
         assert len(res[0]) == 0
 
-    # TODO
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_single_range_multi_fields(self, connect, collection):
@@ -1543,7 +1534,7 @@ class TestSearchDSL(object):
     ******************************************************************
     """
 
-    # TODO
+    @pytest.mark.tags("0331")
     @pytest.mark.level(2)
     def test_query_single_term_range_has_common(self, connect, collection):
         '''
@@ -1560,7 +1551,7 @@ class TestSearchDSL(object):
         assert len(res) == nq
         assert len(res[0]) == default_top_k
 
-    # TODO
+    @pytest.mark.tags("0331")
     def test_query_single_term_range_no_common(self, connect, collection):
         '''
         method: build query with single term single range
@@ -1582,7 +1573,6 @@ class TestSearchDSL(object):
     ******************************************************************
     """
 
-    # TODO
     @pytest.mark.tags("0331")
     def test_query_multi_vectors_same_field(self, connect, collection):
         '''

tests/python_test/requirements.txt

@@ -10,6 +10,9 @@ allure-pytest==2.7.0
 pytest-print==0.2.1
 pytest-level==0.1.1
 pytest-xdist==2.2.1
-pymilvus-distributed==0.0.50
+pymilvus-distributed==0.0.54
 pytest-rerunfailures==9.1.1
 git+https://github.com/Projectplace/pytest-tags
+ndg-httpsclient
+pyopenssl
+pyasn1

tests/python_test/test_index.py

@@ -57,8 +57,9 @@ class TestIndexBase:
         '''
         ids = connect.insert(collection, default_entities)
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331", "l1")
     def test_create_index_on_field_not_existed(self, connect, collection, get_simple_index):
@@ -94,8 +95,9 @@ class TestIndexBase:
         expected: return search success
         '''
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331")
     @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -108,8 +110,9 @@ class TestIndexBase:
         connect.create_partition(collection, default_tag)
         ids = connect.insert(collection, default_entities, partition_tag=default_tag)
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331", "l1")
     @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -123,8 +126,9 @@ class TestIndexBase:
         ids = connect.insert(collection, default_entities, partition_tag=default_tag)
         connect.flush([collection])
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331")
     def test_create_index_without_connect(self, dis_connect, collection):
@@ -169,8 +173,9 @@ class TestIndexBase:
 
         def build(connect):
             connect.create_index(collection, field_name, default_index)
-            index = connect.describe_index(collection, field_name)
+            if default_index["index_type"] != "FLAT":
-            assert index == default_index
+                index = connect.describe_index(collection, field_name)
+                assert index == default_index
 
         threads_num = 8
         threads = []
@@ -209,8 +214,9 @@ class TestIndexBase:
         connect.flush([collection])
         stats = connect.get_collection_stats(collection)
         assert stats["row_count"] == default_nb
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
@@ -223,8 +229,9 @@ class TestIndexBase:
         '''
         connect.create_index(collection, field_name, get_simple_index)
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
@@ -243,7 +250,8 @@ class TestIndexBase:
             connect.release_collection(collection)
             connect.load_collection(collection)
         index = connect.describe_index(collection, field_name)
-        assert index == indexs[-1]
+        # assert index == indexs[-1]
+        assert not index    # FLAT is the last index_type, drop all indexes in server
 
     @pytest.mark.tags("0331", "l1")
     @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -256,8 +264,9 @@ class TestIndexBase:
         ids = connect.insert(collection, default_entities)
         get_simple_index["metric_type"] = "IP"
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331", "l1")
     @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -269,8 +278,9 @@ class TestIndexBase:
         '''
         get_simple_index["metric_type"] = "IP"
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331")
     @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -284,8 +294,9 @@ class TestIndexBase:
         ids = connect.insert(collection, default_entities, partition_tag=default_tag)
         get_simple_index["metric_type"] = "IP"
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331", "l1")
     @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -300,8 +311,9 @@ class TestIndexBase:
         connect.flush([collection])
         get_simple_index["metric_type"] = "IP"
         connect.create_index(collection, field_name, get_simple_index)
-        index = connect.describe_index(collection, field_name)
+        if get_simple_index["index_type"] != "FLAT":
-        assert index == get_simple_index
+            index = connect.describe_index(collection, field_name)
+            assert index == get_simple_index
 
     @pytest.mark.tags("0331", "l1")
     @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -339,8 +351,9 @@ class TestIndexBase:
         def build(connect):
             default_index["metric_type"] = "IP"
             connect.create_index(collection, field_name, default_index)
-            index = connect.describe_index(collection, field_name)
+            if default_index["index_type"] != "FLAT":
-            assert index == default_index
+                index = connect.describe_index(collection, field_name)
+                assert index == default_index
 
         threads_num = 8
         threads = []
@@ -380,8 +393,9 @@ class TestIndexBase:
         connect.flush([collection])
         stats = connect.get_collection_stats(collection)
         assert stats["row_count"] == default_nb
-        index = connect.describe_index(collection, field_name)
+        if default_index["index_type"] != "FLAT":
-        assert index == default_index
+            index = connect.describe_index(collection, field_name)
+            assert index == default_index
 
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
@@ -395,8 +409,9 @@ class TestIndexBase:
         default_index["metric_type"] = "IP"
         connect.create_index(collection, field_name, default_index)
         connect.create_index(collection, field_name, default_index)
-        index = connect.describe_index(collection, field_name)
+        if default_index["index_type"] != "FLAT":
-        assert index == default_index
+            index = connect.describe_index(collection, field_name)
+            assert index == default_index
 
     @pytest.mark.tags("0331")
     @pytest.mark.level(2)
@@ -419,7 +434,8 @@ class TestIndexBase:
             connect.release_collection(collection)
             connect.load_collection(collection)
         index = connect.describe_index(collection, field_name)
-        assert index == indexs[-1]
+        # assert index == indexs[-1]
+        assert not index
 
     """
     ******************************************************************