diff --git a/configs/advanced/channel.yaml b/configs/advanced/channel.yaml index 0f8fd22d8a..63cc9f1cd3 100644 --- a/configs/advanced/channel.yaml +++ b/configs/advanced/channel.yaml @@ -22,11 +22,9 @@ msgChannel: k2s: "k2s" proxyTimeTick: "proxyTimeTick" proxyServiceTimeTick: "proxyServiceTimeTick" - writeNodeTimeTick: "writeNodeTimeTick" # GOOSE TODO: remove this dataNodeTimeTick: "dataNodeTimeTick" queryTimeTick: "queryTimeTick" dataNodeSegStatistics: "dataNodeSegStatistics" - # old name: statsChannels: "statistic" queryNodeStats: "query-node-stats" # cmd for loadIndex, flush, etc... cmd: "cmd" @@ -40,7 +38,6 @@ msgChannel: masterSubNamePrefix: "master" proxySubNamePrefix: "proxy" queryNodeSubNamePrefix: "queryNode" - writeNodeSubNamePrefix: "writeNode" # GOOSE TODO: remove this dataNodeSubNamePrefix: "dataNode" dataServiceSubNamePrefix: "dataService" diff --git a/configs/advanced/common.yaml b/configs/advanced/common.yaml index f1f5a25ffe..5613c44c88 100644 --- a/configs/advanced/common.yaml +++ b/configs/advanced/common.yaml @@ -10,6 +10,5 @@ # or implied. See the License for the specific language governing permissions and limitations under the License. common: - defaultPartitionTag: _default #TODO, remove defaultPartitionName: "_default" defaultIndexName: "_default_idx" diff --git a/configs/advanced/data_node.yaml b/configs/advanced/data_node.yaml index b2919eb478..9a2bdf095b 100644 --- a/configs/advanced/data_node.yaml +++ b/configs/advanced/data_node.yaml @@ -23,12 +23,10 @@ dataNode: pulsarBufSize: 64 # pulsar chan buffer size insert: - #streamBufSize: 1024 # msgPack chan buffer size recvBufSize: 1024 # msgPack chan buffer size pulsarBufSize: 1024 # pulsar chan buffer size delete: - #streamBufSize: 1024 # msgPack chan buffer size recvBufSize: 1024 # msgPack chan buffer size pulsarBufSize: 1024 # pulsar chan buffer size @@ -38,5 +36,5 @@ dataNode: flush: # max buffer size to flush - insertBufSize: 32000 + insertBufSize: 32000 # number of rows ddBufSize: 20 diff --git a/configs/advanced/data_service.yaml b/configs/advanced/data_service.yaml index c82a0cea08..86f8b3324c 100644 --- a/configs/advanced/data_service.yaml +++ b/configs/advanced/data_service.yaml @@ -3,11 +3,9 @@ dataservice: address: "127.0.0.1" port: 13333 segment: - # old name: segmentThreshold: 536870912 size: 512 # MB sizeFactor: 0.75 defaultSizePerRecord: 1024 - # old name: segmentExpireDuration: 2000 IDAssignExpiration: 2000 # ms insertChannelNum: 2 - dataNodeNum: 1 \ No newline at end of file + dataNodeNum: 1 diff --git a/configs/advanced/master.yaml b/configs/advanced/master.yaml index 7307d30272..ec78aa3b71 100644 --- a/configs/advanced/master.yaml +++ b/configs/advanced/master.yaml @@ -10,18 +10,6 @@ # or implied. See the License for the specific language governing permissions and limitations under the License. master: - timeSync: - softTimeTickBarrierInterval: 500 # ms - segment: - # old name: segmentThreshold: 536870912 - size: 512 # MB - sizeFactor: 0.75 - defaultSizePerRecord: 1024 - minIDAssignCnt: 1024 - maxIDAssignCnt: 16384 - # old name: segmentExpireDuration: 2000 - IDAssignExpiration: 2000 # ms - maxPartitionNum: 4096 minSegmentSizeToEnableIndex: 1024 nodeID: 100 diff --git a/configs/advanced/proxy_node.yaml b/configs/advanced/proxy_node.yaml index 4ca7886a6e..c7e23a5e28 100644 --- a/configs/advanced/proxy_node.yaml +++ b/configs/advanced/proxy_node.yaml @@ -14,7 +14,6 @@ proxyNode: msgStream: insert: - #streamBufSize: 1024 # msgPack chan buffer size bufSize: 1024 # msgPack chan buffer size search: diff --git a/configs/advanced/query_node.yaml b/configs/advanced/query_node.yaml index 2250fcacda..33320c851c 100644 --- a/configs/advanced/query_node.yaml +++ b/configs/advanced/query_node.yaml @@ -20,7 +20,6 @@ queryNode: msgStream: insert: - #streamBufSize: 1024 # msgPack chan buffer size recvBufSize: 1024 # msgPack chan buffer size pulsarBufSize: 1024 # pulsar chan buffer size @@ -29,7 +28,6 @@ queryNode: pulsarBufSize: 64 # pulsar chan buffer size delete: - #streamBufSize: 1024 # msgPack chan buffer size recvBufSize: 1024 # msgPack chan buffer size pulsarBufSize: 1024 # pulsar chan buffer size diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 6ed91c11a4..a6b9ee1d5a 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -10,10 +10,9 @@ # or implied. See the License for the specific language governing permissions and limitations under the License. -nodeID: # will be deprecated after v0.2 - proxyIDList: [0] +nodeID: # will be deprecated later queryNodeIDList: [1] - dataNodeIDList: [3] + dataNodeIDList: [3] etcd: address: localhost @@ -22,10 +21,7 @@ etcd: metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath segFlushMetaSubPath: datanode/segment # Full Path = rootPath/metaSubPath/segFlushMetaSubPath - ddlFlushMetaSubPath: datanode/ddl # Full Path = rootPath/metaSubPaht/ddlFlushMetaSubPath - writeNodeSegKvSubPath: writer/segment # GOOSE TODO: remove this - writeNodeDDLKvSubPath: writer/ddl # GOOSE TODO: remove this - segThreshold: 10000 + ddlFlushMetaSubPath: datanode/ddl # Full Path = rootPath/metaSubPath/ddlFlushMetaSubPath minio: address: localhost @@ -38,11 +34,7 @@ minio: pulsar: address: localhost port: 6650 - rest-port: 18080 # keep same with pulsar container - maxMessageSize: 5242880 # 5 * 1024 * 1024 - authentication: false - user: user-default - token: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY + maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes master: address: localhost @@ -60,7 +52,7 @@ proxyService: port: 21122 queryNode: - gracefulTime: 5000 #ms + gracefulTime: 5000 # ms, for search port: 21123 indexService: @@ -78,11 +70,11 @@ dataNode: port: 21124 log: - level: debug + level: debug # info, warn, error, panic, fatal, dpanic, file: - rootPath: "" + rootPath: "" # default to stdout, stderr maxSize: 300 # MB maxAge: 10 # day maxBackups: 20 - dev: true + dev: true # false, change behaviour of dpaniclevel format: text # text/json diff --git a/docs/developer_guides/chap05_proxy.md b/docs/developer_guides/chap05_proxy.md index 4d936a60ca..f075b1acbf 100644 --- a/docs/developer_guides/chap05_proxy.md +++ b/docs/developer_guides/chap05_proxy.md @@ -403,7 +403,7 @@ type GlobalParamsTable struct { MaxNameLength int64 MaxFieldNum int64 MaxDimension int64 - DefaultPartitionTag string + DefaultPartitionName string DefaultIndexName string } diff --git a/internal/proxynode/impl.go b/internal/proxynode/impl.go index 5f0b554a2b..33d50b7050 100644 --- a/internal/proxynode/impl.go +++ b/internal/proxynode/impl.go @@ -946,7 +946,7 @@ func (node *ProxyNode) Insert(ctx context.Context, request *milvuspb.InsertReque rowIDAllocator: node.idAllocator, } if len(it.PartitionName) <= 0 { - it.PartitionName = Params.DefaultPartitionTag + it.PartitionName = Params.DefaultPartitionName } err := node.sched.DmQueue.Enqueue(it) diff --git a/internal/proxynode/paramtable.go b/internal/proxynode/paramtable.go index a1d935e99b..7e6ea194c2 100644 --- a/internal/proxynode/paramtable.go +++ b/internal/proxynode/paramtable.go @@ -54,7 +54,7 @@ type ParamTable struct { MaxNameLength int64 MaxFieldNum int64 MaxDimension int64 - DefaultPartitionTag string + DefaultPartitionName string DefaultIndexName string PulsarMaxMessageSize int @@ -136,7 +136,6 @@ func (pt *ParamTable) initParams() { pt.initPulsarAddress() pt.initQueryNodeIDList() pt.initQueryNodeNum() - // pt.initProxyID() pt.initTimeTickInterval() pt.initInsertChannelNames() pt.initDeleteChannelNames() @@ -154,7 +153,7 @@ func (pt *ParamTable) initParams() { pt.initMaxNameLength() pt.initMaxFieldNum() pt.initMaxDimension() - pt.initDefaultPartitionTag() + pt.initDefaultPartitionName() pt.initDefaultIndexName() pt.initPulsarMaxMessageSize() @@ -191,18 +190,6 @@ func (pt *ParamTable) initQueryNodeIDList() []UniqueID { return ret } -func (pt *ParamTable) initProxyID() { - proxyID, err := pt.Load("_proxyID") - if err != nil { - panic(err) - } - ID, err := strconv.Atoi(proxyID) - if err != nil { - panic(err) - } - pt.ProxyID = UniqueID(ID) -} - func (pt *ParamTable) initTimeTickInterval() { intervalStr, err := pt.Load("proxyNode.timeTickInterval") if err != nil { @@ -311,11 +298,6 @@ func (pt *ParamTable) initProxySubName() { if err != nil { panic(err) } - pt.ProxySubName = prefix - // proxyIDStr, err := pt.Load("_proxyID") - // if err != nil { - // panic(err) - // } pt.ProxySubName = prefix + "-" + strconv.Itoa(int(pt.ProxyID)) } @@ -393,12 +375,12 @@ func (pt *ParamTable) initMaxDimension() { pt.MaxDimension = maxDimension } -func (pt *ParamTable) initDefaultPartitionTag() { - tag, err := pt.Load("common.defaultPartitionTag") +func (pt *ParamTable) initDefaultPartitionName() { + name, err := pt.Load("common.defaultPartitionName") if err != nil { panic(err) } - pt.DefaultPartitionTag = tag + pt.DefaultPartitionName = name } func (pt *ParamTable) initDefaultIndexName() { diff --git a/internal/proxynode/task.go b/internal/proxynode/task.go index e97f45a8c4..ae038ecd9e 100644 --- a/internal/proxynode/task.go +++ b/internal/proxynode/task.go @@ -159,7 +159,7 @@ func (it *InsertTask) Execute(ctx context.Context) error { return err } } else { - partitionID, err = globalMetaCache.GetPartitionID(ctx, collectionName, Params.DefaultPartitionTag) + partitionID, err = globalMetaCache.GetPartitionID(ctx, collectionName, Params.DefaultPartitionName) if err != nil { return err } diff --git a/internal/querynode/load_service_test.go b/internal/querynode/load_service_test.go index ebd9cf44cd..cf0fecc8f1 100644 --- a/internal/querynode/load_service_test.go +++ b/internal/querynode/load_service_test.go @@ -1122,7 +1122,6 @@ func TestSegmentLoad_Search_Vector(t *testing.T) { // mock write insert bin log keyPrefix := path.Join("query-node-seg-manager-test-minio-prefix", strconv.FormatInt(collectionID, 10), strconv.FormatInt(partitionID, 10)) - Params.WriteNodeSegKvSubPath = keyPrefix node := newQueryNodeMock() defer node.Stop() diff --git a/internal/querynode/param_table.go b/internal/querynode/param_table.go index 77c89dcc44..4f7e5be628 100644 --- a/internal/querynode/param_table.go +++ b/internal/querynode/param_table.go @@ -14,10 +14,9 @@ import ( type ParamTable struct { paramtable.BaseTable - PulsarAddress string - ETCDAddress string - MetaRootPath string - WriteNodeSegKvSubPath string + PulsarAddress string + ETCDAddress string + MetaRootPath string QueryNodeIP string QueryNodePort int64 @@ -106,7 +105,6 @@ func (p *ParamTable) Init() { p.initPulsarAddress() p.initETCDAddress() p.initMetaRootPath() - p.initWriteNodeSegKvSubPath() p.initGracefulTime() p.initMsgChannelSubName() @@ -312,14 +310,6 @@ func (p *ParamTable) initMetaRootPath() { p.MetaRootPath = rootPath + "/" + subPath } -func (p *ParamTable) initWriteNodeSegKvSubPath() { - subPath, err := p.Load("etcd.writeNodeSegKvSubPath") - if err != nil { - panic(err) - } - p.WriteNodeSegKvSubPath = subPath + "/" -} - func (p *ParamTable) initGracefulTime() { p.GracefulTime = p.ParseInt64("queryNode.gracefulTime") } diff --git a/internal/util/paramtable/paramtable.go b/internal/util/paramtable/paramtable.go index 5a634a2698..6f6f77beb4 100644 --- a/internal/util/paramtable/paramtable.go +++ b/internal/util/paramtable/paramtable.go @@ -329,32 +329,14 @@ func (gp *BaseTable) ParseInt(key string) int { return value } -// GOOSE TODO: remove writenode -func (gp *BaseTable) WriteNodeIDList() []UniqueID { - proxyIDStr, err := gp.Load("nodeID.dataNodeIDList") - if err != nil { - panic(err) - } - var ret []UniqueID - proxyIDs := strings.Split(proxyIDStr, ",") - for _, i := range proxyIDs { - v, err := strconv.Atoi(i) - if err != nil { - log.Panicf("load write node id list error, %s", err.Error()) - } - ret = append(ret, UniqueID(v)) - } - return ret -} - func (gp *BaseTable) DataNodeIDList() []UniqueID { - proxyIDStr, err := gp.Load("nodeID.dataNodeIDList") + datanodeIDStr, err := gp.Load("nodeID.dataNodeIDList") if err != nil { panic(err) } var ret []UniqueID - proxyIDs := strings.Split(proxyIDStr, ",") - for _, i := range proxyIDs { + datanodeIDs := strings.Split(datanodeIDStr, ",") + for _, i := range datanodeIDs { v, err := strconv.Atoi(i) if err != nil { log.Panicf("load write node id list error, %s", err.Error()) @@ -364,23 +346,6 @@ func (gp *BaseTable) DataNodeIDList() []UniqueID { return ret } -func (gp *BaseTable) ProxyIDList() []UniqueID { - proxyIDStr, err := gp.Load("nodeID.proxyIDList") - if err != nil { - panic(err) - } - var ret []UniqueID - proxyIDs := strings.Split(proxyIDStr, ",") - for _, i := range proxyIDs { - v, err := strconv.Atoi(i) - if err != nil { - log.Panicf("load proxy id list error, %s", err.Error()) - } - ret = append(ret, UniqueID(v)) - } - return ret -} - func (gp *BaseTable) QueryNodeIDList() []UniqueID { queryNodeIDStr, err := gp.Load("nodeID.queryNodeIDList") if err != nil {