milvus/pkg/proto/messages.proto

syntax = "proto3";

package milvus.proto.messages;

option go_package = "github.com/milvus-io/milvus/pkg/v2/proto/messagespb";

import "schema.proto";

// MessageID is the unique identifier of a message.
message MessageID {
    string id = 1;
}

// Message is the basic unit of communication between publisher and consumer.
message Message {
    bytes payload                  = 1;  // message body
    map<string, string> properties = 2;  // message properties
}

// ImmutableMessage is the message that can not be modified anymore.
message ImmutableMessage {
    MessageID id                   = 1;
    bytes payload                  = 2;  // message body
    map<string, string> properties = 3;  // message properties
}

// MessageType is the type of message.
enum MessageType {
    Unknown          = 0;
    TimeTick         = 1;
    Insert           = 2;
    Delete           = 3;
    Flush            = 4;
    CreateCollection = 5;
    DropCollection   = 6;
    CreatePartition  = 7;
    DropPartition    = 8;
    ManualFlush      = 9;
    CreateSegment    = 10;
    Import           = 11;
    SchemaChange     = 12;
    // begin transaction message is only used for transaction, once a begin
    // transaction message is received, all messages combined with the
    // transaction message cannot be consumed until a CommitTxn message
    // is received.
    BeginTxn = 900;
    // commit transaction message is only used for transaction, once a commit
    // transaction message is received, all messages combined with the
    // transaction message can be consumed, the message combined with the
    // transaction which is received after the commit transaction message will
    // be drop.
    CommitTxn = 901;
    // rollback transaction message is only used for transaction, once a
    // rollback transaction message is received, all messages combined with the
    // transaction message can be discarded, the message combined with the
    // transaction which is received after the rollback transaction message will
    // be drop.
    RollbackTxn = 902;
    // txn message is a set of messages combined by multiple messages in a
    // transaction. the txn properties is consist of the begin txn message and
    // commit txn message.
    Txn = 999;
}

///
/// Message Payload Definitions
/// Some message payload is defined at msg.proto at milvus-proto for
/// compatibility.
/// 1. InsertRequest
/// 2. DeleteRequest
/// 3. TimeTickRequest
/// 4. CreateCollectionRequest
/// 5. DropCollectionRequest
/// 6. CreatePartitionRequest
/// 7. DropPartitionRequest
///

// FlushMessageBody is the body of flush message.
message FlushMessageBody {}

// ManualFlushMessageBody is the body of manual flush message.
message ManualFlushMessageBody {}

// CreateSegmentMessageBody is the body of create segment message.
message CreateSegmentMessageBody {}

// BeginTxnMessageBody is the body of begin transaction message.
// Just do nothing now.
message BeginTxnMessageBody {}

// CommitTxnMessageBody is the body of commit transaction message.
// Just do nothing now.
message CommitTxnMessageBody {}

// RollbackTxnMessageBody is the body of rollback transaction message.
// Just do nothing now.
message RollbackTxnMessageBody {}

// TxnMessageBody is the body of transaction message.
// A transaction message is combined by multiple messages.
// It's only can be seen at consume side.
// All message in a transaction message only has same timetick which is equal to
// the CommitTransationMessage.
message TxnMessageBody {
    repeated Message messages = 1;
}

///
/// Message Header Definitions
/// Used to fast handling at streaming node write ahead.
/// The header should be simple and light enough to be parsed.
/// Do not put too much information in the header if unnecessary.
///

// TimeTickMessageHeader just nothing.
message TimeTickMessageHeader {}

// InsertMessageHeader is the header of insert message.
message InsertMessageHeader {
    int64 collection_id                            = 1;
    repeated PartitionSegmentAssignment partitions = 2;
}

// PartitionSegmentAssignment is the segment assignment of a partition.
message PartitionSegmentAssignment {
    int64 partition_id                   = 1;
    uint64 rows                          = 2;
    uint64 binary_size                   = 3;
    SegmentAssignment segment_assignment = 4;
}

// SegmentAssignment is the assignment of a segment.
message SegmentAssignment {
    int64 segment_id = 1;
}

// DeleteMessageHeader
message DeleteMessageHeader {
    int64 collection_id = 1;
    uint64 rows = 2;
}

// FlushMessageHeader just nothing.
message FlushMessageHeader {
    int64 collection_id = 1;
    int64 partition_id = 2;
    int64 segment_id = 3;
}

// CreateSegmentMessageHeader just nothing.
message CreateSegmentMessageHeader {
    int64 collection_id     = 1;
    int64 partition_id      = 2;
    int64 segment_id        = 3;
    int64 storage_version   = 4;        // the storage version of the segment.
    uint64 max_segment_size = 5; // the max size bytes of the segment.
}

message ManualFlushMessageHeader {
    int64 collection_id = 1;
    uint64 flush_ts     = 2;
    repeated int64 segment_ids   = 3; // the segment ids to be flushed, will be filled by wal shard manager.
}

// CreateCollectionMessageHeader is the header of create collection message.
message CreateCollectionMessageHeader {
    int64 collection_id          = 1;
    repeated int64 partition_ids = 2;
}

// DropCollectionMessageHeader is the header of drop collection message.
message DropCollectionMessageHeader {
    int64 collection_id = 1;
}

// CreatePartitionMessageHeader is the header of create partition message.
message CreatePartitionMessageHeader {
    int64 collection_id = 1;
    int64 partition_id  = 2;
}

// DropPartitionMessageHeader is the header of drop partition message.
message DropPartitionMessageHeader {
    int64 collection_id = 1;
    int64 partition_id  = 2;
}

// BeginTxnMessageHeader is the header of begin transaction message.
// Just do nothing now.
// Add Channel info here to implement cross pchannel transaction.
message BeginTxnMessageHeader {
    // the max milliseconds to keep alive of the transaction.
    // the keepalive_milliseconds is never changed in a transaction by now,
    int64 keepalive_milliseconds = 1;
}

// CommitTxnMessageHeader is the header of commit transaction message.
// Just do nothing now.
message CommitTxnMessageHeader {}

// RollbackTxnMessageHeader is the header of rollback transaction
// message.
// Just do nothing now.
message RollbackTxnMessageHeader {}

// TxnMessageHeader is the header of transaction message.
// Just do nothing now.
message TxnMessageHeader {}

message ImportMessageHeader {}

// SchemaChangeMessageHeader is the header of CollectionSchema update message.
message SchemaChangeMessageHeader{
    int64 collection_id = 1;
    repeated int64 flushed_segment_ids = 2; // will be filled by wal shard manager.
}

// SchemaChangeMessageBody is the body of CollectionSchema update message.
message SchemaChangeMessageBody{
    schema.CollectionSchema schema = 1;
    uint64 modify_ts = 2;
}

///
/// Message Extra Response
/// Used to add extra information when response to the client.
///
///

// ManualFlushExtraResponse is the extra response of manual flush message.
message ManualFlushExtraResponse {
    repeated int64 segment_ids = 1;
}

// TxnContext is the context of transaction.
// It will be carried by every message in a transaction.
message TxnContext {
    // the unique id of the transaction.
    // the txn_id is never changed in a transaction.
    int64 txn_id = 1;
    // the next keep alive timeout of the transaction.
    // after the keep alive timeout, the transaction will be expired.
    int64 keepalive_milliseconds = 2;
}

enum TxnState {
    // should never be used.
    TxnUnknown = 0;
    // the transaction is in flight.
    TxnInFlight = 1;
    // the transaction is on commit.
    TxnOnCommit = 2;
    // the transaction is committed.
    TxnCommitted = 3;
    // the transaction is on rollback.
    TxnOnRollback = 4;
    // the transaction is rollbacked.
    TxnRollbacked = 5;
}

// RMQMessageLayout is the layout of message for RMQ.
message RMQMessageLayout {
    bytes payload                  = 1;  // message body
    map<string, string> properties = 2;  // message properties
}

// BroadcastHeader is the common header of broadcast message.
message BroadcastHeader {
    uint64 broadcast_id = 1;
    repeated string vchannels = 2;
    repeated ResourceKey Resource_keys = 3; // the resource key of the broadcast message.
    // Once the broadcast is sent, the resource of resource key will be hold.
    // New broadcast message with the same resource key will be rejected.
    // And the user can watch the resource key to known when the resource is released.
}

// ResourceDomain is the domain of resource hold.
enum ResourceDomain {
    ResourceDomainUnknown = 0; // should never be used.
    ResourceDomainImportJobID = 1; // the domain of import job id.
    ResourceDomainCollectionName = 2; // the domain of collection name.
}

// ResourceKey is the key for resource hold.
// It's used to implement the resource acquirition mechanism for broadcast message.
// The key should be a unique identifier of the resource for different domain.
message ResourceKey {
    ResourceDomain domain = 1;
    string key = 2;
}

// CipherHeader is the header of a message that is encrypted.
message CipherHeader {
    int64 ez_id = 1; // related to the encryption zone id
    int64 collection_id = 2; // related to the collection id
    bytes safe_key = 3; // the safe key
    int64 payload_bytes = 4; // the size of the payload before encryption
}