mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-12-24 17:47:57 +08:00
636 lines
16 KiB
Protocol Buffer
636 lines
16 KiB
Protocol Buffer
syntax = "proto3";
|
|
|
|
package volume_server_pb;
|
|
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
|
|
|
|
import "remote.proto";
|
|
|
|
//////////////////////////////////////////////////
|
|
|
|
service VolumeServer {
|
|
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
|
|
rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
|
|
}
|
|
|
|
rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
|
|
}
|
|
rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
|
|
}
|
|
rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
|
|
}
|
|
rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
|
|
}
|
|
|
|
rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
|
|
}
|
|
rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
|
|
}
|
|
|
|
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
|
|
}
|
|
rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
|
|
}
|
|
|
|
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
|
|
}
|
|
rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
|
|
}
|
|
rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
|
|
}
|
|
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
|
|
}
|
|
rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
|
|
}
|
|
rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
|
|
}
|
|
rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
|
|
}
|
|
|
|
// copy the .idx .dat files, and mount this volume
|
|
rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
|
|
}
|
|
rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
|
|
}
|
|
rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
|
|
}
|
|
|
|
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
|
|
}
|
|
rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
|
|
}
|
|
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
|
|
}
|
|
rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
|
|
}
|
|
|
|
rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
|
|
}
|
|
rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
|
|
}
|
|
|
|
// erasure coding
|
|
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
|
|
}
|
|
rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
|
|
}
|
|
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
|
|
}
|
|
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
|
|
}
|
|
rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
|
|
}
|
|
rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
|
|
}
|
|
rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
|
|
}
|
|
rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
|
|
}
|
|
rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
|
|
}
|
|
|
|
// tiered storage
|
|
rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
|
|
}
|
|
rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
|
|
}
|
|
|
|
rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
|
|
}
|
|
rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
|
|
}
|
|
|
|
// remote storage
|
|
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
|
|
}
|
|
|
|
// <experimental> query
|
|
rpc Query (QueryRequest) returns (stream QueriedStripe) {
|
|
}
|
|
|
|
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
|
|
}
|
|
|
|
rpc Ping (PingRequest) returns (PingResponse) {
|
|
}
|
|
|
|
}
|
|
|
|
//////////////////////////////////////////////////
|
|
|
|
message BatchDeleteRequest {
|
|
repeated string file_ids = 1;
|
|
bool skip_cookie_check = 2;
|
|
}
|
|
|
|
message BatchDeleteResponse {
|
|
repeated DeleteResult results = 1;
|
|
}
|
|
message DeleteResult {
|
|
string file_id = 1;
|
|
int32 status = 2;
|
|
string error = 3;
|
|
uint32 size = 4;
|
|
uint32 version = 5;
|
|
}
|
|
|
|
message Empty {
|
|
}
|
|
|
|
message VacuumVolumeCheckRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VacuumVolumeCheckResponse {
|
|
double garbage_ratio = 1;
|
|
}
|
|
|
|
message VacuumVolumeCompactRequest {
|
|
uint32 volume_id = 1;
|
|
int64 preallocate = 2;
|
|
}
|
|
message VacuumVolumeCompactResponse {
|
|
int64 processed_bytes = 1;
|
|
float load_avg_1m = 2;
|
|
}
|
|
|
|
message VacuumVolumeCommitRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VacuumVolumeCommitResponse {
|
|
bool is_read_only = 1;
|
|
uint64 volume_size = 2;
|
|
}
|
|
|
|
message VacuumVolumeCleanupRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VacuumVolumeCleanupResponse {
|
|
}
|
|
|
|
message DeleteCollectionRequest {
|
|
string collection = 1;
|
|
}
|
|
message DeleteCollectionResponse {
|
|
}
|
|
|
|
message AllocateVolumeRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
int64 preallocate = 3;
|
|
string replication = 4;
|
|
string ttl = 5;
|
|
uint32 memory_map_max_size_mb = 6;
|
|
string disk_type = 7;
|
|
}
|
|
message AllocateVolumeResponse {
|
|
}
|
|
|
|
message VolumeSyncStatusRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VolumeSyncStatusResponse {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
string replication = 4;
|
|
string ttl = 5;
|
|
uint64 tail_offset = 6;
|
|
uint32 compact_revision = 7;
|
|
uint64 idx_file_size = 8;
|
|
}
|
|
|
|
message VolumeIncrementalCopyRequest {
|
|
uint32 volume_id = 1;
|
|
uint64 since_ns = 2;
|
|
}
|
|
message VolumeIncrementalCopyResponse {
|
|
bytes file_content = 1;
|
|
}
|
|
|
|
message VolumeMountRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VolumeMountResponse {
|
|
}
|
|
|
|
message VolumeUnmountRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VolumeUnmountResponse {
|
|
}
|
|
|
|
message VolumeDeleteRequest {
|
|
uint32 volume_id = 1;
|
|
bool only_empty = 2;
|
|
}
|
|
message VolumeDeleteResponse {
|
|
}
|
|
|
|
message VolumeMarkReadonlyRequest {
|
|
uint32 volume_id = 1;
|
|
bool persist = 2;
|
|
}
|
|
message VolumeMarkReadonlyResponse {
|
|
}
|
|
|
|
message VolumeMarkWritableRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VolumeMarkWritableResponse {
|
|
}
|
|
|
|
message VolumeConfigureRequest {
|
|
uint32 volume_id = 1;
|
|
string replication = 2;
|
|
}
|
|
message VolumeConfigureResponse {
|
|
string error = 1;
|
|
}
|
|
|
|
message VolumeStatusRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message VolumeStatusResponse {
|
|
bool is_read_only = 1;
|
|
uint64 volume_size = 2;
|
|
uint64 file_count = 3;
|
|
uint64 file_deleted_count = 4;
|
|
}
|
|
|
|
message VolumeCopyRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
string replication = 3;
|
|
string ttl = 4;
|
|
string source_data_node = 5;
|
|
string disk_type = 6;
|
|
int64 io_byte_per_second = 7;
|
|
}
|
|
message VolumeCopyResponse {
|
|
uint64 last_append_at_ns = 1;
|
|
int64 processed_bytes = 2;
|
|
}
|
|
|
|
message CopyFileRequest {
|
|
uint32 volume_id = 1;
|
|
string ext = 2;
|
|
uint32 compaction_revision = 3;
|
|
uint64 stop_offset = 4;
|
|
string collection = 5;
|
|
bool is_ec_volume = 6;
|
|
bool ignore_source_file_not_found = 7;
|
|
}
|
|
message CopyFileResponse {
|
|
bytes file_content = 1;
|
|
int64 modified_ts_ns = 2;
|
|
}
|
|
|
|
message ReadNeedleBlobRequest {
|
|
uint32 volume_id = 1;
|
|
int64 offset = 3; // actual offset
|
|
int32 size = 4;
|
|
}
|
|
message ReadNeedleBlobResponse {
|
|
bytes needle_blob = 1;
|
|
}
|
|
|
|
message ReadNeedleMetaRequest {
|
|
uint32 volume_id = 1;
|
|
uint64 needle_id = 2;
|
|
int64 offset = 3; // actual offset
|
|
int32 size = 4;
|
|
}
|
|
message ReadNeedleMetaResponse {
|
|
uint32 cookie = 1;
|
|
uint64 last_modified = 2;
|
|
uint32 crc = 3;
|
|
string ttl = 4;
|
|
uint64 append_at_ns = 5;
|
|
}
|
|
|
|
message WriteNeedleBlobRequest {
|
|
uint32 volume_id = 1;
|
|
uint64 needle_id = 2;
|
|
int32 size = 3;
|
|
bytes needle_blob = 4;
|
|
}
|
|
message WriteNeedleBlobResponse {
|
|
}
|
|
|
|
message ReadAllNeedlesRequest {
|
|
repeated uint32 volume_ids = 1;
|
|
}
|
|
message ReadAllNeedlesResponse {
|
|
uint32 volume_id = 1;
|
|
uint64 needle_id = 2;
|
|
uint32 cookie = 3;
|
|
bytes needle_blob = 5;
|
|
bool needle_blob_compressed = 6;
|
|
uint64 last_modified = 7;
|
|
uint32 crc = 8;
|
|
bytes name = 9;
|
|
bytes mime = 10;
|
|
}
|
|
|
|
message VolumeTailSenderRequest {
|
|
uint32 volume_id = 1;
|
|
uint64 since_ns = 2;
|
|
uint32 idle_timeout_seconds = 3;
|
|
}
|
|
message VolumeTailSenderResponse {
|
|
bytes needle_header = 1;
|
|
bytes needle_body = 2;
|
|
bool is_last_chunk = 3;
|
|
}
|
|
|
|
message VolumeTailReceiverRequest {
|
|
uint32 volume_id = 1;
|
|
uint64 since_ns = 2;
|
|
uint32 idle_timeout_seconds = 3;
|
|
string source_volume_server = 4;
|
|
}
|
|
message VolumeTailReceiverResponse {
|
|
}
|
|
|
|
message VolumeEcShardsGenerateRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
}
|
|
message VolumeEcShardsGenerateResponse {
|
|
}
|
|
|
|
message VolumeEcShardsRebuildRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
}
|
|
message VolumeEcShardsRebuildResponse {
|
|
repeated uint32 rebuilt_shard_ids = 1;
|
|
}
|
|
|
|
message VolumeEcShardsCopyRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
repeated uint32 shard_ids = 3;
|
|
bool copy_ecx_file = 4;
|
|
string source_data_node = 5;
|
|
bool copy_ecj_file = 6;
|
|
bool copy_vif_file = 7;
|
|
}
|
|
message VolumeEcShardsCopyResponse {
|
|
}
|
|
|
|
message VolumeEcShardsDeleteRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
repeated uint32 shard_ids = 3;
|
|
}
|
|
message VolumeEcShardsDeleteResponse {
|
|
}
|
|
|
|
message VolumeEcShardsMountRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
repeated uint32 shard_ids = 3;
|
|
}
|
|
message VolumeEcShardsMountResponse {
|
|
}
|
|
|
|
message VolumeEcShardsUnmountRequest {
|
|
uint32 volume_id = 1;
|
|
repeated uint32 shard_ids = 3;
|
|
}
|
|
message VolumeEcShardsUnmountResponse {
|
|
}
|
|
|
|
message VolumeEcShardReadRequest {
|
|
uint32 volume_id = 1;
|
|
uint32 shard_id = 2;
|
|
int64 offset = 3;
|
|
int64 size = 4;
|
|
uint64 file_key = 5;
|
|
}
|
|
message VolumeEcShardReadResponse {
|
|
bytes data = 1;
|
|
bool is_deleted = 2;
|
|
}
|
|
|
|
message VolumeEcBlobDeleteRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
uint64 file_key = 3;
|
|
uint32 version = 4;
|
|
}
|
|
message VolumeEcBlobDeleteResponse {
|
|
}
|
|
|
|
message VolumeEcShardsToVolumeRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
}
|
|
message VolumeEcShardsToVolumeResponse {
|
|
}
|
|
|
|
message ReadVolumeFileStatusRequest {
|
|
uint32 volume_id = 1;
|
|
}
|
|
message ReadVolumeFileStatusResponse {
|
|
uint32 volume_id = 1;
|
|
uint64 idx_file_timestamp_seconds = 2;
|
|
uint64 idx_file_size = 3;
|
|
uint64 dat_file_timestamp_seconds = 4;
|
|
uint64 dat_file_size = 5;
|
|
uint64 file_count = 6;
|
|
uint32 compaction_revision = 7;
|
|
string collection = 8;
|
|
string disk_type = 9;
|
|
VolumeInfo volume_info = 10;
|
|
}
|
|
|
|
message DiskStatus {
|
|
string dir = 1;
|
|
uint64 all = 2;
|
|
uint64 used = 3;
|
|
uint64 free = 4;
|
|
float percent_free = 5;
|
|
float percent_used = 6;
|
|
string disk_type = 7;
|
|
}
|
|
|
|
message MemStatus {
|
|
int32 goroutines = 1;
|
|
uint64 all = 2;
|
|
uint64 used = 3;
|
|
uint64 free = 4;
|
|
uint64 self = 5;
|
|
uint64 heap = 6;
|
|
uint64 stack = 7;
|
|
}
|
|
|
|
// tired storage on volume servers
|
|
message RemoteFile {
|
|
string backend_type = 1;
|
|
string backend_id = 2;
|
|
string key = 3;
|
|
uint64 offset = 4;
|
|
uint64 file_size = 5;
|
|
uint64 modified_time = 6;
|
|
string extension = 7;
|
|
}
|
|
message VolumeInfo {
|
|
repeated RemoteFile files = 1;
|
|
uint32 version = 2;
|
|
string replication = 3;
|
|
uint32 bytes_offset = 4;
|
|
int64 dat_file_size = 5; // store the original dat file size
|
|
uint64 expire_at_sec = 6; // expiration time of ec volume
|
|
bool read_only = 7;
|
|
}
|
|
message OldVersionVolumeInfo {
|
|
repeated RemoteFile files = 1;
|
|
uint32 version = 2;
|
|
string replication = 3;
|
|
uint32 BytesOffset = 4;
|
|
int64 dat_file_size = 5; // store the original dat file size
|
|
uint64 DestroyTime = 6; // expiration time of ec volume
|
|
bool read_only = 7;
|
|
}
|
|
|
|
// tiered storage
|
|
message VolumeTierMoveDatToRemoteRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
string destination_backend_name = 3;
|
|
bool keep_local_dat_file = 4;
|
|
}
|
|
message VolumeTierMoveDatToRemoteResponse {
|
|
int64 processed = 1;
|
|
float processedPercentage = 2;
|
|
}
|
|
|
|
message VolumeTierMoveDatFromRemoteRequest {
|
|
uint32 volume_id = 1;
|
|
string collection = 2;
|
|
bool keep_remote_dat_file = 3;
|
|
}
|
|
message VolumeTierMoveDatFromRemoteResponse {
|
|
int64 processed = 1;
|
|
float processedPercentage = 2;
|
|
}
|
|
|
|
message VolumeServerStatusRequest {
|
|
|
|
}
|
|
message VolumeServerStatusResponse {
|
|
repeated DiskStatus disk_statuses = 1;
|
|
MemStatus memory_status = 2;
|
|
string version = 3;
|
|
string data_center = 4;
|
|
string rack = 5;
|
|
}
|
|
|
|
message VolumeServerLeaveRequest {
|
|
}
|
|
message VolumeServerLeaveResponse {
|
|
}
|
|
|
|
// remote storage
|
|
message FetchAndWriteNeedleRequest {
|
|
uint32 volume_id = 1;
|
|
uint64 needle_id = 2;
|
|
uint32 cookie = 3;
|
|
int64 offset = 4;
|
|
int64 size = 5;
|
|
message Replica {
|
|
string url = 1;
|
|
string public_url = 2;
|
|
int32 grpc_port = 3;
|
|
}
|
|
repeated Replica replicas = 6;
|
|
string auth = 7;
|
|
// remote conf
|
|
remote_pb.RemoteConf remote_conf = 15;
|
|
remote_pb.RemoteStorageLocation remote_location = 16;
|
|
}
|
|
message FetchAndWriteNeedleResponse {
|
|
string e_tag = 1;
|
|
}
|
|
|
|
// select on volume servers
|
|
message QueryRequest {
|
|
repeated string selections = 1;
|
|
repeated string from_file_ids = 2;
|
|
message Filter {
|
|
string field = 1;
|
|
string operand = 2;
|
|
string value = 3;
|
|
}
|
|
Filter filter = 3;
|
|
|
|
message InputSerialization {
|
|
// NONE | GZIP | BZIP2
|
|
string compression_type = 1;
|
|
message CSVInput {
|
|
string file_header_info = 1; // Valid values: NONE | USE | IGNORE
|
|
string record_delimiter = 2; // Default: \n
|
|
string field_delimiter = 3; // Default: ,
|
|
string quote_character = 4; // Default: "
|
|
string quote_escape_character = 5; // Default: "
|
|
string comments = 6; // Default: #
|
|
// If true, records might contain record delimiters within quote characters
|
|
bool allow_quoted_record_delimiter = 7; // default False.
|
|
}
|
|
message JSONInput {
|
|
string type = 1; // Valid values: DOCUMENT | LINES
|
|
}
|
|
message ParquetInput {
|
|
}
|
|
|
|
CSVInput csv_input = 2;
|
|
JSONInput json_input = 3;
|
|
ParquetInput parquet_input = 4;
|
|
}
|
|
InputSerialization input_serialization = 4;
|
|
|
|
message OutputSerialization {
|
|
message CSVOutput {
|
|
string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
|
|
string record_delimiter = 2; // Default: \n
|
|
string field_delimiter = 3; // Default: ,
|
|
string quote_character = 4; // Default: "
|
|
string quote_escape_character = 5; // Default: "
|
|
}
|
|
message JSONOutput {
|
|
string record_delimiter = 1;
|
|
}
|
|
|
|
CSVOutput csv_output = 2;
|
|
JSONOutput json_output = 3;
|
|
}
|
|
|
|
OutputSerialization output_serialization = 5;
|
|
}
|
|
message QueriedStripe {
|
|
bytes records = 1;
|
|
}
|
|
|
|
message VolumeNeedleStatusRequest {
|
|
uint32 volume_id = 1;
|
|
uint64 needle_id = 2;
|
|
}
|
|
message VolumeNeedleStatusResponse {
|
|
uint64 needle_id = 1;
|
|
uint32 cookie = 2;
|
|
uint32 size = 3;
|
|
uint64 last_modified = 4;
|
|
uint32 crc = 5;
|
|
string ttl = 6;
|
|
}
|
|
|
|
message PingRequest {
|
|
string target = 1; // default to ping itself
|
|
string target_type = 2;
|
|
}
|
|
message PingResponse {
|
|
int64 start_time_ns = 1;
|
|
int64 remote_time_ns = 2;
|
|
int64 stop_time_ns = 3;
|
|
}
|