diff --git a/api/pb/caddy_storage.pb.go b/api/pb/caddy_storage.pb.go index b591cadc..34ddb200 100644 --- a/api/pb/caddy_storage.pb.go +++ b/api/pb/caddy_storage.pb.go @@ -129,8 +129,8 @@ type LoadCaddyStorageResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Must contain only one repeated messages field to allow broadcasting Load requests to multiple machines. - Messages []*MachineCaddyStorageValue `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"` + Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` } func (x *LoadCaddyStorageResponse) Reset() { @@ -165,70 +165,14 @@ func (*LoadCaddyStorageResponse) Descriptor() ([]byte, []int) { return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{2} } -func (x *LoadCaddyStorageResponse) GetMessages() []*MachineCaddyStorageValue { - if x != nil { - return x.Messages - } - return nil -} - -type MachineCaddyStorageValue struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` -} - -func (x *MachineCaddyStorageValue) Reset() { - *x = MachineCaddyStorageValue{} - if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MachineCaddyStorageValue) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MachineCaddyStorageValue) ProtoMessage() {} - -func (x *MachineCaddyStorageValue) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MachineCaddyStorageValue.ProtoReflect.Descriptor instead. -func (*MachineCaddyStorageValue) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{3} -} - -func (x *MachineCaddyStorageValue) GetMetadata() *Metadata { - if x != nil { - return x.Metadata - } - return nil -} - -func (x *MachineCaddyStorageValue) GetValue() []byte { +func (x *LoadCaddyStorageResponse) GetValue() []byte { if x != nil { return x.Value } return nil } -func (x *MachineCaddyStorageValue) GetUpdatedAt() *timestamppb.Timestamp { +func (x *LoadCaddyStorageResponse) GetUpdatedAt() *timestamppb.Timestamp { if x != nil { return x.UpdatedAt } @@ -246,7 +190,7 @@ type DeleteCaddyStorageRequest struct { func (x *DeleteCaddyStorageRequest) Reset() { *x = DeleteCaddyStorageRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[4] + mi := &file_api_pb_caddy_storage_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -259,7 +203,7 @@ func (x *DeleteCaddyStorageRequest) String() string { func (*DeleteCaddyStorageRequest) ProtoMessage() {} func (x *DeleteCaddyStorageRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[4] + mi := &file_api_pb_caddy_storage_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -272,7 +216,7 @@ func (x *DeleteCaddyStorageRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteCaddyStorageRequest.ProtoReflect.Descriptor instead. func (*DeleteCaddyStorageRequest) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{4} + return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{3} } func (x *DeleteCaddyStorageRequest) GetKey() string { @@ -294,7 +238,7 @@ type ListCaddyStorageRequest struct { func (x *ListCaddyStorageRequest) Reset() { *x = ListCaddyStorageRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[5] + mi := &file_api_pb_caddy_storage_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -307,7 +251,7 @@ func (x *ListCaddyStorageRequest) String() string { func (*ListCaddyStorageRequest) ProtoMessage() {} func (x *ListCaddyStorageRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[5] + mi := &file_api_pb_caddy_storage_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -320,7 +264,7 @@ func (x *ListCaddyStorageRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListCaddyStorageRequest.ProtoReflect.Descriptor instead. func (*ListCaddyStorageRequest) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{5} + return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{4} } func (x *ListCaddyStorageRequest) GetPrefix() string { @@ -342,14 +286,13 @@ type ListCaddyStorageResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Must contain only one repeated messages field to allow broadcasting List requests to multiple machines. - Messages []*MachineCaddyStorageKeys `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"` + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` } func (x *ListCaddyStorageResponse) Reset() { *x = ListCaddyStorageResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[6] + mi := &file_api_pb_caddy_storage_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -362,7 +305,7 @@ func (x *ListCaddyStorageResponse) String() string { func (*ListCaddyStorageResponse) ProtoMessage() {} func (x *ListCaddyStorageResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[6] + mi := &file_api_pb_caddy_storage_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -375,65 +318,10 @@ func (x *ListCaddyStorageResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListCaddyStorageResponse.ProtoReflect.Descriptor instead. func (*ListCaddyStorageResponse) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{6} + return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{5} } -func (x *ListCaddyStorageResponse) GetMessages() []*MachineCaddyStorageKeys { - if x != nil { - return x.Messages - } - return nil -} - -type MachineCaddyStorageKeys struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` - Keys []string `protobuf:"bytes,2,rep,name=keys,proto3" json:"keys,omitempty"` -} - -func (x *MachineCaddyStorageKeys) Reset() { - *x = MachineCaddyStorageKeys{} - if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MachineCaddyStorageKeys) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MachineCaddyStorageKeys) ProtoMessage() {} - -func (x *MachineCaddyStorageKeys) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MachineCaddyStorageKeys.ProtoReflect.Descriptor instead. -func (*MachineCaddyStorageKeys) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{7} -} - -func (x *MachineCaddyStorageKeys) GetMetadata() *Metadata { - if x != nil { - return x.Metadata - } - return nil -} - -func (x *MachineCaddyStorageKeys) GetKeys() []string { +func (x *ListCaddyStorageResponse) GetKeys() []string { if x != nil { return x.Keys } @@ -451,7 +339,7 @@ type StatCaddyStorageRequest struct { func (x *StatCaddyStorageRequest) Reset() { *x = StatCaddyStorageRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[8] + mi := &file_api_pb_caddy_storage_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -464,7 +352,7 @@ func (x *StatCaddyStorageRequest) String() string { func (*StatCaddyStorageRequest) ProtoMessage() {} func (x *StatCaddyStorageRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[8] + mi := &file_api_pb_caddy_storage_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -477,7 +365,7 @@ func (x *StatCaddyStorageRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StatCaddyStorageRequest.ProtoReflect.Descriptor instead. func (*StatCaddyStorageRequest) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{8} + return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{6} } func (x *StatCaddyStorageRequest) GetKey() string { @@ -492,14 +380,16 @@ type StatCaddyStorageResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Must contain only one repeated messages field to allow broadcasting Stat requests to multiple machines. - Messages []*MachineCaddyStorageKeyInfo `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` + Size int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + IsTerminal bool `protobuf:"varint,4,opt,name=is_terminal,json=isTerminal,proto3" json:"is_terminal,omitempty"` } func (x *StatCaddyStorageResponse) Reset() { *x = StatCaddyStorageResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[9] + mi := &file_api_pb_caddy_storage_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -512,7 +402,7 @@ func (x *StatCaddyStorageResponse) String() string { func (*StatCaddyStorageResponse) ProtoMessage() {} func (x *StatCaddyStorageResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[9] + mi := &file_api_pb_caddy_storage_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -525,89 +415,31 @@ func (x *StatCaddyStorageResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StatCaddyStorageResponse.ProtoReflect.Descriptor instead. func (*StatCaddyStorageResponse) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{9} + return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{7} } -func (x *StatCaddyStorageResponse) GetMessages() []*MachineCaddyStorageKeyInfo { - if x != nil { - return x.Messages - } - return nil -} - -type MachineCaddyStorageKeyInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` - Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` - Size int64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` - IsTerminal bool `protobuf:"varint,5,opt,name=is_terminal,json=isTerminal,proto3" json:"is_terminal,omitempty"` -} - -func (x *MachineCaddyStorageKeyInfo) Reset() { - *x = MachineCaddyStorageKeyInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_api_pb_caddy_storage_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MachineCaddyStorageKeyInfo) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MachineCaddyStorageKeyInfo) ProtoMessage() {} - -func (x *MachineCaddyStorageKeyInfo) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_caddy_storage_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MachineCaddyStorageKeyInfo.ProtoReflect.Descriptor instead. -func (*MachineCaddyStorageKeyInfo) Descriptor() ([]byte, []int) { - return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{10} -} - -func (x *MachineCaddyStorageKeyInfo) GetMetadata() *Metadata { - if x != nil { - return x.Metadata - } - return nil -} - -func (x *MachineCaddyStorageKeyInfo) GetKey() string { +func (x *StatCaddyStorageResponse) GetKey() string { if x != nil { return x.Key } return "" } -func (x *MachineCaddyStorageKeyInfo) GetUpdatedAt() *timestamppb.Timestamp { +func (x *StatCaddyStorageResponse) GetUpdatedAt() *timestamppb.Timestamp { if x != nil { return x.UpdatedAt } return nil } -func (x *MachineCaddyStorageKeyInfo) GetSize() int64 { +func (x *StatCaddyStorageResponse) GetSize() int64 { if x != nil { return x.Size } return 0 } -func (x *MachineCaddyStorageKeyInfo) GetIsTerminal() bool { +func (x *StatCaddyStorageResponse) GetIsTerminal() bool { if x != nil { return x.IsTerminal } @@ -622,96 +454,70 @@ var file_api_pb_caddy_storage_proto_rawDesc = []byte{ 0x69, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, - 0x13, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x42, 0x0a, 0x18, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x61, 0x64, - 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2b, 0x0a, 0x17, 0x4c, 0x6f, 0x61, 0x64, - 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x55, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, - 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x39, 0x0a, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, - 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, 0x96, 0x01, 0x0a, - 0x18, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, - 0x72, 0x61, 0x67, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x2d, 0x0a, 0x19, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, - 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x22, 0x4f, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, - 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, - 0x73, 0x69, 0x76, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, - 0x72, 0x73, 0x69, 0x76, 0x65, 0x22, 0x54, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, - 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x38, 0x0a, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, - 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, - 0x73, 0x52, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, 0x58, 0x0a, 0x17, 0x4d, - 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x2b, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, - 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x22, 0x57, 0x0a, 0x18, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, - 0x0a, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x43, 0x61, - 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, 0xc9, 0x01, 0x0a, 0x1a, - 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, - 0x41, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x73, 0x5f, 0x74, 0x65, 0x72, - 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x54, - 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x32, 0xdb, 0x02, 0x0a, 0x0c, 0x43, 0x61, 0x64, 0x64, - 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x3e, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, - 0x65, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x61, 0x64, - 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x6f, 0x61, 0x64, - 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, 0x64, 0x79, - 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, - 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, - 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x4c, - 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, - 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, - 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x43, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x42, 0x0a, 0x18, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x22, 0x2b, 0x0a, 0x17, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, 0x64, 0x79, + 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x22, 0x6b, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x2d, 0x0a, + 0x19, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x4f, 0x0a, 0x17, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, + 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, + 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x22, 0x2e, 0x0a, + 0x18, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x2b, 0x0a, + 0x17, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x9c, 0x01, 0x0a, 0x18, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, - 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x64, 0x41, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x73, 0x5f, 0x74, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, + 0x73, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x32, 0xdf, 0x02, 0x0a, 0x0c, 0x43, 0x61, + 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x3e, 0x0a, 0x05, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, + 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x6f, + 0x61, 0x64, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, + 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, 0x64, 0x79, + 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x40, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x1c, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, + 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -726,48 +532,37 @@ func file_api_pb_caddy_storage_proto_rawDescGZIP() []byte { return file_api_pb_caddy_storage_proto_rawDescData } -var file_api_pb_caddy_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 11) +var file_api_pb_caddy_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_api_pb_caddy_storage_proto_goTypes = []any{ - (*StoreCaddyStorageRequest)(nil), // 0: api.StoreCaddyStorageRequest - (*LoadCaddyStorageRequest)(nil), // 1: api.LoadCaddyStorageRequest - (*LoadCaddyStorageResponse)(nil), // 2: api.LoadCaddyStorageResponse - (*MachineCaddyStorageValue)(nil), // 3: api.MachineCaddyStorageValue - (*DeleteCaddyStorageRequest)(nil), // 4: api.DeleteCaddyStorageRequest - (*ListCaddyStorageRequest)(nil), // 5: api.ListCaddyStorageRequest - (*ListCaddyStorageResponse)(nil), // 6: api.ListCaddyStorageResponse - (*MachineCaddyStorageKeys)(nil), // 7: api.MachineCaddyStorageKeys - (*StatCaddyStorageRequest)(nil), // 8: api.StatCaddyStorageRequest - (*StatCaddyStorageResponse)(nil), // 9: api.StatCaddyStorageResponse - (*MachineCaddyStorageKeyInfo)(nil), // 10: api.MachineCaddyStorageKeyInfo - (*Metadata)(nil), // 11: api.Metadata - (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 13: google.protobuf.Empty - (*EmptyResponse)(nil), // 14: api.EmptyResponse + (*StoreCaddyStorageRequest)(nil), // 0: api.StoreCaddyStorageRequest + (*LoadCaddyStorageRequest)(nil), // 1: api.LoadCaddyStorageRequest + (*LoadCaddyStorageResponse)(nil), // 2: api.LoadCaddyStorageResponse + (*DeleteCaddyStorageRequest)(nil), // 3: api.DeleteCaddyStorageRequest + (*ListCaddyStorageRequest)(nil), // 4: api.ListCaddyStorageRequest + (*ListCaddyStorageResponse)(nil), // 5: api.ListCaddyStorageResponse + (*StatCaddyStorageRequest)(nil), // 6: api.StatCaddyStorageRequest + (*StatCaddyStorageResponse)(nil), // 7: api.StatCaddyStorageResponse + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_api_pb_caddy_storage_proto_depIdxs = []int32{ - 3, // 0: api.LoadCaddyStorageResponse.messages:type_name -> api.MachineCaddyStorageValue - 11, // 1: api.MachineCaddyStorageValue.metadata:type_name -> api.Metadata - 12, // 2: api.MachineCaddyStorageValue.updated_at:type_name -> google.protobuf.Timestamp - 7, // 3: api.ListCaddyStorageResponse.messages:type_name -> api.MachineCaddyStorageKeys - 11, // 4: api.MachineCaddyStorageKeys.metadata:type_name -> api.Metadata - 10, // 5: api.StatCaddyStorageResponse.messages:type_name -> api.MachineCaddyStorageKeyInfo - 11, // 6: api.MachineCaddyStorageKeyInfo.metadata:type_name -> api.Metadata - 12, // 7: api.MachineCaddyStorageKeyInfo.updated_at:type_name -> google.protobuf.Timestamp - 0, // 8: api.CaddyStorage.Store:input_type -> api.StoreCaddyStorageRequest - 1, // 9: api.CaddyStorage.Load:input_type -> api.LoadCaddyStorageRequest - 4, // 10: api.CaddyStorage.Delete:input_type -> api.DeleteCaddyStorageRequest - 5, // 11: api.CaddyStorage.List:input_type -> api.ListCaddyStorageRequest - 8, // 12: api.CaddyStorage.Stat:input_type -> api.StatCaddyStorageRequest - 13, // 13: api.CaddyStorage.Store:output_type -> google.protobuf.Empty - 2, // 14: api.CaddyStorage.Load:output_type -> api.LoadCaddyStorageResponse - 14, // 15: api.CaddyStorage.Delete:output_type -> api.EmptyResponse - 6, // 16: api.CaddyStorage.List:output_type -> api.ListCaddyStorageResponse - 9, // 17: api.CaddyStorage.Stat:output_type -> api.StatCaddyStorageResponse - 13, // [13:18] is the sub-list for method output_type - 8, // [8:13] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 8, // 0: api.LoadCaddyStorageResponse.updated_at:type_name -> google.protobuf.Timestamp + 8, // 1: api.StatCaddyStorageResponse.updated_at:type_name -> google.protobuf.Timestamp + 0, // 2: api.CaddyStorage.Store:input_type -> api.StoreCaddyStorageRequest + 1, // 3: api.CaddyStorage.Load:input_type -> api.LoadCaddyStorageRequest + 3, // 4: api.CaddyStorage.Delete:input_type -> api.DeleteCaddyStorageRequest + 4, // 5: api.CaddyStorage.List:input_type -> api.ListCaddyStorageRequest + 6, // 6: api.CaddyStorage.Stat:input_type -> api.StatCaddyStorageRequest + 9, // 7: api.CaddyStorage.Store:output_type -> google.protobuf.Empty + 2, // 8: api.CaddyStorage.Load:output_type -> api.LoadCaddyStorageResponse + 9, // 9: api.CaddyStorage.Delete:output_type -> google.protobuf.Empty + 5, // 10: api.CaddyStorage.List:output_type -> api.ListCaddyStorageResponse + 7, // 11: api.CaddyStorage.Stat:output_type -> api.StatCaddyStorageResponse + 7, // [7:12] is the sub-list for method output_type + 2, // [2:7] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_api_pb_caddy_storage_proto_init() } @@ -775,7 +570,6 @@ func file_api_pb_caddy_storage_proto_init() { if File_api_pb_caddy_storage_proto != nil { return } - file_api_pb_common_proto_init() if !protoimpl.UnsafeEnabled { file_api_pb_caddy_storage_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*StoreCaddyStorageRequest); i { @@ -814,18 +608,6 @@ func file_api_pb_caddy_storage_proto_init() { } } file_api_pb_caddy_storage_proto_msgTypes[3].Exporter = func(v any, i int) any { - switch v := v.(*MachineCaddyStorageValue); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_api_pb_caddy_storage_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*DeleteCaddyStorageRequest); i { case 0: return &v.state @@ -837,7 +619,7 @@ func file_api_pb_caddy_storage_proto_init() { return nil } } - file_api_pb_caddy_storage_proto_msgTypes[5].Exporter = func(v any, i int) any { + file_api_pb_caddy_storage_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*ListCaddyStorageRequest); i { case 0: return &v.state @@ -849,7 +631,7 @@ func file_api_pb_caddy_storage_proto_init() { return nil } } - file_api_pb_caddy_storage_proto_msgTypes[6].Exporter = func(v any, i int) any { + file_api_pb_caddy_storage_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*ListCaddyStorageResponse); i { case 0: return &v.state @@ -861,19 +643,7 @@ func file_api_pb_caddy_storage_proto_init() { return nil } } - file_api_pb_caddy_storage_proto_msgTypes[7].Exporter = func(v any, i int) any { - switch v := v.(*MachineCaddyStorageKeys); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_api_pb_caddy_storage_proto_msgTypes[8].Exporter = func(v any, i int) any { + file_api_pb_caddy_storage_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*StatCaddyStorageRequest); i { case 0: return &v.state @@ -885,7 +655,7 @@ func file_api_pb_caddy_storage_proto_init() { return nil } } - file_api_pb_caddy_storage_proto_msgTypes[9].Exporter = func(v any, i int) any { + file_api_pb_caddy_storage_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*StatCaddyStorageResponse); i { case 0: return &v.state @@ -897,18 +667,6 @@ func file_api_pb_caddy_storage_proto_init() { return nil } } - file_api_pb_caddy_storage_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*MachineCaddyStorageKeyInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } } type x struct{} out := protoimpl.TypeBuilder{ @@ -916,7 +674,7 @@ func file_api_pb_caddy_storage_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_api_pb_caddy_storage_proto_rawDesc, NumEnums: 0, - NumMessages: 11, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/api/pb/caddy_storage.proto b/api/pb/caddy_storage.proto index a18a82b4..f62e28f8 100644 --- a/api/pb/caddy_storage.proto +++ b/api/pb/caddy_storage.proto @@ -6,14 +6,16 @@ option go_package = "github.com/psviderski/uncloud/api/pb"; import "google/protobuf/empty.proto"; import "google/protobuf/timestamp.proto"; -import "api/pb/common.proto"; // CaddyStorage exposes the CertMagic storage operations backed by the distributed cluster store. +// Each request operates on the receiving machine's store replica. Callers that need replication to catch up before +// reading should capture store versions with Machine.InspectMachine and call Machine.WaitForStoreVersion on the +// receiving machine, subject to that RPC's data-availability limitations (see WaitForStoreVersion docs). // See Storage interface in https://github.com/caddyserver/certmagic/blob/master/storage.go. service CaddyStorage { rpc Store(StoreCaddyStorageRequest) returns (google.protobuf.Empty); rpc Load(LoadCaddyStorageRequest) returns (LoadCaddyStorageResponse); - rpc Delete(DeleteCaddyStorageRequest) returns (EmptyResponse); + rpc Delete(DeleteCaddyStorageRequest) returns (google.protobuf.Empty); rpc List(ListCaddyStorageRequest) returns (ListCaddyStorageResponse); rpc Stat(StatCaddyStorageRequest) returns (StatCaddyStorageResponse); } @@ -28,14 +30,8 @@ message LoadCaddyStorageRequest { } message LoadCaddyStorageResponse { - // Must contain only one repeated messages field to allow broadcasting Load requests to multiple machines. - repeated MachineCaddyStorageValue messages = 1; -} - -message MachineCaddyStorageValue { - Metadata metadata = 1; - bytes value = 2; - google.protobuf.Timestamp updated_at = 3; + bytes value = 1; + google.protobuf.Timestamp updated_at = 2; } message DeleteCaddyStorageRequest { @@ -48,13 +44,7 @@ message ListCaddyStorageRequest { } message ListCaddyStorageResponse { - // Must contain only one repeated messages field to allow broadcasting List requests to multiple machines. - repeated MachineCaddyStorageKeys messages = 1; -} - -message MachineCaddyStorageKeys { - Metadata metadata = 1; - repeated string keys = 2; + repeated string keys = 1; } message StatCaddyStorageRequest { @@ -62,14 +52,8 @@ message StatCaddyStorageRequest { } message StatCaddyStorageResponse { - // Must contain only one repeated messages field to allow broadcasting Stat requests to multiple machines. - repeated MachineCaddyStorageKeyInfo messages = 1; -} - -message MachineCaddyStorageKeyInfo { - Metadata metadata = 1; - string key = 2; - google.protobuf.Timestamp updated_at = 3; - int64 size = 4; - bool is_terminal = 5; + string key = 1; + google.protobuf.Timestamp updated_at = 2; + int64 size = 3; + bool is_terminal = 4; } diff --git a/api/pb/caddy_storage_grpc.pb.go b/api/pb/caddy_storage_grpc.pb.go index dff90feb..56c488ab 100644 --- a/api/pb/caddy_storage_grpc.pb.go +++ b/api/pb/caddy_storage_grpc.pb.go @@ -32,11 +32,14 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // CaddyStorage exposes the CertMagic storage operations backed by the distributed cluster store. +// Each request operates on the receiving machine's store replica. Callers that need replication to catch up before +// reading should capture store versions with Machine.InspectMachine and call Machine.WaitForStoreVersion on the +// receiving machine, subject to that RPC's data-availability limitations (see WaitForStoreVersion docs). // See Storage interface in https://github.com/caddyserver/certmagic/blob/master/storage.go. type CaddyStorageClient interface { Store(ctx context.Context, in *StoreCaddyStorageRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) Load(ctx context.Context, in *LoadCaddyStorageRequest, opts ...grpc.CallOption) (*LoadCaddyStorageResponse, error) - Delete(ctx context.Context, in *DeleteCaddyStorageRequest, opts ...grpc.CallOption) (*EmptyResponse, error) + Delete(ctx context.Context, in *DeleteCaddyStorageRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) List(ctx context.Context, in *ListCaddyStorageRequest, opts ...grpc.CallOption) (*ListCaddyStorageResponse, error) Stat(ctx context.Context, in *StatCaddyStorageRequest, opts ...grpc.CallOption) (*StatCaddyStorageResponse, error) } @@ -69,9 +72,9 @@ func (c *caddyStorageClient) Load(ctx context.Context, in *LoadCaddyStorageReque return out, nil } -func (c *caddyStorageClient) Delete(ctx context.Context, in *DeleteCaddyStorageRequest, opts ...grpc.CallOption) (*EmptyResponse, error) { +func (c *caddyStorageClient) Delete(ctx context.Context, in *DeleteCaddyStorageRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(EmptyResponse) + out := new(emptypb.Empty) err := c.cc.Invoke(ctx, CaddyStorage_Delete_FullMethodName, in, out, cOpts...) if err != nil { return nil, err @@ -104,11 +107,14 @@ func (c *caddyStorageClient) Stat(ctx context.Context, in *StatCaddyStorageReque // for forward compatibility. // // CaddyStorage exposes the CertMagic storage operations backed by the distributed cluster store. +// Each request operates on the receiving machine's store replica. Callers that need replication to catch up before +// reading should capture store versions with Machine.InspectMachine and call Machine.WaitForStoreVersion on the +// receiving machine, subject to that RPC's data-availability limitations (see WaitForStoreVersion docs). // See Storage interface in https://github.com/caddyserver/certmagic/blob/master/storage.go. type CaddyStorageServer interface { Store(context.Context, *StoreCaddyStorageRequest) (*emptypb.Empty, error) Load(context.Context, *LoadCaddyStorageRequest) (*LoadCaddyStorageResponse, error) - Delete(context.Context, *DeleteCaddyStorageRequest) (*EmptyResponse, error) + Delete(context.Context, *DeleteCaddyStorageRequest) (*emptypb.Empty, error) List(context.Context, *ListCaddyStorageRequest) (*ListCaddyStorageResponse, error) Stat(context.Context, *StatCaddyStorageRequest) (*StatCaddyStorageResponse, error) mustEmbedUnimplementedCaddyStorageServer() @@ -127,7 +133,7 @@ func (UnimplementedCaddyStorageServer) Store(context.Context, *StoreCaddyStorage func (UnimplementedCaddyStorageServer) Load(context.Context, *LoadCaddyStorageRequest) (*LoadCaddyStorageResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Load not implemented") } -func (UnimplementedCaddyStorageServer) Delete(context.Context, *DeleteCaddyStorageRequest) (*EmptyResponse, error) { +func (UnimplementedCaddyStorageServer) Delete(context.Context, *DeleteCaddyStorageRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") } func (UnimplementedCaddyStorageServer) List(context.Context, *ListCaddyStorageRequest) (*ListCaddyStorageResponse, error) { diff --git a/internal/machine/caddystorage/server.go b/internal/machine/caddystorage/server.go index c29204f9..db48d10e 100644 --- a/internal/machine/caddystorage/server.go +++ b/internal/machine/caddystorage/server.go @@ -57,14 +57,12 @@ func (s *Server) Load(ctx context.Context, req *pb.LoadCaddyStorageRequest) (*pb } return &pb.LoadCaddyStorageResponse{ - Messages: []*pb.MachineCaddyStorageValue{{ - Value: record.Value, - UpdatedAt: timestamppb.New(record.UpdatedAt), - }}, + Value: record.Value, + UpdatedAt: timestamppb.New(record.UpdatedAt), }, nil } -func (s *Server) Delete(ctx context.Context, req *pb.DeleteCaddyStorageRequest) (*pb.EmptyResponse, error) { +func (s *Server) Delete(ctx context.Context, req *pb.DeleteCaddyStorageRequest) (*emptypb.Empty, error) { if err := validateKey(req.Key); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -77,9 +75,7 @@ func (s *Server) Delete(ctx context.Context, req *pb.DeleteCaddyStorageRequest) return nil, status.Errorf(codes.Internal, "delete key: %v", err) } - return &pb.EmptyResponse{ - Messages: []*pb.Empty{{}}, - }, nil + return &emptypb.Empty{}, nil } func (s *Server) List(ctx context.Context, req *pb.ListCaddyStorageRequest) (*pb.ListCaddyStorageResponse, error) { @@ -108,9 +104,7 @@ func (s *Server) List(ctx context.Context, req *pb.ListCaddyStorageRequest) (*pb } return &pb.ListCaddyStorageResponse{ - Messages: []*pb.MachineCaddyStorageKeys{{ - Keys: listKeys(records, req.Prefix, req.Recursive), - }}, + Keys: listKeys(records, req.Prefix, req.Recursive), }, nil } @@ -143,12 +137,10 @@ func (s *Server) Stat(ctx context.Context, req *pb.StatCaddyStorageRequest) (*pb record, err := s.store.Get(ctx, req.Key) if err == nil { return &pb.StatCaddyStorageResponse{ - Messages: []*pb.MachineCaddyStorageKeyInfo{{ - Key: req.Key, - UpdatedAt: timestamppb.New(record.UpdatedAt), - Size: int64(len(record.Value)), - IsTerminal: true, - }}, + Key: req.Key, + UpdatedAt: timestamppb.New(record.UpdatedAt), + Size: int64(len(record.Value)), + IsTerminal: true, }, nil } if !errors.Is(err, store.ErrKeyNotFound) { @@ -165,10 +157,8 @@ func (s *Server) Stat(ctx context.Context, req *pb.StatCaddyStorageRequest) (*pb } return &pb.StatCaddyStorageResponse{ - Messages: []*pb.MachineCaddyStorageKeyInfo{{ - Key: req.Key, - IsTerminal: false, - }}, + Key: req.Key, + IsTerminal: false, }, nil } diff --git a/pkg/client/machine.go b/pkg/client/machine.go index e3f9424a..8a84de8f 100644 --- a/pkg/client/machine.go +++ b/pkg/client/machine.go @@ -141,3 +141,23 @@ func (cli *Client) WaitClusterReady(ctx context.Context, timeout time.Duration) } return backoff.Retry(listMachines, boff) } + +// WaitForStoreVersion waits until the cluster store on the target machine has reached each requested actor version +// in minVersion, with no known missing or pending transactions through those versions. +// The context controls cancellation and the deadline. An empty minVersion requires no replication. +// This method observes replication without initiating synchronisation. +// +// Corrosion may satisfy a version by applying its surviving changes or by marking it complete because its changes +// have been superseded. +// +// Waiting normally makes the captured data available on the machine. However, another write may replace some of that +// data before it arrives. Corrosion can then complete the older version without transferring the replaced data. +// If the replacement is outside the requested versions, this method can succeed while the affected data is still +// missing or outdated. This can happen during concurrent updates even when all machines are well connected. +// +// Success does not guarantee an exact snapshot or delivery of every historical value. +// Callers that require a specific record or condition should verify it after waiting. +func (cli *Client) WaitForStoreVersion(ctx context.Context, minVersion map[string]uint64) error { + _, err := cli.MachineClient.WaitForStoreVersion(ctx, &pb.WaitForStoreVersionRequest{MinVersion: minVersion}) + return err +} diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index 6ee4856d..2422f3d1 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -1,9 +1,9 @@ package e2e import ( - "bytes" "context" "errors" + "maps" "os" "strings" "testing" @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" ) func createTestCluster( @@ -178,174 +179,144 @@ func TestClusterLifecycle(t *testing.T) { }) t.Run("Caddy storage replication", func(t *testing.T) { - cli0, err := c.Machines[0].Connect(ctx) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, cli0.Close()) - }) + clients := make([]*client.Client, len(c.Machines)) + for i, m := range c.Machines { + cli, err := m.Connect(ctx) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, cli.Close()) + }) + clients[i] = cli + } - cli1, err := c.Machines[1].Connect(ctx) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, cli1.Close()) - }) + storeVersion := func(clis ...*client.Client) map[string]uint64 { + t.Helper() + callCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + version := make(map[string]uint64) + for _, cli := range clis { + resp, err := cli.MachineClient.InspectMachine(callCtx, &emptypb.Empty{}) + require.NoError(t, err) + require.Len(t, resp.Machines, 1) + m := resp.Machines[0] + require.Len(t, m.StoreVersion, 3) + for actor, v := range m.StoreVersion { + version[actor] = max(version[actor], v) + } + } + return version + } + waitForStoreVersion := func(cli *client.Client, version map[string]uint64) { + t.Helper() + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + err := cli.WaitForStoreVersion(waitCtx, version) + require.NoError(t, err) + } prefix := "e2e/caddy-storage/" + uuid.NewString() key := prefix + "/key/path" + otherKey := prefix + "/other-key" // Keep reused test clusters clean if an assertion stops the test before its explicit deletes. t.Cleanup(func() { cleanupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - _, _ = cli1.CaddyStorage.Delete(client.ProxyMachinesContext(cleanupCtx, nil), - &pb.DeleteCaddyStorageRequest{Key: prefix}) + _, _ = clients[0].CaddyStorage.Delete(cleanupCtx, &pb.DeleteCaddyStorageRequest{Key: prefix}) + _, _ = clients[1].CaddyStorage.Delete(cleanupCtx, &pb.DeleteCaddyStorageRequest{Key: prefix}) }) - // Verify both creation and overwrite, allowing each value to replicate before writing the next. + // Create and overwrite a key on the first machine before waiting for replication. var updatedAt time.Time for _, value := range [][]byte{[]byte("test-value"), []byte("replacement-value")} { - _, err = cli0.CaddyStorage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: key, Value: value}) + _, err := clients[0].CaddyStorage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: key, Value: value}) require.NoError(t, err) // A Load through another machine must find the value on the machine that accepted the local write, // regardless of whether Corrosion has replicated it to the other machines yet. - loadResp, err := cli1.CaddyStorage.Load(client.ProxySingleMachineContext(ctx, c.Machines[0].ID), + loadResp, err := clients[1].CaddyStorage.Load(client.ProxySingleMachineContext(ctx, c.Machines[0].ID), &pb.LoadCaddyStorageRequest{Key: key}) require.NoError(t, err) - require.Len(t, loadResp.Messages, 1) - originResult := loadResp.Messages[0] - require.Nil(t, originResult.Metadata, - "Proxy to a single machine should not inject metadata into the response") - require.Equal(t, value, originResult.Value) - require.NoError(t, originResult.UpdatedAt.CheckValid()) - modified := originResult.UpdatedAt.AsTime() + require.Equal(t, value, loadResp.Value) + require.NoError(t, loadResp.UpdatedAt.CheckValid()) + modified := loadResp.UpdatedAt.AsTime() require.False(t, modified.IsZero(), "Stored value should have a valid updated_at timestamp") if !updatedAt.IsZero() { - require.True(t, modified.After(updatedAt), "Overwriting a value should advance updated_at") + // Back-to-back writes can have the same timestamp. + require.False(t, modified.Before(updatedAt), "Overwriting a value should not move updated_at backwards") } updatedAt = modified + } - require.Eventually(t, func() bool { - callCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() + // Write a distinct key on the second machine, then capture the combined store version from both machines. + otherValue := []byte("second-value") + _, err := clients[1].CaddyStorage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: otherKey, Value: otherValue}) + require.NoError(t, err) + version := storeVersion(clients[0], clients[1]) + values := map[string][]byte{key: []byte("replacement-value"), otherKey: otherValue} - resp, err := cli1.CaddyStorage.Load(client.ProxyMachinesContext(callCtx, nil), - &pb.LoadCaddyStorageRequest{Key: key}) - if err != nil { - return false + // On each machine, wait for the store version to be reached, then verify that the final values are readable + // and that the Stat and List endpoints return the expected results. + for _, cli := range clients { + waitForStoreVersion(cli, version) + + // Both final values must be readable locally as soon as the wait returns. + for k, v := range values { + resp, err := cli.CaddyStorage.Load(ctx, &pb.LoadCaddyStorageRequest{Key: k}) + require.NoError(t, err) + assert.Equal(t, v, resp.Value) + require.NoError(t, resp.UpdatedAt.CheckValid()) + if k == key { + assert.True(t, resp.UpdatedAt.AsTime().Equal(updatedAt)) } - require.Len(t, resp.Messages, 3) - for _, m := range resp.Messages { - require.NotNil(t, m.Metadata) - if m.Metadata.Error != "" || !bytes.Equal(m.Value, - value) || !m.UpdatedAt.AsTime().Equal(updatedAt) { - return false - } - } - return true - }, 30*time.Second, 100*time.Millisecond, "Caddy storage value %q should replicate to every machine", value) - - // Every machine should report the same Caddy storage key information. - statResp, err := cli1.CaddyStorage.Stat(client.ProxyMachinesContext(ctx, nil), - &pb.StatCaddyStorageRequest{Key: key}) - require.NoError(t, err) - require.Len(t, statResp.Messages, 3) - for _, m := range statResp.Messages { - require.NotNil(t, m.Metadata) - assert.Equal(t, "", m.Metadata.Error) - assert.Equal(t, key, m.Key) - assert.True(t, m.UpdatedAt.AsTime().Equal(updatedAt)) - assert.EqualValues(t, len(value), m.Size) - assert.True(t, m.IsTerminal) + statResp, err := cli.CaddyStorage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: k}) + require.NoError(t, err) + assert.Equal(t, k, statResp.Key) + assert.True(t, statResp.UpdatedAt.AsTime().Equal(resp.UpdatedAt.AsTime())) + assert.EqualValues(t, len(v), statResp.Size) + assert.True(t, statResp.IsTerminal) } - } - // A path with descendants should exist as a directory even though no value is stored at that key. - statResp, err := cli1.CaddyStorage.Stat(client.ProxyMachinesContext(ctx, nil), - &pb.StatCaddyStorageRequest{Key: prefix + "/key"}) - require.NoError(t, err) - require.Len(t, statResp.Messages, 3) - for _, m := range statResp.Messages { - require.NotNil(t, m.Metadata) - assert.Equal(t, "", m.Metadata.Error) - assert.Equal(t, prefix+"/key", m.Key) - assert.Nil(t, m.UpdatedAt) - assert.EqualValues(t, 0, m.Size) - assert.False(t, m.IsTerminal) - } + // A path with descendants should exist as a directory even though no value is stored at that key. + statResp, err := cli.CaddyStorage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: prefix + "/key"}) + require.NoError(t, err) + assert.Equal(t, prefix+"/key", statResp.Key) + assert.Nil(t, statResp.UpdatedAt) + assert.EqualValues(t, 0, statResp.Size) + assert.False(t, statResp.IsTerminal) - listResp, err := cli1.CaddyStorage.List(client.ProxyMachinesContext(ctx, nil), - &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true}) - require.NoError(t, err) - require.Len(t, listResp.Messages, 3) - for _, m := range listResp.Messages { - require.NotNil(t, m.Metadata) - assert.Equal(t, "", m.Metadata.Error) - assert.Equal(t, []string{prefix + "/key", key}, m.Keys) - } + listResp, err := cli.CaddyStorage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true}) + require.NoError(t, err) + assert.Equal(t, []string{prefix + "/key", key, otherKey}, listResp.Keys) - // A non-recursive list should only return the immediate child keys. - listResp, err = cli1.CaddyStorage.List(client.ProxyMachinesContext(ctx, nil), - &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: false}) - require.NoError(t, err) - require.Len(t, listResp.Messages, 3) - for _, m := range listResp.Messages { - require.NotNil(t, m.Metadata) - assert.Equal(t, "", m.Metadata.Error) - assert.Equal(t, []string{prefix + "/key"}, m.Keys) + // A non-recursive list should only return the immediate child keys. + listResp, err = cli.CaddyStorage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: false}) + require.NoError(t, err) + assert.Equal(t, []string{prefix + "/key", otherKey}, listResp.Keys) } // Delete through one machine. The deletion must reach the other replicas through Corrosion. - _, err = cli1.CaddyStorage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: prefix}) + _, err = clients[2].CaddyStorage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: prefix}) require.NoError(t, err) + version = storeVersion(clients[2]) - require.Eventually(t, func() bool { - callCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() + for _, cli := range clients { + waitForStoreVersion(cli, version) - resp, err := cli0.CaddyStorage.Load(client.ProxyMachinesContext(callCtx, nil), - &pb.LoadCaddyStorageRequest{Key: key}) - if err != nil { - return false + for k := range values { + _, err = cli.CaddyStorage.Load(ctx, &pb.LoadCaddyStorageRequest{Key: k}) + assert.Equal(t, codes.NotFound, status.Code(err)) + _, err = cli.CaddyStorage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: k}) + assert.Equal(t, codes.NotFound, status.Code(err)) } + _, err = cli.CaddyStorage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true}) + assert.Equal(t, codes.NotFound, status.Code(err)) - require.Len(t, resp.Messages, 3) - for _, m := range resp.Messages { - require.NotNil(t, m.Metadata) - if codes.Code(m.Metadata.Status.GetCode()) != codes.NotFound { - return false - } - } - return true - }, 30*time.Second, 100*time.Millisecond, "Caddy storage deletion should replicate to every machine") - - statResp, err = cli0.CaddyStorage.Stat(client.ProxyMachinesContext(ctx, nil), - &pb.StatCaddyStorageRequest{Key: key}) - require.NoError(t, err) - require.Len(t, statResp.Messages, 3) - for _, m := range statResp.Messages { - require.NotNil(t, m.Metadata) - assert.Equal(t, codes.NotFound, codes.Code(m.Metadata.Status.GetCode())) - } - - listResp, err = cli0.CaddyStorage.List(client.ProxyMachinesContext(ctx, nil), - &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true}) - require.NoError(t, err) - require.Len(t, listResp.Messages, 3) - for _, m := range listResp.Messages { - require.NotNil(t, m.Metadata) - assert.Equal(t, codes.NotFound, codes.Code(m.Metadata.Status.GetCode())) - } - - // Delete is idempotent, so every machine should still return a successful response. - deleteResp, err := cli1.CaddyStorage.Delete(client.ProxyMachinesContext(ctx, nil), - &pb.DeleteCaddyStorageRequest{Key: prefix}) - require.NoError(t, err) - require.Len(t, deleteResp.Messages, 3) - for _, m := range deleteResp.Messages { - require.NotNil(t, m.Metadata) - assert.Equal(t, "", m.Metadata.Error) + // Delete is idempotent on each machine. + _, err = cli.CaddyStorage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: prefix}) + require.NoError(t, err) } }) @@ -359,22 +330,28 @@ func TestClusterLifecycle(t *testing.T) { require.NoError(t, cli.Close()) }) + resp, err := cli.MachineClient.InspectMachine(ctx, &emptypb.Empty{}) + require.NoError(t, err) + version := resp.Machines[0].StoreVersion + require.Len(t, version, 3) + + t.Run("already satisfied vector", func(t *testing.T) { + err := cli.WaitForStoreVersion(ctx, version) + require.NoError(t, err) + }) + t.Run("empty vector", func(t *testing.T) { - _, err := cli.WaitForStoreVersion(ctx, &pb.WaitForStoreVersionRequest{}) + err := cli.WaitForStoreVersion(ctx, nil) require.NoError(t, err) }) t.Run("zero version for unknown actor", func(t *testing.T) { - _, err := cli.WaitForStoreVersion(ctx, &pb.WaitForStoreVersionRequest{ - MinVersion: map[string]uint64{uuid.NewString(): 0}, - }) + err := cli.WaitForStoreVersion(ctx, map[string]uint64{uuid.NewString(): 0}) require.NoError(t, err) }) t.Run("invalid actor UUID", func(t *testing.T) { - _, err := cli.WaitForStoreVersion(ctx, &pb.WaitForStoreVersionRequest{ - MinVersion: map[string]uint64{"not-a-uuid": 1}, - }) + err := cli.WaitForStoreVersion(ctx, map[string]uint64{"not-a-uuid": 1}) require.Equal(t, codes.InvalidArgument, status.Code(err)) }) @@ -383,9 +360,17 @@ func TestClusterLifecycle(t *testing.T) { defer cancel() // Background writes cannot satisfy a target for an actor that does not exist. - _, err := cli.WaitForStoreVersion(waitCtx, &pb.WaitForStoreVersionRequest{ - MinVersion: map[string]uint64{uuid.NewString(): 1}, - }) + err := cli.WaitForStoreVersion(waitCtx, map[string]uint64{uuid.NewString(): 1}) + require.Equal(t, codes.DeadlineExceeded, status.Code(err)) + }) + + t.Run("partially satisfied vector times out", func(t *testing.T) { + waitCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + + minVersion := maps.Clone(version) + minVersion[uuid.NewString()] = 1 + err := cli.WaitForStoreVersion(waitCtx, minVersion) require.Equal(t, codes.DeadlineExceeded, status.Code(err)) }) @@ -395,9 +380,7 @@ func TestClusterLifecycle(t *testing.T) { timer := time.AfterFunc(500*time.Millisecond, cancel) defer timer.Stop() - _, err := cli.WaitForStoreVersion(waitCtx, &pb.WaitForStoreVersionRequest{ - MinVersion: map[string]uint64{uuid.NewString(): 1}, - }) + err := cli.WaitForStoreVersion(waitCtx, map[string]uint64{uuid.NewString(): 1}) require.Equal(t, codes.Canceled, status.Code(err)) }) })