From 11a4e39e56b2ff11bbe03641884faf12b0470ce7 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:43:57 +0200 Subject: [PATCH 01/17] Add bounded experiment readiness protocol and shared result types --- pkg/wire/experiment.go | 24 ++ protocol/protocol.pb.go | 474 +++++++++++++++++++++++++---------- protocol/protocol.proto | 19 ++ protocol/protocol_grpc.pb.go | 38 +++ 4 files changed, 424 insertions(+), 131 deletions(-) create mode 100644 pkg/wire/experiment.go diff --git a/pkg/wire/experiment.go b/pkg/wire/experiment.go new file mode 100644 index 00000000..1a8c5f93 --- /dev/null +++ b/pkg/wire/experiment.go @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package wire + +const ( + MaxExperimentMetadata = 4096 + MaxExperimentParticipants = 128 +) + +// Experiment is a one-shot readiness result for one admitted batch. Metadata +// is opaque application data and grants no permission to contact a peer. +type Experiment struct { + ID string `json:"experiment_id"` + StartTimeNS int64 `json:"start_time_ns"` + Participants []ExperimentParticipant `json:"participants"` +} + +type ExperimentParticipant struct { + ID string `json:"id"` + ExecutorID string `json:"executor_id"` + Metadata []byte `json:"metadata"` + ReadyAtNS int64 `json:"ready_at_ns"` +} diff --git a/protocol/protocol.pb.go b/protocol/protocol.pb.go index afd55c68..f3d1807e 100644 --- a/protocol/protocol.pb.go +++ b/protocol/protocol.pb.go @@ -2590,6 +2590,195 @@ func (x *VerifyTagsResponse) GetReason() string { return "" } +// Bounded, authenticated readiness for the admitted transaction's fixed runs. +type ExperimentReadyRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + DebugletId string `protobuf:"bytes,1,opt,name=debuglet_id,json=debugletId,proto3" json:"debuglet_id,omitempty"` + ExecutorId string `protobuf:"bytes,2,opt,name=executor_id,json=executorId,proto3" json:"executor_id,omitempty"` + Metadata []byte `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExperimentReadyRequest) Reset() { + *x = ExperimentReadyRequest{} + mi := &file_protocol_protocol_proto_msgTypes[39] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExperimentReadyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExperimentReadyRequest) ProtoMessage() {} + +func (x *ExperimentReadyRequest) ProtoReflect() protoreflect.Message { + mi := &file_protocol_protocol_proto_msgTypes[39] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExperimentReadyRequest.ProtoReflect.Descriptor instead. +func (*ExperimentReadyRequest) Descriptor() ([]byte, []int) { + return file_protocol_protocol_proto_rawDescGZIP(), []int{39} +} + +func (x *ExperimentReadyRequest) GetDebugletId() string { + if x != nil { + return x.DebugletId + } + return "" +} + +func (x *ExperimentReadyRequest) GetExecutorId() string { + if x != nil { + return x.ExecutorId + } + return "" +} + +func (x *ExperimentReadyRequest) GetMetadata() []byte { + if x != nil { + return x.Metadata + } + return nil +} + +type ExperimentParticipant struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + ExecutorId string `protobuf:"bytes,2,opt,name=executor_id,json=executorId,proto3" json:"executor_id,omitempty"` + Metadata []byte `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` + ReadyAtNs int64 `protobuf:"varint,4,opt,name=ready_at_ns,json=readyAtNs,proto3" json:"ready_at_ns,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExperimentParticipant) Reset() { + *x = ExperimentParticipant{} + mi := &file_protocol_protocol_proto_msgTypes[40] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExperimentParticipant) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExperimentParticipant) ProtoMessage() {} + +func (x *ExperimentParticipant) ProtoReflect() protoreflect.Message { + mi := &file_protocol_protocol_proto_msgTypes[40] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExperimentParticipant.ProtoReflect.Descriptor instead. +func (*ExperimentParticipant) Descriptor() ([]byte, []int) { + return file_protocol_protocol_proto_rawDescGZIP(), []int{40} +} + +func (x *ExperimentParticipant) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ExperimentParticipant) GetExecutorId() string { + if x != nil { + return x.ExecutorId + } + return "" +} + +func (x *ExperimentParticipant) GetMetadata() []byte { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ExperimentParticipant) GetReadyAtNs() int64 { + if x != nil { + return x.ReadyAtNs + } + return 0 +} + +type ExperimentReadyResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ExperimentId string `protobuf:"bytes,1,opt,name=experiment_id,json=experimentId,proto3" json:"experiment_id,omitempty"` + StartTimeNs int64 `protobuf:"varint,2,opt,name=start_time_ns,json=startTimeNs,proto3" json:"start_time_ns,omitempty"` // Zero means pending; callers poll without holding a mutation. + Participants []*ExperimentParticipant `protobuf:"bytes,3,rep,name=participants,proto3" json:"participants,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExperimentReadyResponse) Reset() { + *x = ExperimentReadyResponse{} + mi := &file_protocol_protocol_proto_msgTypes[41] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExperimentReadyResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExperimentReadyResponse) ProtoMessage() {} + +func (x *ExperimentReadyResponse) ProtoReflect() protoreflect.Message { + mi := &file_protocol_protocol_proto_msgTypes[41] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExperimentReadyResponse.ProtoReflect.Descriptor instead. +func (*ExperimentReadyResponse) Descriptor() ([]byte, []int) { + return file_protocol_protocol_proto_rawDescGZIP(), []int{41} +} + +func (x *ExperimentReadyResponse) GetExperimentId() string { + if x != nil { + return x.ExperimentId + } + return "" +} + +func (x *ExperimentReadyResponse) GetStartTimeNs() int64 { + if x != nil { + return x.StartTimeNs + } + return 0 +} + +func (x *ExperimentReadyResponse) GetParticipants() []*ExperimentParticipant { + if x != nil { + return x.Participants + } + return nil +} + // Positive process capabilities observed by this executor. These are not // destination reachability or admission guarantees. Unknown versions are ignored. type ExecutorCapabilities struct { @@ -2614,7 +2803,7 @@ type ExecutorCapabilities struct { func (x *ExecutorCapabilities) Reset() { *x = ExecutorCapabilities{} - mi := &file_protocol_protocol_proto_msgTypes[39] + mi := &file_protocol_protocol_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2626,7 +2815,7 @@ func (x *ExecutorCapabilities) String() string { func (*ExecutorCapabilities) ProtoMessage() {} func (x *ExecutorCapabilities) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[39] + mi := &file_protocol_protocol_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2639,7 +2828,7 @@ func (x *ExecutorCapabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecutorCapabilities.ProtoReflect.Descriptor instead. func (*ExecutorCapabilities) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{39} + return file_protocol_protocol_proto_rawDescGZIP(), []int{42} } func (x *ExecutorCapabilities) GetSchemaVersion() uint32 { @@ -2702,7 +2891,7 @@ type ProbeState struct { func (x *ProbeState) Reset() { *x = ProbeState{} - mi := &file_protocol_protocol_proto_msgTypes[40] + mi := &file_protocol_protocol_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2714,7 +2903,7 @@ func (x *ProbeState) String() string { func (*ProbeState) ProtoMessage() {} func (x *ProbeState) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[40] + mi := &file_protocol_protocol_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2727,7 +2916,7 @@ func (x *ProbeState) ProtoReflect() protoreflect.Message { // Deprecated: Use ProbeState.ProtoReflect.Descriptor instead. func (*ProbeState) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{40} + return file_protocol_protocol_proto_rawDescGZIP(), []int{43} } func (x *ProbeState) GetState() string { @@ -2766,7 +2955,7 @@ type TaggingMode struct { func (x *TaggingMode) Reset() { *x = TaggingMode{} - mi := &file_protocol_protocol_proto_msgTypes[41] + mi := &file_protocol_protocol_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2778,7 +2967,7 @@ func (x *TaggingMode) String() string { func (*TaggingMode) ProtoMessage() {} func (x *TaggingMode) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[41] + mi := &file_protocol_protocol_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2791,7 +2980,7 @@ func (x *TaggingMode) ProtoReflect() protoreflect.Message { // Deprecated: Use TaggingMode.ProtoReflect.Descriptor instead. func (*TaggingMode) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{41} + return file_protocol_protocol_proto_rawDescGZIP(), []int{44} } func (x *TaggingMode) GetIpv4() string { @@ -2840,7 +3029,7 @@ type AttributionState struct { func (x *AttributionState) Reset() { *x = AttributionState{} - mi := &file_protocol_protocol_proto_msgTypes[42] + mi := &file_protocol_protocol_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2852,7 +3041,7 @@ func (x *AttributionState) String() string { func (*AttributionState) ProtoMessage() {} func (x *AttributionState) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[42] + mi := &file_protocol_protocol_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2865,7 +3054,7 @@ func (x *AttributionState) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributionState.ProtoReflect.Descriptor instead. func (*AttributionState) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{42} + return file_protocol_protocol_proto_rawDescGZIP(), []int{45} } func (x *AttributionState) GetState() string { @@ -2950,7 +3139,7 @@ type VantagePointReport struct { func (x *VantagePointReport) Reset() { *x = VantagePointReport{} - mi := &file_protocol_protocol_proto_msgTypes[43] + mi := &file_protocol_protocol_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2962,7 +3151,7 @@ func (x *VantagePointReport) String() string { func (*VantagePointReport) ProtoMessage() {} func (x *VantagePointReport) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[43] + mi := &file_protocol_protocol_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2975,7 +3164,7 @@ func (x *VantagePointReport) ProtoReflect() protoreflect.Message { // Deprecated: Use VantagePointReport.ProtoReflect.Descriptor instead. func (*VantagePointReport) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{43} + return file_protocol_protocol_proto_rawDescGZIP(), []int{46} } func (x *VantagePointReport) GetSchemaVersion() uint32 { @@ -3100,7 +3289,7 @@ type AddressSelfCheck struct { func (x *AddressSelfCheck) Reset() { *x = AddressSelfCheck{} - mi := &file_protocol_protocol_proto_msgTypes[44] + mi := &file_protocol_protocol_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3112,7 +3301,7 @@ func (x *AddressSelfCheck) String() string { func (*AddressSelfCheck) ProtoMessage() {} func (x *AddressSelfCheck) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[44] + mi := &file_protocol_protocol_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3125,7 +3314,7 @@ func (x *AddressSelfCheck) ProtoReflect() protoreflect.Message { // Deprecated: Use AddressSelfCheck.ProtoReflect.Descriptor instead. func (*AddressSelfCheck) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{44} + return file_protocol_protocol_proto_rawDescGZIP(), []int{47} } func (x *AddressSelfCheck) GetIpv4LocalPrivate() bool { @@ -3159,7 +3348,7 @@ type HostResourceValue struct { func (x *HostResourceValue) Reset() { *x = HostResourceValue{} - mi := &file_protocol_protocol_proto_msgTypes[45] + mi := &file_protocol_protocol_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3171,7 +3360,7 @@ func (x *HostResourceValue) String() string { func (*HostResourceValue) ProtoMessage() {} func (x *HostResourceValue) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[45] + mi := &file_protocol_protocol_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3184,7 +3373,7 @@ func (x *HostResourceValue) ProtoReflect() protoreflect.Message { // Deprecated: Use HostResourceValue.ProtoReflect.Descriptor instead. func (*HostResourceValue) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{45} + return file_protocol_protocol_proto_rawDescGZIP(), []int{48} } func (x *HostResourceValue) GetValue() uint64 { @@ -3214,7 +3403,7 @@ type HostResources struct { func (x *HostResources) Reset() { *x = HostResources{} - mi := &file_protocol_protocol_proto_msgTypes[46] + mi := &file_protocol_protocol_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3226,7 +3415,7 @@ func (x *HostResources) String() string { func (*HostResources) ProtoMessage() {} func (x *HostResources) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[46] + mi := &file_protocol_protocol_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3239,7 +3428,7 @@ func (x *HostResources) ProtoReflect() protoreflect.Message { // Deprecated: Use HostResources.ProtoReflect.Descriptor instead. func (*HostResources) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{46} + return file_protocol_protocol_proto_rawDescGZIP(), []int{49} } func (x *HostResources) GetProcessRssBytes() *HostResourceValue { @@ -3280,7 +3469,7 @@ type ReflectAddressRequest struct { func (x *ReflectAddressRequest) Reset() { *x = ReflectAddressRequest{} - mi := &file_protocol_protocol_proto_msgTypes[47] + mi := &file_protocol_protocol_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3292,7 +3481,7 @@ func (x *ReflectAddressRequest) String() string { func (*ReflectAddressRequest) ProtoMessage() {} func (x *ReflectAddressRequest) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[47] + mi := &file_protocol_protocol_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3305,7 +3494,7 @@ func (x *ReflectAddressRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReflectAddressRequest.ProtoReflect.Descriptor instead. func (*ReflectAddressRequest) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{47} + return file_protocol_protocol_proto_rawDescGZIP(), []int{50} } func (x *ReflectAddressRequest) GetExecutorId() string { @@ -3331,7 +3520,7 @@ type ReflectAddressResponse struct { func (x *ReflectAddressResponse) Reset() { *x = ReflectAddressResponse{} - mi := &file_protocol_protocol_proto_msgTypes[48] + mi := &file_protocol_protocol_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3343,7 +3532,7 @@ func (x *ReflectAddressResponse) String() string { func (*ReflectAddressResponse) ProtoMessage() {} func (x *ReflectAddressResponse) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[48] + mi := &file_protocol_protocol_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3356,7 +3545,7 @@ func (x *ReflectAddressResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReflectAddressResponse.ProtoReflect.Descriptor instead. func (*ReflectAddressResponse) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{48} + return file_protocol_protocol_proto_rawDescGZIP(), []int{51} } func (x *ReflectAddressResponse) GetAddress() string { @@ -3377,7 +3566,7 @@ type EgressTest struct { func (x *EgressTest) Reset() { *x = EgressTest{} - mi := &file_protocol_protocol_proto_msgTypes[49] + mi := &file_protocol_protocol_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3389,7 +3578,7 @@ func (x *EgressTest) String() string { func (*EgressTest) ProtoMessage() {} func (x *EgressTest) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[49] + mi := &file_protocol_protocol_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3402,7 +3591,7 @@ func (x *EgressTest) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressTest.ProtoReflect.Descriptor instead. func (*EgressTest) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{49} + return file_protocol_protocol_proto_rawDescGZIP(), []int{52} } func (x *EgressTest) GetState() string { @@ -3437,7 +3626,7 @@ type ListenerChallenge struct { func (x *ListenerChallenge) Reset() { *x = ListenerChallenge{} - mi := &file_protocol_protocol_proto_msgTypes[50] + mi := &file_protocol_protocol_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3449,7 +3638,7 @@ func (x *ListenerChallenge) String() string { func (*ListenerChallenge) ProtoMessage() {} func (x *ListenerChallenge) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[50] + mi := &file_protocol_protocol_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3462,7 +3651,7 @@ func (x *ListenerChallenge) ProtoReflect() protoreflect.Message { // Deprecated: Use ListenerChallenge.ProtoReflect.Descriptor instead. func (*ListenerChallenge) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{50} + return file_protocol_protocol_proto_rawDescGZIP(), []int{53} } func (x *ListenerChallenge) GetTransport() string { @@ -3500,7 +3689,7 @@ type ConnectivityReport struct { func (x *ConnectivityReport) Reset() { *x = ConnectivityReport{} - mi := &file_protocol_protocol_proto_msgTypes[51] + mi := &file_protocol_protocol_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3512,7 +3701,7 @@ func (x *ConnectivityReport) String() string { func (*ConnectivityReport) ProtoMessage() {} func (x *ConnectivityReport) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[51] + mi := &file_protocol_protocol_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3525,7 +3714,7 @@ func (x *ConnectivityReport) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectivityReport.ProtoReflect.Descriptor instead. func (*ConnectivityReport) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{51} + return file_protocol_protocol_proto_rawDescGZIP(), []int{54} } func (x *ConnectivityReport) GetIpv4() *EgressTest { @@ -3585,7 +3774,7 @@ type ClockState struct { func (x *ClockState) Reset() { *x = ClockState{} - mi := &file_protocol_protocol_proto_msgTypes[52] + mi := &file_protocol_protocol_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3597,7 +3786,7 @@ func (x *ClockState) String() string { func (*ClockState) ProtoMessage() {} func (x *ClockState) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[52] + mi := &file_protocol_protocol_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3610,7 +3799,7 @@ func (x *ClockState) ProtoReflect() protoreflect.Message { // Deprecated: Use ClockState.ProtoReflect.Descriptor instead. func (*ClockState) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{52} + return file_protocol_protocol_proto_rawDescGZIP(), []int{55} } func (x *ClockState) GetState() string { @@ -3670,7 +3859,7 @@ type HostPlatform struct { func (x *HostPlatform) Reset() { *x = HostPlatform{} - mi := &file_protocol_protocol_proto_msgTypes[53] + mi := &file_protocol_protocol_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3682,7 +3871,7 @@ func (x *HostPlatform) String() string { func (*HostPlatform) ProtoMessage() {} func (x *HostPlatform) ProtoReflect() protoreflect.Message { - mi := &file_protocol_protocol_proto_msgTypes[53] + mi := &file_protocol_protocol_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3695,7 +3884,7 @@ func (x *HostPlatform) ProtoReflect() protoreflect.Message { // Deprecated: Use HostPlatform.ProtoReflect.Descriptor instead. func (*HostPlatform) Descriptor() ([]byte, []int) { - return file_protocol_protocol_proto_rawDescGZIP(), []int{53} + return file_protocol_protocol_proto_rawDescGZIP(), []int{56} } func (x *HostPlatform) GetOs() string { @@ -3937,7 +4126,23 @@ const file_protocol_protocol_proto_rawDesc = "" + "\x12VerifyTagsResponse\x12\x18\n" + "\averdict\x18\x01 \x01(\tR\averdict\x12\x15\n" + "\x06run_id\x18\x02 \x01(\tR\x05runId\x12\x16\n" + - "\x06reason\x18\x03 \x01(\tR\x06reason\"\xe9\x02\n" + + "\x06reason\x18\x03 \x01(\tR\x06reason\"v\n" + + "\x16ExperimentReadyRequest\x12\x1f\n" + + "\vdebuglet_id\x18\x01 \x01(\tR\n" + + "debugletId\x12\x1f\n" + + "\vexecutor_id\x18\x02 \x01(\tR\n" + + "executorId\x12\x1a\n" + + "\bmetadata\x18\x03 \x01(\fR\bmetadata\"\x84\x01\n" + + "\x15ExperimentParticipant\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1f\n" + + "\vexecutor_id\x18\x02 \x01(\tR\n" + + "executorId\x12\x1a\n" + + "\bmetadata\x18\x03 \x01(\fR\bmetadata\x12\x1e\n" + + "\vready_at_ns\x18\x04 \x01(\x03R\treadyAtNs\"\xb0\x01\n" + + "\x17ExperimentReadyResponse\x12#\n" + + "\rexperiment_id\x18\x01 \x01(\tR\fexperimentId\x12\"\n" + + "\rstart_time_ns\x18\x02 \x01(\x03R\vstartTimeNs\x12L\n" + + "\fparticipants\x18\x03 \x03(\v2(.debuglet.protocol.ExperimentParticipantR\fparticipants\"\xe9\x02\n" + "\x14ExecutorCapabilities\x12%\n" + "\x0eschema_version\x18\x01 \x01(\rR\rschemaVersion\x12\x1c\n" + "\tprotocols\x18\x02 \x03(\tR\tprotocols\x12)\n" + @@ -4056,13 +4261,14 @@ const file_protocol_protocol_proto_rawDesc = "" + "\x1fRETAINED_RUN_STATUS_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19RETAINED_RUN_STATUS_FOUND\x10\x01\x12 \n" + "\x1cRETAINED_RUN_STATUS_FILTERED\x10\x02\x12\x1e\n" + - "\x1aRETAINED_RUN_STATUS_ABSENT\x10\x032\x80\a\n" + + "\x1aRETAINED_RUN_STATUS_ABSENT\x10\x032\xea\a\n" + "\x11DispatcherService\x12e\n" + "\x0eReflectAddress\x12(.debuglet.protocol.ReflectAddressRequest\x1a).debuglet.protocol.ReflectAddressResponse\x12V\n" + "\tHeartbeat\x12#.debuglet.protocol.HeartbeatRequest\x1a$.debuglet.protocol.HeartbeatResponse\x12V\n" + "\tResources\x12#.debuglet.protocol.ResourcesRequest\x1a$.debuglet.protocol.ResourcesResponse\x12b\n" + "\rDebugletState\x12'.debuglet.protocol.DebugletStateRequest\x1a(.debuglet.protocol.DebugletStateResponse\x12k\n" + - "\x10DebugletAllocate\x12*.debuglet.protocol.DebugletAllocateRequest\x1a+.debuglet.protocol.DebugletAllocateResponse\x12_\n" + + "\x10DebugletAllocate\x12*.debuglet.protocol.DebugletAllocateRequest\x1a+.debuglet.protocol.DebugletAllocateResponse\x12h\n" + + "\x0fExperimentReady\x12).debuglet.protocol.ExperimentReadyRequest\x1a*.debuglet.protocol.ExperimentReadyResponse\x12_\n" + "\fDebugletExit\x12&.debuglet.protocol.DebugletExitRequest\x1a'.debuglet.protocol.DebugletExitResponse\x12i\n" + "\x0eDebugletStream\x12(.debuglet.protocol.DebugletStreamRequest\x1a).debuglet.protocol.DebugletStreamResponse(\x010\x01\x12\\\n" + "\vBindSession\x12%.debuglet.protocol.BindSessionRequest\x1a&.debuglet.protocol.BindSessionResponse\x12Y\n" + @@ -4091,7 +4297,7 @@ func file_protocol_protocol_proto_rawDescGZIP() []byte { } var file_protocol_protocol_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_protocol_protocol_proto_msgTypes = make([]protoimpl.MessageInfo, 54) +var file_protocol_protocol_proto_msgTypes = make([]protoimpl.MessageInfo, 57) var file_protocol_protocol_proto_goTypes = []any{ (RunState)(0), // 0: debuglet.protocol.RunState (DebugletOutputStatus)(0), // 1: debuglet.protocol.DebugletOutputStatus @@ -4135,104 +4341,110 @@ var file_protocol_protocol_proto_goTypes = []any{ (*InspectRetainedRunResponse)(nil), // 39: debuglet.protocol.InspectRetainedRunResponse (*VerifyTagsRequest)(nil), // 40: debuglet.protocol.VerifyTagsRequest (*VerifyTagsResponse)(nil), // 41: debuglet.protocol.VerifyTagsResponse - (*ExecutorCapabilities)(nil), // 42: debuglet.protocol.ExecutorCapabilities - (*ProbeState)(nil), // 43: debuglet.protocol.ProbeState - (*TaggingMode)(nil), // 44: debuglet.protocol.TaggingMode - (*AttributionState)(nil), // 45: debuglet.protocol.AttributionState - (*VantagePointReport)(nil), // 46: debuglet.protocol.VantagePointReport - (*AddressSelfCheck)(nil), // 47: debuglet.protocol.AddressSelfCheck - (*HostResourceValue)(nil), // 48: debuglet.protocol.HostResourceValue - (*HostResources)(nil), // 49: debuglet.protocol.HostResources - (*ReflectAddressRequest)(nil), // 50: debuglet.protocol.ReflectAddressRequest - (*ReflectAddressResponse)(nil), // 51: debuglet.protocol.ReflectAddressResponse - (*EgressTest)(nil), // 52: debuglet.protocol.EgressTest - (*ListenerChallenge)(nil), // 53: debuglet.protocol.ListenerChallenge - (*ConnectivityReport)(nil), // 54: debuglet.protocol.ConnectivityReport - (*ClockState)(nil), // 55: debuglet.protocol.ClockState - (*HostPlatform)(nil), // 56: debuglet.protocol.HostPlatform - (*timestamppb.Timestamp)(nil), // 57: google.protobuf.Timestamp + (*ExperimentReadyRequest)(nil), // 42: debuglet.protocol.ExperimentReadyRequest + (*ExperimentParticipant)(nil), // 43: debuglet.protocol.ExperimentParticipant + (*ExperimentReadyResponse)(nil), // 44: debuglet.protocol.ExperimentReadyResponse + (*ExecutorCapabilities)(nil), // 45: debuglet.protocol.ExecutorCapabilities + (*ProbeState)(nil), // 46: debuglet.protocol.ProbeState + (*TaggingMode)(nil), // 47: debuglet.protocol.TaggingMode + (*AttributionState)(nil), // 48: debuglet.protocol.AttributionState + (*VantagePointReport)(nil), // 49: debuglet.protocol.VantagePointReport + (*AddressSelfCheck)(nil), // 50: debuglet.protocol.AddressSelfCheck + (*HostResourceValue)(nil), // 51: debuglet.protocol.HostResourceValue + (*HostResources)(nil), // 52: debuglet.protocol.HostResources + (*ReflectAddressRequest)(nil), // 53: debuglet.protocol.ReflectAddressRequest + (*ReflectAddressResponse)(nil), // 54: debuglet.protocol.ReflectAddressResponse + (*EgressTest)(nil), // 55: debuglet.protocol.EgressTest + (*ListenerChallenge)(nil), // 56: debuglet.protocol.ListenerChallenge + (*ConnectivityReport)(nil), // 57: debuglet.protocol.ConnectivityReport + (*ClockState)(nil), // 58: debuglet.protocol.ClockState + (*HostPlatform)(nil), // 59: debuglet.protocol.HostPlatform + (*timestamppb.Timestamp)(nil), // 60: google.protobuf.Timestamp } var file_protocol_protocol_proto_depIdxs = []int32{ - 42, // 0: debuglet.protocol.HelloResponse.capabilities:type_name -> debuglet.protocol.ExecutorCapabilities - 46, // 1: debuglet.protocol.HelloResponse.vantage_point:type_name -> debuglet.protocol.VantagePointReport - 57, // 2: debuglet.protocol.UploadRequest.start_time:type_name -> google.protobuf.Timestamp + 45, // 0: debuglet.protocol.HelloResponse.capabilities:type_name -> debuglet.protocol.ExecutorCapabilities + 49, // 1: debuglet.protocol.HelloResponse.vantage_point:type_name -> debuglet.protocol.VantagePointReport + 60, // 2: debuglet.protocol.UploadRequest.start_time:type_name -> google.protobuf.Timestamp 5, // 3: debuglet.protocol.UploadRequest.policy:type_name -> debuglet.protocol.DebugletPolicy 30, // 4: debuglet.protocol.UploadRequest.control_binding:type_name -> debuglet.protocol.ControlBinding 10, // 5: debuglet.protocol.BandwidthRequest.limits:type_name -> debuglet.protocol.DestinationLimit - 42, // 6: debuglet.protocol.HeartbeatRequest.capabilities:type_name -> debuglet.protocol.ExecutorCapabilities - 46, // 7: debuglet.protocol.HeartbeatRequest.vantage_point:type_name -> debuglet.protocol.VantagePointReport + 45, // 6: debuglet.protocol.HeartbeatRequest.capabilities:type_name -> debuglet.protocol.ExecutorCapabilities + 49, // 7: debuglet.protocol.HeartbeatRequest.vantage_point:type_name -> debuglet.protocol.VantagePointReport 14, // 8: debuglet.protocol.HeartbeatRequest.extra_disclosures:type_name -> debuglet.protocol.TeslaDisclosure 0, // 9: debuglet.protocol.DebugletStateRequest.state:type_name -> debuglet.protocol.RunState 19, // 10: debuglet.protocol.DebugletStateRequest.tcp_listener:type_name -> debuglet.protocol.ListenerEndpoint 5, // 11: debuglet.protocol.DebugletAllocateRequest.policy:type_name -> debuglet.protocol.DebugletPolicy 10, // 12: debuglet.protocol.DebugletAllocateResponse.allocated_limits:type_name -> debuglet.protocol.DestinationLimit 30, // 13: debuglet.protocol.DebugletIdent.original_binding:type_name -> debuglet.protocol.ControlBinding - 57, // 14: debuglet.protocol.DebugletOutput.timestamp:type_name -> google.protobuf.Timestamp + 60, // 14: debuglet.protocol.DebugletOutput.timestamp:type_name -> google.protobuf.Timestamp 1, // 15: debuglet.protocol.DebugletOutputEnd.status:type_name -> debuglet.protocol.DebugletOutputStatus 25, // 16: debuglet.protocol.DebugletStreamRequest.ident:type_name -> debuglet.protocol.DebugletIdent 26, // 17: debuglet.protocol.DebugletStreamRequest.output:type_name -> debuglet.protocol.DebugletOutput 27, // 18: debuglet.protocol.DebugletStreamRequest.end:type_name -> debuglet.protocol.DebugletOutputEnd 27, // 19: debuglet.protocol.DebugletStreamResponse.end:type_name -> debuglet.protocol.DebugletOutputEnd 30, // 20: debuglet.protocol.RetainedRun.original_binding:type_name -> debuglet.protocol.ControlBinding - 57, // 21: debuglet.protocol.RetainedRun.started_at:type_name -> google.protobuf.Timestamp - 57, // 22: debuglet.protocol.RetainedRun.start_time:type_name -> google.protobuf.Timestamp + 60, // 21: debuglet.protocol.RetainedRun.started_at:type_name -> google.protobuf.Timestamp + 60, // 22: debuglet.protocol.RetainedRun.start_time:type_name -> google.protobuf.Timestamp 30, // 23: debuglet.protocol.InspectRetainedRunRequest.control_binding:type_name -> debuglet.protocol.ControlBinding 2, // 24: debuglet.protocol.InspectRetainedRunResponse.status:type_name -> debuglet.protocol.RetainedRunStatus 37, // 25: debuglet.protocol.InspectRetainedRunResponse.run:type_name -> debuglet.protocol.RetainedRun 30, // 26: debuglet.protocol.VerifyTagsRequest.control_binding:type_name -> debuglet.protocol.ControlBinding - 45, // 27: debuglet.protocol.ExecutorCapabilities.attribution:type_name -> debuglet.protocol.AttributionState - 44, // 28: debuglet.protocol.ExecutorCapabilities.tagging:type_name -> debuglet.protocol.TaggingMode - 43, // 29: debuglet.protocol.ExecutorCapabilities.icmp:type_name -> debuglet.protocol.ProbeState - 55, // 30: debuglet.protocol.VantagePointReport.clock:type_name -> debuglet.protocol.ClockState - 56, // 31: debuglet.protocol.VantagePointReport.platform:type_name -> debuglet.protocol.HostPlatform - 54, // 32: debuglet.protocol.VantagePointReport.connectivity:type_name -> debuglet.protocol.ConnectivityReport - 43, // 33: debuglet.protocol.VantagePointReport.scion_paths:type_name -> debuglet.protocol.ProbeState - 49, // 34: debuglet.protocol.VantagePointReport.resources:type_name -> debuglet.protocol.HostResources - 47, // 35: debuglet.protocol.VantagePointReport.address_check:type_name -> debuglet.protocol.AddressSelfCheck - 48, // 36: debuglet.protocol.HostResources.process_rss_bytes:type_name -> debuglet.protocol.HostResourceValue - 48, // 37: debuglet.protocol.HostResources.open_fds:type_name -> debuglet.protocol.HostResourceValue - 48, // 38: debuglet.protocol.HostResources.state_available_bytes:type_name -> debuglet.protocol.HostResourceValue - 48, // 39: debuglet.protocol.HostResources.state_capacity_bytes:type_name -> debuglet.protocol.HostResourceValue - 52, // 40: debuglet.protocol.ConnectivityReport.ipv4:type_name -> debuglet.protocol.EgressTest - 52, // 41: debuglet.protocol.ConnectivityReport.ipv6:type_name -> debuglet.protocol.EgressTest - 53, // 42: debuglet.protocol.ConnectivityReport.listeners:type_name -> debuglet.protocol.ListenerChallenge - 50, // 43: debuglet.protocol.DispatcherService.ReflectAddress:input_type -> debuglet.protocol.ReflectAddressRequest - 13, // 44: debuglet.protocol.DispatcherService.Heartbeat:input_type -> debuglet.protocol.HeartbeatRequest - 16, // 45: debuglet.protocol.DispatcherService.Resources:input_type -> debuglet.protocol.ResourcesRequest - 18, // 46: debuglet.protocol.DispatcherService.DebugletState:input_type -> debuglet.protocol.DebugletStateRequest - 21, // 47: debuglet.protocol.DispatcherService.DebugletAllocate:input_type -> debuglet.protocol.DebugletAllocateRequest - 23, // 48: debuglet.protocol.DispatcherService.DebugletExit:input_type -> debuglet.protocol.DebugletExitRequest - 28, // 49: debuglet.protocol.DispatcherService.DebugletStream:input_type -> debuglet.protocol.DebugletStreamRequest - 31, // 50: debuglet.protocol.DispatcherService.BindSession:input_type -> debuglet.protocol.BindSessionRequest - 33, // 51: debuglet.protocol.DispatcherService.RenewLease:input_type -> debuglet.protocol.RenewLeaseRequest - 3, // 52: debuglet.protocol.ExecutorService.Hello:input_type -> debuglet.protocol.HelloRequest - 6, // 53: debuglet.protocol.ExecutorService.Upload:input_type -> debuglet.protocol.UploadRequest - 8, // 54: debuglet.protocol.ExecutorService.Abort:input_type -> debuglet.protocol.AbortRequest - 11, // 55: debuglet.protocol.ExecutorService.Bandwidth:input_type -> debuglet.protocol.BandwidthRequest - 35, // 56: debuglet.protocol.ExecutorService.ProbeSession:input_type -> debuglet.protocol.ProbeSessionRequest - 38, // 57: debuglet.protocol.ExecutorService.InspectRetainedRun:input_type -> debuglet.protocol.InspectRetainedRunRequest - 40, // 58: debuglet.protocol.ExecutorService.VerifyTags:input_type -> debuglet.protocol.VerifyTagsRequest - 51, // 59: debuglet.protocol.DispatcherService.ReflectAddress:output_type -> debuglet.protocol.ReflectAddressResponse - 15, // 60: debuglet.protocol.DispatcherService.Heartbeat:output_type -> debuglet.protocol.HeartbeatResponse - 17, // 61: debuglet.protocol.DispatcherService.Resources:output_type -> debuglet.protocol.ResourcesResponse - 20, // 62: debuglet.protocol.DispatcherService.DebugletState:output_type -> debuglet.protocol.DebugletStateResponse - 22, // 63: debuglet.protocol.DispatcherService.DebugletAllocate:output_type -> debuglet.protocol.DebugletAllocateResponse - 24, // 64: debuglet.protocol.DispatcherService.DebugletExit:output_type -> debuglet.protocol.DebugletExitResponse - 29, // 65: debuglet.protocol.DispatcherService.DebugletStream:output_type -> debuglet.protocol.DebugletStreamResponse - 32, // 66: debuglet.protocol.DispatcherService.BindSession:output_type -> debuglet.protocol.BindSessionResponse - 34, // 67: debuglet.protocol.DispatcherService.RenewLease:output_type -> debuglet.protocol.RenewLeaseResponse - 4, // 68: debuglet.protocol.ExecutorService.Hello:output_type -> debuglet.protocol.HelloResponse - 7, // 69: debuglet.protocol.ExecutorService.Upload:output_type -> debuglet.protocol.UploadResponse - 9, // 70: debuglet.protocol.ExecutorService.Abort:output_type -> debuglet.protocol.AbortResponse - 12, // 71: debuglet.protocol.ExecutorService.Bandwidth:output_type -> debuglet.protocol.BandwidthResponse - 36, // 72: debuglet.protocol.ExecutorService.ProbeSession:output_type -> debuglet.protocol.ProbeSessionResponse - 39, // 73: debuglet.protocol.ExecutorService.InspectRetainedRun:output_type -> debuglet.protocol.InspectRetainedRunResponse - 41, // 74: debuglet.protocol.ExecutorService.VerifyTags:output_type -> debuglet.protocol.VerifyTagsResponse - 59, // [59:75] is the sub-list for method output_type - 43, // [43:59] is the sub-list for method input_type - 43, // [43:43] is the sub-list for extension type_name - 43, // [43:43] is the sub-list for extension extendee - 0, // [0:43] is the sub-list for field type_name + 43, // 27: debuglet.protocol.ExperimentReadyResponse.participants:type_name -> debuglet.protocol.ExperimentParticipant + 48, // 28: debuglet.protocol.ExecutorCapabilities.attribution:type_name -> debuglet.protocol.AttributionState + 47, // 29: debuglet.protocol.ExecutorCapabilities.tagging:type_name -> debuglet.protocol.TaggingMode + 46, // 30: debuglet.protocol.ExecutorCapabilities.icmp:type_name -> debuglet.protocol.ProbeState + 58, // 31: debuglet.protocol.VantagePointReport.clock:type_name -> debuglet.protocol.ClockState + 59, // 32: debuglet.protocol.VantagePointReport.platform:type_name -> debuglet.protocol.HostPlatform + 57, // 33: debuglet.protocol.VantagePointReport.connectivity:type_name -> debuglet.protocol.ConnectivityReport + 46, // 34: debuglet.protocol.VantagePointReport.scion_paths:type_name -> debuglet.protocol.ProbeState + 52, // 35: debuglet.protocol.VantagePointReport.resources:type_name -> debuglet.protocol.HostResources + 50, // 36: debuglet.protocol.VantagePointReport.address_check:type_name -> debuglet.protocol.AddressSelfCheck + 51, // 37: debuglet.protocol.HostResources.process_rss_bytes:type_name -> debuglet.protocol.HostResourceValue + 51, // 38: debuglet.protocol.HostResources.open_fds:type_name -> debuglet.protocol.HostResourceValue + 51, // 39: debuglet.protocol.HostResources.state_available_bytes:type_name -> debuglet.protocol.HostResourceValue + 51, // 40: debuglet.protocol.HostResources.state_capacity_bytes:type_name -> debuglet.protocol.HostResourceValue + 55, // 41: debuglet.protocol.ConnectivityReport.ipv4:type_name -> debuglet.protocol.EgressTest + 55, // 42: debuglet.protocol.ConnectivityReport.ipv6:type_name -> debuglet.protocol.EgressTest + 56, // 43: debuglet.protocol.ConnectivityReport.listeners:type_name -> debuglet.protocol.ListenerChallenge + 53, // 44: debuglet.protocol.DispatcherService.ReflectAddress:input_type -> debuglet.protocol.ReflectAddressRequest + 13, // 45: debuglet.protocol.DispatcherService.Heartbeat:input_type -> debuglet.protocol.HeartbeatRequest + 16, // 46: debuglet.protocol.DispatcherService.Resources:input_type -> debuglet.protocol.ResourcesRequest + 18, // 47: debuglet.protocol.DispatcherService.DebugletState:input_type -> debuglet.protocol.DebugletStateRequest + 21, // 48: debuglet.protocol.DispatcherService.DebugletAllocate:input_type -> debuglet.protocol.DebugletAllocateRequest + 42, // 49: debuglet.protocol.DispatcherService.ExperimentReady:input_type -> debuglet.protocol.ExperimentReadyRequest + 23, // 50: debuglet.protocol.DispatcherService.DebugletExit:input_type -> debuglet.protocol.DebugletExitRequest + 28, // 51: debuglet.protocol.DispatcherService.DebugletStream:input_type -> debuglet.protocol.DebugletStreamRequest + 31, // 52: debuglet.protocol.DispatcherService.BindSession:input_type -> debuglet.protocol.BindSessionRequest + 33, // 53: debuglet.protocol.DispatcherService.RenewLease:input_type -> debuglet.protocol.RenewLeaseRequest + 3, // 54: debuglet.protocol.ExecutorService.Hello:input_type -> debuglet.protocol.HelloRequest + 6, // 55: debuglet.protocol.ExecutorService.Upload:input_type -> debuglet.protocol.UploadRequest + 8, // 56: debuglet.protocol.ExecutorService.Abort:input_type -> debuglet.protocol.AbortRequest + 11, // 57: debuglet.protocol.ExecutorService.Bandwidth:input_type -> debuglet.protocol.BandwidthRequest + 35, // 58: debuglet.protocol.ExecutorService.ProbeSession:input_type -> debuglet.protocol.ProbeSessionRequest + 38, // 59: debuglet.protocol.ExecutorService.InspectRetainedRun:input_type -> debuglet.protocol.InspectRetainedRunRequest + 40, // 60: debuglet.protocol.ExecutorService.VerifyTags:input_type -> debuglet.protocol.VerifyTagsRequest + 54, // 61: debuglet.protocol.DispatcherService.ReflectAddress:output_type -> debuglet.protocol.ReflectAddressResponse + 15, // 62: debuglet.protocol.DispatcherService.Heartbeat:output_type -> debuglet.protocol.HeartbeatResponse + 17, // 63: debuglet.protocol.DispatcherService.Resources:output_type -> debuglet.protocol.ResourcesResponse + 20, // 64: debuglet.protocol.DispatcherService.DebugletState:output_type -> debuglet.protocol.DebugletStateResponse + 22, // 65: debuglet.protocol.DispatcherService.DebugletAllocate:output_type -> debuglet.protocol.DebugletAllocateResponse + 44, // 66: debuglet.protocol.DispatcherService.ExperimentReady:output_type -> debuglet.protocol.ExperimentReadyResponse + 24, // 67: debuglet.protocol.DispatcherService.DebugletExit:output_type -> debuglet.protocol.DebugletExitResponse + 29, // 68: debuglet.protocol.DispatcherService.DebugletStream:output_type -> debuglet.protocol.DebugletStreamResponse + 32, // 69: debuglet.protocol.DispatcherService.BindSession:output_type -> debuglet.protocol.BindSessionResponse + 34, // 70: debuglet.protocol.DispatcherService.RenewLease:output_type -> debuglet.protocol.RenewLeaseResponse + 4, // 71: debuglet.protocol.ExecutorService.Hello:output_type -> debuglet.protocol.HelloResponse + 7, // 72: debuglet.protocol.ExecutorService.Upload:output_type -> debuglet.protocol.UploadResponse + 9, // 73: debuglet.protocol.ExecutorService.Abort:output_type -> debuglet.protocol.AbortResponse + 12, // 74: debuglet.protocol.ExecutorService.Bandwidth:output_type -> debuglet.protocol.BandwidthResponse + 36, // 75: debuglet.protocol.ExecutorService.ProbeSession:output_type -> debuglet.protocol.ProbeSessionResponse + 39, // 76: debuglet.protocol.ExecutorService.InspectRetainedRun:output_type -> debuglet.protocol.InspectRetainedRunResponse + 41, // 77: debuglet.protocol.ExecutorService.VerifyTags:output_type -> debuglet.protocol.VerifyTagsResponse + 61, // [61:78] is the sub-list for method output_type + 44, // [44:61] is the sub-list for method input_type + 44, // [44:44] is the sub-list for extension type_name + 44, // [44:44] is the sub-list for extension extendee + 0, // [0:44] is the sub-list for field type_name } func init() { file_protocol_protocol_proto_init() } @@ -4250,18 +4462,18 @@ func file_protocol_protocol_proto_init() { } file_protocol_protocol_proto_msgTypes[34].OneofWrappers = []any{} file_protocol_protocol_proto_msgTypes[36].OneofWrappers = []any{} - file_protocol_protocol_proto_msgTypes[42].OneofWrappers = []any{} - file_protocol_protocol_proto_msgTypes[44].OneofWrappers = []any{} file_protocol_protocol_proto_msgTypes[45].OneofWrappers = []any{} - file_protocol_protocol_proto_msgTypes[52].OneofWrappers = []any{} - file_protocol_protocol_proto_msgTypes[53].OneofWrappers = []any{} + file_protocol_protocol_proto_msgTypes[47].OneofWrappers = []any{} + file_protocol_protocol_proto_msgTypes[48].OneofWrappers = []any{} + file_protocol_protocol_proto_msgTypes[55].OneofWrappers = []any{} + file_protocol_protocol_proto_msgTypes[56].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_protocol_protocol_proto_rawDesc), len(file_protocol_protocol_proto_rawDesc)), NumEnums: 3, - NumMessages: 54, + NumMessages: 57, NumExtensions: 0, NumServices: 2, }, diff --git a/protocol/protocol.proto b/protocol/protocol.proto index 44390aa0..e0ec45ff 100644 --- a/protocol/protocol.proto +++ b/protocol/protocol.proto @@ -285,6 +285,24 @@ message VerifyTagsResponse { // ====================== SERVICE DEFS ======================== // ============================================================ +// Bounded, authenticated readiness for the admitted transaction's fixed runs. +message ExperimentReadyRequest { + string debuglet_id = 1; + string executor_id = 2; + bytes metadata = 3; +} +message ExperimentParticipant { + string id = 1; + string executor_id = 2; + bytes metadata = 3; + int64 ready_at_ns = 4; +} +message ExperimentReadyResponse { + string experiment_id = 1; + int64 start_time_ns = 2; // Zero means pending; callers poll without holding a mutation. + repeated ExperimentParticipant participants = 3; +} + service DispatcherService { // Optional address reflection on this dispatcher's authenticated listener. // It returns only the caller's peer address; it cannot initiate a connection. @@ -295,6 +313,7 @@ service DispatcherService { rpc DebugletState(DebugletStateRequest) returns (DebugletStateResponse); // Request resource allocation before a debuglet starts. The dispatcher can then have the caller wait until the resources have been allocated. rpc DebugletAllocate(DebugletAllocateRequest) returns (DebugletAllocateResponse); + rpc ExperimentReady(ExperimentReadyRequest) returns (ExperimentReadyResponse); rpc DebugletExit(DebugletExitRequest) returns (DebugletExitResponse); rpc DebugletStream(stream DebugletStreamRequest) returns (stream DebugletStreamResponse); rpc BindSession(BindSessionRequest) returns (BindSessionResponse); diff --git a/protocol/protocol_grpc.pb.go b/protocol/protocol_grpc.pb.go index 1b23516d..746799df 100644 --- a/protocol/protocol_grpc.pb.go +++ b/protocol/protocol_grpc.pb.go @@ -27,6 +27,7 @@ const ( DispatcherService_Resources_FullMethodName = "/debuglet.protocol.DispatcherService/Resources" DispatcherService_DebugletState_FullMethodName = "/debuglet.protocol.DispatcherService/DebugletState" DispatcherService_DebugletAllocate_FullMethodName = "/debuglet.protocol.DispatcherService/DebugletAllocate" + DispatcherService_ExperimentReady_FullMethodName = "/debuglet.protocol.DispatcherService/ExperimentReady" DispatcherService_DebugletExit_FullMethodName = "/debuglet.protocol.DispatcherService/DebugletExit" DispatcherService_DebugletStream_FullMethodName = "/debuglet.protocol.DispatcherService/DebugletStream" DispatcherService_BindSession_FullMethodName = "/debuglet.protocol.DispatcherService/BindSession" @@ -45,6 +46,7 @@ type DispatcherServiceClient interface { DebugletState(ctx context.Context, in *DebugletStateRequest, opts ...grpc.CallOption) (*DebugletStateResponse, error) // Request resource allocation before a debuglet starts. The dispatcher can then have the caller wait until the resources have been allocated. DebugletAllocate(ctx context.Context, in *DebugletAllocateRequest, opts ...grpc.CallOption) (*DebugletAllocateResponse, error) + ExperimentReady(ctx context.Context, in *ExperimentReadyRequest, opts ...grpc.CallOption) (*ExperimentReadyResponse, error) DebugletExit(ctx context.Context, in *DebugletExitRequest, opts ...grpc.CallOption) (*DebugletExitResponse, error) DebugletStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[DebugletStreamRequest, DebugletStreamResponse], error) BindSession(ctx context.Context, in *BindSessionRequest, opts ...grpc.CallOption) (*BindSessionResponse, error) @@ -109,6 +111,16 @@ func (c *dispatcherServiceClient) DebugletAllocate(ctx context.Context, in *Debu return out, nil } +func (c *dispatcherServiceClient) ExperimentReady(ctx context.Context, in *ExperimentReadyRequest, opts ...grpc.CallOption) (*ExperimentReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ExperimentReadyResponse) + err := c.cc.Invoke(ctx, DispatcherService_ExperimentReady_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *dispatcherServiceClient) DebugletExit(ctx context.Context, in *DebugletExitRequest, opts ...grpc.CallOption) (*DebugletExitResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DebugletExitResponse) @@ -164,6 +176,7 @@ type DispatcherServiceServer interface { DebugletState(context.Context, *DebugletStateRequest) (*DebugletStateResponse, error) // Request resource allocation before a debuglet starts. The dispatcher can then have the caller wait until the resources have been allocated. DebugletAllocate(context.Context, *DebugletAllocateRequest) (*DebugletAllocateResponse, error) + ExperimentReady(context.Context, *ExperimentReadyRequest) (*ExperimentReadyResponse, error) DebugletExit(context.Context, *DebugletExitRequest) (*DebugletExitResponse, error) DebugletStream(grpc.BidiStreamingServer[DebugletStreamRequest, DebugletStreamResponse]) error BindSession(context.Context, *BindSessionRequest) (*BindSessionResponse, error) @@ -193,6 +206,9 @@ func (UnimplementedDispatcherServiceServer) DebugletState(context.Context, *Debu func (UnimplementedDispatcherServiceServer) DebugletAllocate(context.Context, *DebugletAllocateRequest) (*DebugletAllocateResponse, error) { return nil, status.Error(codes.Unimplemented, "method DebugletAllocate not implemented") } +func (UnimplementedDispatcherServiceServer) ExperimentReady(context.Context, *ExperimentReadyRequest) (*ExperimentReadyResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ExperimentReady not implemented") +} func (UnimplementedDispatcherServiceServer) DebugletExit(context.Context, *DebugletExitRequest) (*DebugletExitResponse, error) { return nil, status.Error(codes.Unimplemented, "method DebugletExit not implemented") } @@ -316,6 +332,24 @@ func _DispatcherService_DebugletAllocate_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } +func _DispatcherService_ExperimentReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ExperimentReadyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DispatcherServiceServer).ExperimentReady(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DispatcherService_ExperimentReady_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DispatcherServiceServer).ExperimentReady(ctx, req.(*ExperimentReadyRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _DispatcherService_DebugletExit_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DebugletExitRequest) if err := dec(in); err != nil { @@ -404,6 +438,10 @@ var DispatcherService_ServiceDesc = grpc.ServiceDesc{ MethodName: "DebugletAllocate", Handler: _DispatcherService_DebugletAllocate_Handler, }, + { + MethodName: "ExperimentReady", + Handler: _DispatcherService_ExperimentReady_Handler, + }, { MethodName: "DebugletExit", Handler: _DispatcherService_DebugletExit_Handler, From cf9a006cf20fddabc0c27c736c236bc866a21090 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:49:11 +0200 Subject: [PATCH 02/17] Add reproducible experiment submission and five-executor peer example --- examples/experiments/README.md | 71 ++++++++++++++++++++ examples/experiments/five.json | 84 ++++++++++++++++++++++++ examples/experiments/peer/main.go | 79 +++++++++++++++++++++++ examples/experiments/run/main.go | 101 +++++++++++++++++++++++++++++ pkg/client/experiment.go | 104 ++++++++++++++++++++++++++++++ pkg/client/experiment_test.go | 71 ++++++++++++++++++++ 6 files changed, 510 insertions(+) create mode 100644 examples/experiments/README.md create mode 100644 examples/experiments/five.json create mode 100644 examples/experiments/peer/main.go create mode 100644 examples/experiments/run/main.go create mode 100644 pkg/client/experiment.go create mode 100644 pkg/client/experiment_test.go diff --git a/examples/experiments/README.md b/examples/experiments/README.md new file mode 100644 index 00000000..15938e82 --- /dev/null +++ b/examples/experiments/README.md @@ -0,0 +1,71 @@ +# Five-executor experiment + +This example submits five ordinary TEST runs as one batch. The batch transaction +is the experiment ID; the saved receipt records each order ID, run ID, executor, +WASM path and SHA256, arguments and requested policy. Each participant may use a +different WASM file and arguments. There is no role or topology configuration. + +The included `peer` guest obtains its UDP listener before announcing readiness. +The dispatcher exchanges the participants' opaque endpoint metadata and assigns +one future start time after all five are ready. Each guest waits for that time, +sends a datagram directly to each of the other four executors, and reports all +four arrivals. The output contains the requested start, actual local start, +send/receive timestamps and final peer count. No experiment traffic passes +through the dispatcher. + +Use a dispatcher and five distinct connected executors built from this revision. +Configure UDP listeners with reachable public hosts and available port ranges on +each executor. Both operator destination policy and the per-run `addresses` must +permit the peer hosts; metadata does not grant network access. `dbl up` has only +one executor and is not sufficient for this example. + +```sh +make wasm SAMPLE_DIR=examples/experiments/peer +# Use your existing saved connection and login: +dbl executor list +cp examples/experiments/five.json experiment-definition.json +# Replace executor-1 through executor-5 with actual IDs and the example +# addresses with peer IPs/CIDRs allowed by your operator policy. +go run ./examples/experiments/run -manifest experiment-definition.json \ + -receipt experiment-receipt.json > experiment-results.json +``` + +The WASM paths in the definition are relative to your working directory. Set +`sha256` to a known lower-case digest to reject changed files before submission; +leave it empty to record the submitted digest. The runner uses the same saved +connection and credentials as `dbl`; `-config` and `-dispatcher` select another +profile. `-endpoint http://127.0.0.1:9000` explicitly selects a local development +dispatcher without reading credentials. Remote TEST use also needs +`-allow-remote-test` and server authorization. No payment activation is performed. + +The receipt file must not already exist. It is written even when submission +returns an error, preserving known identities for inspection. An uncertain +submission must not be blindly resubmitted. Existing output and cancellation +routes remain usable independently: + +```sh +go run ./examples/experiments/run -action results -receipt experiment-receipt.json \ + > experiment-results.json +go run ./examples/experiments/run -action cancel -receipt experiment-receipt.json +# Decode the ordinary result exports' base64 guest output: +jq -r '.results[].output.entries[].output | @base64d' experiment-results.json +``` + +`results` captures a snapshot without waiting. `submit` waits up to `-timeout` +(default 90s), then requests cancellation of known runs. Cancellation +acknowledgement is not proof of termination. A missing participant is bounded by +the guest/dispatcher deadlines; a lost UDP datagram is bounded by each run's +`timeout_ms`. This example fails rather than retrying packets or replacing members. +The guest uses a 45s readiness deadline; keep its policy budget longer (the sample +uses 60s). Ready metadata is limited to 4KiB per participant. + +The shared start is a requested time, not atomic distributed execution. Host +clocks may differ, and these guest-reported timestamps are not a clock +synchronization or authenticated packet-evidence claim. UDP can lose packets; +network reachability, NAT traversal and clock setup remain operator concerns. + +Native applications can call `client.SubmitExperimentTEST`, +`client.ExportExperiment` and `client.CancelExperiment`. Guest authors call +`debuglet.Ready(ctx, metadata)` after their setup and +`debuglet.WaitStart(ctx, experiment)` before beginning their own algorithm. +All participants must call Ready; membership is fixed by the submitted batch. diff --git a/examples/experiments/five.json b/examples/experiments/five.json new file mode 100644 index 00000000..53fd6267 --- /dev/null +++ b/examples/experiments/five.json @@ -0,0 +1,84 @@ +{ + "participants": [ + { + "order_id": 0, + "executor_id": "executor-1", + "wasm_path": "examples/experiments/peer/debuglet.wasm", + "sha256": "", + "args": [], + "policy": { + "floor_bw": 1000000, + "ceil_bw": 1000000, + "timeout_ms": 60000, + "addresses": [ + "192.0.2.0/24" + ], + "listen_udp": true + } + }, + { + "order_id": 1, + "executor_id": "executor-2", + "wasm_path": "examples/experiments/peer/debuglet.wasm", + "sha256": "", + "args": [], + "policy": { + "floor_bw": 1000000, + "ceil_bw": 1000000, + "timeout_ms": 60000, + "addresses": [ + "192.0.2.0/24" + ], + "listen_udp": true + } + }, + { + "order_id": 2, + "executor_id": "executor-3", + "wasm_path": "examples/experiments/peer/debuglet.wasm", + "sha256": "", + "args": [], + "policy": { + "floor_bw": 1000000, + "ceil_bw": 1000000, + "timeout_ms": 60000, + "addresses": [ + "192.0.2.0/24" + ], + "listen_udp": true + } + }, + { + "order_id": 3, + "executor_id": "executor-4", + "wasm_path": "examples/experiments/peer/debuglet.wasm", + "sha256": "", + "args": [], + "policy": { + "floor_bw": 1000000, + "ceil_bw": 1000000, + "timeout_ms": 60000, + "addresses": [ + "192.0.2.0/24" + ], + "listen_udp": true + } + }, + { + "order_id": 4, + "executor_id": "executor-5", + "wasm_path": "examples/experiments/peer/debuglet.wasm", + "sha256": "", + "args": [], + "policy": { + "floor_bw": 1000000, + "ceil_bw": 1000000, + "timeout_ms": 60000, + "addresses": [ + "192.0.2.0/24" + ], + "listen_udp": true + } + } + ] +} diff --git a/examples/experiments/peer/main.go b/examples/experiments/peer/main.go new file mode 100644 index 00000000..97c3447d --- /dev/null +++ b/examples/experiments/peer/main.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +// peer exchanges one UDP datagram with every other experiment participant. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "github.com/netsec-ethz/debuglet/pkg/debuglet" +) + +type endpoint struct { Address string `json:"endpoint"` } +type packet struct { + ExperimentID string `json:"experiment_id"` + RunID string `json:"run_id"` + SentAtNS int64 `json:"sent_at_ns"` +} + +func main() { + if err := run(); err != nil { fmt.Fprintln(os.Stderr, err); os.Exit(1) } +} + +func run() error { + address, err := debuglet.ListenUDPAddr() + if err != nil { return err } + metadata, err := json.Marshal(endpoint{Address: address}) + if err != nil { return err } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + readyAt := time.Now().UnixNano() + experiment, err := debuglet.Ready(ctx, metadata) + if err != nil { return err } + if len(experiment.Participants) != 5 { return errors.New("peer example requires five participants") } + seen := map[string]bool{} + peers := map[string]string{} + self := "" + for _, member := range experiment.Participants { + if seen[member.ExecutorID] { return errors.New("peer example requires five distinct executors") } + seen[member.ExecutorID] = true + var listener endpoint + if err := json.Unmarshal(member.Metadata, &listener); err != nil { return err } + if listener.Address == "" { return errors.New("participant has no endpoint") } + if listener.Address == address { self = member.ID } else { peers[member.ID] = listener.Address } + } + if self == "" || len(peers) != 4 { return errors.New("listener endpoints must be distinct") } + if err := debuglet.WaitStart(ctx, experiment); err != nil { return err } + startedAt := time.Now().UnixNano() + packets := []map[string]any{} + for id, destination := range peers { + conn, err := debuglet.ConnectUDP(destination) + if err != nil { return err } + sentAt := time.Now().UnixNano() + data, err := json.Marshal(packet{ExperimentID:experiment.ID, RunID:self, SentAtNS:sentAt}) + if err != nil { conn.Close(); return err } + err = conn.Write(data) + conn.Close() + if err != nil { return err } + packets = append(packets, map[string]any{"direction":"sent", "peer_run_id":id, "endpoint":destination, "sent_at_ns":sentAt}) + } + buffer := make([]byte, 1024) + received := map[string]bool{} + for len(received) < len(peers) { + n, from, err := debuglet.ReadFromUDP(buffer) + if err != nil { return err } + receivedAt := time.Now().UnixNano() + var message packet + if err := json.Unmarshal(buffer[:n], &message); err != nil { return err } + if message.ExperimentID != experiment.ID || peers[message.RunID] == "" || received[message.RunID] { return errors.New("unexpected peer datagram") } + received[message.RunID] = true + packets = append(packets, map[string]any{"direction":"received", "peer_run_id":message.RunID, "endpoint":from, "sent_at_ns":message.SentAtNS, "received_at_ns":receivedAt}) + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"experiment_id":experiment.ID, "run_id":self, "endpoint":address, "ready_at_ns":readyAt, "start_time_ns":experiment.StartTimeNS, "actual_start_ns":startedAt, "lateness_ns":startedAt-experiment.StartTimeNS, "sent":len(peers), "received":len(received), "finished_at_ns":time.Now().UnixNano(), "packets":packets}) +} diff --git a/examples/experiments/run/main.go b/examples/experiments/run/main.go new file mode 100644 index 00000000..2c5f439d --- /dev/null +++ b/examples/experiments/run/main.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +// run submits a saved experiment definition and collects its existing run results. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "time" + + "github.com/netsec-ethz/debuglet/internal/connections" + "github.com/netsec-ethz/debuglet/pkg/client" +) + +func main() { + if err := run(); err != nil { fmt.Fprintln(os.Stderr, "experiment:", err); os.Exit(1) } +} + +func run() (failure error) { + action := flag.String("action", "submit", "submit, results or cancel") + manifest := flag.String("manifest", "", "experiment definition JSON (paths relative to working directory)") + receiptPath := flag.String("receipt", "experiment.json", "new receipt for submit; existing receipt for results/cancel") + endpoint := flag.String("endpoint", "", "explicit dispatcher URL; otherwise use saved dbl connection") + config := flag.String("config", "", "saved dbl connections file") + dispatcher := flag.String("dispatcher", "", "saved dbl connection name") + remote := flag.Bool("allow-remote-test", false, "permit remote TEST submission") + timeout := flag.Duration("timeout", 90*time.Second, "whole operation deadline") + flag.Parse() + if *action != "submit" && *action != "results" && *action != "cancel" { return errors.New("unknown action") } + if *timeout <= 0 { return errors.New("timeout must be positive") } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + ctx, cancel := context.WithTimeout(ctx, *timeout) + defer cancel() + profile := connections.Profile{Endpoint:*endpoint} + var err error + if *endpoint == "" { profile, err = connections.Resolve(*config, *dispatcher); if err != nil { return err } } + credential, err := connections.CredentialFor(ctx, *config, profile.Name, profile.Endpoint) + if err != nil { return err } + c, err := client.New(profile.Endpoint, client.Options{Credential:credential.Token, AllowRemoteTEST:*remote}) + if err != nil { return err } + var receipt client.ExperimentSubmission + defer func() { + if *action == "submit" && ctx.Err() != nil { + cleanup, done := context.WithTimeout(context.Background(), 10*time.Second) + defer done() + failure = errors.Join(failure, c.CancelExperiment(cleanup, receipt)) + } + }() + if *action == "submit" { + var definition client.ExperimentDefinition + if err := readJSON(*manifest, &definition); err != nil { return err } + file, err := os.OpenFile(*receiptPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { return err } + receipt, err = c.SubmitExperimentTEST(ctx, definition) + writeErr := json.NewEncoder(file).Encode(receipt) + closeErr := file.Close() + if err != nil || writeErr != nil || closeErr != nil { return errors.Join(err, writeErr, closeErr) } + fmt.Fprintf(os.Stderr, "Experiment %s; receipt %s\n", receipt.ExperimentID, *receiptPath) + } else if err := readJSON(*receiptPath, &receipt); err != nil { return err } + if receipt.ExperimentID == "" || len(receipt.Participants) == 0 { return errors.New("receipt has no admitted experiment") } + if *action == "cancel" { return c.CancelExperiment(ctx, receipt) } + for { + group, err := c.ExportExperiment(ctx, receipt) + if err != nil { return err } + complete := true + var failures []error + for _, result := range group.Results { + if result.Outcome.State != client.StateExited || (result.Output.Status.State != "complete" && result.Output.Status.State != "truncated") { complete = false } + if result.Outcome.Error != "" { failures = append(failures, fmt.Errorf("run %s: %s", result.RunID, result.Outcome.Error)) } + } + if complete || *action == "results" { + if err := json.NewEncoder(os.Stdout).Encode(group); err != nil { return err } + return errors.Join(failures...) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500*time.Millisecond): + } + } +} + +func readJSON(path string, destination any) error { + file, err := os.Open(path) + if err != nil { return err } + defer file.Close() + decoder := json.NewDecoder(io.LimitReader(file, 1<<20)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(destination); err != nil { return err } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { return errors.New("expected one JSON document") } + return nil +} diff --git a/pkg/client/experiment.go b/pkg/client/experiment.go new file mode 100644 index 00000000..b7e1a360 --- /dev/null +++ b/pkg/client/experiment.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package client + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" +) + +// ExperimentDefinition is a reproducible batch. WASM paths are local to the +// submitting process; SHA256 may be omitted initially and is filled in the receipt. +type ExperimentDefinition struct { + Participants []ExperimentRun `json:"participants"` +} + +// ExperimentRun binds a manifest entry to its admitted run. A definition omits +// RunID; a submission receipt retains the input, digest and assigned run ID. +type ExperimentRun struct { + OrderID int64 `json:"order_id"` + ExecutorID string `json:"executor_id"` + WASMPath string `json:"wasm_path"` + SHA256 string `json:"sha256"` + Args []string `json:"args"` + Policy Policy `json:"policy"` + RunID string `json:"run_id,omitempty"` +} + +type ExperimentSubmission struct { + ExperimentID string `json:"experiment_id"` + Participants []ExperimentRun `json:"participants"` +} + +type ExperimentResults struct { + Submission ExperimentSubmission `json:"submission"` + Results []Result `json:"results"` +} + +// SubmitExperimentTEST submits one existing TEST batch. Its transaction is the +// experiment identity, and each OrderID identifies a member within that batch. +// A supplied digest must match before any network request. On submission errors +// the returned receipt preserves known identities; the error's outcome-unknown +// classification still applies. Never retry an uncertain submission blindly. +func (c *Client) SubmitExperimentTEST(ctx context.Context, def ExperimentDefinition) (ExperimentSubmission, error) { + receipt := ExperimentSubmission{Participants: append([]ExperimentRun(nil), def.Participants...)} + requests := make([]Request, len(receipt.Participants)) + for i := range receipt.Participants { + p := &receipt.Participants[i] + if p.RunID != "" { return receipt, errors.New("client: definition already contains a run id") } + wasm, err := os.ReadFile(p.WASMPath) + if err != nil { return receipt, fmt.Errorf("participant %d: %w", p.OrderID, err) } + digest := sha256.Sum256(wasm) + hash := hex.EncodeToString(digest[:]) + if p.SHA256 != "" && p.SHA256 != hash { return receipt, fmt.Errorf("participant %d: WASM SHA256 mismatch", p.OrderID) } + p.SHA256 = hash + p.Args = append([]string{}, p.Args...) + p.Policy.Addresses = append([]string{}, p.Policy.Addresses...) + requests[i] = Request{OrderID: p.OrderID, ExecutorID: p.ExecutorID, Wasm: wasm, Args: p.Args, Policy: p.Policy} + } + batch, err := Prepare(requests) + if err != nil { return receipt, err } + submission, err := c.SubmitTEST(ctx, batch) + if err != nil { + var se *SubmissionError + if errors.As(err, &se) { submission = Submission{TransactionID: se.TransactionID, IDs: se.AdmittedIDs} } + } + receipt.ExperimentID = submission.TransactionID + if len(submission.IDs) == len(receipt.Participants) { + for i, id := range submission.IDs { receipt.Participants[i].RunID = id } + } + return receipt, err +} + +// ExportExperiment groups existing result snapshots in manifest order. It does +// not wait for completion. On error it returns the successfully read prefix. +func (c *Client) ExportExperiment(ctx context.Context, receipt ExperimentSubmission) (ExperimentResults, error) { + group := ExperimentResults{Submission: receipt, Results: []Result{}} + for _, participant := range receipt.Participants { + result, err := c.Export(ctx, participant.RunID) + if err != nil { return group, err } + if result.ExecutorID != participant.ExecutorID || result.Provenance != nil && result.Provenance.WorkloadSHA256 != participant.SHA256 { + return group, errors.New("client: experiment result disagrees with submission receipt") + } + group.Results = append(group.Results, result) + } + return group, nil +} + +// CancelExperiment requests cancellation for all known runs. An acknowledgement +// is not proof that execution stopped; inspect their existing result endpoints. +func (c *Client) CancelExperiment(ctx context.Context, receipt ExperimentSubmission) error { + var failures []error + for _, participant := range receipt.Participants { + if participant.RunID == "" { continue } + if err := c.Cancel(ctx, participant.RunID, participant.ExecutorID); err != nil { + failures = append(failures, fmt.Errorf("participant %d: %w", participant.OrderID, err)) + } + } + return errors.Join(failures...) +} diff --git a/pkg/client/experiment_test.go b/pkg/client/experiment_test.go new file mode 100644 index 00000000..ab2e343e --- /dev/null +++ b/pkg/client/experiment_test.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package client + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func experimentDefinition(t *testing.T) ExperimentDefinition { + t.Helper() + path := filepath.Join(t.TempDir(), "peer.wasm") + if err := os.WriteFile(path, []byte("test-wasm"), 0600); err != nil { t.Fatal(err) } + return ExperimentDefinition{Participants: []ExperimentRun{ + {OrderID:42, ExecutorID:"node-a", WASMPath:path, Args:[]string{"a"}, Policy:Policy{TimeoutMS:1000}}, + {OrderID:7, ExecutorID:"node-b", WASMPath:path, Args:[]string{"b"}, Policy:Policy{TimeoutMS:2000}}, + }} +} + +func TestExperimentRetainsManifestOrderAndHashes(t *testing.T) { + f := newFakeServer(t, "") + f.handle("PUT /payment/intent", intentHandler(fixtureTx, "")) + ids := []string{fixtureID, "00000000-0000-4000-8000-000000000002"} + body, _ := json.Marshal(ids) + f.handle("PUT /debuglet", jsonHandler(http.StatusOK, string(body))) + definition := experimentDefinition(t) + receipt, err := f.client(t, Options{}).SubmitExperimentTEST(t.Context(), definition) + if err != nil { t.Fatal(err) } + digest := sha256.Sum256([]byte("test-wasm")) + if receipt.ExperimentID != fixtureTx || receipt.Participants[0].OrderID != 42 || receipt.Participants[1].OrderID != 7 || receipt.Participants[0].RunID != ids[0] || receipt.Participants[1].RunID != ids[1] || receipt.Participants[0].SHA256 != hex.EncodeToString(digest[:]) { t.Fatalf("receipt %+v", receipt) } + definition.Participants[0].Args[0] = "changed" + if receipt.Participants[0].Args[0] != "a" || definition.Participants[0].SHA256 != "" { t.Fatal("receipt aliases or mutates caller input") } + if len(f.requests()) != 2 { t.Fatal("submission made unexpected requests") } +} + +func TestExperimentRejectsChangedWASMBeforeSubmission(t *testing.T) { + f := newFakeServer(t, "") + definition := experimentDefinition(t) + definition.Participants[1].SHA256 = strings.Repeat("0", 64) + _, err := f.client(t, Options{}).SubmitExperimentTEST(t.Context(), definition) + if err == nil || len(f.requests()) != 0 { t.Fatal("changed artifact reached dispatcher") } +} + +func TestExperimentPreservesUncertainAdmission(t *testing.T) { + f := newFakeServer(t, "") + f.handle("PUT /payment/intent", intentHandler(fixtureTx, "")) + ids := []string{fixtureID, "00000000-0000-4000-8000-000000000002"} + body, _ := json.Marshal(map[string]any{"code":CodeInternal, "message":"failed", "admitted_ids":ids}) + f.handle("PUT /debuglet", jsonHandler(http.StatusInternalServerError, string(body))) + receipt, err := f.client(t, Options{}).SubmitExperimentTEST(t.Context(), experimentDefinition(t)) + se := asSubmissionError(t, err) + if !se.OutcomeUnknown || receipt.ExperimentID != fixtureTx || !reflect.DeepEqual([]string{receipt.Participants[0].RunID, receipt.Participants[1].RunID}, ids) { t.Fatalf("receipt %+v error %v", receipt, err) } +} + +func TestExperimentExportRejectsWrongExecutor(t *testing.T) { + f := newFakeServer(t, "") + doc := resultFixture(t) + body, _ := json.Marshal(doc) + f.handle("GET /debuglet/"+doc.RunID+"/result", jsonHandler(http.StatusOK, string(body))) + receipt := ExperimentSubmission{ExperimentID:fixtureTx, Participants:[]ExperimentRun{{RunID:doc.RunID, ExecutorID:"wrong-executor"}}} + result, err := f.client(t, Options{}).ExportExperiment(t.Context(), receipt) + if err == nil || len(result.Results) != 0 { t.Fatal("mismatched result was grouped") } +} From 7bbe62e3278a15aecc4f6ceae97a530a9aa768fd Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:50:26 +0200 Subject: [PATCH 03/17] Add run-bound guest experiment readiness and common start helpers --- internal/executor/debuglet/debuglet.go | 13 ++++ internal/executor/debuglet/wasm/env.go | 2 + internal/executor/debuglet/wasm/experiment.go | 61 ++++++++++++++++ .../executor/debuglet/wasm/experiment_test.go | 49 +++++++++++++ internal/executor/debuglet/wasm/functions.go | 1 + internal/executor/debuglet/worker_child.go | 2 +- internal/executor/experiment.go | 43 +++++++++++ internal/executor/experiment_test.go | 41 +++++++++++ internal/executor/handle_debuglet.go | 1 + .../executor/transport/rpc/bound_client.go | 7 ++ internal/executor/transport/rpc/lease_test.go | 10 ++- pkg/debuglet/experiment.go | 73 +++++++++++++++++++ pkg/debuglet/experiment_stub.go | 8 ++ pkg/debuglet/experiment_test.go | 56 ++++++++++++++ pkg/debuglet/experiment_wasip1.go | 15 ++++ pkg/debuglet/hostengine_test.go | 5 +- pkg/debuglet/testdata/experiment/main.go | 25 +++++++ 17 files changed, 408 insertions(+), 4 deletions(-) create mode 100644 internal/executor/debuglet/wasm/experiment.go create mode 100644 internal/executor/debuglet/wasm/experiment_test.go create mode 100644 internal/executor/experiment.go create mode 100644 internal/executor/experiment_test.go create mode 100644 pkg/debuglet/experiment.go create mode 100644 pkg/debuglet/experiment_stub.go create mode 100644 pkg/debuglet/experiment_test.go create mode 100644 pkg/debuglet/experiment_wasip1.go create mode 100644 pkg/debuglet/testdata/experiment/main.go diff --git a/internal/executor/debuglet/debuglet.go b/internal/executor/debuglet/debuglet.go index d7d3c51e..3d647189 100644 --- a/internal/executor/debuglet/debuglet.go +++ b/internal/executor/debuglet/debuglet.go @@ -155,6 +155,15 @@ func newWithBPFTagger(logger *zap.Logger, debugletID uuid.UUID, transactionID st } } +// SetExperimentControl installs the run-bound dispatcher capability before the +// runtime is initialized. It is shared by local and isolated worker host calls. +func (d *Debuglet) SetExperimentControl(control wasm.ExperimentControl) { + d.mu.Lock() + defer d.mu.Unlock() + if d.initialized || d.closed { panic("experiment control must be set before initialization") } + d.env.Experiment = control +} + // UserspaceTagging is the IPv4 mode the pure-Go tagger gives a run on this // host: userspace where it can send tagged datagrams through raw sockets // (Linux with CAP_NET_RAW), none elsewhere. @@ -422,6 +431,10 @@ func (d *Debuglet) createWASMInstance(ctx context.Context, wasmBytes []byte) (er return fmt.Errorf("createWASMInstance: recoverable I/O module instantiation: %w", err) } + if _, err := wasm.Register(rt.NewHostModuleBuilder(wasm.ExperimentModule), wasm.ExperimentModule, wasm.Functions(d.env), nil).Instantiate(ctx); err != nil { + return fmt.Errorf("createWASMInstance: experiment module instantiation: %w", err) + } + d.mu.Lock() closed := d.closed d.mu.Unlock() diff --git a/internal/executor/debuglet/wasm/env.go b/internal/executor/debuglet/wasm/env.go index 53c0e484..e21409ec 100644 --- a/internal/executor/debuglet/wasm/env.go +++ b/internal/executor/debuglet/wasm/env.go @@ -31,6 +31,8 @@ type WasmEnv struct { closeErr error lateCloseErr error + Experiment ExperimentControl + DebugletID uuid.UUID Policy scheduler.Policy // Net is the network policy every transport is checked against. It is the diff --git a/internal/executor/debuglet/wasm/experiment.go b/internal/executor/debuglet/wasm/experiment.go new file mode 100644 index 00000000..1f8b5825 --- /dev/null +++ b/internal/executor/debuglet/wasm/experiment.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package wasm + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/netsec-ethz/debuglet/pkg/wire" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ExperimentModule is the optional readiness ABI; the base env ABI is unchanged. +const ExperimentModule = "debuglet_experiment_v1" + +// ExperimentControl is the run-bound capability supplied by the executor. +// The guest can publish metadata but cannot select a run, account or session. +type ExperimentControl interface { + Ready(context.Context, []byte) (wire.Experiment, error) +} + +func hostExperimentReady(env *WasmEnv) func(context.Context, Memory, uint32, uint32, uint32, uint32, int64) int32 { + return func(ctx context.Context, mem Memory, ptr, size, out, capacity uint32, deadlineNS int64) int32 { + if size > wire.MaxExperimentMetadata || capacity > 1<<20 { return -4 } + // Memory is contiguous. Validate both ends before any readiness mutation, + // without copying a megabyte through the isolated worker's memory bridge. + if capacity == 0 || uint64(out)+uint64(capacity) > 1<<32 { panic("invalid experiment output buffer") } + if _, ok := mem.Read(out, 1); !ok { panic("invalid experiment output buffer") } + if _, ok := mem.Read(out+capacity-1, 1); !ok { panic("invalid experiment output buffer") } + metadata, ok := mem.Read(ptr, size) + if !ok { panic("invalid experiment metadata buffer") } + if env.Experiment == nil { return -1 } + if deadlineNS <= 0 { return -3 } + callCtx, cancel := context.WithDeadline(ctx, time.Unix(0, deadlineNS)) + defer cancel() + if err := callCtx.Err(); err != nil { return experimentError(err) } + result, err := env.Experiment.Ready(callCtx, append([]byte(nil), metadata...)) + if err != nil { return experimentError(err) } + if err := callCtx.Err(); err != nil { return experimentError(err) } + data, err := json.Marshal(result) + if err != nil { return -1 } + if len(data) > int(capacity) { return -4 } + for offset := 0; offset < len(data); { + end := min(offset+MAX_SLICE_LENGTH, len(data)) + if !mem.Write(out+uint32(offset), data[offset:end]) { panic("invalid experiment output buffer") } + offset = end + } + return int32(len(data)) + } +} + +func experimentError(err error) int32 { + if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { return -2 } + if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded { return -3 } + if status.Code(err) == codes.ResourceExhausted { return -4 } + return -1 // Never expose transport diagnostics or authentication material. +} diff --git a/internal/executor/debuglet/wasm/experiment_test.go b/internal/executor/debuglet/wasm/experiment_test.go new file mode 100644 index 00000000..109c7013 --- /dev/null +++ b/internal/executor/debuglet/wasm/experiment_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package wasm + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/netsec-ethz/debuglet/pkg/wire" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type experimentControl func(context.Context, []byte) (wire.Experiment, error) +func (f experimentControl) Ready(ctx context.Context, metadata []byte) (wire.Experiment,error) { return f(ctx,metadata) } + +func TestExperimentHostValidatesBeforePublication(t *testing.T) { + mod := newGuestModule(t) + calls := 0 + env := &WasmEnv{Experiment:experimentControl(func(context.Context,[]byte)(wire.Experiment,error){calls++;return wire.Experiment{},nil})} + ready := hostExperimentReady(env) + ctx := context.Background() + deadline := time.Now().Add(time.Second).UnixNano() + if got:=ready(ctx,mod.Memory(),0,4097,0,1024,deadline); got != -4 { t.Fatal(got) } + for _, call := range []func(){ + func(){ready(ctx,mod.Memory(),0,1,wasmPageSize-1,10,deadline)}, + func(){ready(ctx,mod.Memory(),wasmPageSize,1,0,1024,deadline)}, + func(){ready(ctx,mod.Memory(),0,1,^uint32(0),10,deadline)}, + } { if hostTrap(call)==nil { t.Fatal("invalid memory accepted") } } + if got:=ready(ctx,mod.Memory(),0,0,0,1024,time.Now().Add(-time.Second).UnixNano()); got != -3 { t.Fatal(got) } + if calls != 0 { t.Fatalf("invalid input published %d times",calls) } +} + +func TestExperimentHostChunkedResponseAndPrivateErrors(t *testing.T) { + mod := newGuestModule(t) + result := wire.Experiment{ID:"batch",StartTimeNS:10,Participants:[]wire.ExperimentParticipant{ + {ID:"one",Metadata:make([]byte,4096)},{ID:"two",Metadata:make([]byte,4096)}, + }} + env := &WasmEnv{Experiment:experimentControl(func(context.Context,[]byte)(wire.Experiment,error){return result,nil})} + got:=hostExperimentReady(env)(context.Background(),mod.Memory(),0,0,0,32000,time.Now().Add(time.Second).UnixNano()) + if got <= MAX_SLICE_LENGTH { t.Fatalf("response not chunked: %d",got) } + data,_:=mod.Memory().Read(0,uint32(got)); var decoded wire.Experiment + if err:=json.Unmarshal(data,&decoded); err != nil || len(decoded.Participants)!=2 || len(decoded.Participants[1].Metadata)!=4096 { t.Fatalf("invalid result %v",err) } + env.Experiment=experimentControl(func(context.Context,[]byte)(wire.Experiment,error){return wire.Experiment{},status.Error(codes.Unauthenticated,"secret credential")}) + if got:=hostExperimentReady(env)(context.Background(),mod.Memory(),0,0,0,32000,time.Now().Add(time.Second).UnixNano()); got != -1 { t.Fatal(got) } +} diff --git a/internal/executor/debuglet/wasm/functions.go b/internal/executor/debuglet/wasm/functions.go index d483a2c0..fbcfc705 100644 --- a/internal/executor/debuglet/wasm/functions.go +++ b/internal/executor/debuglet/wasm/functions.go @@ -122,5 +122,6 @@ func Functions(env *WasmEnv) []Function { function(guestio.Module, "write", hostIOWrite(env)), function(guestio.Module, "close", HostIOClose(env)), function(guestio.Module, "deadline", HostIODeadline(env)), + function(ExperimentModule, "ready", hostExperimentReady(env)), } } diff --git a/internal/executor/debuglet/worker_child.go b/internal/executor/debuglet/worker_child.go index 99aee6a6..70fa4073 100644 --- a/internal/executor/debuglet/worker_child.go +++ b/internal/executor/debuglet/worker_child.go @@ -72,7 +72,7 @@ func serveWorker(b bridge) error { return b.send(frameCompiled, []byte{1}) } functions := wasm.Functions(nil) - for _, moduleName := range []string{"env", guestio.Module} { + for _, moduleName := range []string{"env", guestio.Module, wasm.ExperimentModule} { if _, err = wasm.Register(rt.NewHostModuleBuilder(moduleName), moduleName, functions, b.proxy).Instantiate(ctx); err != nil { return b.send(frameCompiled, []byte{1}) } diff --git a/internal/executor/experiment.go b/internal/executor/experiment.go new file mode 100644 index 00000000..2871fc93 --- /dev/null +++ b/internal/executor/experiment.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package executor + +import ( + "context" + "time" + + "github.com/netsec-ethz/debuglet/internal/executor/scheduler" + "github.com/netsec-ethz/debuglet/pkg/wire" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type runExperiment struct { + executor *Executor + spec scheduler.Spec +} + +func (r runExperiment) Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { + var result wire.Experiment + if len(metadata) > wire.MaxExperimentMetadata { return result, status.Error(codes.ResourceExhausted, "experiment metadata exceeds limit") } + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + client, err := r.executor.dispatcherClient(ctx, r.spec.Binding) + if err != nil { return result, err } + reply, err := client.ExperimentReady(ctx, &pb.ExperimentReadyRequest{ + DebugletId: r.spec.DebugletID.String(), ExecutorId: r.executor.cfg.Identity.ExecutorID, Metadata: metadata, + }) + if err != nil { return result, err } + if err := r.executor.checkExecutionLease(ctx, r.spec.Binding); err != nil { return result, err } + if reply == nil || len(reply.GetParticipants()) > wire.MaxExperimentParticipants { return result, status.Error(codes.ResourceExhausted, "invalid experiment response") } + result.ID, result.StartTimeNS = reply.GetExperimentId(), reply.GetStartTimeNs() + for _, p := range reply.GetParticipants() { + if len(p.GetMetadata()) > wire.MaxExperimentMetadata { return wire.Experiment{}, status.Error(codes.ResourceExhausted, "experiment metadata exceeds limit") } + result.Participants = append(result.Participants, wire.ExperimentParticipant{ + ID: p.GetId(), ExecutorID: p.GetExecutorId(), Metadata: p.GetMetadata(), ReadyAtNS: p.GetReadyAtNs(), + }) + } + return result, nil +} diff --git a/internal/executor/experiment_test.go b/internal/executor/experiment_test.go new file mode 100644 index 00000000..a7d01c77 --- /dev/null +++ b/internal/executor/experiment_test.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package executor + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/netsec-ethz/debuglet/internal/executor/scheduler" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type experimentPeer struct { + pb.UnimplementedDispatcherServiceServer + ready func(context.Context,*pb.ExperimentReadyRequest)(*pb.ExperimentReadyResponse,error) +} +func (p experimentPeer) ExperimentReady(ctx context.Context, req *pb.ExperimentReadyRequest)(*pb.ExperimentReadyResponse,error) { return p.ready(ctx,req) } + +func TestExperimentRunBindingAndCancellation(t *testing.T) { + id:=uuid.New() + calls:=make(chan *pb.ExperimentReadyRequest,2) + peer:=experimentPeer{ready:func(ctx context.Context,req *pb.ExperimentReadyRequest)(*pb.ExperimentReadyResponse,error){ + calls<-req + if string(req.Metadata)=="wait" { <-ctx.Done(); return nil,ctx.Err() } + return &pb.ExperimentReadyResponse{ExperimentId:"batch",StartTimeNs:time.Now().Add(time.Second).UnixNano(),Participants:[]*pb.ExperimentParticipant{{Id:id.String(),ExecutorId:req.ExecutorId,Metadata:req.Metadata,ReadyAtNs:123}}},nil + }} + e,_:=newExecutorRPCFixture(t,peer,newFixtureMemoryStorage(t)) + control:=runExperiment{executor:e,spec:scheduler.Spec{DebugletID:id,Binding:operationBinding()}} + result,err:=control.Ready(context.Background(),[]byte("opaque")) + if err != nil { t.Fatal(err) } + req:=<-calls + if req.DebugletId != id.String() || req.ExecutorId != e.cfg.Identity.ExecutorID || result.ID != "batch" || len(result.Participants)!=1 || result.Participants[0].ReadyAtNS!=123 || string(result.Participants[0].Metadata)!="opaque" { t.Fatalf("incorrect request/result: %v %v",req,result) } + ctx,cancel:=context.WithTimeout(context.Background(),20*time.Millisecond);defer cancel() + if _,err:=control.Ready(ctx,[]byte("wait")); status.Code(err)!=codes.DeadlineExceeded && ctx.Err()==nil { t.Fatalf("cancellation ignored: %v",err) } + <-calls +} diff --git a/internal/executor/handle_debuglet.go b/internal/executor/handle_debuglet.go index c2dc4f3d..1b3f7e3e 100644 --- a/internal/executor/handle_debuglet.go +++ b/internal/executor/handle_debuglet.go @@ -151,6 +151,7 @@ func (e *Executor) registerDebuglet(spec scheduler.Spec, op *debugletOperation) operator = operator.WithRevocations(e.revoked) local := debuglet.New(e.logger, spec.DebugletID, spec.TransactionID, spec.Policy, operator, e.teslaSchedule, e.limiter, e.packetCount, e.iface, e.portManager, socket.NewBudget(socket.DefaultLimits(), e.socketBudget)) + local.SetExperimentControl(runExperiment{executor: e, spec: spec}) deb = local if e.supervisor != nil { deb = debuglet.NewWorker(local, e.supervisor) diff --git a/internal/executor/transport/rpc/bound_client.go b/internal/executor/transport/rpc/bound_client.go index cf8aaa6a..dd016408 100644 --- a/internal/executor/transport/rpc/bound_client.go +++ b/internal/executor/transport/rpc/bound_client.go @@ -51,6 +51,13 @@ func (c *boundDispatcherClient) DebugletExit(ctx context.Context, in *pb.Debugle out, err := c.client.DebugletExit(c.credentials.Outgoing(ctx), in, opts...) return out, c.credentials.RedactError(err) } +func (c *boundDispatcherClient) ExperimentReady(ctx context.Context, in *pb.ExperimentReadyRequest, opts ...grpc.CallOption) (*pb.ExperimentReadyResponse, error) { + if err := c.owner.CheckLease(c.credentials.Binding); err != nil { return nil, err } + out, err := c.client.ExperimentReady(c.credentials.Outgoing(ctx), in, opts...) + if err != nil { return nil, c.credentials.RedactError(err) } + if err := c.owner.CheckLease(c.credentials.Binding); err != nil { return nil, err } + return out, nil +} func (c *boundDispatcherClient) BindSession(ctx context.Context, in *pb.BindSessionRequest, opts ...grpc.CallOption) (*pb.BindSessionResponse, error) { return nil, controlrpc.Unavailable() } diff --git a/internal/executor/transport/rpc/lease_test.go b/internal/executor/transport/rpc/lease_test.go index 8da5424a..7a9e93cf 100644 --- a/internal/executor/transport/rpc/lease_test.go +++ b/internal/executor/transport/rpc/lease_test.go @@ -31,6 +31,10 @@ func (c *leaseScriptClient) Heartbeat(context.Context, *pb.HeartbeatRequest, ... c.calls.Add(1) return &pb.HeartbeatResponse{}, nil } +func (c *leaseScriptClient) ExperimentReady(context.Context, *pb.ExperimentReadyRequest, ...grpc.CallOption) (*pb.ExperimentReadyResponse, error) { + c.calls.Add(1) + return &pb.ExperimentReadyResponse{}, nil +} func (c *leaseScriptClient) BindSession(ctx context.Context, in *pb.BindSessionRequest, _ ...grpc.CallOption) (*pb.BindSessionResponse, error) { c.calls.Add(1) return c.bind(ctx, in) @@ -92,7 +96,8 @@ func TestClientLeaseGuardsWithoutWatchdog(t *testing.T) { if _, err = cached.Heartbeat(context.Background(), &pb.HeartbeatRequest{}); err != nil { t.Fatal(err) } - before := raw.calls.Load() + if _, err = cached.ExperimentReady(context.Background(), &pb.ExperimentReadyRequest{}); err != nil { t.Fatal(err) } + before := raw.calls.Load() offset.Store(int64(time.Second)) committed := false if err = b.CommitLease(binding, func() { committed = true }); err == nil || committed { @@ -113,7 +118,8 @@ func TestClientLeaseGuardsWithoutWatchdog(t *testing.T) { if err = stream.SendMsg(&pb.DebugletStreamRequest{}); err == nil { t.Fatal("cached stream SendMsg escaped lease guard") } - if raw.calls.Load() != before || raw.stream.sends.Load() != 1 { + if _, err = cached.ExperimentReady(context.Background(), &pb.ExperimentReadyRequest{}); err == nil { t.Fatal("cached experiment call escaped lease guard") } + if raw.calls.Load() != before || raw.stream.sends.Load() != 1 { t.Fatal("expired operation reached transport") } if err = stream.CloseSend(); err != nil { diff --git a/pkg/debuglet/experiment.go b/pkg/debuglet/experiment.go new file mode 100644 index 00000000..5aea8471 --- /dev/null +++ b/pkg/debuglet/experiment.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package debuglet + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/netsec-ethz/debuglet/pkg/wire" +) + +// MaxExperimentMetadata is the opaque metadata allowance for one participant. +const MaxExperimentMetadata = wire.MaxExperimentMetadata +const experimentBufferBytes = 1 << 20 + +// ErrExperimentLate means the agreed start passed before the guest could wait. +var ErrExperimentLate = errors.New("debuglet: experiment start already passed") +var ErrExperimentUnavailable = errors.New("debuglet: experiment readiness unavailable or refused") + +// Ready publishes immutable metadata for this run and waits for every run in its +// submitted batch. Listeners and application setup should be ready first. The +// host supplies run identity and authorization; metadata grants no network access. +// Waiting is limited to 30 seconds, the caller's deadline and the run's lifetime. +// Cancellation is checked between bounded host calls (at most one second each). +func Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { + var result wire.Experiment + if len(metadata) > MaxExperimentMetadata { return result, ErrTooLarge } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + buf := make([]byte, experimentBufferBytes) + for { + if err := ctx.Err(); err != nil { return result, err } + deadline, _ := ctx.Deadline() + n := experimentReady(metadata, buf, deadline.UnixNano()) + if err := ctx.Err(); err != nil { return result, err } + switch n { + case -2: return result, context.Canceled + case -3: return result, context.DeadlineExceeded + case -4: return result, ErrTooLarge + } + if n < 0 { return result, ErrExperimentUnavailable } + if int(n) > len(buf) { return result, fmt.Errorf("debuglet: invalid experiment response length") } + if err := json.Unmarshal(buf[:n], &result); err != nil { return result, fmt.Errorf("debuglet: invalid experiment response: %w", err) } + if result.StartTimeNS != 0 { + if result.StartTimeNS <= time.Now().UnixNano() { return result, ErrExperimentLate } + return result, nil + } + timer := time.NewTimer(100*time.Millisecond) + select { + case <-ctx.Done(): timer.Stop(); return result, ctx.Err() + case <-timer.C: + } + } +} + +// WaitStart waits for the agreed wall-clock time. It rejects a start already in +// the past; callers should record their observed start themselves. This helper +// cannot guarantee synchronized clocks or simultaneous execution across hosts. +func WaitStart(ctx context.Context, experiment wire.Experiment) error { + if err := ctx.Err(); err != nil { return err } + delay := time.Until(time.Unix(0, experiment.StartTimeNS)) + if experiment.StartTimeNS <= 0 || delay <= 0 { return ErrExperimentLate } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): return ctx.Err() + case <-timer.C: return ctx.Err() + } +} diff --git a/pkg/debuglet/experiment_stub.go b/pkg/debuglet/experiment_stub.go new file mode 100644 index 00000000..8ea631fd --- /dev/null +++ b/pkg/debuglet/experiment_stub.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +//go:build !wasip1 + +package debuglet + +func experimentReady(metadata, result []byte, deadlineNS int64) int32 { panic(offTarget) } diff --git a/pkg/debuglet/experiment_test.go b/pkg/debuglet/experiment_test.go new file mode 100644 index 00000000..eb934ae6 --- /dev/null +++ b/pkg/debuglet/experiment_test.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package debuglet_test + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/netsec-ethz/debuglet/pkg/debuglet" + "github.com/netsec-ethz/debuglet/pkg/wire" +) + +type experimentControl func(context.Context, []byte) (wire.Experiment, error) +func (f experimentControl) Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { return f(ctx, metadata) } + +func TestExperimentGuest(t *testing.T) { + module := buildGuest(t, "./pkg/debuglet/testdata/experiment") + t.Run("poll metadata and wait", func(t *testing.T) { + var calls atomic.Int32 + var target atomic.Int64 + control := experimentControl(func(ctx context.Context, metadata []byte) (wire.Experiment, error) { + if string(metadata) != "guest endpoint" { return wire.Experiment{}, fmt.Errorf("wrong metadata %q", metadata) } + if calls.Add(1) == 1 { return wire.Experiment{ID:"batch"}, nil } + target.Store(time.Now().Add(400*time.Millisecond).UnixNano()) + return wire.Experiment{ID:"batch", StartTimeNS:target.Load(), Participants:[]wire.ExperimentParticipant{{ID:"peer", ExecutorID:"executor", Metadata:[]byte("peer endpoint")}}}, nil + }) + g := runGuest(t, module, hostOptions{experiment:control,args:[]string{"ready"}}) + requireSuccess(t,g) + requireContains(t,g,"experiment=batch members=1 peer=peer endpoint", "wait=") + var observed, requested int64 + line := g.waitFor("wait=",0) + if _, err := fmt.Sscanf(line,"wait= observed=%d target=%d", &observed, &requested); err != nil { t.Fatal(err) } + if calls.Load() != 2 || requested != target.Load() || observed < requested { t.Fatalf("calls=%d observed=%d target=%d",calls.Load(),observed,requested) } + }) + t.Run("deadline", func(t *testing.T) { + control := experimentControl(func(ctx context.Context, _ []byte) (wire.Experiment,error) { <-ctx.Done(); return wire.Experiment{},ctx.Err() }) + g := runGuest(t,module,hostOptions{experiment:control,args:[]string{"deadline"}}) + requireSuccess(t,g); requireContains(t,g,"deadline=true late=false") + }) + t.Run("late", func(t *testing.T) { + control := experimentControl(func(context.Context, []byte) (wire.Experiment,error) { return wire.Experiment{StartTimeNS:time.Now().Add(-time.Second).UnixNano()},nil }) + g := runGuest(t,module,hostOptions{experiment:control,args:[]string{"late"}}) + requireSuccess(t,g); requireContains(t,g,"deadline=false late=true") + }) +} + +func TestWaitStartCancellationAndLate(t *testing.T) { + if err := debuglet.WaitStart(context.Background(),wire.Experiment{StartTimeNS:time.Now().Add(-time.Second).UnixNano()}); !errors.Is(err,debuglet.ErrExperimentLate) { t.Fatal(err) } + ctx,cancel := context.WithCancel(context.Background()); cancel() + if err := debuglet.WaitStart(ctx,wire.Experiment{StartTimeNS:time.Now().Add(time.Hour).UnixNano()}); !errors.Is(err,context.Canceled) { t.Fatal(err) } +} diff --git a/pkg/debuglet/experiment_wasip1.go b/pkg/debuglet/experiment_wasip1.go new file mode 100644 index 00000000..3b75223e --- /dev/null +++ b/pkg/debuglet/experiment_wasip1.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +//go:build wasip1 + +package debuglet + +//go:wasmimport debuglet_experiment_v1 ready +func experimentReadyHost(metadataPtr, metadataLen, resultPtr, resultLen uint32, deadlineNS int64) int32 + +func experimentReady(metadata, result []byte, deadlineNS int64) int32 { + var ptr uint32 + if len(metadata) > 0 { ptr = bytePtr(metadata) } + return experimentReadyHost(ptr, uint32(len(metadata)), bytePtr(result), uint32(len(result)), deadlineNS) +} diff --git a/pkg/debuglet/hostengine_test.go b/pkg/debuglet/hostengine_test.go index 0be52d0a..15cda5c5 100644 --- a/pkg/debuglet/hostengine_test.go +++ b/pkg/debuglet/hostengine_test.go @@ -40,6 +40,7 @@ import ( hostdebuglet "github.com/netsec-ethz/debuglet/internal/executor/debuglet" "github.com/netsec-ethz/debuglet/internal/executor/debuglet/netpolicy" "github.com/netsec-ethz/debuglet/internal/executor/debuglet/socket" + "github.com/netsec-ethz/debuglet/internal/executor/debuglet/wasm" "github.com/netsec-ethz/debuglet/internal/executor/ratelimit/app" "github.com/netsec-ethz/debuglet/internal/executor/ratelimit/fallback" "github.com/netsec-ethz/debuglet/internal/executor/scheduler" @@ -117,6 +118,7 @@ func buildGuest(t *testing.T, pkgPath string) []byte { // hostOptions describes one job: its destination policy, its listeners, its // guest arguments and its execution budget. type hostOptions struct { + experiment wasm.ExperimentControl addresses []string listenTCP bool listenUDP bool @@ -223,7 +225,8 @@ func startGuest(t *testing.T, wasm []byte, opts hostOptions) *guestRun { t.Fatalf("netpolicy.Parse: %v", err) } deb := hostdebuglet.New(logger, id, "guest-compatibility", policy, operator, schedule, limiter, packetCount, nil, ports, socket.NewBudget(socket.DefaultLimits(), socket.NewDescriptorBudget(socket.DefaultNodeDescriptors))) - initCtx, cancelInit := context.WithTimeout(context.Background(), initTimeout) + deb.SetExperimentControl(opts.experiment) + initCtx, cancelInit := context.WithTimeout(context.Background(), initTimeout) g := &guestRun{t: t, deb: deb, cancelInit: cancelInit, errCh: make(chan error, 1)} t.Cleanup(g.stop) diff --git a/pkg/debuglet/testdata/experiment/main.go b/pkg/debuglet/testdata/experiment/main.go new file mode 100644 index 00000000..0dbd572c --- /dev/null +++ b/pkg/debuglet/testdata/experiment/main.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package main + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/netsec-ethz/debuglet/pkg/debuglet" +) + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if os.Args[0] == "deadline" { ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond); defer cancel() } + exp, err := debuglet.Ready(ctx, []byte("guest endpoint")) + if err != nil { fmt.Printf("deadline=%v late=%v\n", errors.Is(err, context.DeadlineExceeded), errors.Is(err, debuglet.ErrExperimentLate)); return } + fmt.Printf("experiment=%s members=%d peer=%s\n", exp.ID, len(exp.Participants), exp.Participants[0].Metadata) + err = debuglet.WaitStart(ctx, exp) + fmt.Printf("wait=%v observed=%d target=%d\n", err, time.Now().UnixNano(), exp.StartTimeNS) +} From 93525102012e5f60b17cf2e85e26efc2ef683480 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:52:58 +0200 Subject: [PATCH 04/17] Persist authenticated bounded experiment readiness and one-shot release --- .../database/migrations/00028_experiments.sql | 14 ++ .../database/queries/experiments.sql | 34 +++++ internal/dispatcher/experiments.go | 102 ++++++++++++++ internal/dispatcher/experiments_test.go | 132 ++++++++++++++++++ .../transport/rpc/experiment_test.go | 45 ++++++ internal/dispatcher/transport/rpc/server.go | 10 ++ internal/dispatcher/transport/rpc/state.go | 1 + internal/dispatcher/window_expiry.go | 3 + internal/storagecheck/init_test.go | 4 +- internal/storagecheck/policy.go | 6 +- 10 files changed, 348 insertions(+), 3 deletions(-) create mode 100644 internal/dispatcher/database/migrations/00028_experiments.sql create mode 100644 internal/dispatcher/database/queries/experiments.sql create mode 100644 internal/dispatcher/experiments.go create mode 100644 internal/dispatcher/experiments_test.go create mode 100644 internal/dispatcher/transport/rpc/experiment_test.go diff --git a/internal/dispatcher/database/migrations/00028_experiments.sql b/internal/dispatcher/database/migrations/00028_experiments.sql new file mode 100644 index 00000000..2915ff99 --- /dev/null +++ b/internal/dispatcher/database/migrations/00028_experiments.sql @@ -0,0 +1,14 @@ +-- +goose up +CREATE TABLE experiment_barriers ( + transaction_id TEXT PRIMARY KEY NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, + deadline_ns INTEGER NOT NULL, + start_time_ns INTEGER NOT NULL DEFAULT 0 +); +CREATE TABLE experiment_readiness ( + debuglet_id INTEGER PRIMARY KEY NOT NULL REFERENCES debuglets(id) ON DELETE CASCADE, + metadata BLOB NOT NULL CHECK(length(metadata) <= 4096), + ready_at_ns INTEGER NOT NULL +); +-- +goose down +DROP TABLE experiment_readiness; +DROP TABLE experiment_barriers; diff --git a/internal/dispatcher/database/queries/experiments.sql b/internal/dispatcher/database/queries/experiments.sql new file mode 100644 index 00000000..8a5b42d4 --- /dev/null +++ b/internal/dispatcher/database/queries/experiments.sql @@ -0,0 +1,34 @@ +-- name: GetExperimentBarrier :one +SELECT * FROM experiment_barriers WHERE transaction_id = ?; + +-- name: CreateExperimentBarrier :exec +INSERT INTO experiment_barriers(transaction_id, deadline_ns) VALUES (?, ?) +ON CONFLICT(transaction_id) DO NOTHING; + +-- name: ReleaseExperiment :exec +UPDATE experiment_barriers SET start_time_ns = ? +WHERE transaction_id = ? AND start_time_ns = 0; + +-- name: RecordExperimentReady :exec +INSERT INTO experiment_readiness(debuglet_id, metadata, ready_at_ns) VALUES (?, ?, ?) +ON CONFLICT(debuglet_id) DO NOTHING; + +-- name: CountExperimentOrders :one +SELECT count(*) FROM debuglet_order WHERE transaction_id = ?; + +-- name: GetExperimentMembers :many +SELECT sqlc.embed(d), r.metadata, r.ready_at_ns, + EXISTS(SELECT 1 FROM debuglet_cancellations c WHERE c.debuglet_id = d.id) AS cancelled, + EXISTS(SELECT 1 FROM allocation_reclamations a WHERE a.debuglet_id = d.id) AS reclaimed +FROM debuglet_order o JOIN debuglets d ON d.id = o.debuglet_id + AND d.transaction_id = o.transaction_id AND d.order_id = o.order_id +LEFT JOIN experiment_readiness r ON r.debuglet_id = d.id +WHERE o.transaction_id = ? ORDER BY o.order_id LIMIT 129; + +-- name: PruneExperimentMetadata :exec +DELETE FROM experiment_readiness WHERE debuglet_id IN ( + SELECT r.debuglet_id FROM experiment_readiness r + JOIN debuglets d ON d.id = r.debuglet_id + JOIN experiment_barriers b ON b.transaction_id = d.transaction_id + WHERE b.deadline_ns <= ? LIMIT 512 +); diff --git a/internal/dispatcher/experiments.go b/internal/dispatcher/experiments.go new file mode 100644 index 00000000..25b3625b --- /dev/null +++ b/internal/dispatcher/experiments.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package dispatcher + +import ( + "bytes" + "context" + "time" + + "github.com/netsec-ethz/debuglet/internal/dispatcher/database" + "github.com/netsec-ethz/debuglet/internal/dispatcher/models" + "github.com/netsec-ethz/debuglet/internal/dispatcher/transport/rpc" + "github.com/netsec-ethz/debuglet/pkg/wire" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const experimentLead = 2 * time.Second +const experimentWait = 30 * time.Second + +// OnExperimentReady performs one bounded poll. SQLite serializes readiness, +// release and cancellation; a committed release is immutable across restarts. +func (d *Dispatcher) OnExperimentReady(ctx context.Context, mutation *rpc.Mutation, req *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { + if req.GetExecutorId() == "" { return nil, status.Error(codes.PermissionDenied, "executor identity required") } + owner, err := requireMutation(mutation, req.GetExecutorId()) + if err != nil { return nil, err } + if len(req.GetMetadata()) > wire.MaxExperimentMetadata { return nil, status.Error(codes.ResourceExhausted, "experiment metadata exceeds 4096 bytes") } + id, err := parseRunID(req.GetDebugletId()) + if err != nil { return nil, err } + // Classify ownership before reading any transaction or peer metadata. + run, err := d.ownedDebuglet(ctx, owner, id) + if err != nil { return nil, err } + d.mu.Lock() + defer d.mu.Unlock() + tx, err := d.db.BeginTx(ctx, nil) + if err != nil { return nil, err } + defer tx.Rollback() + q := database.New(tx) + members, err := q.GetExperimentMembers(ctx, run.TransactionID) + if err != nil { return nil, err } + orders, err := q.CountExperimentOrders(ctx, run.TransactionID) + if err != nil { return nil, err } + if len(members) == 0 || len(members) > wire.MaxExperimentParticipants || int64(len(members)) != orders { + return nil, status.Error(codes.FailedPrecondition, "experiment membership is not fully admitted") + } + now := d.now() + deadline := now.Add(experimentWait) + ready := 0 + found := false + for _, member := range members { + row := member.Debuglet + entry := d.executors[row.ExecutorID] + if d.closed || entry == nil || !entry.owner.Available() || entry.owner.Binding().Incarnation != row.DispatcherIncarnation || entry.owner.Binding().SessionID != row.SessionID { + return nil, status.Error(codes.FailedPrecondition, "experiment participant session is unavailable") + } + if row.State == models.RunStateExited || member.Cancelled || member.Reclaimed { + return nil, status.Error(codes.FailedPrecondition, "experiment participant is no longer active") + } + if end := row.EndTime.Time.Add(-experimentLead); end.Before(deadline) { deadline = end } + if member.ReadyAtNs.Valid { ready++ } + if row.ID == run.ID { + found = true + if row.State != models.RunStateStarted || now.Before(row.StartTime.Time) { return nil, status.Error(codes.FailedPrecondition, "participant has not started") } + if entry.owner != owner { return nil, status.Error(codes.PermissionDenied, "run does not belong to session") } + if member.ReadyAtNs.Valid && !bytes.Equal(member.Metadata, req.GetMetadata()) { + return nil, status.Error(codes.AlreadyExists, "readiness metadata is immutable") + } + if !member.ReadyAtNs.Valid { ready++ } + } + } + if !found { return nil, status.Error(codes.PermissionDenied, "run is not an experiment participant") } + if err := q.CreateExperimentBarrier(ctx, database.CreateExperimentBarrierParams{TransactionID: run.TransactionID, DeadlineNs: deadline.UnixNano()}); err != nil { return nil, err } + barrier, err := q.GetExperimentBarrier(ctx, run.TransactionID) + if err != nil { return nil, err } + if now.UnixNano() >= barrier.DeadlineNs || !now.Before(deadline) { + // Preserve the original deadline even when the first call already expired. + if err := tx.Commit(); err != nil { return nil, err } + return nil, status.Error(codes.DeadlineExceeded, "experiment readiness deadline expired") + } + metadata := req.GetMetadata() + if metadata == nil { metadata = []byte{} } + if err := q.RecordExperimentReady(ctx, database.RecordExperimentReadyParams{DebugletID: run.ID, Metadata: metadata, ReadyAtNs: now.UnixNano()}); err != nil { return nil, err } + start := barrier.StartTimeNs + if start == 0 && ready == len(members) { + start = now.Add(experimentLead).UnixNano() + if err := q.ReleaseExperiment(ctx, database.ReleaseExperimentParams{StartTimeNs: start, TransactionID: run.TransactionID}); err != nil { return nil, err } + } + out := &pb.ExperimentReadyResponse{ExperimentId: run.TransactionID, StartTimeNs: start} + // Pending callers learn no partial metadata and never wait inside a mutation. + if start != 0 { + for _, member := range members { + m := &pb.ExperimentParticipant{Id: member.Debuglet.Uuid.String(), ExecutorId: member.Debuglet.ExecutorID, Metadata: member.Metadata, ReadyAtNs: member.ReadyAtNs.Int64} + if member.Debuglet.ID == run.ID && !member.ReadyAtNs.Valid { m.Metadata, m.ReadyAtNs = metadata, now.UnixNano() } + out.Participants = append(out.Participants, m) + } + } + if err := tx.Commit(); err != nil { return nil, err } + return out, nil +} + diff --git a/internal/dispatcher/experiments_test.go b/internal/dispatcher/experiments_test.go new file mode 100644 index 00000000..74c3e74c --- /dev/null +++ b/internal/dispatcher/experiments_test.go @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package dispatcher + +import ( + "bytes" + "testing" + "time" + + "github.com/google/uuid" + "github.com/netsec-ethz/debuglet/internal/dispatcher/database" + "github.com/netsec-ethz/debuglet/internal/dispatcher/models" + "github.com/netsec-ethz/debuglet/internal/dispatcher/transport/rpc" + "github.com/netsec-ethz/debuglet/pkg/wire" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func experimentFixture(t *testing.T, count int) (*tgFixture, []uuid.UUID, string, *time.Time) { + t.Helper() + f := newTGFixture(t, nil) + first := f.seedDirect(t, tgFloorA) + now := time.Now() + f.d.now = func() time.Time { return now } + ids := []uuid.UUID{first.id} + if _, err := f.db.Exec("UPDATE debuglets SET start_time = ?, end_time = ?, state = ? WHERE id = ?", models.NewUTCTime(now.Add(-time.Second)), models.NewUTCTime(now.Add(2*time.Minute)), models.RunStateStarted, first.row.ID); err != nil { t.Fatal(err) } + if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ?", first.row.ID, first.txID); err != nil { t.Fatal(err) } + for i := 1; i < count; i++ { + id := uuid.New() + row, err := f.q.CreateDebuglet(f.ctx, database.CreateDebugletParams{Uuid:id, StartTime:models.NewUTCTime(now.Add(-time.Second)), EndTime:models.NewUTCTime(now.Add(2*time.Minute)), ExecutorID:tgExecutorID, State:models.RunStateStarted, TransactionID:first.txID, OrderID:int64(i+1), DispatcherIncarnation:first.row.DispatcherIncarnation, SessionID:first.row.SessionID}) + if err != nil { t.Fatal(err) } + if _, err := f.q.CreateDebugletOrder(f.ctx, database.CreateDebugletOrderParams{TransactionID:first.txID, OrderID:int64(i+1), ExecutorID:tgExecutorID, State:int64(models.Outstanding)}); err != nil { t.Fatal(err) } + if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ? AND order_id = ?", row.ID, first.txID, i+1); err != nil { t.Fatal(err) } + ids = append(ids, id) + } + return f, ids, first.txID, &now +} + +func experimentReady(t *testing.T, f *tgFixture, id uuid.UUID, metadata []byte) (*pb.ExperimentReadyResponse, error) { + t.Helper() + mutation := effectTestMutation(t, f.d, tgExecutorID) + defer mutation.Finish() + return f.d.OnExperimentReady(f.ctx, mutation, &pb.ExperimentReadyRequest{DebugletId:id.String(), ExecutorId:tgExecutorID, Metadata:metadata}) +} + +func TestExperimentFiveMemberReleaseAndRestart(t *testing.T) { + f, ids, group, now := experimentFixture(t, 5) + for i, id := range ids { + got, err := experimentReady(t, f, id, []byte{byte(i)}) + if err != nil { t.Fatal(err) } + if i < 4 && (got.StartTimeNs != 0 || len(got.Participants) != 0) { t.Fatalf("early release: %+v", got) } + } + first, err := experimentReady(t, f, ids[0], []byte{0}) + if err != nil || first.ExperimentId != group || first.StartTimeNs != now.Add(experimentLead).UnixNano() || len(first.Participants) != 5 { t.Fatalf("release=%+v err=%v", first, err) } + for i, member := range first.Participants { + if member.Id != ids[i].String() || !bytes.Equal(member.Metadata, []byte{byte(i)}) || member.ReadyAtNs != now.UnixNano() { t.Fatalf("participant=%+v", member) } + } + *now = now.Add(time.Second) + again, err := experimentReady(t, f, ids[0], []byte{0}) + if err != nil || again.StartTimeNs != first.StartTimeNs { t.Fatalf("duplicate reset release: %+v %v", again, err) } + if _, err := experimentReady(t, f, ids[0], []byte("changed")); status.Code(err) != codes.AlreadyExists { t.Fatalf("metadata replacement: %v", err) } + // A new dispatcher reconstructs no live original sessions. Durable release + // survives, but a replacement control identity cannot replay the old run. + restarted, err := New(f.d.logger, f.db, "restart", time.Minute, time.Minute, f.ph) + if err != nil { t.Fatal(err) } + defer restarted.Close() + barrier, err := database.New(f.db).GetExperimentBarrier(f.ctx, group) + if err != nil || barrier.StartTimeNs != first.StartTimeNs { t.Fatalf("lost persisted release: %+v %v", barrier, err) } + owner, err := rpc.NewSessionOwner(tgExecutorID, effectTestBinding(t), time.Minute) + if err != nil { t.Fatal(err) } + if !owner.MarkRegistered() { t.Fatal("register replacement") } + mutation, err := owner.AdmitMutation(f.ctx) + if err != nil { t.Fatal(err) } + defer mutation.Finish() + _, err = restarted.OnExperimentReady(f.ctx, mutation, &pb.ExperimentReadyRequest{DebugletId:ids[0].String(), ExecutorId:tgExecutorID}) + if status.Code(err) != codes.PermissionDenied { t.Fatalf("restart replay: %v", err) } +} + +func TestExperimentDeadlineCancellationAndBounds(t *testing.T) { + for _, kind := range []string{"deadline", "cancelled", "terminal", "retired", "oversize", "unstarted", "unadmitted"} { + t.Run(kind, func(t *testing.T) { + f, ids, group, now := experimentFixture(t, 2) + if _, err := experimentReady(t, f, ids[0], nil); err != nil { t.Fatal(err) } + want := codes.FailedPrecondition + var metadata []byte + switch kind { + case "deadline": *now = now.Add(experimentWait); want = codes.DeadlineExceeded + case "cancelled": + row, _ := f.q.GetDebugletByUUID(f.ctx, ids[0]); if _, err := f.d.requestCancellation(f.ctx, row.ID, "test"); err != nil { t.Fatal(err) } + case "terminal": if _, err := f.db.Exec("UPDATE debuglets SET state = ? WHERE uuid = ?", models.RunStateExited, ids[0]); err != nil { t.Fatal(err) } + case "retired": f.d.executors[tgExecutorID].owner.Retire() + case "oversize": metadata = make([]byte, wire.MaxExperimentMetadata+1); want = codes.ResourceExhausted + case "unstarted": if _, err := f.db.Exec("UPDATE debuglets SET state = ? WHERE uuid = ?", models.RunStateUploaded, ids[1]); err != nil { t.Fatal(err) } + case "unadmitted": if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = NULL WHERE transaction_id = ? AND order_id = 1", group); err != nil { t.Fatal(err) } + } + var err error + if kind == "retired" { + _, err = f.d.OnExperimentReady(f.ctx, nil, &pb.ExperimentReadyRequest{DebugletId:ids[1].String(), ExecutorId:tgExecutorID}) + } else { _, err = experimentReady(t, f, ids[1], metadata) } + if status.Code(err) != want { t.Fatalf("got=%v want=%v", err, want) } + barrier, err := f.q.GetExperimentBarrier(f.ctx, group) + if err != nil || barrier.StartTimeNs != 0 { t.Fatalf("late release=%+v %v", barrier, err) } + *now = now.Add(2*time.Minute) + f.d.sweepEndedWindows(time.Time{}) + var retained int + if err := f.db.QueryRow("SELECT COUNT(*) FROM experiment_readiness").Scan(&retained); err != nil || retained != 0 { t.Fatalf("retained=%d %v", retained, err) } + }) + } +} + +func TestExperimentForeignOwnerAndGroupIsolation(t *testing.T) { + f, ids, group, _ := experimentFixture(t, 2) + foreign := f.seedDirect(t, tgFloorA) + mutation := effectTestMutation(t, f.d, tgExecutorID) + defer mutation.Finish() + for _, req := range []*pb.ExperimentReadyRequest{ + {DebugletId:ids[0].String(), ExecutorId:"foreign"}, + {DebugletId:ids[0].String()}, + } { + if _, err := f.d.OnExperimentReady(f.ctx, mutation, req); status.Code(err) != codes.PermissionDenied { t.Fatalf("foreign owner: %v", err) } + } + if _, err := experimentReady(t, f, ids[0], []byte("private")); err != nil { t.Fatal(err) } + // Another admitted transaction receives only its own fixed participant. + if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ?", foreign.row.ID, foreign.txID); err != nil { t.Fatal(err) } + if _, err := f.db.Exec("UPDATE debuglets SET state = ?, start_time = ? WHERE id = ?", models.RunStateStarted, models.NewUTCTime(f.d.now().Add(-time.Second)), foreign.row.ID); err != nil { t.Fatal(err) } + other, err := experimentReady(t, f, foreign.id, nil) + if err != nil || other.ExperimentId != foreign.txID || len(other.Participants) != 1 || other.Participants[0].Id != foreign.id.String() || len(other.Participants[0].Metadata) != 0 { t.Fatalf("group leak: %+v %v", other, err) } + got, err := f.q.GetExperimentBarrier(f.ctx, group) + if err != nil || got.StartTimeNs != 0 { t.Fatalf("foreign readiness released group: %v %v", got, err) } +} diff --git a/internal/dispatcher/transport/rpc/experiment_test.go b/internal/dispatcher/transport/rpc/experiment_test.go new file mode 100644 index 00000000..5d22f426 --- /dev/null +++ b/internal/dispatcher/transport/rpc/experiment_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package rpc + +import ( + "context" + "testing" + + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +func (s *lifecycleState) OnExperimentReady(ctx context.Context, mutation *Mutation, req *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { + if !mutation.Live() || mutation.IsSetup() || mutation.Owner().ExecutorID() != req.ExecutorId { return nil, status.Error(codes.Internal, "invalid readiness mutation") } + return &pb.ExperimentReadyResponse{ExperimentId:"authenticated"}, nil +} + +func TestExperimentReadyUsesBoundAuthenticatedSession(t *testing.T) { + f := newControlTLS(t, controlTLSOptions{requireClientIdentity:true}) + owned := f.connectedPeer(&lifecyclePeer{id:"executor", version:"A"}, f.ca.ClientConfig(f.client, "")) + owner := f.registered("A") + if err := owned.client.WaitReadyContext(f.ctx); err != nil { t.Fatal(err) } + for _, tc := range []struct{name, executor string; strip bool; want codes.Code}{ + {"owner", "executor", false, codes.OK}, + {"foreign executor", "foreign", false, codes.PermissionDenied}, + {"empty executor", "", false, codes.PermissionDenied}, + {"missing credentials", "executor", true, codes.PermissionDenied}, + } { + t.Run(tc.name, func(t *testing.T) { + err := f.directCall(f.client, owner, func(ctx context.Context, client pb.DispatcherServiceClient) error { + if tc.strip { ctx = metadata.NewOutgoingContext(ctx, metadata.MD{}) } + _, err := client.ExperimentReady(ctx, &pb.ExperimentReadyRequest{DebugletId:"run", ExecutorId:tc.executor}) + return err + }) + if status.Code(err) != tc.want { t.Fatalf("got %v want %v", err, tc.want) } + }) + } + // Each poll has finished its mutation; retiring the session need not wait on + // an application barrier or a guest's next poll. + owner.Retire() + awaitSessionSignal(t, owner.MutationsDrained()) +} diff --git a/internal/dispatcher/transport/rpc/server.go b/internal/dispatcher/transport/rpc/server.go index 7ee176ce..92f957cf 100644 --- a/internal/dispatcher/transport/rpc/server.go +++ b/internal/dispatcher/transport/rpc/server.go @@ -88,3 +88,13 @@ func (s *server) DebugletStream(stream grpc.BidiStreamingServer[pb.DebugletStrea func (s *server) RenewLease(ctx context.Context, in *pb.RenewLeaseRequest) (*pb.RenewLeaseResponse, error) { return s.bidi.renewLease(ctx, in) } + +// ExperimentReady records or polls once and always releases its mutation before +// the guest waits. No participant can pin another session's replacement. +func (s *server) ExperimentReady(ctx context.Context, in *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { + ticket, err := s.bidi.admitted(ctx, in.GetExecutorId()) + if err != nil { return nil, err } + defer ticket.Finish() + if in.GetExecutorId() == "" { return nil, controlrpc.Denied() } + return s.state.OnExperimentReady(ticket.Context(), ticket, in) +} diff --git a/internal/dispatcher/transport/rpc/state.go b/internal/dispatcher/transport/rpc/state.go index d8e81514..58bd97dd 100644 --- a/internal/dispatcher/transport/rpc/state.go +++ b/internal/dispatcher/transport/rpc/state.go @@ -19,6 +19,7 @@ type DispatcherState interface { OnResources(ctx context.Context, mutation *Mutation, req *pb.ResourcesRequest) (*pb.ResourcesResponse, error) OnDebugletState(ctx context.Context, mutation *Mutation, req *pb.DebugletStateRequest) (*pb.DebugletStateResponse, error) OnDebugletAllocate(ctx context.Context, mutation *Mutation, req *pb.DebugletAllocateRequest) (*pb.DebugletAllocateResponse, error) + OnExperimentReady(ctx context.Context, mutation *Mutation, req *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) OnDebugletExit(ctx context.Context, mutation *Mutation, req *pb.DebugletExitRequest) (*pb.DebugletExitResponse, error) OnDebugletStream(owner *SessionOwner, stream grpc.BidiStreamingServer[pb.DebugletStreamRequest, pb.DebugletStreamResponse]) error diff --git a/internal/dispatcher/window_expiry.go b/internal/dispatcher/window_expiry.go index f2ce09f5..232ae49c 100644 --- a/internal/dispatcher/window_expiry.go +++ b/internal/dispatcher/window_expiry.go @@ -52,6 +52,9 @@ func (d *Dispatcher) sweepEndedWindows(last time.Time) time.Time { ctx, cancel := context.WithTimeout(context.Background(), windowSweepBound) defer cancel() queries := database.New(d.db) + if err := queries.PruneExperimentMetadata(ctx, now.UnixNano()); err != nil { + d.logger.Warn("Experiment metadata cleanup remains pending") + } destinations := map[string]struct{}{} release := func(run database.Debuglet) { d.releaseTerminal(run) diff --git a/internal/storagecheck/init_test.go b/internal/storagecheck/init_test.go index 3a9bef9a..964a06f5 100644 --- a/internal/storagecheck/init_test.go +++ b/internal/storagecheck/init_test.go @@ -51,6 +51,8 @@ func TestBootstrapFresh(t *testing.T) { "measurement_requests": "debuglet_id document", "measurement_execution": "debuglet_id started_observed_ns terminal_observed_ns exit_code tcp_endpoint", "retry_requests": "caller_scope request_id parent_run_id transaction_id request_hash intent_metadata", + "experiment_barriers": "transaction_id deadline_ns start_time_ns", + "experiment_readiness": "debuglet_id metadata ready_at_ns", "allocation_reclamations": "debuglet_id reclaimed_at", "allowance_grants": "id user_id amount currency granted_by reason idempotency_key granted_at", "probe_status": "executor_id first_connected last_connected connected status_since total_uptime is_public host_tags version", @@ -84,7 +86,7 @@ func TestBootstrapFresh(t *testing.T) { "oauth_login_attempts": "state_hash provider verifier nonce purpose session_selector expires_at", "pending_identity_links": "user_id provider issuer subject login session_selector expires_at", "device_logins": "selector verifier_hash user_code_hash audience scopes label expires_at next_poll_at poll_interval state approver_session user_id", - }, []string{"account_recovery_pending", "account_run_reservations_live", "attribution_runs_source_idx", "chain_transfers_executor_idx", "debuglet_logs_sequence_idx", "debuglets_uuid_idx", "device_logins_expiry", "executor_enrollment_tokens_executor_idx", "measurement_profiles_owner", "oauth_login_expiry", "owned_executors_user_idx", "payment_receipts_nonce_idx", "sessions_user_idx", "users_uuid_idx"}, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}) + }, []string{"account_recovery_pending", "account_run_reservations_live", "attribution_runs_source_idx", "chain_transfers_executor_idx", "debuglet_logs_sequence_idx", "debuglets_uuid_idx", "device_logins_expiry", "executor_enrollment_tokens_executor_idx", "measurement_profiles_owner", "oauth_login_expiry", "owned_executors_user_idx", "payment_receipts_nonce_idx", "sessions_user_idx", "users_uuid_idx"}, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}) dispatcherSchemaRoundTrip(t, db) } else { assertSchema(t, db, map[string]string{ diff --git a/internal/storagecheck/policy.go b/internal/storagecheck/policy.go index d85e4765..86640d89 100644 --- a/internal/storagecheck/policy.go +++ b/internal/storagecheck/policy.go @@ -31,8 +31,8 @@ const ( // an older database can then no longer answer them and must be refused instead // of failing later during service. const ( - // Schema 27: cancellation terminal decisions are read for refund recovery. - MinimumDispatcherVersion int64 = 27 + // Schema 28: durable one-shot experiment readiness. + MinimumDispatcherVersion int64 = 28 MinimumExecutorVersion int64 = 8 ) @@ -72,6 +72,8 @@ func PolicyFor(role Role) (Policy, error) { "transactions": nil, }, Tables: map[string][]string{ "account_recovery_audit": {"selector", "user_id", "case_reference", "issued_by_uid", "issued_at", "expires_at", "consumed_at", "revoked_at", "revoked_by_uid", "revocation_reference"}, + "experiment_barriers": {"transaction_id", "deadline_ns", "start_time_ns"}, + "experiment_readiness": {"debuglet_id", "metadata", "ready_at_ns"}, "allocation_reclamations": {"debuglet_id", "reclaimed_at"}, "allowance_grants": {"id", "user_id", "amount", "granted_by", "reason", "idempotency_key", "granted_at"}, "probe_status": {"executor_id", "first_connected", "last_connected", "connected", "status_since", "total_uptime", "is_public", "host_tags", "version"}, From b8728ce893cca613c63f2266e53a4a169a88237b Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:54:24 +0200 Subject: [PATCH 05/17] Exercise coordinated experiments across five installed executors --- .github/workflows/ci.yml | 1 + examples/experiments/README.md | 13 +-- .../roles/experiment_integration_test.go | 89 +++++++++++++++++++ .../roles/rendezvous_integration_test.go | 14 ++- scripts/ci-fmt.sh | 1 + scripts/ci-roles.sh | 5 +- 6 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 internal/acceptance/roles/experiment_integration_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a09ecfd..5a854360 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,7 @@ jobs: name: evidence-${{ matrix.lane }}-${{ github.sha }}-${{ github.run_attempt }} path: | .cache/ci/gofmt.txt + .cache/ci/gofmt.diff .cache/ci/tests.json .cache/ci/race-tests.json .cache/ci/kernel-tests.json diff --git a/examples/experiments/README.md b/examples/experiments/README.md index 15938e82..0cd6ce1b 100644 --- a/examples/experiments/README.md +++ b/examples/experiments/README.md @@ -20,9 +20,9 @@ permit the peer hosts; metadata does not grant network access. `dbl up` has only one executor and is not sufficient for this example. ```sh -make wasm SAMPLE_DIR=examples/experiments/peer +GOOS=wasip1 GOARCH=wasm go build -o examples/experiments/peer/debuglet.wasm ./examples/experiments/peer # Use your existing saved connection and login: -dbl executor list +dbl nodes cp examples/experiments/five.json experiment-definition.json # Replace executor-1 through executor-5 with actual IDs and the example # addresses with peer IPs/CIDRs allowed by your operator policy. @@ -35,7 +35,7 @@ The WASM paths in the definition are relative to your working directory. Set leave it empty to record the submitted digest. The runner uses the same saved connection and credentials as `dbl`; `-config` and `-dispatcher` select another profile. `-endpoint http://127.0.0.1:9000` explicitly selects a local development -dispatcher without reading credentials. Remote TEST use also needs +dispatcher; saved credentials are only used for that exact endpoint. Remote TEST use also needs `-allow-remote-test` and server authorization. No payment activation is performed. The receipt file must not already exist. It is written even when submission @@ -56,8 +56,11 @@ jq -r '.results[].output.entries[].output | @base64d' experiment-results.json acknowledgement is not proof of termination. A missing participant is bounded by the guest/dispatcher deadlines; a lost UDP datagram is bounded by each run's `timeout_ms`. This example fails rather than retrying packets or replacing members. -The guest uses a 45s readiness deadline; keep its policy budget longer (the sample -uses 60s). Ready metadata is limited to 4KiB per participant. +Ready waits for at most 30 seconds and is also bounded by the run lifetime and +caller deadline. Keep the policy budget longer (the sample uses 60s). Metadata +is limited to 4KiB per participant. The guest requires the optional +`debuglet_experiment_v1.ready` host extension. Declaring readiness is persistent; +cancelling the local wait does not withdraw it. Cancel the run to withdraw. The shared start is a requested time, not atomic distributed execution. Host clocks may differ, and these guest-reported timestamps are not a clock diff --git a/internal/acceptance/roles/experiment_integration_test.go b/internal/acceptance/roles/experiment_integration_test.go new file mode 100644 index 00000000..11fd1881 --- /dev/null +++ b/internal/acceptance/roles/experiment_integration_test.go @@ -0,0 +1,89 @@ +//go:build linux && roles_integration + +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package roles + +import ( + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/google/uuid" + dispatcherconfig "github.com/netsec-ethz/debuglet/internal/dispatcher/config" + executorconfig "github.com/netsec-ethz/debuglet/internal/executor/config" + "github.com/netsec-ethz/debuglet/internal/storagecheck" + "github.com/netsec-ethz/debuglet/pkg/client" +) + +// TestInstalledExperiment runs the same coordinated guest in five independent +// executor processes and checks the retained output of every admitted run. +func TestInstalledExperiment(t *testing.T) { + ctx, assets, work, launch := installedRendezvous(t) + wasm, err := os.ReadFile(os.Getenv("DEBUGLET_EXPERIMENT_WASM")) + if err != nil { t.Fatal(err) } + d := launch("dispatcher", assets.Dispatcher, storagecheck.Dispatcher, dispatcherconfig.DispatcherConfig{ + Admission: dispatcherconfig.DefaultAdmissionConfig(), Attribution: dispatcherconfig.DefaultAttributionConfig(), + Server: dispatcherconfig.ServerConfig{BindHost:"127.0.0.1", LocalDevelopment:true, Version:assets.Manifest.Version}, + TLS: dispatcherconfig.TLSConfig{Disable:true}, Sui:dispatcherconfig.SuiConfig{Disabled:true}, + Scheduler:dispatcherconfig.SchedulerConfig{ExecutorTimeout:10, SchedulerGranularityMs:100}, Output:dispatcherconfig.DefaultOutputConfig(), + Database:dispatcherconfig.DatabaseConfig{Path:filepath.Join(work,"dispatcher","state.sqlite")}, Logging:dispatcherconfig.LoggingConfig{LogLevel:"info",JSONLogs:true}, + }) + c := sdk(t,"http://"+d.HTTPAddr) + requests := make([]client.Request,5) + for i := range requests { + id, name := uuid.NewString(), fmt.Sprintf("executor-%d",i) + conn, err := net.ListenPacket("udp","127.0.0.1:0") + if err != nil { t.Fatal(err) } + port := conn.LocalAddr().(*net.UDPAddr).Port + conn.Close() + local := true + launch(name,assets.Executor,storagecheck.Executor,executorconfig.ExecutorConfig{ + Identity:executorconfig.IdentityConfig{ExecutorID:id,Version:assets.Manifest.Version}, + Dispatcher:executorconfig.DispatcherConfig{Addr:d.GRPCAddr,YamuxAddr:d.HTTPAddr},TLS:executorconfig.TLSConfig{Disable:true}, + Resources:executorconfig.ResourcesConfig{Capacity:1_000_000,MaxDebuglets:4},Tesla:executorconfig.TeslaConfig{Delay:1,ChainLength:300}, + Network:executorconfig.NetworkConfig{PublicHost:"127.0.0.1",PublicPorts:strconv.Itoa(port),PacketCounter:"fallback",DisableSCIONEnvironment:true,Policy:executorconfig.PolicyConfig{LocalTargets:&local}}, + Database:executorconfig.DatabaseConfig{Path:filepath.Join(work,name,"state.sqlite")},Logging:executorconfig.LoggingConfig{LogLevel:"info",JSONLogs:true},Pricing:executorconfig.PricingConfig{Currency:"TEST",PricePerBwS:1}, + }) + requests[i] = client.Request{OrderID:int64(i),ExecutorID:id,Wasm:wasm,Args:[]string{strconv.Itoa(i)},Policy:client.Policy{FloorBW:100_000,CeilBW:200_000,TimeoutMS:30_000,Addresses:[]string{"127.0.0.1"},ListenUDP:true}} + } + for { + nodes, err := c.Nodes(ctx) + if err != nil { t.Fatal(err) } + ready := 0 + for _, n := range nodes { if n.Ready { ready++ } } + if ready==len(requests) { break } + select { case <-ctx.Done(): t.Fatal(ctx.Err()); case <-time.After(50*time.Millisecond): } + } + batch, err := client.Prepare(requests) + if err != nil { t.Fatal(err) } + submission, err := c.SubmitTEST(ctx,batch) + if err != nil { t.Fatal(err) } + var sharedStart int64 + for _, id := range submission.IDs { + var output []byte + var after int64 + for { + page, err := c.Logs(ctx,id,client.LogOptions{After:after,Limit:100}) + if err != nil { t.Fatal(err) } + for _, entry := range page.Logs { output=append(output,entry.Output...) } + after=page.After + if page.Error!="" { t.Fatalf("participant %s: %s output=%s",id,page.Error,output) } + if len(output)>64<<10 { t.Fatal("excessive guest output") } + if page.State==client.StateExited && !page.HasMore { break } + select { case <-ctx.Done(): t.Fatalf("waiting %s: %v output=%s",id,ctx.Err(),output); case <-time.After(50*time.Millisecond): } + } + var result struct { ExperimentID string `json:"experiment_id"`; StartTimeNS int64 `json:"start_time_ns"`; Sent int `json:"sent"`; Received int `json:"received"` } + if err:=json.Unmarshal(output,&result);err!=nil { t.Fatalf("participant %s output=%s: %v",id,output,err) } + if result.ExperimentID!=submission.TransactionID || result.StartTimeNS==0 || result.Sent!=4 || result.Received!=4 { t.Fatalf("participant %s: %+v",id,result) } + if sharedStart!=0 && sharedStart!=result.StartTimeNS { t.Fatal("participants received different start times") } + sharedStart=result.StartTimeNS + t.Logf("run=%s group=%s start=%d sent=%d received=%d",id,result.ExperimentID,result.StartTimeNS,result.Sent,result.Received) + } +} diff --git a/internal/acceptance/roles/rendezvous_integration_test.go b/internal/acceptance/roles/rendezvous_integration_test.go index a12b7092..9cbbf426 100644 --- a/internal/acceptance/roles/rendezvous_integration_test.go +++ b/internal/acceptance/roles/rendezvous_integration_test.go @@ -29,9 +29,10 @@ import ( "github.com/pelletier/go-toml/v2" ) -// TestInstalledRendezvous runs the installed command and exact packaged guest -// across two real executor processes. Every listener and target is loopback. -func TestInstalledRendezvous(t *testing.T) { +type installedLaunch func(string, string, storagecheck.Role, any) readiness.Record + +func installedRendezvous(t *testing.T) (context.Context, demo.Assets, string, installedLaunch) { + t.Helper() root, evidence := os.Getenv("DEBUGLET_LOCAL_INSTALL_ROOT"), os.Getenv("DEBUGLET_ROLE_EVIDENCE_DIR") if !filepath.IsAbs(root) || !filepath.IsAbs(evidence) { t.Fatal("absolute installed and evidence directories required") @@ -119,6 +120,13 @@ func TestInstalledRendezvous(t *testing.T) { } } } + return ctx, assets, work, launch +} + +// TestInstalledRendezvous runs the installed command and exact packaged guest +// across two real executor processes. Every listener and target is loopback. +func TestInstalledRendezvous(t *testing.T) { + ctx, assets, work, launch := installedRendezvous(t) d := launch("dispatcher", assets.Dispatcher, storagecheck.Dispatcher, dispatcherconfig.DispatcherConfig{ Admission: dispatcherconfig.DefaultAdmissionConfig(), Attribution: dispatcherconfig.DefaultAttributionConfig(), diff --git a/scripts/ci-fmt.sh b/scripts/ci-fmt.sh index 86fce8b6..dc60ccdd 100644 --- a/scripts/ci-fmt.sh +++ b/scripts/ci-fmt.sh @@ -49,6 +49,7 @@ if [[ -s "$errors" ]]; then fi if [[ -s "$report" ]]; then + "$gofmt_bin" -d -- "${files[@]}" > .cache/ci/gofmt.diff echo "Unformatted tracked Go files ($(wc -l <"$report" | tr -d ' ')), also listed in $report:" >&2 sed 's/^/ /' "$report" >&2 echo 'Format them with the supported toolchain, then commit the result:' >&2 diff --git a/scripts/ci-roles.sh b/scripts/ci-roles.sh index 1b298510..1a67a705 100644 --- a/scripts/ci-roles.sh +++ b/scripts/ci-roles.sh @@ -10,6 +10,8 @@ mkdir -p .cache/ci evidence="$(realpath .cache/ci)/role-evidence" mkdir -p -m 0700 "$evidence" export DEBUGLET_ROLE_EVIDENCE_DIR="$evidence" +GOOS=wasip1 GOARCH=wasm "${GO:-go}" build -mod=readonly -trimpath -o "$evidence/experiment-peer.wasm" ./examples/experiments/peer +export DEBUGLET_EXPERIMENT_WASM="$evidence/experiment-peer.wasm" # Exercise the actual previous release, not a rebuild with a similar version. previous="$(realpath .cache/ci)/previous-release" mkdir -p "$previous" @@ -27,7 +29,7 @@ sh "$previous/install.sh" --archive "$previous/debuglet-v0.2.0-linux-amd64.tar.g > .cache/ci/role-previous-install.log export DEBUGLET_PREVIOUS_INSTALL_ROOT="$previous/installed/lib/debuglet/v0.2.0" "${GO:-go}" test -mod=readonly -json -tags=roles_integration -count=1 -timeout=5m \ - ./internal/acceptance/roles -run '^TestInstalled(Roles|BackupRestore|Recovery|Output|ReleasedCompatibility|ReleasedUpgrade|Rendezvous)$' | tee .cache/ci/role-tests.json + ./internal/acceptance/roles -run '^TestInstalled(Roles|BackupRestore|Recovery|Output|ReleasedCompatibility|ReleasedUpgrade|Rendezvous|Experiment)$' | tee .cache/ci/role-tests.json python3 tools/check-evidence.py --test ./internal/acceptance/roles:TestInstalledRoles \ --test ./internal/acceptance/roles:TestInstalledBackupRestore \ --test ./internal/acceptance/roles:TestInstalledRecovery \ @@ -35,6 +37,7 @@ python3 tools/check-evidence.py --test ./internal/acceptance/roles:TestInstalled --test ./internal/acceptance/roles:TestInstalledReleasedCompatibility \ --test ./internal/acceptance/roles:TestInstalledReleasedUpgrade \ --test ./internal/acceptance/roles:TestInstalledRendezvous \ + --test ./internal/acceptance/roles:TestInstalledExperiment \ --test ./internal/acceptance/roles:TestInstalledRecovery/graceful \ --test ./internal/acceptance/roles:TestInstalledRecovery/terminated \ --evidence .cache/ci/role-tests.json From 80920232cb1ab947ff9ad6072eee136a0df9c4a3 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:54:13 +0200 Subject: [PATCH 06/17] Exercise bounded worker memory transfers for experiment responses --- internal/executor/debuglet/wasm/experiment_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/executor/debuglet/wasm/experiment_test.go b/internal/executor/debuglet/wasm/experiment_test.go index 109c7013..a83f65b2 100644 --- a/internal/executor/debuglet/wasm/experiment_test.go +++ b/internal/executor/debuglet/wasm/experiment_test.go @@ -34,13 +34,23 @@ func TestExperimentHostValidatesBeforePublication(t *testing.T) { if calls != 0 { t.Fatalf("invalid input published %d times",calls) } } +type boundedExperimentMemory struct { Memory } +func (m boundedExperimentMemory) Read(ptr,size uint32)([]byte,bool) { + if size > MAX_SLICE_LENGTH { return nil,false } + return m.Memory.Read(ptr,size) +} +func (m boundedExperimentMemory) Write(ptr uint32, data []byte)bool { + if len(data)>MAX_SLICE_LENGTH { return false } + return m.Memory.Write(ptr,data) +} + func TestExperimentHostChunkedResponseAndPrivateErrors(t *testing.T) { mod := newGuestModule(t) result := wire.Experiment{ID:"batch",StartTimeNS:10,Participants:[]wire.ExperimentParticipant{ {ID:"one",Metadata:make([]byte,4096)},{ID:"two",Metadata:make([]byte,4096)}, }} env := &WasmEnv{Experiment:experimentControl(func(context.Context,[]byte)(wire.Experiment,error){return result,nil})} - got:=hostExperimentReady(env)(context.Background(),mod.Memory(),0,0,0,32000,time.Now().Add(time.Second).UnixNano()) + got:=hostExperimentReady(env)(context.Background(),boundedExperimentMemory{mod.Memory()},0,0,0,32000,time.Now().Add(time.Second).UnixNano()) if got <= MAX_SLICE_LENGTH { t.Fatalf("response not chunked: %d",got) } data,_:=mod.Memory().Read(0,uint32(got)); var decoded wire.Experiment if err:=json.Unmarshal(data,&decoded); err != nil || len(decoded.Participants)!=2 || len(decoded.Participants[1].Metadata)!=4096 { t.Fatalf("invalid result %v",err) } From beaacd4e2620a02d058dbe8c547bec3ccc2af6db Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:55:54 +0200 Subject: [PATCH 07/17] Verify readiness persistence through a reopened database --- internal/dispatcher/experiments_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/dispatcher/experiments_test.go b/internal/dispatcher/experiments_test.go index 74c3e74c..3288c98e 100644 --- a/internal/dispatcher/experiments_test.go +++ b/internal/dispatcher/experiments_test.go @@ -12,6 +12,7 @@ import ( "github.com/netsec-ethz/debuglet/internal/dispatcher/database" "github.com/netsec-ethz/debuglet/internal/dispatcher/models" "github.com/netsec-ethz/debuglet/internal/dispatcher/transport/rpc" + "github.com/netsec-ethz/debuglet/internal/sqlitedb" "github.com/netsec-ethz/debuglet/pkg/wire" pb "github.com/netsec-ethz/debuglet/protocol" "google.golang.org/grpc/codes" @@ -63,10 +64,15 @@ func TestExperimentFiveMemberReleaseAndRestart(t *testing.T) { if _, err := experimentReady(t, f, ids[0], []byte("changed")); status.Code(err) != codes.AlreadyExists { t.Fatalf("metadata replacement: %v", err) } // A new dispatcher reconstructs no live original sessions. Durable release // survives, but a replacement control identity cannot replay the old run. - restarted, err := New(f.d.logger, f.db, "restart", time.Minute, time.Minute, f.ph) + var path string + if err := f.db.QueryRow("SELECT file FROM pragma_database_list WHERE name = 'main'").Scan(&path); err != nil { t.Fatal(err) } + reopened, err := sqlitedb.Open(path) + if err != nil { t.Fatal(err) } + defer reopened.Close() + restarted, err := New(f.d.logger, reopened, "restart", time.Minute, time.Minute, f.ph) if err != nil { t.Fatal(err) } defer restarted.Close() - barrier, err := database.New(f.db).GetExperimentBarrier(f.ctx, group) + barrier, err := database.New(reopened).GetExperimentBarrier(f.ctx, group) if err != nil || barrier.StartTimeNs != first.StartTimeNs { t.Fatalf("lost persisted release: %+v %v", barrier, err) } owner, err := rpc.NewSessionOwner(tgExecutorID, effectTestBinding(t), time.Minute) if err != nil { t.Fatal(err) } From 276a2b873fa1783ae201a76242c172eb976a1db0 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:58:57 +0200 Subject: [PATCH 08/17] Document experiment schema and preserve formatting diagnostics --- docs/operations/configuration.md | 2 +- examples/experiments/README.md | 2 +- scripts/ci-fmt.sh | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/operations/configuration.md b/docs/operations/configuration.md index 8ab33733..8f18fb24 100644 --- a/docs/operations/configuration.md +++ b/docs/operations/configuration.md @@ -203,7 +203,7 @@ OAuth, external TLS and SCION state need the deployment's complete backup plan; a database snapshot alone does not include every required credential or config. Never start original and restored copies with the same identity simultaneously. -Dispatcher schema 27 and executor schema 8 are the current schema boundaries. +Dispatcher schema 28 and executor schema 8 are the current schema boundaries. Recognized older databases require the explicit upgrade below. Dispatcher schemas below 3 and executor schemas below 2 lose recorded `debuglets` and `debuglet_logs` on upgrade and require explicit acceptance. Preserved paid rows diff --git a/examples/experiments/README.md b/examples/experiments/README.md index 0cd6ce1b..674b90d4 100644 --- a/examples/experiments/README.md +++ b/examples/experiments/README.md @@ -35,7 +35,7 @@ The WASM paths in the definition are relative to your working directory. Set leave it empty to record the submitted digest. The runner uses the same saved connection and credentials as `dbl`; `-config` and `-dispatcher` select another profile. `-endpoint http://127.0.0.1:9000` explicitly selects a local development -dispatcher; saved credentials are only used for that exact endpoint. Remote TEST use also needs +dispatcher without loading a saved credential. Remote TEST use also needs `-allow-remote-test` and server authorization. No payment activation is performed. The receipt file must not already exist. It is written even when submission diff --git a/scripts/ci-fmt.sh b/scripts/ci-fmt.sh index dc60ccdd..39abef87 100644 --- a/scripts/ci-fmt.sh +++ b/scripts/ci-fmt.sh @@ -49,7 +49,9 @@ if [[ -s "$errors" ]]; then fi if [[ -s "$report" ]]; then - "$gofmt_bin" -d -- "${files[@]}" > .cache/ci/gofmt.diff + if ! "$gofmt_bin" -d -- "${files[@]}" > .cache/ci/gofmt.diff; then + echo "Could not produce a formatting patch." >&2 + fi echo "Unformatted tracked Go files ($(wc -l <"$report" | tr -d ' ')), also listed in $report:" >&2 sed 's/^/ /' "$report" >&2 echo 'Format them with the supported toolchain, then commit the result:' >&2 From caa170645c72dd79f15f37483bee8c1f2bdaacd2 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 09:59:52 +0200 Subject: [PATCH 09/17] Regenerate experiment database bindings --- .../dispatcher/database/experiments.sql.go | 153 ++++++++++++++++++ internal/dispatcher/database/models.go | 12 ++ 2 files changed, 165 insertions(+) create mode 100644 internal/dispatcher/database/experiments.sql.go diff --git a/internal/dispatcher/database/experiments.sql.go b/internal/dispatcher/database/experiments.sql.go new file mode 100644 index 00000000..2aca434c --- /dev/null +++ b/internal/dispatcher/database/experiments.sql.go @@ -0,0 +1,153 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: experiments.sql + +package database + +import ( + "context" + "database/sql" +) + +const countExperimentOrders = `-- name: CountExperimentOrders :one +SELECT count(*) FROM debuglet_order WHERE transaction_id = ? +` + +func (q *Queries) CountExperimentOrders(ctx context.Context, transactionID string) (int64, error) { + row := q.db.QueryRowContext(ctx, countExperimentOrders, transactionID) + var count int64 + err := row.Scan(&count) + return count, err +} + +const createExperimentBarrier = `-- name: CreateExperimentBarrier :exec +INSERT INTO experiment_barriers(transaction_id, deadline_ns) VALUES (?, ?) +ON CONFLICT(transaction_id) DO NOTHING +` + +type CreateExperimentBarrierParams struct { + TransactionID string + DeadlineNs int64 +} + +func (q *Queries) CreateExperimentBarrier(ctx context.Context, arg CreateExperimentBarrierParams) error { + _, err := q.db.ExecContext(ctx, createExperimentBarrier, arg.TransactionID, arg.DeadlineNs) + return err +} + +const getExperimentBarrier = `-- name: GetExperimentBarrier :one +SELECT transaction_id, deadline_ns, start_time_ns FROM experiment_barriers WHERE transaction_id = ? +` + +func (q *Queries) GetExperimentBarrier(ctx context.Context, transactionID string) (ExperimentBarrier, error) { + row := q.db.QueryRowContext(ctx, getExperimentBarrier, transactionID) + var i ExperimentBarrier + err := row.Scan(&i.TransactionID, &i.DeadlineNs, &i.StartTimeNs) + return i, err +} + +const getExperimentMembers = `-- name: GetExperimentMembers :many +SELECT d.id, d.uuid, d.start_time, d.end_time, d.usage, d.ceil_bw, d.executor_id, d.addresses, d.state, d.error, d.transaction_id, d.order_id, d.dispatcher_incarnation, d.session_id, r.metadata, r.ready_at_ns, + EXISTS(SELECT 1 FROM debuglet_cancellations c WHERE c.debuglet_id = d.id) AS cancelled, + EXISTS(SELECT 1 FROM allocation_reclamations a WHERE a.debuglet_id = d.id) AS reclaimed +FROM debuglet_order o JOIN debuglets d ON d.id = o.debuglet_id + AND d.transaction_id = o.transaction_id AND d.order_id = o.order_id +LEFT JOIN experiment_readiness r ON r.debuglet_id = d.id +WHERE o.transaction_id = ? ORDER BY o.order_id LIMIT 129 +` + +type GetExperimentMembersRow struct { + Debuglet Debuglet + Metadata []byte + ReadyAtNs sql.NullInt64 + Cancelled bool + Reclaimed bool +} + +func (q *Queries) GetExperimentMembers(ctx context.Context, transactionID string) ([]GetExperimentMembersRow, error) { + rows, err := q.db.QueryContext(ctx, getExperimentMembers, transactionID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetExperimentMembersRow + for rows.Next() { + var i GetExperimentMembersRow + if err := rows.Scan( + &i.Debuglet.ID, + &i.Debuglet.Uuid, + &i.Debuglet.StartTime, + &i.Debuglet.EndTime, + &i.Debuglet.Usage, + &i.Debuglet.CeilBw, + &i.Debuglet.ExecutorID, + &i.Debuglet.Addresses, + &i.Debuglet.State, + &i.Debuglet.Error, + &i.Debuglet.TransactionID, + &i.Debuglet.OrderID, + &i.Debuglet.DispatcherIncarnation, + &i.Debuglet.SessionID, + &i.Metadata, + &i.ReadyAtNs, + &i.Cancelled, + &i.Reclaimed, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const pruneExperimentMetadata = `-- name: PruneExperimentMetadata :exec +DELETE FROM experiment_readiness WHERE debuglet_id IN ( + SELECT r.debuglet_id FROM experiment_readiness r + JOIN debuglets d ON d.id = r.debuglet_id + JOIN experiment_barriers b ON b.transaction_id = d.transaction_id + WHERE b.deadline_ns <= ? LIMIT 512 +) +` + +func (q *Queries) PruneExperimentMetadata(ctx context.Context, deadlineNs int64) error { + _, err := q.db.ExecContext(ctx, pruneExperimentMetadata, deadlineNs) + return err +} + +const recordExperimentReady = `-- name: RecordExperimentReady :exec +INSERT INTO experiment_readiness(debuglet_id, metadata, ready_at_ns) VALUES (?, ?, ?) +ON CONFLICT(debuglet_id) DO NOTHING +` + +type RecordExperimentReadyParams struct { + DebugletID int64 + Metadata []byte + ReadyAtNs int64 +} + +func (q *Queries) RecordExperimentReady(ctx context.Context, arg RecordExperimentReadyParams) error { + _, err := q.db.ExecContext(ctx, recordExperimentReady, arg.DebugletID, arg.Metadata, arg.ReadyAtNs) + return err +} + +const releaseExperiment = `-- name: ReleaseExperiment :exec +UPDATE experiment_barriers SET start_time_ns = ? +WHERE transaction_id = ? AND start_time_ns = 0 +` + +type ReleaseExperimentParams struct { + StartTimeNs int64 + TransactionID string +} + +func (q *Queries) ReleaseExperiment(ctx context.Context, arg ReleaseExperimentParams) error { + _, err := q.db.ExecContext(ctx, releaseExperiment, arg.StartTimeNs, arg.TransactionID) + return err +} diff --git a/internal/dispatcher/database/models.go b/internal/dispatcher/database/models.go index 11c60ab7..d0018874 100644 --- a/internal/dispatcher/database/models.go +++ b/internal/dispatcher/database/models.go @@ -236,6 +236,18 @@ type ExecutorEnrollmentToken struct { ExpiresAt models.UTCTime } +type ExperimentBarrier struct { + TransactionID string + DeadlineNs int64 + StartTimeNs int64 +} + +type ExperimentReadiness struct { + DebugletID int64 + Metadata []byte + ReadyAtNs int64 +} + type MeasurementExecution struct { DebugletID int64 StartedObservedNs sql.NullInt64 From 642b6602f76a26ec54c6c04e4f76b6485944a681 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:06:18 +0200 Subject: [PATCH 10/17] Format coordinated experiment implementation and update recovery fixture --- examples/experiments/peer/main.go | 88 +++-- examples/experiments/run/main.go | 86 +++-- .../roles/experiment_integration_test.go | 121 ++++--- internal/dispatcher/experiments.go | 215 +++++++----- internal/dispatcher/experiments_test.go | 307 +++++++++++------- .../transport/rpc/experiment_test.go | 74 +++-- internal/dispatcher/transport/rpc/server.go | 8 +- internal/executor/debuglet/debuglet.go | 16 +- internal/executor/debuglet/wasm/experiment.go | 110 ++++--- .../executor/debuglet/wasm/experiment_test.go | 109 ++++--- internal/executor/experiment.go | 70 ++-- internal/executor/experiment_test.go | 67 ++-- .../executor/recovery_integration_test.go | 3 + .../executor/transport/rpc/bound_client.go | 16 +- internal/executor/transport/rpc/lease_test.go | 16 +- internal/storagecheck/init_test.go | 4 +- internal/storagecheck/policy.go | 4 +- pkg/client/experiment.go | 50 ++- pkg/client/experiment_test.go | 40 ++- pkg/debuglet/experiment.go | 111 ++++--- pkg/debuglet/experiment_test.go | 103 +++--- pkg/debuglet/experiment_wasip1.go | 8 +- pkg/debuglet/hostengine_test.go | 16 +- pkg/debuglet/testdata/experiment/main.go | 34 +- 24 files changed, 1063 insertions(+), 613 deletions(-) diff --git a/examples/experiments/peer/main.go b/examples/experiments/peer/main.go index 97c3447d..498b27c6 100644 --- a/examples/experiments/peer/main.go +++ b/examples/experiments/peer/main.go @@ -15,65 +15,105 @@ import ( "github.com/netsec-ethz/debuglet/pkg/debuglet" ) -type endpoint struct { Address string `json:"endpoint"` } +type endpoint struct { + Address string `json:"endpoint"` +} type packet struct { ExperimentID string `json:"experiment_id"` - RunID string `json:"run_id"` - SentAtNS int64 `json:"sent_at_ns"` + RunID string `json:"run_id"` + SentAtNS int64 `json:"sent_at_ns"` } func main() { - if err := run(); err != nil { fmt.Fprintln(os.Stderr, err); os.Exit(1) } + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } } func run() error { address, err := debuglet.ListenUDPAddr() - if err != nil { return err } + if err != nil { + return err + } metadata, err := json.Marshal(endpoint{Address: address}) - if err != nil { return err } + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() readyAt := time.Now().UnixNano() experiment, err := debuglet.Ready(ctx, metadata) - if err != nil { return err } - if len(experiment.Participants) != 5 { return errors.New("peer example requires five participants") } + if err != nil { + return err + } + if len(experiment.Participants) != 5 { + return errors.New("peer example requires five participants") + } seen := map[string]bool{} peers := map[string]string{} self := "" for _, member := range experiment.Participants { - if seen[member.ExecutorID] { return errors.New("peer example requires five distinct executors") } + if seen[member.ExecutorID] { + return errors.New("peer example requires five distinct executors") + } seen[member.ExecutorID] = true var listener endpoint - if err := json.Unmarshal(member.Metadata, &listener); err != nil { return err } - if listener.Address == "" { return errors.New("participant has no endpoint") } - if listener.Address == address { self = member.ID } else { peers[member.ID] = listener.Address } + if err := json.Unmarshal(member.Metadata, &listener); err != nil { + return err + } + if listener.Address == "" { + return errors.New("participant has no endpoint") + } + if listener.Address == address { + self = member.ID + } else { + peers[member.ID] = listener.Address + } + } + if self == "" || len(peers) != 4 { + return errors.New("listener endpoints must be distinct") + } + if err := debuglet.WaitStart(ctx, experiment); err != nil { + return err } - if self == "" || len(peers) != 4 { return errors.New("listener endpoints must be distinct") } - if err := debuglet.WaitStart(ctx, experiment); err != nil { return err } startedAt := time.Now().UnixNano() packets := []map[string]any{} for id, destination := range peers { conn, err := debuglet.ConnectUDP(destination) - if err != nil { return err } + if err != nil { + return err + } sentAt := time.Now().UnixNano() - data, err := json.Marshal(packet{ExperimentID:experiment.ID, RunID:self, SentAtNS:sentAt}) - if err != nil { conn.Close(); return err } + data, err := json.Marshal(packet{ExperimentID: experiment.ID, RunID: self, SentAtNS: sentAt}) + if err != nil { + conn.Close() + return err + } err = conn.Write(data) conn.Close() - if err != nil { return err } - packets = append(packets, map[string]any{"direction":"sent", "peer_run_id":id, "endpoint":destination, "sent_at_ns":sentAt}) + if err != nil { + return err + } + packets = append(packets, map[string]any{"direction": "sent", "peer_run_id": id, "endpoint": destination, "sent_at_ns": sentAt}) } buffer := make([]byte, 1024) received := map[string]bool{} for len(received) < len(peers) { n, from, err := debuglet.ReadFromUDP(buffer) - if err != nil { return err } + if err != nil { + return err + } receivedAt := time.Now().UnixNano() var message packet - if err := json.Unmarshal(buffer[:n], &message); err != nil { return err } - if message.ExperimentID != experiment.ID || peers[message.RunID] == "" || received[message.RunID] { return errors.New("unexpected peer datagram") } + if err := json.Unmarshal(buffer[:n], &message); err != nil { + return err + } + if message.ExperimentID != experiment.ID || peers[message.RunID] == "" || received[message.RunID] { + return errors.New("unexpected peer datagram") + } received[message.RunID] = true - packets = append(packets, map[string]any{"direction":"received", "peer_run_id":message.RunID, "endpoint":from, "sent_at_ns":message.SentAtNS, "received_at_ns":receivedAt}) + packets = append(packets, map[string]any{"direction": "received", "peer_run_id": message.RunID, "endpoint": from, "sent_at_ns": message.SentAtNS, "received_at_ns": receivedAt}) } - return json.NewEncoder(os.Stdout).Encode(map[string]any{"experiment_id":experiment.ID, "run_id":self, "endpoint":address, "ready_at_ns":readyAt, "start_time_ns":experiment.StartTimeNS, "actual_start_ns":startedAt, "lateness_ns":startedAt-experiment.StartTimeNS, "sent":len(peers), "received":len(received), "finished_at_ns":time.Now().UnixNano(), "packets":packets}) + return json.NewEncoder(os.Stdout).Encode(map[string]any{"experiment_id": experiment.ID, "run_id": self, "endpoint": address, "ready_at_ns": readyAt, "start_time_ns": experiment.StartTimeNS, "actual_start_ns": startedAt, "lateness_ns": startedAt - experiment.StartTimeNS, "sent": len(peers), "received": len(received), "finished_at_ns": time.Now().UnixNano(), "packets": packets}) } diff --git a/examples/experiments/run/main.go b/examples/experiments/run/main.go index 2c5f439d..53e0b140 100644 --- a/examples/experiments/run/main.go +++ b/examples/experiments/run/main.go @@ -20,7 +20,10 @@ import ( ) func main() { - if err := run(); err != nil { fmt.Fprintln(os.Stderr, "experiment:", err); os.Exit(1) } + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "experiment:", err) + os.Exit(1) + } } func run() (failure error) { @@ -33,19 +36,32 @@ func run() (failure error) { remote := flag.Bool("allow-remote-test", false, "permit remote TEST submission") timeout := flag.Duration("timeout", 90*time.Second, "whole operation deadline") flag.Parse() - if *action != "submit" && *action != "results" && *action != "cancel" { return errors.New("unknown action") } - if *timeout <= 0 { return errors.New("timeout must be positive") } + if *action != "submit" && *action != "results" && *action != "cancel" { + return errors.New("unknown action") + } + if *timeout <= 0 { + return errors.New("timeout must be positive") + } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() ctx, cancel := context.WithTimeout(ctx, *timeout) defer cancel() - profile := connections.Profile{Endpoint:*endpoint} + profile := connections.Profile{Endpoint: *endpoint} var err error - if *endpoint == "" { profile, err = connections.Resolve(*config, *dispatcher); if err != nil { return err } } + if *endpoint == "" { + profile, err = connections.Resolve(*config, *dispatcher) + if err != nil { + return err + } + } credential, err := connections.CredentialFor(ctx, *config, profile.Name, profile.Endpoint) - if err != nil { return err } - c, err := client.New(profile.Endpoint, client.Options{Credential:credential.Token, AllowRemoteTEST:*remote}) - if err != nil { return err } + if err != nil { + return err + } + c, err := client.New(profile.Endpoint, client.Options{Credential: credential.Token, AllowRemoteTEST: *remote}) + if err != nil { + return err + } var receipt client.ExperimentSubmission defer func() { if *action == "submit" && ctx.Err() != nil { @@ -56,46 +72,72 @@ func run() (failure error) { }() if *action == "submit" { var definition client.ExperimentDefinition - if err := readJSON(*manifest, &definition); err != nil { return err } + if err := readJSON(*manifest, &definition); err != nil { + return err + } file, err := os.OpenFile(*receiptPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) - if err != nil { return err } + if err != nil { + return err + } receipt, err = c.SubmitExperimentTEST(ctx, definition) writeErr := json.NewEncoder(file).Encode(receipt) closeErr := file.Close() - if err != nil || writeErr != nil || closeErr != nil { return errors.Join(err, writeErr, closeErr) } + if err != nil || writeErr != nil || closeErr != nil { + return errors.Join(err, writeErr, closeErr) + } fmt.Fprintf(os.Stderr, "Experiment %s; receipt %s\n", receipt.ExperimentID, *receiptPath) - } else if err := readJSON(*receiptPath, &receipt); err != nil { return err } - if receipt.ExperimentID == "" || len(receipt.Participants) == 0 { return errors.New("receipt has no admitted experiment") } - if *action == "cancel" { return c.CancelExperiment(ctx, receipt) } + } else if err := readJSON(*receiptPath, &receipt); err != nil { + return err + } + if receipt.ExperimentID == "" || len(receipt.Participants) == 0 { + return errors.New("receipt has no admitted experiment") + } + if *action == "cancel" { + return c.CancelExperiment(ctx, receipt) + } for { group, err := c.ExportExperiment(ctx, receipt) - if err != nil { return err } + if err != nil { + return err + } complete := true var failures []error for _, result := range group.Results { - if result.Outcome.State != client.StateExited || (result.Output.Status.State != "complete" && result.Output.Status.State != "truncated") { complete = false } - if result.Outcome.Error != "" { failures = append(failures, fmt.Errorf("run %s: %s", result.RunID, result.Outcome.Error)) } + if result.Outcome.State != client.StateExited || (result.Output.Status.State != "complete" && result.Output.Status.State != "truncated") { + complete = false + } + if result.Outcome.Error != "" { + failures = append(failures, fmt.Errorf("run %s: %s", result.RunID, result.Outcome.Error)) + } } if complete || *action == "results" { - if err := json.NewEncoder(os.Stdout).Encode(group); err != nil { return err } + if err := json.NewEncoder(os.Stdout).Encode(group); err != nil { + return err + } return errors.Join(failures...) } select { case <-ctx.Done(): return ctx.Err() - case <-time.After(500*time.Millisecond): + case <-time.After(500 * time.Millisecond): } } } func readJSON(path string, destination any) error { file, err := os.Open(path) - if err != nil { return err } + if err != nil { + return err + } defer file.Close() decoder := json.NewDecoder(io.LimitReader(file, 1<<20)) decoder.DisallowUnknownFields() - if err := decoder.Decode(destination); err != nil { return err } + if err := decoder.Decode(destination); err != nil { + return err + } var extra any - if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { return errors.New("expected one JSON document") } + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return errors.New("expected one JSON document") + } return nil } diff --git a/internal/acceptance/roles/experiment_integration_test.go b/internal/acceptance/roles/experiment_integration_test.go index 11fd1881..ae2b1551 100644 --- a/internal/acceptance/roles/experiment_integration_test.go +++ b/internal/acceptance/roles/experiment_integration_test.go @@ -27,63 +27,108 @@ import ( func TestInstalledExperiment(t *testing.T) { ctx, assets, work, launch := installedRendezvous(t) wasm, err := os.ReadFile(os.Getenv("DEBUGLET_EXPERIMENT_WASM")) - if err != nil { t.Fatal(err) } + if err != nil { + t.Fatal(err) + } d := launch("dispatcher", assets.Dispatcher, storagecheck.Dispatcher, dispatcherconfig.DispatcherConfig{ Admission: dispatcherconfig.DefaultAdmissionConfig(), Attribution: dispatcherconfig.DefaultAttributionConfig(), - Server: dispatcherconfig.ServerConfig{BindHost:"127.0.0.1", LocalDevelopment:true, Version:assets.Manifest.Version}, - TLS: dispatcherconfig.TLSConfig{Disable:true}, Sui:dispatcherconfig.SuiConfig{Disabled:true}, - Scheduler:dispatcherconfig.SchedulerConfig{ExecutorTimeout:10, SchedulerGranularityMs:100}, Output:dispatcherconfig.DefaultOutputConfig(), - Database:dispatcherconfig.DatabaseConfig{Path:filepath.Join(work,"dispatcher","state.sqlite")}, Logging:dispatcherconfig.LoggingConfig{LogLevel:"info",JSONLogs:true}, + Server: dispatcherconfig.ServerConfig{BindHost: "127.0.0.1", LocalDevelopment: true, Version: assets.Manifest.Version}, + TLS: dispatcherconfig.TLSConfig{Disable: true}, Sui: dispatcherconfig.SuiConfig{Disabled: true}, + Scheduler: dispatcherconfig.SchedulerConfig{ExecutorTimeout: 10, SchedulerGranularityMs: 100}, Output: dispatcherconfig.DefaultOutputConfig(), + Database: dispatcherconfig.DatabaseConfig{Path: filepath.Join(work, "dispatcher", "state.sqlite")}, Logging: dispatcherconfig.LoggingConfig{LogLevel: "info", JSONLogs: true}, }) - c := sdk(t,"http://"+d.HTTPAddr) - requests := make([]client.Request,5) + c := sdk(t, "http://"+d.HTTPAddr) + requests := make([]client.Request, 5) for i := range requests { - id, name := uuid.NewString(), fmt.Sprintf("executor-%d",i) - conn, err := net.ListenPacket("udp","127.0.0.1:0") - if err != nil { t.Fatal(err) } + id, name := uuid.NewString(), fmt.Sprintf("executor-%d", i) + conn, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } port := conn.LocalAddr().(*net.UDPAddr).Port conn.Close() local := true - launch(name,assets.Executor,storagecheck.Executor,executorconfig.ExecutorConfig{ - Identity:executorconfig.IdentityConfig{ExecutorID:id,Version:assets.Manifest.Version}, - Dispatcher:executorconfig.DispatcherConfig{Addr:d.GRPCAddr,YamuxAddr:d.HTTPAddr},TLS:executorconfig.TLSConfig{Disable:true}, - Resources:executorconfig.ResourcesConfig{Capacity:1_000_000,MaxDebuglets:4},Tesla:executorconfig.TeslaConfig{Delay:1,ChainLength:300}, - Network:executorconfig.NetworkConfig{PublicHost:"127.0.0.1",PublicPorts:strconv.Itoa(port),PacketCounter:"fallback",DisableSCIONEnvironment:true,Policy:executorconfig.PolicyConfig{LocalTargets:&local}}, - Database:executorconfig.DatabaseConfig{Path:filepath.Join(work,name,"state.sqlite")},Logging:executorconfig.LoggingConfig{LogLevel:"info",JSONLogs:true},Pricing:executorconfig.PricingConfig{Currency:"TEST",PricePerBwS:1}, + launch(name, assets.Executor, storagecheck.Executor, executorconfig.ExecutorConfig{ + Identity: executorconfig.IdentityConfig{ExecutorID: id, Version: assets.Manifest.Version}, + Dispatcher: executorconfig.DispatcherConfig{Addr: d.GRPCAddr, YamuxAddr: d.HTTPAddr}, TLS: executorconfig.TLSConfig{Disable: true}, + Resources: executorconfig.ResourcesConfig{Capacity: 1_000_000, MaxDebuglets: 4}, Tesla: executorconfig.TeslaConfig{Delay: 1, ChainLength: 300}, + Network: executorconfig.NetworkConfig{PublicHost: "127.0.0.1", PublicPorts: strconv.Itoa(port), PacketCounter: "fallback", DisableSCIONEnvironment: true, Policy: executorconfig.PolicyConfig{LocalTargets: &local}}, + Database: executorconfig.DatabaseConfig{Path: filepath.Join(work, name, "state.sqlite")}, Logging: executorconfig.LoggingConfig{LogLevel: "info", JSONLogs: true}, Pricing: executorconfig.PricingConfig{Currency: "TEST", PricePerBwS: 1}, }) - requests[i] = client.Request{OrderID:int64(i),ExecutorID:id,Wasm:wasm,Args:[]string{strconv.Itoa(i)},Policy:client.Policy{FloorBW:100_000,CeilBW:200_000,TimeoutMS:30_000,Addresses:[]string{"127.0.0.1"},ListenUDP:true}} + requests[i] = client.Request{OrderID: int64(i), ExecutorID: id, Wasm: wasm, Args: []string{strconv.Itoa(i)}, Policy: client.Policy{FloorBW: 100_000, CeilBW: 200_000, TimeoutMS: 30_000, Addresses: []string{"127.0.0.1"}, ListenUDP: true}} } for { nodes, err := c.Nodes(ctx) - if err != nil { t.Fatal(err) } + if err != nil { + t.Fatal(err) + } ready := 0 - for _, n := range nodes { if n.Ready { ready++ } } - if ready==len(requests) { break } - select { case <-ctx.Done(): t.Fatal(ctx.Err()); case <-time.After(50*time.Millisecond): } + for _, n := range nodes { + if n.Ready { + ready++ + } + } + if ready == len(requests) { + break + } + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case <-time.After(50 * time.Millisecond): + } } batch, err := client.Prepare(requests) - if err != nil { t.Fatal(err) } - submission, err := c.SubmitTEST(ctx,batch) - if err != nil { t.Fatal(err) } + if err != nil { + t.Fatal(err) + } + submission, err := c.SubmitTEST(ctx, batch) + if err != nil { + t.Fatal(err) + } var sharedStart int64 for _, id := range submission.IDs { var output []byte var after int64 for { - page, err := c.Logs(ctx,id,client.LogOptions{After:after,Limit:100}) - if err != nil { t.Fatal(err) } - for _, entry := range page.Logs { output=append(output,entry.Output...) } - after=page.After - if page.Error!="" { t.Fatalf("participant %s: %s output=%s",id,page.Error,output) } - if len(output)>64<<10 { t.Fatal("excessive guest output") } - if page.State==client.StateExited && !page.HasMore { break } - select { case <-ctx.Done(): t.Fatalf("waiting %s: %v output=%s",id,ctx.Err(),output); case <-time.After(50*time.Millisecond): } + page, err := c.Logs(ctx, id, client.LogOptions{After: after, Limit: 100}) + if err != nil { + t.Fatal(err) + } + for _, entry := range page.Logs { + output = append(output, entry.Output...) + } + after = page.After + if page.Error != "" { + t.Fatalf("participant %s: %s output=%s", id, page.Error, output) + } + if len(output) > 64<<10 { + t.Fatal("excessive guest output") + } + if page.State == client.StateExited && !page.HasMore { + break + } + select { + case <-ctx.Done(): + t.Fatalf("waiting %s: %v output=%s", id, ctx.Err(), output) + case <-time.After(50 * time.Millisecond): + } + } + var result struct { + ExperimentID string `json:"experiment_id"` + StartTimeNS int64 `json:"start_time_ns"` + Sent int `json:"sent"` + Received int `json:"received"` + } + if err := json.Unmarshal(output, &result); err != nil { + t.Fatalf("participant %s output=%s: %v", id, output, err) + } + if result.ExperimentID != submission.TransactionID || result.StartTimeNS == 0 || result.Sent != 4 || result.Received != 4 { + t.Fatalf("participant %s: %+v", id, result) + } + if sharedStart != 0 && sharedStart != result.StartTimeNS { + t.Fatal("participants received different start times") } - var result struct { ExperimentID string `json:"experiment_id"`; StartTimeNS int64 `json:"start_time_ns"`; Sent int `json:"sent"`; Received int `json:"received"` } - if err:=json.Unmarshal(output,&result);err!=nil { t.Fatalf("participant %s output=%s: %v",id,output,err) } - if result.ExperimentID!=submission.TransactionID || result.StartTimeNS==0 || result.Sent!=4 || result.Received!=4 { t.Fatalf("participant %s: %+v",id,result) } - if sharedStart!=0 && sharedStart!=result.StartTimeNS { t.Fatal("participants received different start times") } - sharedStart=result.StartTimeNS - t.Logf("run=%s group=%s start=%d sent=%d received=%d",id,result.ExperimentID,result.StartTimeNS,result.Sent,result.Received) + sharedStart = result.StartTimeNS + t.Logf("run=%s group=%s start=%d sent=%d received=%d", id, result.ExperimentID, result.StartTimeNS, result.Sent, result.Received) } } diff --git a/internal/dispatcher/experiments.go b/internal/dispatcher/experiments.go index 25b3625b..ab95fff2 100644 --- a/internal/dispatcher/experiments.go +++ b/internal/dispatcher/experiments.go @@ -4,17 +4,17 @@ package dispatcher import ( - "bytes" - "context" - "time" + "bytes" + "context" + "time" - "github.com/netsec-ethz/debuglet/internal/dispatcher/database" - "github.com/netsec-ethz/debuglet/internal/dispatcher/models" - "github.com/netsec-ethz/debuglet/internal/dispatcher/transport/rpc" - "github.com/netsec-ethz/debuglet/pkg/wire" - pb "github.com/netsec-ethz/debuglet/protocol" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/netsec-ethz/debuglet/internal/dispatcher/database" + "github.com/netsec-ethz/debuglet/internal/dispatcher/models" + "github.com/netsec-ethz/debuglet/internal/dispatcher/transport/rpc" + "github.com/netsec-ethz/debuglet/pkg/wire" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const experimentLead = 2 * time.Second @@ -23,80 +23,123 @@ const experimentWait = 30 * time.Second // OnExperimentReady performs one bounded poll. SQLite serializes readiness, // release and cancellation; a committed release is immutable across restarts. func (d *Dispatcher) OnExperimentReady(ctx context.Context, mutation *rpc.Mutation, req *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { - if req.GetExecutorId() == "" { return nil, status.Error(codes.PermissionDenied, "executor identity required") } - owner, err := requireMutation(mutation, req.GetExecutorId()) - if err != nil { return nil, err } - if len(req.GetMetadata()) > wire.MaxExperimentMetadata { return nil, status.Error(codes.ResourceExhausted, "experiment metadata exceeds 4096 bytes") } - id, err := parseRunID(req.GetDebugletId()) - if err != nil { return nil, err } - // Classify ownership before reading any transaction or peer metadata. - run, err := d.ownedDebuglet(ctx, owner, id) - if err != nil { return nil, err } - d.mu.Lock() - defer d.mu.Unlock() - tx, err := d.db.BeginTx(ctx, nil) - if err != nil { return nil, err } - defer tx.Rollback() - q := database.New(tx) - members, err := q.GetExperimentMembers(ctx, run.TransactionID) - if err != nil { return nil, err } - orders, err := q.CountExperimentOrders(ctx, run.TransactionID) - if err != nil { return nil, err } - if len(members) == 0 || len(members) > wire.MaxExperimentParticipants || int64(len(members)) != orders { - return nil, status.Error(codes.FailedPrecondition, "experiment membership is not fully admitted") - } - now := d.now() - deadline := now.Add(experimentWait) - ready := 0 - found := false - for _, member := range members { - row := member.Debuglet - entry := d.executors[row.ExecutorID] - if d.closed || entry == nil || !entry.owner.Available() || entry.owner.Binding().Incarnation != row.DispatcherIncarnation || entry.owner.Binding().SessionID != row.SessionID { - return nil, status.Error(codes.FailedPrecondition, "experiment participant session is unavailable") - } - if row.State == models.RunStateExited || member.Cancelled || member.Reclaimed { - return nil, status.Error(codes.FailedPrecondition, "experiment participant is no longer active") - } - if end := row.EndTime.Time.Add(-experimentLead); end.Before(deadline) { deadline = end } - if member.ReadyAtNs.Valid { ready++ } - if row.ID == run.ID { - found = true - if row.State != models.RunStateStarted || now.Before(row.StartTime.Time) { return nil, status.Error(codes.FailedPrecondition, "participant has not started") } - if entry.owner != owner { return nil, status.Error(codes.PermissionDenied, "run does not belong to session") } - if member.ReadyAtNs.Valid && !bytes.Equal(member.Metadata, req.GetMetadata()) { - return nil, status.Error(codes.AlreadyExists, "readiness metadata is immutable") - } - if !member.ReadyAtNs.Valid { ready++ } - } - } - if !found { return nil, status.Error(codes.PermissionDenied, "run is not an experiment participant") } - if err := q.CreateExperimentBarrier(ctx, database.CreateExperimentBarrierParams{TransactionID: run.TransactionID, DeadlineNs: deadline.UnixNano()}); err != nil { return nil, err } - barrier, err := q.GetExperimentBarrier(ctx, run.TransactionID) - if err != nil { return nil, err } - if now.UnixNano() >= barrier.DeadlineNs || !now.Before(deadline) { - // Preserve the original deadline even when the first call already expired. - if err := tx.Commit(); err != nil { return nil, err } - return nil, status.Error(codes.DeadlineExceeded, "experiment readiness deadline expired") - } - metadata := req.GetMetadata() - if metadata == nil { metadata = []byte{} } - if err := q.RecordExperimentReady(ctx, database.RecordExperimentReadyParams{DebugletID: run.ID, Metadata: metadata, ReadyAtNs: now.UnixNano()}); err != nil { return nil, err } - start := barrier.StartTimeNs - if start == 0 && ready == len(members) { - start = now.Add(experimentLead).UnixNano() - if err := q.ReleaseExperiment(ctx, database.ReleaseExperimentParams{StartTimeNs: start, TransactionID: run.TransactionID}); err != nil { return nil, err } - } - out := &pb.ExperimentReadyResponse{ExperimentId: run.TransactionID, StartTimeNs: start} - // Pending callers learn no partial metadata and never wait inside a mutation. - if start != 0 { - for _, member := range members { - m := &pb.ExperimentParticipant{Id: member.Debuglet.Uuid.String(), ExecutorId: member.Debuglet.ExecutorID, Metadata: member.Metadata, ReadyAtNs: member.ReadyAtNs.Int64} - if member.Debuglet.ID == run.ID && !member.ReadyAtNs.Valid { m.Metadata, m.ReadyAtNs = metadata, now.UnixNano() } - out.Participants = append(out.Participants, m) - } - } - if err := tx.Commit(); err != nil { return nil, err } - return out, nil + if req.GetExecutorId() == "" { + return nil, status.Error(codes.PermissionDenied, "executor identity required") + } + owner, err := requireMutation(mutation, req.GetExecutorId()) + if err != nil { + return nil, err + } + if len(req.GetMetadata()) > wire.MaxExperimentMetadata { + return nil, status.Error(codes.ResourceExhausted, "experiment metadata exceeds 4096 bytes") + } + id, err := parseRunID(req.GetDebugletId()) + if err != nil { + return nil, err + } + // Classify ownership before reading any transaction or peer metadata. + run, err := d.ownedDebuglet(ctx, owner, id) + if err != nil { + return nil, err + } + d.mu.Lock() + defer d.mu.Unlock() + tx, err := d.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + q := database.New(tx) + members, err := q.GetExperimentMembers(ctx, run.TransactionID) + if err != nil { + return nil, err + } + orders, err := q.CountExperimentOrders(ctx, run.TransactionID) + if err != nil { + return nil, err + } + if len(members) == 0 || len(members) > wire.MaxExperimentParticipants || int64(len(members)) != orders { + return nil, status.Error(codes.FailedPrecondition, "experiment membership is not fully admitted") + } + now := d.now() + deadline := now.Add(experimentWait) + ready := 0 + found := false + for _, member := range members { + row := member.Debuglet + entry := d.executors[row.ExecutorID] + if d.closed || entry == nil || !entry.owner.Available() || entry.owner.Binding().Incarnation != row.DispatcherIncarnation || entry.owner.Binding().SessionID != row.SessionID { + return nil, status.Error(codes.FailedPrecondition, "experiment participant session is unavailable") + } + if row.State == models.RunStateExited || member.Cancelled || member.Reclaimed { + return nil, status.Error(codes.FailedPrecondition, "experiment participant is no longer active") + } + if end := row.EndTime.Time.Add(-experimentLead); end.Before(deadline) { + deadline = end + } + if member.ReadyAtNs.Valid { + ready++ + } + if row.ID == run.ID { + found = true + if row.State != models.RunStateStarted || now.Before(row.StartTime.Time) { + return nil, status.Error(codes.FailedPrecondition, "participant has not started") + } + if entry.owner != owner { + return nil, status.Error(codes.PermissionDenied, "run does not belong to session") + } + if member.ReadyAtNs.Valid && !bytes.Equal(member.Metadata, req.GetMetadata()) { + return nil, status.Error(codes.AlreadyExists, "readiness metadata is immutable") + } + if !member.ReadyAtNs.Valid { + ready++ + } + } + } + if !found { + return nil, status.Error(codes.PermissionDenied, "run is not an experiment participant") + } + if err := q.CreateExperimentBarrier(ctx, database.CreateExperimentBarrierParams{TransactionID: run.TransactionID, DeadlineNs: deadline.UnixNano()}); err != nil { + return nil, err + } + barrier, err := q.GetExperimentBarrier(ctx, run.TransactionID) + if err != nil { + return nil, err + } + if now.UnixNano() >= barrier.DeadlineNs || !now.Before(deadline) { + // Preserve the original deadline even when the first call already expired. + if err := tx.Commit(); err != nil { + return nil, err + } + return nil, status.Error(codes.DeadlineExceeded, "experiment readiness deadline expired") + } + metadata := req.GetMetadata() + if metadata == nil { + metadata = []byte{} + } + if err := q.RecordExperimentReady(ctx, database.RecordExperimentReadyParams{DebugletID: run.ID, Metadata: metadata, ReadyAtNs: now.UnixNano()}); err != nil { + return nil, err + } + start := barrier.StartTimeNs + if start == 0 && ready == len(members) { + start = now.Add(experimentLead).UnixNano() + if err := q.ReleaseExperiment(ctx, database.ReleaseExperimentParams{StartTimeNs: start, TransactionID: run.TransactionID}); err != nil { + return nil, err + } + } + out := &pb.ExperimentReadyResponse{ExperimentId: run.TransactionID, StartTimeNs: start} + // Pending callers learn no partial metadata and never wait inside a mutation. + if start != 0 { + for _, member := range members { + m := &pb.ExperimentParticipant{Id: member.Debuglet.Uuid.String(), ExecutorId: member.Debuglet.ExecutorID, Metadata: member.Metadata, ReadyAtNs: member.ReadyAtNs.Int64} + if member.Debuglet.ID == run.ID && !member.ReadyAtNs.Valid { + m.Metadata, m.ReadyAtNs = metadata, now.UnixNano() + } + out.Participants = append(out.Participants, m) + } + } + if err := tx.Commit(); err != nil { + return nil, err + } + return out, nil } - diff --git a/internal/dispatcher/experiments_test.go b/internal/dispatcher/experiments_test.go index 3288c98e..ecff2ffc 100644 --- a/internal/dispatcher/experiments_test.go +++ b/internal/dispatcher/experiments_test.go @@ -4,135 +4,212 @@ package dispatcher import ( - "bytes" - "testing" - "time" + "bytes" + "testing" + "time" - "github.com/google/uuid" - "github.com/netsec-ethz/debuglet/internal/dispatcher/database" - "github.com/netsec-ethz/debuglet/internal/dispatcher/models" - "github.com/netsec-ethz/debuglet/internal/dispatcher/transport/rpc" - "github.com/netsec-ethz/debuglet/internal/sqlitedb" - "github.com/netsec-ethz/debuglet/pkg/wire" - pb "github.com/netsec-ethz/debuglet/protocol" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/google/uuid" + "github.com/netsec-ethz/debuglet/internal/dispatcher/database" + "github.com/netsec-ethz/debuglet/internal/dispatcher/models" + "github.com/netsec-ethz/debuglet/internal/dispatcher/transport/rpc" + "github.com/netsec-ethz/debuglet/internal/sqlitedb" + "github.com/netsec-ethz/debuglet/pkg/wire" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func experimentFixture(t *testing.T, count int) (*tgFixture, []uuid.UUID, string, *time.Time) { - t.Helper() - f := newTGFixture(t, nil) - first := f.seedDirect(t, tgFloorA) - now := time.Now() - f.d.now = func() time.Time { return now } - ids := []uuid.UUID{first.id} - if _, err := f.db.Exec("UPDATE debuglets SET start_time = ?, end_time = ?, state = ? WHERE id = ?", models.NewUTCTime(now.Add(-time.Second)), models.NewUTCTime(now.Add(2*time.Minute)), models.RunStateStarted, first.row.ID); err != nil { t.Fatal(err) } - if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ?", first.row.ID, first.txID); err != nil { t.Fatal(err) } - for i := 1; i < count; i++ { - id := uuid.New() - row, err := f.q.CreateDebuglet(f.ctx, database.CreateDebugletParams{Uuid:id, StartTime:models.NewUTCTime(now.Add(-time.Second)), EndTime:models.NewUTCTime(now.Add(2*time.Minute)), ExecutorID:tgExecutorID, State:models.RunStateStarted, TransactionID:first.txID, OrderID:int64(i+1), DispatcherIncarnation:first.row.DispatcherIncarnation, SessionID:first.row.SessionID}) - if err != nil { t.Fatal(err) } - if _, err := f.q.CreateDebugletOrder(f.ctx, database.CreateDebugletOrderParams{TransactionID:first.txID, OrderID:int64(i+1), ExecutorID:tgExecutorID, State:int64(models.Outstanding)}); err != nil { t.Fatal(err) } - if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ? AND order_id = ?", row.ID, first.txID, i+1); err != nil { t.Fatal(err) } - ids = append(ids, id) - } - return f, ids, first.txID, &now + t.Helper() + f := newTGFixture(t, nil) + first := f.seedDirect(t, tgFloorA) + now := time.Now() + f.d.now = func() time.Time { return now } + ids := []uuid.UUID{first.id} + if _, err := f.db.Exec("UPDATE debuglets SET start_time = ?, end_time = ?, state = ? WHERE id = ?", models.NewUTCTime(now.Add(-time.Second)), models.NewUTCTime(now.Add(2*time.Minute)), models.RunStateStarted, first.row.ID); err != nil { + t.Fatal(err) + } + if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ?", first.row.ID, first.txID); err != nil { + t.Fatal(err) + } + for i := 1; i < count; i++ { + id := uuid.New() + row, err := f.q.CreateDebuglet(f.ctx, database.CreateDebugletParams{Uuid: id, StartTime: models.NewUTCTime(now.Add(-time.Second)), EndTime: models.NewUTCTime(now.Add(2 * time.Minute)), ExecutorID: tgExecutorID, State: models.RunStateStarted, TransactionID: first.txID, OrderID: int64(i + 1), DispatcherIncarnation: first.row.DispatcherIncarnation, SessionID: first.row.SessionID}) + if err != nil { + t.Fatal(err) + } + if _, err := f.q.CreateDebugletOrder(f.ctx, database.CreateDebugletOrderParams{TransactionID: first.txID, OrderID: int64(i + 1), ExecutorID: tgExecutorID, State: int64(models.Outstanding)}); err != nil { + t.Fatal(err) + } + if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ? AND order_id = ?", row.ID, first.txID, i+1); err != nil { + t.Fatal(err) + } + ids = append(ids, id) + } + return f, ids, first.txID, &now } func experimentReady(t *testing.T, f *tgFixture, id uuid.UUID, metadata []byte) (*pb.ExperimentReadyResponse, error) { - t.Helper() - mutation := effectTestMutation(t, f.d, tgExecutorID) - defer mutation.Finish() - return f.d.OnExperimentReady(f.ctx, mutation, &pb.ExperimentReadyRequest{DebugletId:id.String(), ExecutorId:tgExecutorID, Metadata:metadata}) + t.Helper() + mutation := effectTestMutation(t, f.d, tgExecutorID) + defer mutation.Finish() + return f.d.OnExperimentReady(f.ctx, mutation, &pb.ExperimentReadyRequest{DebugletId: id.String(), ExecutorId: tgExecutorID, Metadata: metadata}) } func TestExperimentFiveMemberReleaseAndRestart(t *testing.T) { - f, ids, group, now := experimentFixture(t, 5) - for i, id := range ids { - got, err := experimentReady(t, f, id, []byte{byte(i)}) - if err != nil { t.Fatal(err) } - if i < 4 && (got.StartTimeNs != 0 || len(got.Participants) != 0) { t.Fatalf("early release: %+v", got) } - } - first, err := experimentReady(t, f, ids[0], []byte{0}) - if err != nil || first.ExperimentId != group || first.StartTimeNs != now.Add(experimentLead).UnixNano() || len(first.Participants) != 5 { t.Fatalf("release=%+v err=%v", first, err) } - for i, member := range first.Participants { - if member.Id != ids[i].String() || !bytes.Equal(member.Metadata, []byte{byte(i)}) || member.ReadyAtNs != now.UnixNano() { t.Fatalf("participant=%+v", member) } - } - *now = now.Add(time.Second) - again, err := experimentReady(t, f, ids[0], []byte{0}) - if err != nil || again.StartTimeNs != first.StartTimeNs { t.Fatalf("duplicate reset release: %+v %v", again, err) } - if _, err := experimentReady(t, f, ids[0], []byte("changed")); status.Code(err) != codes.AlreadyExists { t.Fatalf("metadata replacement: %v", err) } - // A new dispatcher reconstructs no live original sessions. Durable release - // survives, but a replacement control identity cannot replay the old run. - var path string - if err := f.db.QueryRow("SELECT file FROM pragma_database_list WHERE name = 'main'").Scan(&path); err != nil { t.Fatal(err) } - reopened, err := sqlitedb.Open(path) - if err != nil { t.Fatal(err) } - defer reopened.Close() - restarted, err := New(f.d.logger, reopened, "restart", time.Minute, time.Minute, f.ph) - if err != nil { t.Fatal(err) } - defer restarted.Close() - barrier, err := database.New(reopened).GetExperimentBarrier(f.ctx, group) - if err != nil || barrier.StartTimeNs != first.StartTimeNs { t.Fatalf("lost persisted release: %+v %v", barrier, err) } - owner, err := rpc.NewSessionOwner(tgExecutorID, effectTestBinding(t), time.Minute) - if err != nil { t.Fatal(err) } - if !owner.MarkRegistered() { t.Fatal("register replacement") } - mutation, err := owner.AdmitMutation(f.ctx) - if err != nil { t.Fatal(err) } - defer mutation.Finish() - _, err = restarted.OnExperimentReady(f.ctx, mutation, &pb.ExperimentReadyRequest{DebugletId:ids[0].String(), ExecutorId:tgExecutorID}) - if status.Code(err) != codes.PermissionDenied { t.Fatalf("restart replay: %v", err) } + f, ids, group, now := experimentFixture(t, 5) + for i, id := range ids { + got, err := experimentReady(t, f, id, []byte{byte(i)}) + if err != nil { + t.Fatal(err) + } + if i < 4 && (got.StartTimeNs != 0 || len(got.Participants) != 0) { + t.Fatalf("early release: %+v", got) + } + } + first, err := experimentReady(t, f, ids[0], []byte{0}) + if err != nil || first.ExperimentId != group || first.StartTimeNs != now.Add(experimentLead).UnixNano() || len(first.Participants) != 5 { + t.Fatalf("release=%+v err=%v", first, err) + } + for i, member := range first.Participants { + if member.Id != ids[i].String() || !bytes.Equal(member.Metadata, []byte{byte(i)}) || member.ReadyAtNs != now.UnixNano() { + t.Fatalf("participant=%+v", member) + } + } + *now = now.Add(time.Second) + again, err := experimentReady(t, f, ids[0], []byte{0}) + if err != nil || again.StartTimeNs != first.StartTimeNs { + t.Fatalf("duplicate reset release: %+v %v", again, err) + } + if _, err := experimentReady(t, f, ids[0], []byte("changed")); status.Code(err) != codes.AlreadyExists { + t.Fatalf("metadata replacement: %v", err) + } + // A new dispatcher reconstructs no live original sessions. Durable release + // survives, but a replacement control identity cannot replay the old run. + var path string + if err := f.db.QueryRow("SELECT file FROM pragma_database_list WHERE name = 'main'").Scan(&path); err != nil { + t.Fatal(err) + } + reopened, err := sqlitedb.Open(path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + restarted, err := New(f.d.logger, reopened, "restart", time.Minute, time.Minute, f.ph) + if err != nil { + t.Fatal(err) + } + defer restarted.Close() + barrier, err := database.New(reopened).GetExperimentBarrier(f.ctx, group) + if err != nil || barrier.StartTimeNs != first.StartTimeNs { + t.Fatalf("lost persisted release: %+v %v", barrier, err) + } + owner, err := rpc.NewSessionOwner(tgExecutorID, effectTestBinding(t), time.Minute) + if err != nil { + t.Fatal(err) + } + if !owner.MarkRegistered() { + t.Fatal("register replacement") + } + mutation, err := owner.AdmitMutation(f.ctx) + if err != nil { + t.Fatal(err) + } + defer mutation.Finish() + _, err = restarted.OnExperimentReady(f.ctx, mutation, &pb.ExperimentReadyRequest{DebugletId: ids[0].String(), ExecutorId: tgExecutorID}) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("restart replay: %v", err) + } } func TestExperimentDeadlineCancellationAndBounds(t *testing.T) { - for _, kind := range []string{"deadline", "cancelled", "terminal", "retired", "oversize", "unstarted", "unadmitted"} { - t.Run(kind, func(t *testing.T) { - f, ids, group, now := experimentFixture(t, 2) - if _, err := experimentReady(t, f, ids[0], nil); err != nil { t.Fatal(err) } - want := codes.FailedPrecondition - var metadata []byte - switch kind { - case "deadline": *now = now.Add(experimentWait); want = codes.DeadlineExceeded - case "cancelled": - row, _ := f.q.GetDebugletByUUID(f.ctx, ids[0]); if _, err := f.d.requestCancellation(f.ctx, row.ID, "test"); err != nil { t.Fatal(err) } - case "terminal": if _, err := f.db.Exec("UPDATE debuglets SET state = ? WHERE uuid = ?", models.RunStateExited, ids[0]); err != nil { t.Fatal(err) } - case "retired": f.d.executors[tgExecutorID].owner.Retire() - case "oversize": metadata = make([]byte, wire.MaxExperimentMetadata+1); want = codes.ResourceExhausted - case "unstarted": if _, err := f.db.Exec("UPDATE debuglets SET state = ? WHERE uuid = ?", models.RunStateUploaded, ids[1]); err != nil { t.Fatal(err) } - case "unadmitted": if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = NULL WHERE transaction_id = ? AND order_id = 1", group); err != nil { t.Fatal(err) } - } - var err error - if kind == "retired" { - _, err = f.d.OnExperimentReady(f.ctx, nil, &pb.ExperimentReadyRequest{DebugletId:ids[1].String(), ExecutorId:tgExecutorID}) - } else { _, err = experimentReady(t, f, ids[1], metadata) } - if status.Code(err) != want { t.Fatalf("got=%v want=%v", err, want) } - barrier, err := f.q.GetExperimentBarrier(f.ctx, group) - if err != nil || barrier.StartTimeNs != 0 { t.Fatalf("late release=%+v %v", barrier, err) } - *now = now.Add(2*time.Minute) - f.d.sweepEndedWindows(time.Time{}) - var retained int - if err := f.db.QueryRow("SELECT COUNT(*) FROM experiment_readiness").Scan(&retained); err != nil || retained != 0 { t.Fatalf("retained=%d %v", retained, err) } - }) - } + for _, kind := range []string{"deadline", "cancelled", "terminal", "retired", "oversize", "unstarted", "unadmitted"} { + t.Run(kind, func(t *testing.T) { + f, ids, group, now := experimentFixture(t, 2) + if _, err := experimentReady(t, f, ids[0], nil); err != nil { + t.Fatal(err) + } + want := codes.FailedPrecondition + var metadata []byte + switch kind { + case "deadline": + *now = now.Add(experimentWait) + want = codes.DeadlineExceeded + case "cancelled": + row, _ := f.q.GetDebugletByUUID(f.ctx, ids[0]) + if _, err := f.d.requestCancellation(f.ctx, row.ID, "test"); err != nil { + t.Fatal(err) + } + case "terminal": + if _, err := f.db.Exec("UPDATE debuglets SET state = ? WHERE uuid = ?", models.RunStateExited, ids[0]); err != nil { + t.Fatal(err) + } + case "retired": + f.d.executors[tgExecutorID].owner.Retire() + case "oversize": + metadata = make([]byte, wire.MaxExperimentMetadata+1) + want = codes.ResourceExhausted + case "unstarted": + if _, err := f.db.Exec("UPDATE debuglets SET state = ? WHERE uuid = ?", models.RunStateUploaded, ids[1]); err != nil { + t.Fatal(err) + } + case "unadmitted": + if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = NULL WHERE transaction_id = ? AND order_id = 1", group); err != nil { + t.Fatal(err) + } + } + var err error + if kind == "retired" { + _, err = f.d.OnExperimentReady(f.ctx, nil, &pb.ExperimentReadyRequest{DebugletId: ids[1].String(), ExecutorId: tgExecutorID}) + } else { + _, err = experimentReady(t, f, ids[1], metadata) + } + if status.Code(err) != want { + t.Fatalf("got=%v want=%v", err, want) + } + barrier, err := f.q.GetExperimentBarrier(f.ctx, group) + if err != nil || barrier.StartTimeNs != 0 { + t.Fatalf("late release=%+v %v", barrier, err) + } + *now = now.Add(2 * time.Minute) + f.d.sweepEndedWindows(time.Time{}) + var retained int + if err := f.db.QueryRow("SELECT COUNT(*) FROM experiment_readiness").Scan(&retained); err != nil || retained != 0 { + t.Fatalf("retained=%d %v", retained, err) + } + }) + } } func TestExperimentForeignOwnerAndGroupIsolation(t *testing.T) { - f, ids, group, _ := experimentFixture(t, 2) - foreign := f.seedDirect(t, tgFloorA) - mutation := effectTestMutation(t, f.d, tgExecutorID) - defer mutation.Finish() - for _, req := range []*pb.ExperimentReadyRequest{ - {DebugletId:ids[0].String(), ExecutorId:"foreign"}, - {DebugletId:ids[0].String()}, - } { - if _, err := f.d.OnExperimentReady(f.ctx, mutation, req); status.Code(err) != codes.PermissionDenied { t.Fatalf("foreign owner: %v", err) } - } - if _, err := experimentReady(t, f, ids[0], []byte("private")); err != nil { t.Fatal(err) } - // Another admitted transaction receives only its own fixed participant. - if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ?", foreign.row.ID, foreign.txID); err != nil { t.Fatal(err) } - if _, err := f.db.Exec("UPDATE debuglets SET state = ?, start_time = ? WHERE id = ?", models.RunStateStarted, models.NewUTCTime(f.d.now().Add(-time.Second)), foreign.row.ID); err != nil { t.Fatal(err) } - other, err := experimentReady(t, f, foreign.id, nil) - if err != nil || other.ExperimentId != foreign.txID || len(other.Participants) != 1 || other.Participants[0].Id != foreign.id.String() || len(other.Participants[0].Metadata) != 0 { t.Fatalf("group leak: %+v %v", other, err) } - got, err := f.q.GetExperimentBarrier(f.ctx, group) - if err != nil || got.StartTimeNs != 0 { t.Fatalf("foreign readiness released group: %v %v", got, err) } + f, ids, group, _ := experimentFixture(t, 2) + foreign := f.seedDirect(t, tgFloorA) + mutation := effectTestMutation(t, f.d, tgExecutorID) + defer mutation.Finish() + for _, req := range []*pb.ExperimentReadyRequest{ + {DebugletId: ids[0].String(), ExecutorId: "foreign"}, + {DebugletId: ids[0].String()}, + } { + if _, err := f.d.OnExperimentReady(f.ctx, mutation, req); status.Code(err) != codes.PermissionDenied { + t.Fatalf("foreign owner: %v", err) + } + } + if _, err := experimentReady(t, f, ids[0], []byte("private")); err != nil { + t.Fatal(err) + } + // Another admitted transaction receives only its own fixed participant. + if _, err := f.db.Exec("UPDATE debuglet_order SET debuglet_id = ? WHERE transaction_id = ?", foreign.row.ID, foreign.txID); err != nil { + t.Fatal(err) + } + if _, err := f.db.Exec("UPDATE debuglets SET state = ?, start_time = ? WHERE id = ?", models.RunStateStarted, models.NewUTCTime(f.d.now().Add(-time.Second)), foreign.row.ID); err != nil { + t.Fatal(err) + } + other, err := experimentReady(t, f, foreign.id, nil) + if err != nil || other.ExperimentId != foreign.txID || len(other.Participants) != 1 || other.Participants[0].Id != foreign.id.String() || len(other.Participants[0].Metadata) != 0 { + t.Fatalf("group leak: %+v %v", other, err) + } + got, err := f.q.GetExperimentBarrier(f.ctx, group) + if err != nil || got.StartTimeNs != 0 { + t.Fatalf("foreign readiness released group: %v %v", got, err) + } } diff --git a/internal/dispatcher/transport/rpc/experiment_test.go b/internal/dispatcher/transport/rpc/experiment_test.go index 5d22f426..bae269ed 100644 --- a/internal/dispatcher/transport/rpc/experiment_test.go +++ b/internal/dispatcher/transport/rpc/experiment_test.go @@ -4,42 +4,54 @@ package rpc import ( - "context" - "testing" + "context" + "testing" - pb "github.com/netsec-ethz/debuglet/protocol" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) func (s *lifecycleState) OnExperimentReady(ctx context.Context, mutation *Mutation, req *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { - if !mutation.Live() || mutation.IsSetup() || mutation.Owner().ExecutorID() != req.ExecutorId { return nil, status.Error(codes.Internal, "invalid readiness mutation") } - return &pb.ExperimentReadyResponse{ExperimentId:"authenticated"}, nil + if !mutation.Live() || mutation.IsSetup() || mutation.Owner().ExecutorID() != req.ExecutorId { + return nil, status.Error(codes.Internal, "invalid readiness mutation") + } + return &pb.ExperimentReadyResponse{ExperimentId: "authenticated"}, nil } func TestExperimentReadyUsesBoundAuthenticatedSession(t *testing.T) { - f := newControlTLS(t, controlTLSOptions{requireClientIdentity:true}) - owned := f.connectedPeer(&lifecyclePeer{id:"executor", version:"A"}, f.ca.ClientConfig(f.client, "")) - owner := f.registered("A") - if err := owned.client.WaitReadyContext(f.ctx); err != nil { t.Fatal(err) } - for _, tc := range []struct{name, executor string; strip bool; want codes.Code}{ - {"owner", "executor", false, codes.OK}, - {"foreign executor", "foreign", false, codes.PermissionDenied}, - {"empty executor", "", false, codes.PermissionDenied}, - {"missing credentials", "executor", true, codes.PermissionDenied}, - } { - t.Run(tc.name, func(t *testing.T) { - err := f.directCall(f.client, owner, func(ctx context.Context, client pb.DispatcherServiceClient) error { - if tc.strip { ctx = metadata.NewOutgoingContext(ctx, metadata.MD{}) } - _, err := client.ExperimentReady(ctx, &pb.ExperimentReadyRequest{DebugletId:"run", ExecutorId:tc.executor}) - return err - }) - if status.Code(err) != tc.want { t.Fatalf("got %v want %v", err, tc.want) } - }) - } - // Each poll has finished its mutation; retiring the session need not wait on - // an application barrier or a guest's next poll. - owner.Retire() - awaitSessionSignal(t, owner.MutationsDrained()) + f := newControlTLS(t, controlTLSOptions{requireClientIdentity: true}) + owned := f.connectedPeer(&lifecyclePeer{id: "executor", version: "A"}, f.ca.ClientConfig(f.client, "")) + owner := f.registered("A") + if err := owned.client.WaitReadyContext(f.ctx); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, executor string + strip bool + want codes.Code + }{ + {"owner", "executor", false, codes.OK}, + {"foreign executor", "foreign", false, codes.PermissionDenied}, + {"empty executor", "", false, codes.PermissionDenied}, + {"missing credentials", "executor", true, codes.PermissionDenied}, + } { + t.Run(tc.name, func(t *testing.T) { + err := f.directCall(f.client, owner, func(ctx context.Context, client pb.DispatcherServiceClient) error { + if tc.strip { + ctx = metadata.NewOutgoingContext(ctx, metadata.MD{}) + } + _, err := client.ExperimentReady(ctx, &pb.ExperimentReadyRequest{DebugletId: "run", ExecutorId: tc.executor}) + return err + }) + if status.Code(err) != tc.want { + t.Fatalf("got %v want %v", err, tc.want) + } + }) + } + // Each poll has finished its mutation; retiring the session need not wait on + // an application barrier or a guest's next poll. + owner.Retire() + awaitSessionSignal(t, owner.MutationsDrained()) } diff --git a/internal/dispatcher/transport/rpc/server.go b/internal/dispatcher/transport/rpc/server.go index 92f957cf..c620740a 100644 --- a/internal/dispatcher/transport/rpc/server.go +++ b/internal/dispatcher/transport/rpc/server.go @@ -93,8 +93,12 @@ func (s *server) RenewLease(ctx context.Context, in *pb.RenewLeaseRequest) (*pb. // the guest waits. No participant can pin another session's replacement. func (s *server) ExperimentReady(ctx context.Context, in *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { ticket, err := s.bidi.admitted(ctx, in.GetExecutorId()) - if err != nil { return nil, err } + if err != nil { + return nil, err + } defer ticket.Finish() - if in.GetExecutorId() == "" { return nil, controlrpc.Denied() } + if in.GetExecutorId() == "" { + return nil, controlrpc.Denied() + } return s.state.OnExperimentReady(ticket.Context(), ticket, in) } diff --git a/internal/executor/debuglet/debuglet.go b/internal/executor/debuglet/debuglet.go index 3d647189..a8cb560d 100644 --- a/internal/executor/debuglet/debuglet.go +++ b/internal/executor/debuglet/debuglet.go @@ -158,10 +158,12 @@ func newWithBPFTagger(logger *zap.Logger, debugletID uuid.UUID, transactionID st // SetExperimentControl installs the run-bound dispatcher capability before the // runtime is initialized. It is shared by local and isolated worker host calls. func (d *Debuglet) SetExperimentControl(control wasm.ExperimentControl) { - d.mu.Lock() - defer d.mu.Unlock() - if d.initialized || d.closed { panic("experiment control must be set before initialization") } - d.env.Experiment = control + d.mu.Lock() + defer d.mu.Unlock() + if d.initialized || d.closed { + panic("experiment control must be set before initialization") + } + d.env.Experiment = control } // UserspaceTagging is the IPv4 mode the pure-Go tagger gives a run on this @@ -431,9 +433,9 @@ func (d *Debuglet) createWASMInstance(ctx context.Context, wasmBytes []byte) (er return fmt.Errorf("createWASMInstance: recoverable I/O module instantiation: %w", err) } - if _, err := wasm.Register(rt.NewHostModuleBuilder(wasm.ExperimentModule), wasm.ExperimentModule, wasm.Functions(d.env), nil).Instantiate(ctx); err != nil { - return fmt.Errorf("createWASMInstance: experiment module instantiation: %w", err) - } + if _, err := wasm.Register(rt.NewHostModuleBuilder(wasm.ExperimentModule), wasm.ExperimentModule, wasm.Functions(d.env), nil).Instantiate(ctx); err != nil { + return fmt.Errorf("createWASMInstance: experiment module instantiation: %w", err) + } d.mu.Lock() closed := d.closed diff --git a/internal/executor/debuglet/wasm/experiment.go b/internal/executor/debuglet/wasm/experiment.go index 1f8b5825..9a8467e8 100644 --- a/internal/executor/debuglet/wasm/experiment.go +++ b/internal/executor/debuglet/wasm/experiment.go @@ -4,14 +4,14 @@ package wasm import ( - "context" - "encoding/json" - "errors" - "time" + "context" + "encoding/json" + "errors" + "time" - "github.com/netsec-ethz/debuglet/pkg/wire" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/netsec-ethz/debuglet/pkg/wire" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // ExperimentModule is the optional readiness ABI; the base env ABI is unchanged. @@ -20,42 +20,74 @@ const ExperimentModule = "debuglet_experiment_v1" // ExperimentControl is the run-bound capability supplied by the executor. // The guest can publish metadata but cannot select a run, account or session. type ExperimentControl interface { - Ready(context.Context, []byte) (wire.Experiment, error) + Ready(context.Context, []byte) (wire.Experiment, error) } func hostExperimentReady(env *WasmEnv) func(context.Context, Memory, uint32, uint32, uint32, uint32, int64) int32 { - return func(ctx context.Context, mem Memory, ptr, size, out, capacity uint32, deadlineNS int64) int32 { - if size > wire.MaxExperimentMetadata || capacity > 1<<20 { return -4 } - // Memory is contiguous. Validate both ends before any readiness mutation, - // without copying a megabyte through the isolated worker's memory bridge. - if capacity == 0 || uint64(out)+uint64(capacity) > 1<<32 { panic("invalid experiment output buffer") } - if _, ok := mem.Read(out, 1); !ok { panic("invalid experiment output buffer") } - if _, ok := mem.Read(out+capacity-1, 1); !ok { panic("invalid experiment output buffer") } - metadata, ok := mem.Read(ptr, size) - if !ok { panic("invalid experiment metadata buffer") } - if env.Experiment == nil { return -1 } - if deadlineNS <= 0 { return -3 } - callCtx, cancel := context.WithDeadline(ctx, time.Unix(0, deadlineNS)) - defer cancel() - if err := callCtx.Err(); err != nil { return experimentError(err) } - result, err := env.Experiment.Ready(callCtx, append([]byte(nil), metadata...)) - if err != nil { return experimentError(err) } - if err := callCtx.Err(); err != nil { return experimentError(err) } - data, err := json.Marshal(result) - if err != nil { return -1 } - if len(data) > int(capacity) { return -4 } - for offset := 0; offset < len(data); { - end := min(offset+MAX_SLICE_LENGTH, len(data)) - if !mem.Write(out+uint32(offset), data[offset:end]) { panic("invalid experiment output buffer") } - offset = end - } - return int32(len(data)) - } + return func(ctx context.Context, mem Memory, ptr, size, out, capacity uint32, deadlineNS int64) int32 { + if size > wire.MaxExperimentMetadata || capacity > 1<<20 { + return -4 + } + // Memory is contiguous. Validate both ends before any readiness mutation, + // without copying a megabyte through the isolated worker's memory bridge. + if capacity == 0 || uint64(out)+uint64(capacity) > 1<<32 { + panic("invalid experiment output buffer") + } + if _, ok := mem.Read(out, 1); !ok { + panic("invalid experiment output buffer") + } + if _, ok := mem.Read(out+capacity-1, 1); !ok { + panic("invalid experiment output buffer") + } + metadata, ok := mem.Read(ptr, size) + if !ok { + panic("invalid experiment metadata buffer") + } + if env.Experiment == nil { + return -1 + } + if deadlineNS <= 0 { + return -3 + } + callCtx, cancel := context.WithDeadline(ctx, time.Unix(0, deadlineNS)) + defer cancel() + if err := callCtx.Err(); err != nil { + return experimentError(err) + } + result, err := env.Experiment.Ready(callCtx, append([]byte(nil), metadata...)) + if err != nil { + return experimentError(err) + } + if err := callCtx.Err(); err != nil { + return experimentError(err) + } + data, err := json.Marshal(result) + if err != nil { + return -1 + } + if len(data) > int(capacity) { + return -4 + } + for offset := 0; offset < len(data); { + end := min(offset+MAX_SLICE_LENGTH, len(data)) + if !mem.Write(out+uint32(offset), data[offset:end]) { + panic("invalid experiment output buffer") + } + offset = end + } + return int32(len(data)) + } } func experimentError(err error) int32 { - if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { return -2 } - if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded { return -3 } - if status.Code(err) == codes.ResourceExhausted { return -4 } - return -1 // Never expose transport diagnostics or authentication material. + if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { + return -2 + } + if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded { + return -3 + } + if status.Code(err) == codes.ResourceExhausted { + return -4 + } + return -1 // Never expose transport diagnostics or authentication material. } diff --git a/internal/executor/debuglet/wasm/experiment_test.go b/internal/executor/debuglet/wasm/experiment_test.go index a83f65b2..602fdd65 100644 --- a/internal/executor/debuglet/wasm/experiment_test.go +++ b/internal/executor/debuglet/wasm/experiment_test.go @@ -4,56 +4,83 @@ package wasm import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/netsec-ethz/debuglet/pkg/wire" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "context" + "encoding/json" + "testing" + "time" + + "github.com/netsec-ethz/debuglet/pkg/wire" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type experimentControl func(context.Context, []byte) (wire.Experiment, error) -func (f experimentControl) Ready(ctx context.Context, metadata []byte) (wire.Experiment,error) { return f(ctx,metadata) } + +func (f experimentControl) Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { + return f(ctx, metadata) +} func TestExperimentHostValidatesBeforePublication(t *testing.T) { - mod := newGuestModule(t) - calls := 0 - env := &WasmEnv{Experiment:experimentControl(func(context.Context,[]byte)(wire.Experiment,error){calls++;return wire.Experiment{},nil})} - ready := hostExperimentReady(env) - ctx := context.Background() - deadline := time.Now().Add(time.Second).UnixNano() - if got:=ready(ctx,mod.Memory(),0,4097,0,1024,deadline); got != -4 { t.Fatal(got) } - for _, call := range []func(){ - func(){ready(ctx,mod.Memory(),0,1,wasmPageSize-1,10,deadline)}, - func(){ready(ctx,mod.Memory(),wasmPageSize,1,0,1024,deadline)}, - func(){ready(ctx,mod.Memory(),0,1,^uint32(0),10,deadline)}, - } { if hostTrap(call)==nil { t.Fatal("invalid memory accepted") } } - if got:=ready(ctx,mod.Memory(),0,0,0,1024,time.Now().Add(-time.Second).UnixNano()); got != -3 { t.Fatal(got) } - if calls != 0 { t.Fatalf("invalid input published %d times",calls) } + mod := newGuestModule(t) + calls := 0 + env := &WasmEnv{Experiment: experimentControl(func(context.Context, []byte) (wire.Experiment, error) { calls++; return wire.Experiment{}, nil })} + ready := hostExperimentReady(env) + ctx := context.Background() + deadline := time.Now().Add(time.Second).UnixNano() + if got := ready(ctx, mod.Memory(), 0, 4097, 0, 1024, deadline); got != -4 { + t.Fatal(got) + } + for _, call := range []func(){ + func() { ready(ctx, mod.Memory(), 0, 1, wasmPageSize-1, 10, deadline) }, + func() { ready(ctx, mod.Memory(), wasmPageSize, 1, 0, 1024, deadline) }, + func() { ready(ctx, mod.Memory(), 0, 1, ^uint32(0), 10, deadline) }, + } { + if hostTrap(call) == nil { + t.Fatal("invalid memory accepted") + } + } + if got := ready(ctx, mod.Memory(), 0, 0, 0, 1024, time.Now().Add(-time.Second).UnixNano()); got != -3 { + t.Fatal(got) + } + if calls != 0 { + t.Fatalf("invalid input published %d times", calls) + } } -type boundedExperimentMemory struct { Memory } -func (m boundedExperimentMemory) Read(ptr,size uint32)([]byte,bool) { - if size > MAX_SLICE_LENGTH { return nil,false } - return m.Memory.Read(ptr,size) +type boundedExperimentMemory struct{ Memory } + +func (m boundedExperimentMemory) Read(ptr, size uint32) ([]byte, bool) { + if size > MAX_SLICE_LENGTH { + return nil, false + } + return m.Memory.Read(ptr, size) } -func (m boundedExperimentMemory) Write(ptr uint32, data []byte)bool { - if len(data)>MAX_SLICE_LENGTH { return false } - return m.Memory.Write(ptr,data) +func (m boundedExperimentMemory) Write(ptr uint32, data []byte) bool { + if len(data) > MAX_SLICE_LENGTH { + return false + } + return m.Memory.Write(ptr, data) } func TestExperimentHostChunkedResponseAndPrivateErrors(t *testing.T) { - mod := newGuestModule(t) - result := wire.Experiment{ID:"batch",StartTimeNS:10,Participants:[]wire.ExperimentParticipant{ - {ID:"one",Metadata:make([]byte,4096)},{ID:"two",Metadata:make([]byte,4096)}, - }} - env := &WasmEnv{Experiment:experimentControl(func(context.Context,[]byte)(wire.Experiment,error){return result,nil})} - got:=hostExperimentReady(env)(context.Background(),boundedExperimentMemory{mod.Memory()},0,0,0,32000,time.Now().Add(time.Second).UnixNano()) - if got <= MAX_SLICE_LENGTH { t.Fatalf("response not chunked: %d",got) } - data,_:=mod.Memory().Read(0,uint32(got)); var decoded wire.Experiment - if err:=json.Unmarshal(data,&decoded); err != nil || len(decoded.Participants)!=2 || len(decoded.Participants[1].Metadata)!=4096 { t.Fatalf("invalid result %v",err) } - env.Experiment=experimentControl(func(context.Context,[]byte)(wire.Experiment,error){return wire.Experiment{},status.Error(codes.Unauthenticated,"secret credential")}) - if got:=hostExperimentReady(env)(context.Background(),mod.Memory(),0,0,0,32000,time.Now().Add(time.Second).UnixNano()); got != -1 { t.Fatal(got) } + mod := newGuestModule(t) + result := wire.Experiment{ID: "batch", StartTimeNS: 10, Participants: []wire.ExperimentParticipant{ + {ID: "one", Metadata: make([]byte, 4096)}, {ID: "two", Metadata: make([]byte, 4096)}, + }} + env := &WasmEnv{Experiment: experimentControl(func(context.Context, []byte) (wire.Experiment, error) { return result, nil })} + got := hostExperimentReady(env)(context.Background(), boundedExperimentMemory{mod.Memory()}, 0, 0, 0, 32000, time.Now().Add(time.Second).UnixNano()) + if got <= MAX_SLICE_LENGTH { + t.Fatalf("response not chunked: %d", got) + } + data, _ := mod.Memory().Read(0, uint32(got)) + var decoded wire.Experiment + if err := json.Unmarshal(data, &decoded); err != nil || len(decoded.Participants) != 2 || len(decoded.Participants[1].Metadata) != 4096 { + t.Fatalf("invalid result %v", err) + } + env.Experiment = experimentControl(func(context.Context, []byte) (wire.Experiment, error) { + return wire.Experiment{}, status.Error(codes.Unauthenticated, "secret credential") + }) + if got := hostExperimentReady(env)(context.Background(), mod.Memory(), 0, 0, 0, 32000, time.Now().Add(time.Second).UnixNano()); got != -1 { + t.Fatal(got) + } } diff --git a/internal/executor/experiment.go b/internal/executor/experiment.go index 2871fc93..af7a0955 100644 --- a/internal/executor/experiment.go +++ b/internal/executor/experiment.go @@ -4,40 +4,52 @@ package executor import ( - "context" - "time" + "context" + "time" - "github.com/netsec-ethz/debuglet/internal/executor/scheduler" - "github.com/netsec-ethz/debuglet/pkg/wire" - pb "github.com/netsec-ethz/debuglet/protocol" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/netsec-ethz/debuglet/internal/executor/scheduler" + "github.com/netsec-ethz/debuglet/pkg/wire" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type runExperiment struct { - executor *Executor - spec scheduler.Spec + executor *Executor + spec scheduler.Spec } func (r runExperiment) Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { - var result wire.Experiment - if len(metadata) > wire.MaxExperimentMetadata { return result, status.Error(codes.ResourceExhausted, "experiment metadata exceeds limit") } - ctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - client, err := r.executor.dispatcherClient(ctx, r.spec.Binding) - if err != nil { return result, err } - reply, err := client.ExperimentReady(ctx, &pb.ExperimentReadyRequest{ - DebugletId: r.spec.DebugletID.String(), ExecutorId: r.executor.cfg.Identity.ExecutorID, Metadata: metadata, - }) - if err != nil { return result, err } - if err := r.executor.checkExecutionLease(ctx, r.spec.Binding); err != nil { return result, err } - if reply == nil || len(reply.GetParticipants()) > wire.MaxExperimentParticipants { return result, status.Error(codes.ResourceExhausted, "invalid experiment response") } - result.ID, result.StartTimeNS = reply.GetExperimentId(), reply.GetStartTimeNs() - for _, p := range reply.GetParticipants() { - if len(p.GetMetadata()) > wire.MaxExperimentMetadata { return wire.Experiment{}, status.Error(codes.ResourceExhausted, "experiment metadata exceeds limit") } - result.Participants = append(result.Participants, wire.ExperimentParticipant{ - ID: p.GetId(), ExecutorID: p.GetExecutorId(), Metadata: p.GetMetadata(), ReadyAtNS: p.GetReadyAtNs(), - }) - } - return result, nil + var result wire.Experiment + if len(metadata) > wire.MaxExperimentMetadata { + return result, status.Error(codes.ResourceExhausted, "experiment metadata exceeds limit") + } + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + client, err := r.executor.dispatcherClient(ctx, r.spec.Binding) + if err != nil { + return result, err + } + reply, err := client.ExperimentReady(ctx, &pb.ExperimentReadyRequest{ + DebugletId: r.spec.DebugletID.String(), ExecutorId: r.executor.cfg.Identity.ExecutorID, Metadata: metadata, + }) + if err != nil { + return result, err + } + if err := r.executor.checkExecutionLease(ctx, r.spec.Binding); err != nil { + return result, err + } + if reply == nil || len(reply.GetParticipants()) > wire.MaxExperimentParticipants { + return result, status.Error(codes.ResourceExhausted, "invalid experiment response") + } + result.ID, result.StartTimeNS = reply.GetExperimentId(), reply.GetStartTimeNs() + for _, p := range reply.GetParticipants() { + if len(p.GetMetadata()) > wire.MaxExperimentMetadata { + return wire.Experiment{}, status.Error(codes.ResourceExhausted, "experiment metadata exceeds limit") + } + result.Participants = append(result.Participants, wire.ExperimentParticipant{ + ID: p.GetId(), ExecutorID: p.GetExecutorId(), Metadata: p.GetMetadata(), ReadyAtNS: p.GetReadyAtNs(), + }) + } + return result, nil } diff --git a/internal/executor/experiment_test.go b/internal/executor/experiment_test.go index a7d01c77..19a039d5 100644 --- a/internal/executor/experiment_test.go +++ b/internal/executor/experiment_test.go @@ -4,38 +4,51 @@ package executor import ( - "context" - "testing" - "time" + "context" + "testing" + "time" - "github.com/google/uuid" - "github.com/netsec-ethz/debuglet/internal/executor/scheduler" - pb "github.com/netsec-ethz/debuglet/protocol" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/google/uuid" + "github.com/netsec-ethz/debuglet/internal/executor/scheduler" + pb "github.com/netsec-ethz/debuglet/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type experimentPeer struct { - pb.UnimplementedDispatcherServiceServer - ready func(context.Context,*pb.ExperimentReadyRequest)(*pb.ExperimentReadyResponse,error) + pb.UnimplementedDispatcherServiceServer + ready func(context.Context, *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) +} + +func (p experimentPeer) ExperimentReady(ctx context.Context, req *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { + return p.ready(ctx, req) } -func (p experimentPeer) ExperimentReady(ctx context.Context, req *pb.ExperimentReadyRequest)(*pb.ExperimentReadyResponse,error) { return p.ready(ctx,req) } func TestExperimentRunBindingAndCancellation(t *testing.T) { - id:=uuid.New() - calls:=make(chan *pb.ExperimentReadyRequest,2) - peer:=experimentPeer{ready:func(ctx context.Context,req *pb.ExperimentReadyRequest)(*pb.ExperimentReadyResponse,error){ - calls<-req - if string(req.Metadata)=="wait" { <-ctx.Done(); return nil,ctx.Err() } - return &pb.ExperimentReadyResponse{ExperimentId:"batch",StartTimeNs:time.Now().Add(time.Second).UnixNano(),Participants:[]*pb.ExperimentParticipant{{Id:id.String(),ExecutorId:req.ExecutorId,Metadata:req.Metadata,ReadyAtNs:123}}},nil - }} - e,_:=newExecutorRPCFixture(t,peer,newFixtureMemoryStorage(t)) - control:=runExperiment{executor:e,spec:scheduler.Spec{DebugletID:id,Binding:operationBinding()}} - result,err:=control.Ready(context.Background(),[]byte("opaque")) - if err != nil { t.Fatal(err) } - req:=<-calls - if req.DebugletId != id.String() || req.ExecutorId != e.cfg.Identity.ExecutorID || result.ID != "batch" || len(result.Participants)!=1 || result.Participants[0].ReadyAtNS!=123 || string(result.Participants[0].Metadata)!="opaque" { t.Fatalf("incorrect request/result: %v %v",req,result) } - ctx,cancel:=context.WithTimeout(context.Background(),20*time.Millisecond);defer cancel() - if _,err:=control.Ready(ctx,[]byte("wait")); status.Code(err)!=codes.DeadlineExceeded && ctx.Err()==nil { t.Fatalf("cancellation ignored: %v",err) } - <-calls + id := uuid.New() + calls := make(chan *pb.ExperimentReadyRequest, 2) + peer := experimentPeer{ready: func(ctx context.Context, req *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { + calls <- req + if string(req.Metadata) == "wait" { + <-ctx.Done() + return nil, ctx.Err() + } + return &pb.ExperimentReadyResponse{ExperimentId: "batch", StartTimeNs: time.Now().Add(time.Second).UnixNano(), Participants: []*pb.ExperimentParticipant{{Id: id.String(), ExecutorId: req.ExecutorId, Metadata: req.Metadata, ReadyAtNs: 123}}}, nil + }} + e, _ := newExecutorRPCFixture(t, peer, newFixtureMemoryStorage(t)) + control := runExperiment{executor: e, spec: scheduler.Spec{DebugletID: id, Binding: operationBinding()}} + result, err := control.Ready(context.Background(), []byte("opaque")) + if err != nil { + t.Fatal(err) + } + req := <-calls + if req.DebugletId != id.String() || req.ExecutorId != e.cfg.Identity.ExecutorID || result.ID != "batch" || len(result.Participants) != 1 || result.Participants[0].ReadyAtNS != 123 || string(result.Participants[0].Metadata) != "opaque" { + t.Fatalf("incorrect request/result: %v %v", req, result) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, err := control.Ready(ctx, []byte("wait")); status.Code(err) != codes.DeadlineExceeded && ctx.Err() == nil { + t.Fatalf("cancellation ignored: %v", err) + } + <-calls } diff --git a/internal/executor/recovery_integration_test.go b/internal/executor/recovery_integration_test.go index cedbab81..d9c48c68 100644 --- a/internal/executor/recovery_integration_test.go +++ b/internal/executor/recovery_integration_test.go @@ -55,6 +55,9 @@ func (p *recoveryPeer) OnDebugletState(ctx context.Context, _ *drpc.Mutation, re func (p *recoveryPeer) OnDebugletExit(ctx context.Context, _ *drpc.Mutation, req *pb.DebugletExitRequest) (*pb.DebugletExitResponse, error) { return p.DebugletExit(ctx, req) } +func (*recoveryPeer) OnExperimentReady(context.Context, *drpc.Mutation, *pb.ExperimentReadyRequest) (*pb.ExperimentReadyResponse, error) { + return nil, errors.New("experiment readiness is not supported by the recovery fixture") +} func (p *recoveryPeer) OnDebugletStream(_ *drpc.SessionOwner, stream grpc.BidiStreamingServer[pb.DebugletStreamRequest, pb.DebugletStreamResponse]) error { // The real Hello negotiates output receipts. These tests script acceptance; // durable dispatcher storage is exercised by the output integration tests. diff --git a/internal/executor/transport/rpc/bound_client.go b/internal/executor/transport/rpc/bound_client.go index dd016408..126d4a3c 100644 --- a/internal/executor/transport/rpc/bound_client.go +++ b/internal/executor/transport/rpc/bound_client.go @@ -52,11 +52,17 @@ func (c *boundDispatcherClient) DebugletExit(ctx context.Context, in *pb.Debugle return out, c.credentials.RedactError(err) } func (c *boundDispatcherClient) ExperimentReady(ctx context.Context, in *pb.ExperimentReadyRequest, opts ...grpc.CallOption) (*pb.ExperimentReadyResponse, error) { - if err := c.owner.CheckLease(c.credentials.Binding); err != nil { return nil, err } - out, err := c.client.ExperimentReady(c.credentials.Outgoing(ctx), in, opts...) - if err != nil { return nil, c.credentials.RedactError(err) } - if err := c.owner.CheckLease(c.credentials.Binding); err != nil { return nil, err } - return out, nil + if err := c.owner.CheckLease(c.credentials.Binding); err != nil { + return nil, err + } + out, err := c.client.ExperimentReady(c.credentials.Outgoing(ctx), in, opts...) + if err != nil { + return nil, c.credentials.RedactError(err) + } + if err := c.owner.CheckLease(c.credentials.Binding); err != nil { + return nil, err + } + return out, nil } func (c *boundDispatcherClient) BindSession(ctx context.Context, in *pb.BindSessionRequest, opts ...grpc.CallOption) (*pb.BindSessionResponse, error) { return nil, controlrpc.Unavailable() diff --git a/internal/executor/transport/rpc/lease_test.go b/internal/executor/transport/rpc/lease_test.go index 7a9e93cf..4e5815fd 100644 --- a/internal/executor/transport/rpc/lease_test.go +++ b/internal/executor/transport/rpc/lease_test.go @@ -32,8 +32,8 @@ func (c *leaseScriptClient) Heartbeat(context.Context, *pb.HeartbeatRequest, ... return &pb.HeartbeatResponse{}, nil } func (c *leaseScriptClient) ExperimentReady(context.Context, *pb.ExperimentReadyRequest, ...grpc.CallOption) (*pb.ExperimentReadyResponse, error) { - c.calls.Add(1) - return &pb.ExperimentReadyResponse{}, nil + c.calls.Add(1) + return &pb.ExperimentReadyResponse{}, nil } func (c *leaseScriptClient) BindSession(ctx context.Context, in *pb.BindSessionRequest, _ ...grpc.CallOption) (*pb.BindSessionResponse, error) { c.calls.Add(1) @@ -96,8 +96,10 @@ func TestClientLeaseGuardsWithoutWatchdog(t *testing.T) { if _, err = cached.Heartbeat(context.Background(), &pb.HeartbeatRequest{}); err != nil { t.Fatal(err) } - if _, err = cached.ExperimentReady(context.Background(), &pb.ExperimentReadyRequest{}); err != nil { t.Fatal(err) } - before := raw.calls.Load() + if _, err = cached.ExperimentReady(context.Background(), &pb.ExperimentReadyRequest{}); err != nil { + t.Fatal(err) + } + before := raw.calls.Load() offset.Store(int64(time.Second)) committed := false if err = b.CommitLease(binding, func() { committed = true }); err == nil || committed { @@ -118,8 +120,10 @@ func TestClientLeaseGuardsWithoutWatchdog(t *testing.T) { if err = stream.SendMsg(&pb.DebugletStreamRequest{}); err == nil { t.Fatal("cached stream SendMsg escaped lease guard") } - if _, err = cached.ExperimentReady(context.Background(), &pb.ExperimentReadyRequest{}); err == nil { t.Fatal("cached experiment call escaped lease guard") } - if raw.calls.Load() != before || raw.stream.sends.Load() != 1 { + if _, err = cached.ExperimentReady(context.Background(), &pb.ExperimentReadyRequest{}); err == nil { + t.Fatal("cached experiment call escaped lease guard") + } + if raw.calls.Load() != before || raw.stream.sends.Load() != 1 { t.Fatal("expired operation reached transport") } if err = stream.CloseSend(); err != nil { diff --git a/internal/storagecheck/init_test.go b/internal/storagecheck/init_test.go index 964a06f5..74e1d3d3 100644 --- a/internal/storagecheck/init_test.go +++ b/internal/storagecheck/init_test.go @@ -51,8 +51,8 @@ func TestBootstrapFresh(t *testing.T) { "measurement_requests": "debuglet_id document", "measurement_execution": "debuglet_id started_observed_ns terminal_observed_ns exit_code tcp_endpoint", "retry_requests": "caller_scope request_id parent_run_id transaction_id request_hash intent_metadata", - "experiment_barriers": "transaction_id deadline_ns start_time_ns", - "experiment_readiness": "debuglet_id metadata ready_at_ns", + "experiment_barriers": "transaction_id deadline_ns start_time_ns", + "experiment_readiness": "debuglet_id metadata ready_at_ns", "allocation_reclamations": "debuglet_id reclaimed_at", "allowance_grants": "id user_id amount currency granted_by reason idempotency_key granted_at", "probe_status": "executor_id first_connected last_connected connected status_since total_uptime is_public host_tags version", diff --git a/internal/storagecheck/policy.go b/internal/storagecheck/policy.go index 86640d89..9eb037bb 100644 --- a/internal/storagecheck/policy.go +++ b/internal/storagecheck/policy.go @@ -72,8 +72,8 @@ func PolicyFor(role Role) (Policy, error) { "transactions": nil, }, Tables: map[string][]string{ "account_recovery_audit": {"selector", "user_id", "case_reference", "issued_by_uid", "issued_at", "expires_at", "consumed_at", "revoked_at", "revoked_by_uid", "revocation_reference"}, - "experiment_barriers": {"transaction_id", "deadline_ns", "start_time_ns"}, - "experiment_readiness": {"debuglet_id", "metadata", "ready_at_ns"}, + "experiment_barriers": {"transaction_id", "deadline_ns", "start_time_ns"}, + "experiment_readiness": {"debuglet_id", "metadata", "ready_at_ns"}, "allocation_reclamations": {"debuglet_id", "reclaimed_at"}, "allowance_grants": {"id", "user_id", "amount", "granted_by", "reason", "idempotency_key", "granted_at"}, "probe_status": {"executor_id", "first_connected", "last_connected", "connected", "status_since", "total_uptime", "is_public", "host_tags", "version"}, diff --git a/pkg/client/experiment.go b/pkg/client/experiment.go index b7e1a360..fa534657 100644 --- a/pkg/client/experiment.go +++ b/pkg/client/experiment.go @@ -21,23 +21,23 @@ type ExperimentDefinition struct { // ExperimentRun binds a manifest entry to its admitted run. A definition omits // RunID; a submission receipt retains the input, digest and assigned run ID. type ExperimentRun struct { - OrderID int64 `json:"order_id"` - ExecutorID string `json:"executor_id"` - WASMPath string `json:"wasm_path"` - SHA256 string `json:"sha256"` - Args []string `json:"args"` - Policy Policy `json:"policy"` - RunID string `json:"run_id,omitempty"` + OrderID int64 `json:"order_id"` + ExecutorID string `json:"executor_id"` + WASMPath string `json:"wasm_path"` + SHA256 string `json:"sha256"` + Args []string `json:"args"` + Policy Policy `json:"policy"` + RunID string `json:"run_id,omitempty"` } type ExperimentSubmission struct { - ExperimentID string `json:"experiment_id"` + ExperimentID string `json:"experiment_id"` Participants []ExperimentRun `json:"participants"` } type ExperimentResults struct { Submission ExperimentSubmission `json:"submission"` - Results []Result `json:"results"` + Results []Result `json:"results"` } // SubmitExperimentTEST submits one existing TEST batch. Its transaction is the @@ -50,27 +50,39 @@ func (c *Client) SubmitExperimentTEST(ctx context.Context, def ExperimentDefinit requests := make([]Request, len(receipt.Participants)) for i := range receipt.Participants { p := &receipt.Participants[i] - if p.RunID != "" { return receipt, errors.New("client: definition already contains a run id") } + if p.RunID != "" { + return receipt, errors.New("client: definition already contains a run id") + } wasm, err := os.ReadFile(p.WASMPath) - if err != nil { return receipt, fmt.Errorf("participant %d: %w", p.OrderID, err) } + if err != nil { + return receipt, fmt.Errorf("participant %d: %w", p.OrderID, err) + } digest := sha256.Sum256(wasm) hash := hex.EncodeToString(digest[:]) - if p.SHA256 != "" && p.SHA256 != hash { return receipt, fmt.Errorf("participant %d: WASM SHA256 mismatch", p.OrderID) } + if p.SHA256 != "" && p.SHA256 != hash { + return receipt, fmt.Errorf("participant %d: WASM SHA256 mismatch", p.OrderID) + } p.SHA256 = hash p.Args = append([]string{}, p.Args...) p.Policy.Addresses = append([]string{}, p.Policy.Addresses...) requests[i] = Request{OrderID: p.OrderID, ExecutorID: p.ExecutorID, Wasm: wasm, Args: p.Args, Policy: p.Policy} } batch, err := Prepare(requests) - if err != nil { return receipt, err } + if err != nil { + return receipt, err + } submission, err := c.SubmitTEST(ctx, batch) if err != nil { var se *SubmissionError - if errors.As(err, &se) { submission = Submission{TransactionID: se.TransactionID, IDs: se.AdmittedIDs} } + if errors.As(err, &se) { + submission = Submission{TransactionID: se.TransactionID, IDs: se.AdmittedIDs} + } } receipt.ExperimentID = submission.TransactionID if len(submission.IDs) == len(receipt.Participants) { - for i, id := range submission.IDs { receipt.Participants[i].RunID = id } + for i, id := range submission.IDs { + receipt.Participants[i].RunID = id + } } return receipt, err } @@ -81,7 +93,9 @@ func (c *Client) ExportExperiment(ctx context.Context, receipt ExperimentSubmiss group := ExperimentResults{Submission: receipt, Results: []Result{}} for _, participant := range receipt.Participants { result, err := c.Export(ctx, participant.RunID) - if err != nil { return group, err } + if err != nil { + return group, err + } if result.ExecutorID != participant.ExecutorID || result.Provenance != nil && result.Provenance.WorkloadSHA256 != participant.SHA256 { return group, errors.New("client: experiment result disagrees with submission receipt") } @@ -95,7 +109,9 @@ func (c *Client) ExportExperiment(ctx context.Context, receipt ExperimentSubmiss func (c *Client) CancelExperiment(ctx context.Context, receipt ExperimentSubmission) error { var failures []error for _, participant := range receipt.Participants { - if participant.RunID == "" { continue } + if participant.RunID == "" { + continue + } if err := c.Cancel(ctx, participant.RunID, participant.ExecutorID); err != nil { failures = append(failures, fmt.Errorf("participant %d: %w", participant.OrderID, err)) } diff --git a/pkg/client/experiment_test.go b/pkg/client/experiment_test.go index ab2e343e..2eb0e9d2 100644 --- a/pkg/client/experiment_test.go +++ b/pkg/client/experiment_test.go @@ -18,10 +18,12 @@ import ( func experimentDefinition(t *testing.T) ExperimentDefinition { t.Helper() path := filepath.Join(t.TempDir(), "peer.wasm") - if err := os.WriteFile(path, []byte("test-wasm"), 0600); err != nil { t.Fatal(err) } + if err := os.WriteFile(path, []byte("test-wasm"), 0600); err != nil { + t.Fatal(err) + } return ExperimentDefinition{Participants: []ExperimentRun{ - {OrderID:42, ExecutorID:"node-a", WASMPath:path, Args:[]string{"a"}, Policy:Policy{TimeoutMS:1000}}, - {OrderID:7, ExecutorID:"node-b", WASMPath:path, Args:[]string{"b"}, Policy:Policy{TimeoutMS:2000}}, + {OrderID: 42, ExecutorID: "node-a", WASMPath: path, Args: []string{"a"}, Policy: Policy{TimeoutMS: 1000}}, + {OrderID: 7, ExecutorID: "node-b", WASMPath: path, Args: []string{"b"}, Policy: Policy{TimeoutMS: 2000}}, }} } @@ -33,12 +35,20 @@ func TestExperimentRetainsManifestOrderAndHashes(t *testing.T) { f.handle("PUT /debuglet", jsonHandler(http.StatusOK, string(body))) definition := experimentDefinition(t) receipt, err := f.client(t, Options{}).SubmitExperimentTEST(t.Context(), definition) - if err != nil { t.Fatal(err) } + if err != nil { + t.Fatal(err) + } digest := sha256.Sum256([]byte("test-wasm")) - if receipt.ExperimentID != fixtureTx || receipt.Participants[0].OrderID != 42 || receipt.Participants[1].OrderID != 7 || receipt.Participants[0].RunID != ids[0] || receipt.Participants[1].RunID != ids[1] || receipt.Participants[0].SHA256 != hex.EncodeToString(digest[:]) { t.Fatalf("receipt %+v", receipt) } + if receipt.ExperimentID != fixtureTx || receipt.Participants[0].OrderID != 42 || receipt.Participants[1].OrderID != 7 || receipt.Participants[0].RunID != ids[0] || receipt.Participants[1].RunID != ids[1] || receipt.Participants[0].SHA256 != hex.EncodeToString(digest[:]) { + t.Fatalf("receipt %+v", receipt) + } definition.Participants[0].Args[0] = "changed" - if receipt.Participants[0].Args[0] != "a" || definition.Participants[0].SHA256 != "" { t.Fatal("receipt aliases or mutates caller input") } - if len(f.requests()) != 2 { t.Fatal("submission made unexpected requests") } + if receipt.Participants[0].Args[0] != "a" || definition.Participants[0].SHA256 != "" { + t.Fatal("receipt aliases or mutates caller input") + } + if len(f.requests()) != 2 { + t.Fatal("submission made unexpected requests") + } } func TestExperimentRejectsChangedWASMBeforeSubmission(t *testing.T) { @@ -46,18 +56,22 @@ func TestExperimentRejectsChangedWASMBeforeSubmission(t *testing.T) { definition := experimentDefinition(t) definition.Participants[1].SHA256 = strings.Repeat("0", 64) _, err := f.client(t, Options{}).SubmitExperimentTEST(t.Context(), definition) - if err == nil || len(f.requests()) != 0 { t.Fatal("changed artifact reached dispatcher") } + if err == nil || len(f.requests()) != 0 { + t.Fatal("changed artifact reached dispatcher") + } } func TestExperimentPreservesUncertainAdmission(t *testing.T) { f := newFakeServer(t, "") f.handle("PUT /payment/intent", intentHandler(fixtureTx, "")) ids := []string{fixtureID, "00000000-0000-4000-8000-000000000002"} - body, _ := json.Marshal(map[string]any{"code":CodeInternal, "message":"failed", "admitted_ids":ids}) + body, _ := json.Marshal(map[string]any{"code": CodeInternal, "message": "failed", "admitted_ids": ids}) f.handle("PUT /debuglet", jsonHandler(http.StatusInternalServerError, string(body))) receipt, err := f.client(t, Options{}).SubmitExperimentTEST(t.Context(), experimentDefinition(t)) se := asSubmissionError(t, err) - if !se.OutcomeUnknown || receipt.ExperimentID != fixtureTx || !reflect.DeepEqual([]string{receipt.Participants[0].RunID, receipt.Participants[1].RunID}, ids) { t.Fatalf("receipt %+v error %v", receipt, err) } + if !se.OutcomeUnknown || receipt.ExperimentID != fixtureTx || !reflect.DeepEqual([]string{receipt.Participants[0].RunID, receipt.Participants[1].RunID}, ids) { + t.Fatalf("receipt %+v error %v", receipt, err) + } } func TestExperimentExportRejectsWrongExecutor(t *testing.T) { @@ -65,7 +79,9 @@ func TestExperimentExportRejectsWrongExecutor(t *testing.T) { doc := resultFixture(t) body, _ := json.Marshal(doc) f.handle("GET /debuglet/"+doc.RunID+"/result", jsonHandler(http.StatusOK, string(body))) - receipt := ExperimentSubmission{ExperimentID:fixtureTx, Participants:[]ExperimentRun{{RunID:doc.RunID, ExecutorID:"wrong-executor"}}} + receipt := ExperimentSubmission{ExperimentID: fixtureTx, Participants: []ExperimentRun{{RunID: doc.RunID, ExecutorID: "wrong-executor"}}} result, err := f.client(t, Options{}).ExportExperiment(t.Context(), receipt) - if err == nil || len(result.Results) != 0 { t.Fatal("mismatched result was grouped") } + if err == nil || len(result.Results) != 0 { + t.Fatal("mismatched result was grouped") + } } diff --git a/pkg/debuglet/experiment.go b/pkg/debuglet/experiment.go index 5aea8471..d27044fa 100644 --- a/pkg/debuglet/experiment.go +++ b/pkg/debuglet/experiment.go @@ -4,13 +4,13 @@ package debuglet import ( - "context" - "encoding/json" - "errors" - "fmt" - "time" + "context" + "encoding/json" + "errors" + "fmt" + "time" - "github.com/netsec-ethz/debuglet/pkg/wire" + "github.com/netsec-ethz/debuglet/pkg/wire" ) // MaxExperimentMetadata is the opaque metadata allowance for one participant. @@ -27,47 +27,72 @@ var ErrExperimentUnavailable = errors.New("debuglet: experiment readiness unavai // Waiting is limited to 30 seconds, the caller's deadline and the run's lifetime. // Cancellation is checked between bounded host calls (at most one second each). func Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { - var result wire.Experiment - if len(metadata) > MaxExperimentMetadata { return result, ErrTooLarge } - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - buf := make([]byte, experimentBufferBytes) - for { - if err := ctx.Err(); err != nil { return result, err } - deadline, _ := ctx.Deadline() - n := experimentReady(metadata, buf, deadline.UnixNano()) - if err := ctx.Err(); err != nil { return result, err } - switch n { - case -2: return result, context.Canceled - case -3: return result, context.DeadlineExceeded - case -4: return result, ErrTooLarge - } - if n < 0 { return result, ErrExperimentUnavailable } - if int(n) > len(buf) { return result, fmt.Errorf("debuglet: invalid experiment response length") } - if err := json.Unmarshal(buf[:n], &result); err != nil { return result, fmt.Errorf("debuglet: invalid experiment response: %w", err) } - if result.StartTimeNS != 0 { - if result.StartTimeNS <= time.Now().UnixNano() { return result, ErrExperimentLate } - return result, nil - } - timer := time.NewTimer(100*time.Millisecond) - select { - case <-ctx.Done(): timer.Stop(); return result, ctx.Err() - case <-timer.C: - } - } + var result wire.Experiment + if len(metadata) > MaxExperimentMetadata { + return result, ErrTooLarge + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + buf := make([]byte, experimentBufferBytes) + for { + if err := ctx.Err(); err != nil { + return result, err + } + deadline, _ := ctx.Deadline() + n := experimentReady(metadata, buf, deadline.UnixNano()) + if err := ctx.Err(); err != nil { + return result, err + } + switch n { + case -2: + return result, context.Canceled + case -3: + return result, context.DeadlineExceeded + case -4: + return result, ErrTooLarge + } + if n < 0 { + return result, ErrExperimentUnavailable + } + if int(n) > len(buf) { + return result, fmt.Errorf("debuglet: invalid experiment response length") + } + if err := json.Unmarshal(buf[:n], &result); err != nil { + return result, fmt.Errorf("debuglet: invalid experiment response: %w", err) + } + if result.StartTimeNS != 0 { + if result.StartTimeNS <= time.Now().UnixNano() { + return result, ErrExperimentLate + } + return result, nil + } + timer := time.NewTimer(100 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return result, ctx.Err() + case <-timer.C: + } + } } // WaitStart waits for the agreed wall-clock time. It rejects a start already in // the past; callers should record their observed start themselves. This helper // cannot guarantee synchronized clocks or simultaneous execution across hosts. func WaitStart(ctx context.Context, experiment wire.Experiment) error { - if err := ctx.Err(); err != nil { return err } - delay := time.Until(time.Unix(0, experiment.StartTimeNS)) - if experiment.StartTimeNS <= 0 || delay <= 0 { return ErrExperimentLate } - timer := time.NewTimer(delay) - defer timer.Stop() - select { - case <-ctx.Done(): return ctx.Err() - case <-timer.C: return ctx.Err() - } + if err := ctx.Err(); err != nil { + return err + } + delay := time.Until(time.Unix(0, experiment.StartTimeNS)) + if experiment.StartTimeNS <= 0 || delay <= 0 { + return ErrExperimentLate + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return ctx.Err() + } } diff --git a/pkg/debuglet/experiment_test.go b/pkg/debuglet/experiment_test.go index eb934ae6..c721c3b9 100644 --- a/pkg/debuglet/experiment_test.go +++ b/pkg/debuglet/experiment_test.go @@ -4,53 +4,76 @@ package debuglet_test import ( - "context" - "errors" - "fmt" - "sync/atomic" - "testing" - "time" + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" - "github.com/netsec-ethz/debuglet/pkg/debuglet" - "github.com/netsec-ethz/debuglet/pkg/wire" + "github.com/netsec-ethz/debuglet/pkg/debuglet" + "github.com/netsec-ethz/debuglet/pkg/wire" ) type experimentControl func(context.Context, []byte) (wire.Experiment, error) -func (f experimentControl) Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { return f(ctx, metadata) } + +func (f experimentControl) Ready(ctx context.Context, metadata []byte) (wire.Experiment, error) { + return f(ctx, metadata) +} func TestExperimentGuest(t *testing.T) { - module := buildGuest(t, "./pkg/debuglet/testdata/experiment") - t.Run("poll metadata and wait", func(t *testing.T) { - var calls atomic.Int32 - var target atomic.Int64 - control := experimentControl(func(ctx context.Context, metadata []byte) (wire.Experiment, error) { - if string(metadata) != "guest endpoint" { return wire.Experiment{}, fmt.Errorf("wrong metadata %q", metadata) } - if calls.Add(1) == 1 { return wire.Experiment{ID:"batch"}, nil } - target.Store(time.Now().Add(400*time.Millisecond).UnixNano()) - return wire.Experiment{ID:"batch", StartTimeNS:target.Load(), Participants:[]wire.ExperimentParticipant{{ID:"peer", ExecutorID:"executor", Metadata:[]byte("peer endpoint")}}}, nil - }) - g := runGuest(t, module, hostOptions{experiment:control,args:[]string{"ready"}}) - requireSuccess(t,g) - requireContains(t,g,"experiment=batch members=1 peer=peer endpoint", "wait=") - var observed, requested int64 - line := g.waitFor("wait=",0) - if _, err := fmt.Sscanf(line,"wait= observed=%d target=%d", &observed, &requested); err != nil { t.Fatal(err) } - if calls.Load() != 2 || requested != target.Load() || observed < requested { t.Fatalf("calls=%d observed=%d target=%d",calls.Load(),observed,requested) } - }) - t.Run("deadline", func(t *testing.T) { - control := experimentControl(func(ctx context.Context, _ []byte) (wire.Experiment,error) { <-ctx.Done(); return wire.Experiment{},ctx.Err() }) - g := runGuest(t,module,hostOptions{experiment:control,args:[]string{"deadline"}}) - requireSuccess(t,g); requireContains(t,g,"deadline=true late=false") - }) - t.Run("late", func(t *testing.T) { - control := experimentControl(func(context.Context, []byte) (wire.Experiment,error) { return wire.Experiment{StartTimeNS:time.Now().Add(-time.Second).UnixNano()},nil }) - g := runGuest(t,module,hostOptions{experiment:control,args:[]string{"late"}}) - requireSuccess(t,g); requireContains(t,g,"deadline=false late=true") - }) + module := buildGuest(t, "./pkg/debuglet/testdata/experiment") + t.Run("poll metadata and wait", func(t *testing.T) { + var calls atomic.Int32 + var target atomic.Int64 + control := experimentControl(func(ctx context.Context, metadata []byte) (wire.Experiment, error) { + if string(metadata) != "guest endpoint" { + return wire.Experiment{}, fmt.Errorf("wrong metadata %q", metadata) + } + if calls.Add(1) == 1 { + return wire.Experiment{ID: "batch"}, nil + } + target.Store(time.Now().Add(400 * time.Millisecond).UnixNano()) + return wire.Experiment{ID: "batch", StartTimeNS: target.Load(), Participants: []wire.ExperimentParticipant{{ID: "peer", ExecutorID: "executor", Metadata: []byte("peer endpoint")}}}, nil + }) + g := runGuest(t, module, hostOptions{experiment: control, args: []string{"ready"}}) + requireSuccess(t, g) + requireContains(t, g, "experiment=batch members=1 peer=peer endpoint", "wait=") + var observed, requested int64 + line := g.waitFor("wait=", 0) + if _, err := fmt.Sscanf(line, "wait= observed=%d target=%d", &observed, &requested); err != nil { + t.Fatal(err) + } + if calls.Load() != 2 || requested != target.Load() || observed < requested { + t.Fatalf("calls=%d observed=%d target=%d", calls.Load(), observed, requested) + } + }) + t.Run("deadline", func(t *testing.T) { + control := experimentControl(func(ctx context.Context, _ []byte) (wire.Experiment, error) { + <-ctx.Done() + return wire.Experiment{}, ctx.Err() + }) + g := runGuest(t, module, hostOptions{experiment: control, args: []string{"deadline"}}) + requireSuccess(t, g) + requireContains(t, g, "deadline=true late=false") + }) + t.Run("late", func(t *testing.T) { + control := experimentControl(func(context.Context, []byte) (wire.Experiment, error) { + return wire.Experiment{StartTimeNS: time.Now().Add(-time.Second).UnixNano()}, nil + }) + g := runGuest(t, module, hostOptions{experiment: control, args: []string{"late"}}) + requireSuccess(t, g) + requireContains(t, g, "deadline=false late=true") + }) } func TestWaitStartCancellationAndLate(t *testing.T) { - if err := debuglet.WaitStart(context.Background(),wire.Experiment{StartTimeNS:time.Now().Add(-time.Second).UnixNano()}); !errors.Is(err,debuglet.ErrExperimentLate) { t.Fatal(err) } - ctx,cancel := context.WithCancel(context.Background()); cancel() - if err := debuglet.WaitStart(ctx,wire.Experiment{StartTimeNS:time.Now().Add(time.Hour).UnixNano()}); !errors.Is(err,context.Canceled) { t.Fatal(err) } + if err := debuglet.WaitStart(context.Background(), wire.Experiment{StartTimeNS: time.Now().Add(-time.Second).UnixNano()}); !errors.Is(err, debuglet.ErrExperimentLate) { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := debuglet.WaitStart(ctx, wire.Experiment{StartTimeNS: time.Now().Add(time.Hour).UnixNano()}); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } } diff --git a/pkg/debuglet/experiment_wasip1.go b/pkg/debuglet/experiment_wasip1.go index 3b75223e..d1d48a07 100644 --- a/pkg/debuglet/experiment_wasip1.go +++ b/pkg/debuglet/experiment_wasip1.go @@ -9,7 +9,9 @@ package debuglet func experimentReadyHost(metadataPtr, metadataLen, resultPtr, resultLen uint32, deadlineNS int64) int32 func experimentReady(metadata, result []byte, deadlineNS int64) int32 { - var ptr uint32 - if len(metadata) > 0 { ptr = bytePtr(metadata) } - return experimentReadyHost(ptr, uint32(len(metadata)), bytePtr(result), uint32(len(result)), deadlineNS) + var ptr uint32 + if len(metadata) > 0 { + ptr = bytePtr(metadata) + } + return experimentReadyHost(ptr, uint32(len(metadata)), bytePtr(result), uint32(len(result)), deadlineNS) } diff --git a/pkg/debuglet/hostengine_test.go b/pkg/debuglet/hostengine_test.go index 15cda5c5..a3ddaffc 100644 --- a/pkg/debuglet/hostengine_test.go +++ b/pkg/debuglet/hostengine_test.go @@ -40,7 +40,7 @@ import ( hostdebuglet "github.com/netsec-ethz/debuglet/internal/executor/debuglet" "github.com/netsec-ethz/debuglet/internal/executor/debuglet/netpolicy" "github.com/netsec-ethz/debuglet/internal/executor/debuglet/socket" - "github.com/netsec-ethz/debuglet/internal/executor/debuglet/wasm" + "github.com/netsec-ethz/debuglet/internal/executor/debuglet/wasm" "github.com/netsec-ethz/debuglet/internal/executor/ratelimit/app" "github.com/netsec-ethz/debuglet/internal/executor/ratelimit/fallback" "github.com/netsec-ethz/debuglet/internal/executor/scheduler" @@ -118,12 +118,12 @@ func buildGuest(t *testing.T, pkgPath string) []byte { // hostOptions describes one job: its destination policy, its listeners, its // guest arguments and its execution budget. type hostOptions struct { - experiment wasm.ExperimentControl - addresses []string - listenTCP bool - listenUDP bool - args []string - budget time.Duration + experiment wasm.ExperimentControl + addresses []string + listenTCP bool + listenUDP bool + args []string + budget time.Duration // operator overrides the executor's network policy. Nil runs the job // under the documented default policy, which is what an executor that // configures nothing applies. @@ -226,7 +226,7 @@ func startGuest(t *testing.T, wasm []byte, opts hostOptions) *guestRun { } deb := hostdebuglet.New(logger, id, "guest-compatibility", policy, operator, schedule, limiter, packetCount, nil, ports, socket.NewBudget(socket.DefaultLimits(), socket.NewDescriptorBudget(socket.DefaultNodeDescriptors))) deb.SetExperimentControl(opts.experiment) - initCtx, cancelInit := context.WithTimeout(context.Background(), initTimeout) + initCtx, cancelInit := context.WithTimeout(context.Background(), initTimeout) g := &guestRun{t: t, deb: deb, cancelInit: cancelInit, errCh: make(chan error, 1)} t.Cleanup(g.stop) diff --git a/pkg/debuglet/testdata/experiment/main.go b/pkg/debuglet/testdata/experiment/main.go index 0dbd572c..08e81266 100644 --- a/pkg/debuglet/testdata/experiment/main.go +++ b/pkg/debuglet/testdata/experiment/main.go @@ -4,22 +4,28 @@ package main import ( - "context" - "errors" - "fmt" - "os" - "time" + "context" + "errors" + "fmt" + "os" + "time" - "github.com/netsec-ethz/debuglet/pkg/debuglet" + "github.com/netsec-ethz/debuglet/pkg/debuglet" ) func main() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if os.Args[0] == "deadline" { ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond); defer cancel() } - exp, err := debuglet.Ready(ctx, []byte("guest endpoint")) - if err != nil { fmt.Printf("deadline=%v late=%v\n", errors.Is(err, context.DeadlineExceeded), errors.Is(err, debuglet.ErrExperimentLate)); return } - fmt.Printf("experiment=%s members=%d peer=%s\n", exp.ID, len(exp.Participants), exp.Participants[0].Metadata) - err = debuglet.WaitStart(ctx, exp) - fmt.Printf("wait=%v observed=%d target=%d\n", err, time.Now().UnixNano(), exp.StartTimeNS) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if os.Args[0] == "deadline" { + ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + } + exp, err := debuglet.Ready(ctx, []byte("guest endpoint")) + if err != nil { + fmt.Printf("deadline=%v late=%v\n", errors.Is(err, context.DeadlineExceeded), errors.Is(err, debuglet.ErrExperimentLate)) + return + } + fmt.Printf("experiment=%s members=%d peer=%s\n", exp.ID, len(exp.Participants), exp.Participants[0].Metadata) + err = debuglet.WaitStart(ctx, exp) + fmt.Printf("wait=%v observed=%d target=%d\n", err, time.Now().UnixNano(), exp.StartTimeNS) } From b50fbc3c20b09f8f6817e216d5dfff9af1751221 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:06:46 +0200 Subject: [PATCH 11/17] Correct readiness rejection assertion and installed fixture lifetime --- internal/acceptance/roles/rendezvous_integration_test.go | 2 +- internal/dispatcher/transport/rpc/experiment_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/acceptance/roles/rendezvous_integration_test.go b/internal/acceptance/roles/rendezvous_integration_test.go index 9cbbf426..e86fcb71 100644 --- a/internal/acceptance/roles/rendezvous_integration_test.go +++ b/internal/acceptance/roles/rendezvous_integration_test.go @@ -46,7 +46,7 @@ func installedRendezvous(t *testing.T) (context.Context, demo.Assets, string, in t.Fatal(err) } ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) - defer cancel() + t.Cleanup(cancel) var children []*demo.Child var logs []*os.File t.Cleanup(func() { diff --git a/internal/dispatcher/transport/rpc/experiment_test.go b/internal/dispatcher/transport/rpc/experiment_test.go index bae269ed..2cb40ba8 100644 --- a/internal/dispatcher/transport/rpc/experiment_test.go +++ b/internal/dispatcher/transport/rpc/experiment_test.go @@ -35,7 +35,7 @@ func TestExperimentReadyUsesBoundAuthenticatedSession(t *testing.T) { {"owner", "executor", false, codes.OK}, {"foreign executor", "foreign", false, codes.PermissionDenied}, {"empty executor", "", false, codes.PermissionDenied}, - {"missing credentials", "executor", true, codes.PermissionDenied}, + {"missing credentials", "executor", true, codes.FailedPrecondition}, } { t.Run(tc.name, func(t *testing.T) { err := f.directCall(f.client, owner, func(ctx context.Context, client pb.DispatcherServiceClient) error { From 42d5e510a0c668bcb5c96765764938594fa80c05 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:08:26 +0200 Subject: [PATCH 12/17] Wait for committed output before checking experiment results --- internal/acceptance/roles/experiment_integration_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/acceptance/roles/experiment_integration_test.go b/internal/acceptance/roles/experiment_integration_test.go index ae2b1551..d83b9f8b 100644 --- a/internal/acceptance/roles/experiment_integration_test.go +++ b/internal/acceptance/roles/experiment_integration_test.go @@ -104,7 +104,10 @@ func TestInstalledExperiment(t *testing.T) { if len(output) > 64<<10 { t.Fatal("excessive guest output") } - if page.State == client.StateExited && !page.HasMore { + if page.Output.State == "truncated" { + t.Fatalf("participant %s output truncated: %+v", id, page.Output) + } + if page.State == client.StateExited && !page.HasMore && page.Output.State == "complete" { break } select { From 019da05f92b030d5e35228f67ead79fd70e55252 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:09:16 +0200 Subject: [PATCH 13/17] Document coordinated measurements and their runtime requirements --- docs/README.md | 1 + docs/debuglets.md | 5 +++++ docs/measurements.md | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/docs/README.md b/docs/README.md index e20c25fe..080a5997 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,7 @@ This directory is the versioned source for the [Debuglet documentation site](htt - [CLI reference](cli.md) — commands, local demo, and scriptable workflows. - [Go client library](client.md) — submit debuglets and read results from an application. +- [Reusable and coordinated measurements](measurements.md) — saved profiles, retries and experiments across executors. - [Portable results](results.md) — export retained output and admission facts for offline analysis. - [Controlled latency experiment](research/latency-evaluation.md) — compare native and WASM probes under known local network faults. - [Write a debuglet](debuglets.md) — execution model and Go authoring interface. diff --git a/docs/debuglets.md b/docs/debuglets.md index f3bab4c4..26161559 100644 --- a/docs/debuglets.md +++ b/docs/debuglets.md @@ -68,6 +68,11 @@ ABI-v1 guests remain supported without recompilation. The baseline ABI label alone does not advertise this extension; use a release that includes it. The [extension contract](development/guest-io.md) records its wire signatures. +Coordinated guests using `Ready` require the optional +`debuglet_experiment_v1.ready` extension on the executor and matching dispatcher +support. See [coordinated measurements](measurements.md#coordinate-a-batch-of-debuglets) +and its five-executor example. Existing guests do not acquire this requirement. + ### Other languages Any WebAssembly module that targets `wasm32-wasip1` and imports only the diff --git a/docs/measurements.md b/docs/measurements.md index c55a7a29..92d40b70 100644 --- a/docs/measurements.md +++ b/docs/measurements.md @@ -107,3 +107,23 @@ never replays an ambiguous submission. On success, failure, deadline or caller cancellation it requests cancellation of every known run, then inspects cleanup under an independent ten-second bound. `confirmed_absent` records a current executor observation; `unconfirmed` requires follow-up using the returned IDs. + +## Coordinate a batch of debuglets + +An experiment submits a fixed set of ordinary runs together. Choose each run's +executor, WASM, arguments and policy independently. The existing transaction ID +groups their results. `Client.SubmitExperimentTEST` accepts a saved definition +and returns a receipt with run IDs and program hashes; `ExportExperiment` reads +the group's results and `CancelExperiment` requests cancellation of its known runs. + +Each guest finishes its setup, then calls `debuglet.Ready(ctx, metadata)`. Once +all batch members have declared readiness, it receives their bounded metadata +and a common future start time. Call `debuglet.WaitStart(ctx, experiment)` before +starting the measurement. Hosts need synchronized clocks for close timing; +this is not a guarantee of simultaneous execution. Experiment traffic uses the +normal permitted sockets, while readiness travels through the dispatcher. + +The [five-executor example](../examples/experiments/README.md) includes a manifest, +runner and UDP guest. It requires the new dispatcher and executor readiness +extension; it does not add a console flow. Missing members, retries and partial +results remain decisions for the experiment's author, within the run deadlines. From b2b5b4b15773a0d596bf4ed03101000805b92baf Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:09:27 +0200 Subject: [PATCH 14/17] Verify experiment receipt membership before export or cancellation --- pkg/client/experiment.go | 27 +++++++++++++++++++--- pkg/client/experiment_test.go | 42 +++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/pkg/client/experiment.go b/pkg/client/experiment.go index fa534657..d909fe72 100644 --- a/pkg/client/experiment.go +++ b/pkg/client/experiment.go @@ -91,14 +91,14 @@ func (c *Client) SubmitExperimentTEST(ctx context.Context, def ExperimentDefinit // not wait for completion. On error it returns the successfully read prefix. func (c *Client) ExportExperiment(ctx context.Context, receipt ExperimentSubmission) (ExperimentResults, error) { group := ExperimentResults{Submission: receipt, Results: []Result{}} + if err := c.checkExperimentReceipt(ctx, receipt); err != nil { + return group, err + } for _, participant := range receipt.Participants { result, err := c.Export(ctx, participant.RunID) if err != nil { return group, err } - if result.ExecutorID != participant.ExecutorID || result.Provenance != nil && result.Provenance.WorkloadSHA256 != participant.SHA256 { - return group, errors.New("client: experiment result disagrees with submission receipt") - } group.Results = append(group.Results, result) } return group, nil @@ -107,6 +107,9 @@ func (c *Client) ExportExperiment(ctx context.Context, receipt ExperimentSubmiss // CancelExperiment requests cancellation for all known runs. An acknowledgement // is not proof that execution stopped; inspect their existing result endpoints. func (c *Client) CancelExperiment(ctx context.Context, receipt ExperimentSubmission) error { + if err := c.checkExperimentReceipt(ctx, receipt); err != nil { + return err + } var failures []error for _, participant := range receipt.Participants { if participant.RunID == "" { @@ -118,3 +121,21 @@ func (c *Client) CancelExperiment(ctx context.Context, receipt ExperimentSubmiss } return errors.Join(failures...) } + +// Check every known run before returning grouped output or cancelling any run. +// Missing IDs remain unknown after uncertain submission and cannot be acted on. +func (c *Client) checkExperimentReceipt(ctx context.Context, receipt ExperimentSubmission) error { + for _, participant := range receipt.Participants { + if participant.RunID == "" { + continue + } + detail, err := c.RunDetail(ctx, participant.RunID) + if err != nil { + return err + } + if receipt.ExperimentID == "" || detail.BatchID != receipt.ExperimentID || detail.OrderID != participant.OrderID || detail.ExecutorID != participant.ExecutorID || detail.Provenance != nil && detail.Provenance.WorkloadSHA256 != participant.SHA256 { + return errors.New("client: experiment receipt disagrees with admitted membership") + } + } + return nil +} diff --git a/pkg/client/experiment_test.go b/pkg/client/experiment_test.go index 2eb0e9d2..c24f273b 100644 --- a/pkg/client/experiment_test.go +++ b/pkg/client/experiment_test.go @@ -77,11 +77,49 @@ func TestExperimentPreservesUncertainAdmission(t *testing.T) { func TestExperimentExportRejectsWrongExecutor(t *testing.T) { f := newFakeServer(t, "") doc := resultFixture(t) - body, _ := json.Marshal(doc) - f.handle("GET /debuglet/"+doc.RunID+"/result", jsonHandler(http.StatusOK, string(body))) + body, _ := json.Marshal(RunDetail{RunID: doc.RunID, ExecutorID: doc.ExecutorID, BatchID: fixtureTx}) + f.handle("GET /debuglet/"+doc.RunID+"/detail", jsonHandler(http.StatusOK, string(body))) receipt := ExperimentSubmission{ExperimentID: fixtureTx, Participants: []ExperimentRun{{RunID: doc.RunID, ExecutorID: "wrong-executor"}}} result, err := f.client(t, Options{}).ExportExperiment(t.Context(), receipt) if err == nil || len(result.Results) != 0 { t.Fatal("mismatched result was grouped") } } + +func TestExperimentActionsRejectWrongBatchBeforeReadingOrCancelling(t *testing.T) { + for _, action := range []string{"export", "cancel"} { + t.Run(action, func(t *testing.T) { + f := newFakeServer(t, "") + doc := resultFixture(t) + receipt := ExperimentSubmission{ExperimentID: fixtureTx, Participants: []ExperimentRun{ + {RunID: doc.RunID, OrderID: 42, ExecutorID: doc.ExecutorID}, + {RunID: "00000000-0000-4000-8000-000000000002", OrderID: 7, ExecutorID: doc.ExecutorID}, + }} + // The first run matches. The later run belongs to another batch, + // so even cancellation of the first run must not begin. + for i, participant := range receipt.Participants { + batch := fixtureTx + if i == 1 { + batch = "another-transaction" + } + detail := RunDetail{RunID: participant.RunID, ExecutorID: participant.ExecutorID, BatchID: batch, OrderID: participant.OrderID} + body, _ := json.Marshal(detail) + f.handle("GET /debuglet/"+participant.RunID+"/detail", jsonHandler(http.StatusOK, string(body))) + } + c := f.client(t, Options{}) + var err error + if action == "export" { + var group ExperimentResults + group, err = c.ExportExperiment(t.Context(), receipt) + if len(group.Results) != 0 { + t.Fatal("mismatched batch returned grouped results") + } + } else { + err = c.CancelExperiment(t.Context(), receipt) + } + if err == nil || len(f.requests()) != 2 { + t.Fatalf("action=%s error=%v requests=%d", action, err, len(f.requests())) + } + }) + } +} From 8c181a6aeff5b50e7d7120eb08092d49a69088c7 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:10:10 +0200 Subject: [PATCH 15/17] Bound experiment membership before loading workload files --- pkg/client/experiment.go | 5 +++++ pkg/client/experiment_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/client/experiment.go b/pkg/client/experiment.go index d909fe72..e2ca3bca 100644 --- a/pkg/client/experiment.go +++ b/pkg/client/experiment.go @@ -10,6 +10,8 @@ import ( "errors" "fmt" "os" + + "github.com/netsec-ethz/debuglet/pkg/wire" ) // ExperimentDefinition is a reproducible batch. WASM paths are local to the @@ -46,6 +48,9 @@ type ExperimentResults struct { // the returned receipt preserves known identities; the error's outcome-unknown // classification still applies. Never retry an uncertain submission blindly. func (c *Client) SubmitExperimentTEST(ctx context.Context, def ExperimentDefinition) (ExperimentSubmission, error) { + if len(def.Participants) > wire.MaxExperimentParticipants { + return ExperimentSubmission{}, fmt.Errorf("client: experiment exceeds %d participants", wire.MaxExperimentParticipants) + } receipt := ExperimentSubmission{Participants: append([]ExperimentRun(nil), def.Participants...)} requests := make([]Request, len(receipt.Participants)) for i := range receipt.Participants { diff --git a/pkg/client/experiment_test.go b/pkg/client/experiment_test.go index c24f273b..130aacd6 100644 --- a/pkg/client/experiment_test.go +++ b/pkg/client/experiment_test.go @@ -13,6 +13,8 @@ import ( "reflect" "strings" "testing" + + "github.com/netsec-ethz/debuglet/pkg/wire" ) func experimentDefinition(t *testing.T) ExperimentDefinition { @@ -123,3 +125,13 @@ func TestExperimentActionsRejectWrongBatchBeforeReadingOrCancelling(t *testing.T }) } } + +func TestExperimentRejectsOversizedMembershipBeforeReadingFiles(t *testing.T) { + f := newFakeServer(t, "") + definition := ExperimentDefinition{Participants: make([]ExperimentRun, wire.MaxExperimentParticipants+1)} + // Every path is empty: a file read would fail with a different error. + _, err := f.client(t, Options{}).SubmitExperimentTEST(t.Context(), definition) + if err == nil || !strings.Contains(err.Error(), "exceeds 128 participants") || len(f.requests()) != 0 { + t.Fatalf("error=%v requests=%d", err, len(f.requests())) + } +} From 20bcd0079ad4e5810c512e949db07cebfd000667 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:11:37 +0200 Subject: [PATCH 16/17] Exercise the manifest submission and grouped export in the installed rehearsal --- .../roles/experiment_integration_test.go | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/internal/acceptance/roles/experiment_integration_test.go b/internal/acceptance/roles/experiment_integration_test.go index d83b9f8b..d1a33747 100644 --- a/internal/acceptance/roles/experiment_integration_test.go +++ b/internal/acceptance/roles/experiment_integration_test.go @@ -26,10 +26,7 @@ import ( // executor processes and checks the retained output of every admitted run. func TestInstalledExperiment(t *testing.T) { ctx, assets, work, launch := installedRendezvous(t) - wasm, err := os.ReadFile(os.Getenv("DEBUGLET_EXPERIMENT_WASM")) - if err != nil { - t.Fatal(err) - } + wasmPath := os.Getenv("DEBUGLET_EXPERIMENT_WASM") d := launch("dispatcher", assets.Dispatcher, storagecheck.Dispatcher, dispatcherconfig.DispatcherConfig{ Admission: dispatcherconfig.DefaultAdmissionConfig(), Attribution: dispatcherconfig.DefaultAttributionConfig(), Server: dispatcherconfig.ServerConfig{BindHost: "127.0.0.1", LocalDevelopment: true, Version: assets.Manifest.Version}, @@ -38,8 +35,8 @@ func TestInstalledExperiment(t *testing.T) { Database: dispatcherconfig.DatabaseConfig{Path: filepath.Join(work, "dispatcher", "state.sqlite")}, Logging: dispatcherconfig.LoggingConfig{LogLevel: "info", JSONLogs: true}, }) c := sdk(t, "http://"+d.HTTPAddr) - requests := make([]client.Request, 5) - for i := range requests { + participants := make([]client.ExperimentRun, 5) + for i := range participants { id, name := uuid.NewString(), fmt.Sprintf("executor-%d", i) conn, err := net.ListenPacket("udp", "127.0.0.1:0") if err != nil { @@ -55,7 +52,7 @@ func TestInstalledExperiment(t *testing.T) { Network: executorconfig.NetworkConfig{PublicHost: "127.0.0.1", PublicPorts: strconv.Itoa(port), PacketCounter: "fallback", DisableSCIONEnvironment: true, Policy: executorconfig.PolicyConfig{LocalTargets: &local}}, Database: executorconfig.DatabaseConfig{Path: filepath.Join(work, name, "state.sqlite")}, Logging: executorconfig.LoggingConfig{LogLevel: "info", JSONLogs: true}, Pricing: executorconfig.PricingConfig{Currency: "TEST", PricePerBwS: 1}, }) - requests[i] = client.Request{OrderID: int64(i), ExecutorID: id, Wasm: wasm, Args: []string{strconv.Itoa(i)}, Policy: client.Policy{FloorBW: 100_000, CeilBW: 200_000, TimeoutMS: 30_000, Addresses: []string{"127.0.0.1"}, ListenUDP: true}} + participants[i] = client.ExperimentRun{OrderID: int64(i), ExecutorID: id, WASMPath: wasmPath, Args: []string{strconv.Itoa(i)}, Policy: client.Policy{FloorBW: 100_000, CeilBW: 200_000, TimeoutMS: 30_000, Addresses: []string{"127.0.0.1"}, ListenUDP: true}} } for { nodes, err := c.Nodes(ctx) @@ -68,7 +65,7 @@ func TestInstalledExperiment(t *testing.T) { ready++ } } - if ready == len(requests) { + if ready == len(participants) { break } select { @@ -77,16 +74,13 @@ func TestInstalledExperiment(t *testing.T) { case <-time.After(50 * time.Millisecond): } } - batch, err := client.Prepare(requests) - if err != nil { - t.Fatal(err) - } - submission, err := c.SubmitTEST(ctx, batch) + submission, err := c.SubmitExperimentTEST(ctx, client.ExperimentDefinition{Participants: participants}) if err != nil { t.Fatal(err) } var sharedStart int64 - for _, id := range submission.IDs { + for _, participant := range submission.Participants { + id := participant.RunID var output []byte var after int64 for { @@ -125,7 +119,7 @@ func TestInstalledExperiment(t *testing.T) { if err := json.Unmarshal(output, &result); err != nil { t.Fatalf("participant %s output=%s: %v", id, output, err) } - if result.ExperimentID != submission.TransactionID || result.StartTimeNS == 0 || result.Sent != 4 || result.Received != 4 { + if result.ExperimentID != submission.ExperimentID || result.StartTimeNS == 0 || result.Sent != 4 || result.Received != 4 { t.Fatalf("participant %s: %+v", id, result) } if sharedStart != 0 && sharedStart != result.StartTimeNS { @@ -134,4 +128,8 @@ func TestInstalledExperiment(t *testing.T) { sharedStart = result.StartTimeNS t.Logf("run=%s group=%s start=%d sent=%d received=%d", id, result.ExperimentID, result.StartTimeNS, result.Sent, result.Received) } + group, err := c.ExportExperiment(ctx, submission) + if err != nil || len(group.Results) != len(participants) { + t.Fatalf("grouped export: %d results, %v", len(group.Results), err) + } } From 4229e61eb83204929db4e4624bfa570183b305da Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 10:18:46 +0200 Subject: [PATCH 17/17] Bound cancellation test synchronization and retain observed experiment timing --- .../roles/experiment_integration_test.go | 2 +- internal/executor/experiment_test.go | 37 ++++++++++++++++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/internal/acceptance/roles/experiment_integration_test.go b/internal/acceptance/roles/experiment_integration_test.go index d1a33747..268c41a4 100644 --- a/internal/acceptance/roles/experiment_integration_test.go +++ b/internal/acceptance/roles/experiment_integration_test.go @@ -126,7 +126,7 @@ func TestInstalledExperiment(t *testing.T) { t.Fatal("participants received different start times") } sharedStart = result.StartTimeNS - t.Logf("run=%s group=%s start=%d sent=%d received=%d", id, result.ExperimentID, result.StartTimeNS, result.Sent, result.Received) + t.Logf("participant result: %s", output) } group, err := c.ExportExperiment(ctx, submission) if err != nil || len(group.Results) != len(participants) { diff --git a/internal/executor/experiment_test.go b/internal/executor/experiment_test.go index 19a039d5..9ddfdede 100644 --- a/internal/executor/experiment_test.go +++ b/internal/executor/experiment_test.go @@ -5,6 +5,7 @@ package executor import ( "context" + "errors" "testing" "time" @@ -45,10 +46,36 @@ func TestExperimentRunBindingAndCancellation(t *testing.T) { if req.DebugletId != id.String() || req.ExecutorId != e.cfg.Identity.ExecutorID || result.ID != "batch" || len(result.Participants) != 1 || result.Participants[0].ReadyAtNS != 123 || string(result.Participants[0].Metadata) != "opaque" { t.Fatalf("incorrect request/result: %v %v", req, result) } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) - defer cancel() - if _, err := control.Ready(ctx, []byte("wait")); status.Code(err) != codes.DeadlineExceeded && ctx.Err() == nil { - t.Fatalf("cancellation ignored: %v", err) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + finished := make(chan error, 1) + joined := make(chan struct{}) + go func() { + defer close(joined) + _, err := control.Ready(ctx, []byte("wait")) + finished <- err + }() + defer func() { + cancel() + select { + case <-joined: + case <-time.After(5 * time.Second): + t.Error("readiness call did not join after cancellation") + } + }() + select { + case <-calls: + cancel() + case err := <-finished: + t.Fatalf("readiness call ended before the handler started: %v", err) + case <-ctx.Done(): + t.Fatalf("readiness handler did not start: %v", ctx.Err()) + } + select { + case err := <-finished: + if !errors.Is(err, context.Canceled) && status.Code(err) != codes.Canceled { + t.Fatalf("cancellation ignored: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("readiness call did not return after cancellation") } - <-calls }