From 9161d306830245f9cf0e27833203b9fb60663406 Mon Sep 17 00:00:00 2001 From: Washington Kigani Kamadi Date: Thu, 1 Aug 2024 16:02:50 +0300 Subject: [PATCH 01/18] PRISM-312 : Fetch Backend Information (#187) * fetch backend info Signed-off-by: WashingtonKK WIP Signed-off-by: WashingtonKK * add id to grpc response Signed-off-by: WashingtonKK * read backend information Signed-off-by: WashingtonKK revert changes in test server Signed-off-by: WashingtonKK * update info json Signed-off-by: WashingtonKK * test on dell machine Signed-off-by: WashingtonKK * update protoc Signed-off-by: WashingtonKK update protoc Signed-off-by: WashingtonKK * refactor fetch backend info Signed-off-by: WashingtonKK * remove computation definition Signed-off-by: WashingtonKK * refactor manager service creation Signed-off-by: WashingtonKK * refactor manager service creation: Signed-off-by: WashingtonKK * return config to main Signed-off-by: WashingtonKK * add tests on test/computation Signed-off-by: WashingtonKK * update backend info path Signed-off-by: WashingtonKK * use sudo Signed-off-by: WashingtonKK * comment out sev testing section Signed-off-by: WashingtonKK * update backend info json location Signed-off-by: WashingtonKK * handle failed execution Signed-off-by: WashingtonKK * return error on failed execution: Signed-off-by: WashingtonKK --------- Signed-off-by: WashingtonKK --- cmd/manager/main.go | 15 +- manager/api/grpc/client.go | 12 + manager/api/logging.go | 9 + manager/api/metrics.go | 9 + manager/manager.proto | 11 + manager/manager_test.go | 4 +- manager/service.go | 48 ++- manager/tracing/tracing.go | 7 + pkg/manager/manager.pb.go | 475 ++++++++++++++++------- scripts/backend_info/backend_info.json | 101 +++++ test/computations/main.go | 12 + test/manual/computation/computation.json | 15 - 12 files changed, 530 insertions(+), 188 deletions(-) create mode 100644 scripts/backend_info/backend_info.json delete mode 100644 test/manual/computation/computation.json diff --git a/cmd/manager/main.go b/cmd/manager/main.go index e5bb3429b..69c338006 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -37,10 +37,11 @@ const ( ) type config struct { - LogLevel string `env:"MANAGER_LOG_LEVEL" envDefault:"info"` - JaegerURL url.URL `env:"COCOS_JAEGER_URL" envDefault:"http://localhost:4318"` - TraceRatio float64 `env:"MG_JAEGER_TRACE_RATIO" envDefault:"1.0"` - InstanceID string `env:"MANAGER_INSTANCE_ID" envDefault:""` + LogLevel string `env:"MANAGER_LOG_LEVEL" envDefault:"info"` + JaegerURL url.URL `env:"COCOS_JAEGER_URL" envDefault:"http://localhost:4318"` + TraceRatio float64 `env:"MG_JAEGER_TRACE_RATIO" envDefault:"1.0"` + InstanceID string `env:"MANAGER_INSTANCE_ID" envDefault:""` + BackendMeasurementBinary string `env:"BACKEND_MEASUREMENT_BINARY" envDefault:"../../build"` } func main() { @@ -103,7 +104,7 @@ func main() { } eventsChan := make(chan *pkgmanager.ClientStreamMessage) - svc := newService(logger, tracer, qemuCfg, eventsChan) + svc := newService(logger, tracer, qemuCfg, eventsChan, cfg.BackendMeasurementBinary) mc := managerapi.NewClient(pc, svc, eventsChan) @@ -120,8 +121,8 @@ func main() { } } -func newService(logger *slog.Logger, tracer trace.Tracer, qemuCfg qemu.Config, eventsChan chan *pkgmanager.ClientStreamMessage) manager.Service { - svc := manager.New(qemuCfg, logger, eventsChan, qemu.NewVM) +func newService(logger *slog.Logger, tracer trace.Tracer, qemuCfg qemu.Config, eventsChan chan *pkgmanager.ClientStreamMessage, backendMeasurementPath string) manager.Service { + svc := manager.New(qemuCfg, backendMeasurementPath, logger, eventsChan, qemu.NewVM) go svc.RetrieveAgentEventsLogs() svc = api.LoggingMiddleware(svc, logger) counter, latency := prometheus.MakeMetrics(svcName, "api") diff --git a/manager/api/grpc/client.go b/manager/api/grpc/client.go index 9159e2e7e..d2b1faf34 100644 --- a/manager/api/grpc/client.go +++ b/manager/api/grpc/client.go @@ -54,6 +54,18 @@ func (client ManagerClient) Process(ctx context.Context, cancel context.CancelFu if err := client.svc.Stop(ctx, mes.StopComputation.ComputationId); err != nil { return err } + case *pkgmanager.ServerStreamMessage_BackendInfoReq: + res, err := client.svc.FetchBackendInfo() + if err != nil { + return err + } + info := &pkgmanager.ClientStreamMessage_BackendInfo{BackendInfo: &pkgmanager.BackendInfo{ + Info: res, + Id: mes.BackendInfoReq.Id, + }} + if err := client.stream.Send(&pkgmanager.ClientStreamMessage{Message: info}); err != nil { + return err + } } } }) diff --git a/manager/api/logging.go b/manager/api/logging.go index 3812b1408..650f4fd2c 100644 --- a/manager/api/logging.go +++ b/manager/api/logging.go @@ -57,3 +57,12 @@ func (lm *loggingMiddleware) Stop(ctx context.Context, computationID string) (er func (lm *loggingMiddleware) RetrieveAgentEventsLogs() { lm.svc.RetrieveAgentEventsLogs() } + +func (lm *loggingMiddleware) FetchBackendInfo() ([]byte, error) { + defer func(begin time.Time) { + message := fmt.Sprintf("Method FetchBackendInfo took %s to complete", time.Since(begin)) + lm.logger.Info(message) + }(time.Now()) + + return lm.svc.FetchBackendInfo() +} diff --git a/manager/api/metrics.go b/manager/api/metrics.go index 30604025a..cffd436da 100644 --- a/manager/api/metrics.go +++ b/manager/api/metrics.go @@ -54,3 +54,12 @@ func (ms *metricsMiddleware) Stop(ctx context.Context, computationID string) err func (ms *metricsMiddleware) RetrieveAgentEventsLogs() { ms.svc.RetrieveAgentEventsLogs() } + +func (ms *metricsMiddleware) FetchBackendInfo() ([]byte, error) { + defer func(begin time.Time) { + ms.counter.With("method", "FetchBackendInfo").Add(1) + ms.latency.With("method", "FetchBackendInfo").Observe(time.Since(begin).Seconds()) + }(time.Now()) + + return ms.svc.FetchBackendInfo() +} diff --git a/manager/manager.proto b/manager/manager.proto index 9207187ea..e4c367552 100644 --- a/manager/manager.proto +++ b/manager/manager.proto @@ -26,6 +26,11 @@ message RunResponse{ string computation_id = 2; } +message BackendInfo{ + bytes info = 1; + string id = 2; +} + message AgentEvent { string event_type = 1; google.protobuf.Timestamp timestamp = 2; @@ -47,6 +52,7 @@ message ClientStreamMessage { AgentLog agent_log = 1; AgentEvent agent_event = 2; RunResponse run_res = 3; + BackendInfo backendInfo = 4; } } @@ -55,6 +61,7 @@ message ServerStreamMessage { ComputationRunReq runReq = 1; Terminate terminateReq = 2; StopComputation stopComputation = 3; + BackendInfoReq backendInfoReq = 4; } } @@ -68,6 +75,10 @@ message ComputationRunReq { AgentConfig agent_config = 7; } +message BackendInfoReq { + string id = 1; +} + message ResultConsumer { bytes userKey = 1; } diff --git a/manager/manager_test.go b/manager/manager_test.go index 6ba524cad..1babee22e 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -17,12 +17,12 @@ import ( ) func TestNew(t *testing.T) { - qemuCfg := qemu.Config{} + cfg := qemu.Config{} logger := slog.Default() eventsChan := make(chan *manager.ClientStreamMessage) vmf := new(mocks.Provider) - service := New(qemuCfg, logger, eventsChan, vmf.Execute) + service := New(cfg, "", logger, eventsChan, vmf.Execute) assert.NotNil(t, service) assert.IsType(t, &managerService{}, service) diff --git a/manager/service.go b/manager/service.go index 465b52289..55526daa1 100644 --- a/manager/service.go +++ b/manager/service.go @@ -9,6 +9,8 @@ import ( "fmt" "log/slog" "net" + "os" + "os/exec" "strconv" "github.com/absmach/magistrala/pkg/errors" @@ -53,28 +55,32 @@ type Service interface { Stop(ctx context.Context, computationID string) error // RetrieveAgentEventsLogs Retrieve and forward agent logs and events via vsock. RetrieveAgentEventsLogs() + // FetchBackendInfo measures and fetches the backend information. + FetchBackendInfo() ([]byte, error) } type managerService struct { - qemuCfg qemu.Config - logger *slog.Logger - agents map[int]string // agent map of vsock cid to computationID. - eventsChan chan *manager.ClientStreamMessage - vms map[string]vm.VM - vmFactory vm.Provider + qemuCfg qemu.Config + backendMeasurementBinaryPath string + logger *slog.Logger + agents map[int]string // agent map of vsock cid to computationID. + eventsChan chan *manager.ClientStreamMessage + vms map[string]vm.VM + vmFactory vm.Provider } var _ Service = (*managerService)(nil) // New instantiates the manager service implementation. -func New(qemuCfg qemu.Config, logger *slog.Logger, eventsChan chan *manager.ClientStreamMessage, vmFactory vm.Provider) Service { +func New(cfg qemu.Config, backendMeasurementBinPath string, logger *slog.Logger, eventsChan chan *manager.ClientStreamMessage, vmFactory vm.Provider) Service { ms := &managerService{ - qemuCfg: qemuCfg, - logger: logger, - agents: make(map[int]string), - vms: make(map[string]vm.VM), - eventsChan: eventsChan, - vmFactory: vmFactory, + qemuCfg: cfg, + logger: logger, + agents: make(map[int]string), + vms: make(map[string]vm.VM), + eventsChan: eventsChan, + vmFactory: vmFactory, + backendMeasurementBinaryPath: backendMeasurementBinPath, } return ms } @@ -160,6 +166,22 @@ func (ms *managerService) Stop(ctx context.Context, computationID string) error return nil } +func (ms *managerService) FetchBackendInfo() ([]byte, error) { + cmd := exec.Command("sudo", fmt.Sprintf("%s/backend_info", ms.backendMeasurementBinaryPath), "--policy", "1966081") + + _, err := cmd.Output() + if err != nil { + return nil, err + } + + f, err := os.ReadFile("./backend_info.json") + if err != nil { + return nil, err + } + + return f, nil +} + func getFreePort() (int, error) { listener, err := net.Listen("tcp", "") if err != nil { diff --git a/manager/tracing/tracing.go b/manager/tracing/tracing.go index 1eadebe1e..6c3ee4e87 100644 --- a/manager/tracing/tracing.go +++ b/manager/tracing/tracing.go @@ -39,3 +39,10 @@ func (tm *tracingMiddleware) Stop(ctx context.Context, computationID string) err func (tm *tracingMiddleware) RetrieveAgentEventsLogs() { tm.svc.RetrieveAgentEventsLogs() } + +func (tm *tracingMiddleware) FetchBackendInfo() ([]byte, error) { + _, span := tm.tracer.Start(context.Background(), "fetch_backend_info") + defer span.End() + + return tm.svc.FetchBackendInfo() +} diff --git a/pkg/manager/manager.pb.go b/pkg/manager/manager.pb.go index 64bbca754..3048e53ac 100644 --- a/pkg/manager/manager.pb.go +++ b/pkg/manager/manager.pb.go @@ -173,6 +173,61 @@ func (x *RunResponse) GetComputationId() string { return "" } +type BackendInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Info []byte `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *BackendInfo) Reset() { + *x = BackendInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_manager_manager_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BackendInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackendInfo) ProtoMessage() {} + +func (x *BackendInfo) ProtoReflect() protoreflect.Message { + mi := &file_manager_manager_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackendInfo.ProtoReflect.Descriptor instead. +func (*BackendInfo) Descriptor() ([]byte, []int) { + return file_manager_manager_proto_rawDescGZIP(), []int{3} +} + +func (x *BackendInfo) GetInfo() []byte { + if x != nil { + return x.Info + } + return nil +} + +func (x *BackendInfo) GetId() string { + if x != nil { + return x.Id + } + return "" +} + type AgentEvent struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -189,7 +244,7 @@ type AgentEvent struct { func (x *AgentEvent) Reset() { *x = AgentEvent{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[3] + mi := &file_manager_manager_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -202,7 +257,7 @@ func (x *AgentEvent) String() string { func (*AgentEvent) ProtoMessage() {} func (x *AgentEvent) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[3] + mi := &file_manager_manager_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -215,7 +270,7 @@ func (x *AgentEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentEvent.ProtoReflect.Descriptor instead. func (*AgentEvent) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{3} + return file_manager_manager_proto_rawDescGZIP(), []int{4} } func (x *AgentEvent) GetEventType() string { @@ -274,7 +329,7 @@ type AgentLog struct { func (x *AgentLog) Reset() { *x = AgentLog{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[4] + mi := &file_manager_manager_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -287,7 +342,7 @@ func (x *AgentLog) String() string { func (*AgentLog) ProtoMessage() {} func (x *AgentLog) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[4] + mi := &file_manager_manager_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -300,7 +355,7 @@ func (x *AgentLog) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentLog.ProtoReflect.Descriptor instead. func (*AgentLog) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{4} + return file_manager_manager_proto_rawDescGZIP(), []int{5} } func (x *AgentLog) GetMessage() string { @@ -341,13 +396,14 @@ type ClientStreamMessage struct { // *ClientStreamMessage_AgentLog // *ClientStreamMessage_AgentEvent // *ClientStreamMessage_RunRes + // *ClientStreamMessage_BackendInfo Message isClientStreamMessage_Message `protobuf_oneof:"message"` } func (x *ClientStreamMessage) Reset() { *x = ClientStreamMessage{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[5] + mi := &file_manager_manager_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -360,7 +416,7 @@ func (x *ClientStreamMessage) String() string { func (*ClientStreamMessage) ProtoMessage() {} func (x *ClientStreamMessage) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[5] + mi := &file_manager_manager_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -373,7 +429,7 @@ func (x *ClientStreamMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ClientStreamMessage.ProtoReflect.Descriptor instead. func (*ClientStreamMessage) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{5} + return file_manager_manager_proto_rawDescGZIP(), []int{6} } func (m *ClientStreamMessage) GetMessage() isClientStreamMessage_Message { @@ -404,6 +460,13 @@ func (x *ClientStreamMessage) GetRunRes() *RunResponse { return nil } +func (x *ClientStreamMessage) GetBackendInfo() *BackendInfo { + if x, ok := x.GetMessage().(*ClientStreamMessage_BackendInfo); ok { + return x.BackendInfo + } + return nil +} + type isClientStreamMessage_Message interface { isClientStreamMessage_Message() } @@ -420,12 +483,18 @@ type ClientStreamMessage_RunRes struct { RunRes *RunResponse `protobuf:"bytes,3,opt,name=run_res,json=runRes,proto3,oneof"` } +type ClientStreamMessage_BackendInfo struct { + BackendInfo *BackendInfo `protobuf:"bytes,4,opt,name=backendInfo,proto3,oneof"` +} + func (*ClientStreamMessage_AgentLog) isClientStreamMessage_Message() {} func (*ClientStreamMessage_AgentEvent) isClientStreamMessage_Message() {} func (*ClientStreamMessage_RunRes) isClientStreamMessage_Message() {} +func (*ClientStreamMessage_BackendInfo) isClientStreamMessage_Message() {} + type ServerStreamMessage struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -436,13 +505,14 @@ type ServerStreamMessage struct { // *ServerStreamMessage_RunReq // *ServerStreamMessage_TerminateReq // *ServerStreamMessage_StopComputation + // *ServerStreamMessage_BackendInfoReq Message isServerStreamMessage_Message `protobuf_oneof:"message"` } func (x *ServerStreamMessage) Reset() { *x = ServerStreamMessage{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[6] + mi := &file_manager_manager_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -455,7 +525,7 @@ func (x *ServerStreamMessage) String() string { func (*ServerStreamMessage) ProtoMessage() {} func (x *ServerStreamMessage) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[6] + mi := &file_manager_manager_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -468,7 +538,7 @@ func (x *ServerStreamMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ServerStreamMessage.ProtoReflect.Descriptor instead. func (*ServerStreamMessage) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{6} + return file_manager_manager_proto_rawDescGZIP(), []int{7} } func (m *ServerStreamMessage) GetMessage() isServerStreamMessage_Message { @@ -499,6 +569,13 @@ func (x *ServerStreamMessage) GetStopComputation() *StopComputation { return nil } +func (x *ServerStreamMessage) GetBackendInfoReq() *BackendInfoReq { + if x, ok := x.GetMessage().(*ServerStreamMessage_BackendInfoReq); ok { + return x.BackendInfoReq + } + return nil +} + type isServerStreamMessage_Message interface { isServerStreamMessage_Message() } @@ -515,12 +592,18 @@ type ServerStreamMessage_StopComputation struct { StopComputation *StopComputation `protobuf:"bytes,3,opt,name=stopComputation,proto3,oneof"` } +type ServerStreamMessage_BackendInfoReq struct { + BackendInfoReq *BackendInfoReq `protobuf:"bytes,4,opt,name=backendInfoReq,proto3,oneof"` +} + func (*ServerStreamMessage_RunReq) isServerStreamMessage_Message() {} func (*ServerStreamMessage_TerminateReq) isServerStreamMessage_Message() {} func (*ServerStreamMessage_StopComputation) isServerStreamMessage_Message() {} +func (*ServerStreamMessage_BackendInfoReq) isServerStreamMessage_Message() {} + type ComputationRunReq struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -538,7 +621,7 @@ type ComputationRunReq struct { func (x *ComputationRunReq) Reset() { *x = ComputationRunReq{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[7] + mi := &file_manager_manager_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -551,7 +634,7 @@ func (x *ComputationRunReq) String() string { func (*ComputationRunReq) ProtoMessage() {} func (x *ComputationRunReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[7] + mi := &file_manager_manager_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -564,7 +647,7 @@ func (x *ComputationRunReq) ProtoReflect() protoreflect.Message { // Deprecated: Use ComputationRunReq.ProtoReflect.Descriptor instead. func (*ComputationRunReq) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{7} + return file_manager_manager_proto_rawDescGZIP(), []int{8} } func (x *ComputationRunReq) GetId() string { @@ -616,6 +699,53 @@ func (x *ComputationRunReq) GetAgentConfig() *AgentConfig { return nil } +type BackendInfoReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *BackendInfoReq) Reset() { + *x = BackendInfoReq{} + if protoimpl.UnsafeEnabled { + mi := &file_manager_manager_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BackendInfoReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackendInfoReq) ProtoMessage() {} + +func (x *BackendInfoReq) ProtoReflect() protoreflect.Message { + mi := &file_manager_manager_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackendInfoReq.ProtoReflect.Descriptor instead. +func (*BackendInfoReq) Descriptor() ([]byte, []int) { + return file_manager_manager_proto_rawDescGZIP(), []int{9} +} + +func (x *BackendInfoReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + type ResultConsumer struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -627,7 +757,7 @@ type ResultConsumer struct { func (x *ResultConsumer) Reset() { *x = ResultConsumer{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[8] + mi := &file_manager_manager_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -640,7 +770,7 @@ func (x *ResultConsumer) String() string { func (*ResultConsumer) ProtoMessage() {} func (x *ResultConsumer) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[8] + mi := &file_manager_manager_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -653,7 +783,7 @@ func (x *ResultConsumer) ProtoReflect() protoreflect.Message { // Deprecated: Use ResultConsumer.ProtoReflect.Descriptor instead. func (*ResultConsumer) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{8} + return file_manager_manager_proto_rawDescGZIP(), []int{10} } func (x *ResultConsumer) GetUserKey() []byte { @@ -675,7 +805,7 @@ type Dataset struct { func (x *Dataset) Reset() { *x = Dataset{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[9] + mi := &file_manager_manager_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -688,7 +818,7 @@ func (x *Dataset) String() string { func (*Dataset) ProtoMessage() {} func (x *Dataset) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[9] + mi := &file_manager_manager_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -701,7 +831,7 @@ func (x *Dataset) ProtoReflect() protoreflect.Message { // Deprecated: Use Dataset.ProtoReflect.Descriptor instead. func (*Dataset) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{9} + return file_manager_manager_proto_rawDescGZIP(), []int{11} } func (x *Dataset) GetHash() []byte { @@ -730,7 +860,7 @@ type Algorithm struct { func (x *Algorithm) Reset() { *x = Algorithm{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[10] + mi := &file_manager_manager_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -743,7 +873,7 @@ func (x *Algorithm) String() string { func (*Algorithm) ProtoMessage() {} func (x *Algorithm) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[10] + mi := &file_manager_manager_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -756,7 +886,7 @@ func (x *Algorithm) ProtoReflect() protoreflect.Message { // Deprecated: Use Algorithm.ProtoReflect.Descriptor instead. func (*Algorithm) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{10} + return file_manager_manager_proto_rawDescGZIP(), []int{12} } func (x *Algorithm) GetHash() []byte { @@ -791,7 +921,7 @@ type AgentConfig struct { func (x *AgentConfig) Reset() { *x = AgentConfig{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[11] + mi := &file_manager_manager_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -804,7 +934,7 @@ func (x *AgentConfig) String() string { func (*AgentConfig) ProtoMessage() {} func (x *AgentConfig) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[11] + mi := &file_manager_manager_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -817,7 +947,7 @@ func (x *AgentConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentConfig.ProtoReflect.Descriptor instead. func (*AgentConfig) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{11} + return file_manager_manager_proto_rawDescGZIP(), []int{13} } func (x *AgentConfig) GetPort() string { @@ -894,56 +1024,67 @@ var file_manager_manager_proto_rawDesc = []byte{ 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xde, 0x01, 0x0a, 0x0a, 0x41, 0x67, 0x65, 0x6e, - 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, - 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, - 0x12, 0x1e, 0x0a, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, - 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x08, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, - 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x38, 0x0a, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xbb, 0x01, 0x0a, 0x13, 0x43, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x30, - 0x0a, 0x09, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, - 0x74, 0x4c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, - 0x12, 0x36, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x72, 0x75, 0x6e, 0x5f, - 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, - 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x22, 0xd6, 0x01, 0x0a, 0x13, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x34, 0x0a, 0x06, - 0x72, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, - 0x65, 0x71, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x0c, - 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x44, 0x0a, 0x0f, - 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, - 0x00, 0x52, 0x0f, 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xb6, 0x02, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x31, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x65, + 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0xde, 0x01, 0x0a, 0x0a, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, + 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x6f, + 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, + 0x74, 0x6f, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x08, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, + 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, + 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, + 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xf5, 0x01, 0x0a, 0x13, 0x43, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x12, 0x30, 0x0a, 0x09, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, + 0x4c, 0x6f, 0x67, 0x12, 0x36, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, + 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x72, + 0x75, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x0b, + 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x42, 0x61, 0x63, 0x6b, + 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, + 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x22, 0x99, 0x02, 0x0a, 0x13, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x75, 0x6e, + 0x52, 0x65, 0x71, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x75, 0x6e, 0x52, 0x65, 0x71, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x12, + 0x38, 0x0a, 0x0c, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x74, 0x65, 0x72, + 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x44, 0x0a, 0x0f, 0x73, 0x74, 0x6f, + 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x6f, + 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0f, + 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x41, 0x0a, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, + 0x71, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, + 0x48, 0x00, 0x52, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x65, 0x71, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xb6, 0x02, 0x0a, 0x11, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, @@ -963,40 +1104,42 @@ var file_manager_manager_proto_rawDesc = []byte{ 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x2a, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, - 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, - 0x65, 0x79, 0x22, 0x37, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, - 0x68, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x39, 0x0a, 0x09, 0x41, - 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, - 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, - 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0xf9, 0x01, 0x0a, 0x0b, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, - 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x1b, - 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6b, - 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6b, - 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x61, 0x46, 0x69, - 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, - 0x21, 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x6c, 0x73, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x54, - 0x6c, 0x73, 0x32, 0x5d, 0x0a, 0x0e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x4b, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, - 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, - 0x01, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x20, 0x0a, 0x0e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, + 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2a, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, + 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, + 0x72, 0x4b, 0x65, 0x79, 0x22, 0x37, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, + 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x39, 0x0a, + 0x09, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, + 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x18, + 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0xf9, 0x01, 0x0a, 0x0b, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, 0x0a, + 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, + 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x61, + 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, + 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x6c, + 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, + 0x64, 0x54, 0x6c, 0x73, 0x32, 0x5d, 0x0a, 0x0e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4b, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, + 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, + 0x01, 0x30, 0x01, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1011,42 +1154,46 @@ func file_manager_manager_proto_rawDescGZIP() []byte { return file_manager_manager_proto_rawDescData } -var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_manager_manager_proto_goTypes = []any{ (*Terminate)(nil), // 0: manager.Terminate (*StopComputation)(nil), // 1: manager.StopComputation (*RunResponse)(nil), // 2: manager.RunResponse - (*AgentEvent)(nil), // 3: manager.AgentEvent - (*AgentLog)(nil), // 4: manager.AgentLog - (*ClientStreamMessage)(nil), // 5: manager.ClientStreamMessage - (*ServerStreamMessage)(nil), // 6: manager.ServerStreamMessage - (*ComputationRunReq)(nil), // 7: manager.ComputationRunReq - (*ResultConsumer)(nil), // 8: manager.ResultConsumer - (*Dataset)(nil), // 9: manager.Dataset - (*Algorithm)(nil), // 10: manager.Algorithm - (*AgentConfig)(nil), // 11: manager.AgentConfig - (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp + (*BackendInfo)(nil), // 3: manager.BackendInfo + (*AgentEvent)(nil), // 4: manager.AgentEvent + (*AgentLog)(nil), // 5: manager.AgentLog + (*ClientStreamMessage)(nil), // 6: manager.ClientStreamMessage + (*ServerStreamMessage)(nil), // 7: manager.ServerStreamMessage + (*ComputationRunReq)(nil), // 8: manager.ComputationRunReq + (*BackendInfoReq)(nil), // 9: manager.BackendInfoReq + (*ResultConsumer)(nil), // 10: manager.ResultConsumer + (*Dataset)(nil), // 11: manager.Dataset + (*Algorithm)(nil), // 12: manager.Algorithm + (*AgentConfig)(nil), // 13: manager.AgentConfig + (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp } var file_manager_manager_proto_depIdxs = []int32{ - 12, // 0: manager.AgentEvent.timestamp:type_name -> google.protobuf.Timestamp - 12, // 1: manager.AgentLog.timestamp:type_name -> google.protobuf.Timestamp - 4, // 2: manager.ClientStreamMessage.agent_log:type_name -> manager.AgentLog - 3, // 3: manager.ClientStreamMessage.agent_event:type_name -> manager.AgentEvent + 14, // 0: manager.AgentEvent.timestamp:type_name -> google.protobuf.Timestamp + 14, // 1: manager.AgentLog.timestamp:type_name -> google.protobuf.Timestamp + 5, // 2: manager.ClientStreamMessage.agent_log:type_name -> manager.AgentLog + 4, // 3: manager.ClientStreamMessage.agent_event:type_name -> manager.AgentEvent 2, // 4: manager.ClientStreamMessage.run_res:type_name -> manager.RunResponse - 7, // 5: manager.ServerStreamMessage.runReq:type_name -> manager.ComputationRunReq - 0, // 6: manager.ServerStreamMessage.terminateReq:type_name -> manager.Terminate - 1, // 7: manager.ServerStreamMessage.stopComputation:type_name -> manager.StopComputation - 9, // 8: manager.ComputationRunReq.datasets:type_name -> manager.Dataset - 10, // 9: manager.ComputationRunReq.algorithm:type_name -> manager.Algorithm - 8, // 10: manager.ComputationRunReq.result_consumers:type_name -> manager.ResultConsumer - 11, // 11: manager.ComputationRunReq.agent_config:type_name -> manager.AgentConfig - 5, // 12: manager.ManagerService.Process:input_type -> manager.ClientStreamMessage - 6, // 13: manager.ManagerService.Process:output_type -> manager.ServerStreamMessage - 13, // [13:14] is the sub-list for method output_type - 12, // [12:13] is the sub-list for method input_type - 12, // [12:12] is the sub-list for extension type_name - 12, // [12:12] is the sub-list for extension extendee - 0, // [0:12] is the sub-list for field type_name + 3, // 5: manager.ClientStreamMessage.backendInfo:type_name -> manager.BackendInfo + 8, // 6: manager.ServerStreamMessage.runReq:type_name -> manager.ComputationRunReq + 0, // 7: manager.ServerStreamMessage.terminateReq:type_name -> manager.Terminate + 1, // 8: manager.ServerStreamMessage.stopComputation:type_name -> manager.StopComputation + 9, // 9: manager.ServerStreamMessage.backendInfoReq:type_name -> manager.BackendInfoReq + 11, // 10: manager.ComputationRunReq.datasets:type_name -> manager.Dataset + 12, // 11: manager.ComputationRunReq.algorithm:type_name -> manager.Algorithm + 10, // 12: manager.ComputationRunReq.result_consumers:type_name -> manager.ResultConsumer + 13, // 13: manager.ComputationRunReq.agent_config:type_name -> manager.AgentConfig + 6, // 14: manager.ManagerService.Process:input_type -> manager.ClientStreamMessage + 7, // 15: manager.ManagerService.Process:output_type -> manager.ServerStreamMessage + 15, // [15:16] is the sub-list for method output_type + 14, // [14:15] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_manager_manager_proto_init() } @@ -1092,7 +1239,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[3].Exporter = func(v any, i int) any { - switch v := v.(*AgentEvent); i { + switch v := v.(*BackendInfo); i { case 0: return &v.state case 1: @@ -1104,7 +1251,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[4].Exporter = func(v any, i int) any { - switch v := v.(*AgentLog); i { + switch v := v.(*AgentEvent); i { case 0: return &v.state case 1: @@ -1116,7 +1263,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[5].Exporter = func(v any, i int) any { - switch v := v.(*ClientStreamMessage); i { + switch v := v.(*AgentLog); i { case 0: return &v.state case 1: @@ -1128,7 +1275,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[6].Exporter = func(v any, i int) any { - switch v := v.(*ServerStreamMessage); i { + switch v := v.(*ClientStreamMessage); i { case 0: return &v.state case 1: @@ -1140,7 +1287,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[7].Exporter = func(v any, i int) any { - switch v := v.(*ComputationRunReq); i { + switch v := v.(*ServerStreamMessage); i { case 0: return &v.state case 1: @@ -1152,7 +1299,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*ResultConsumer); i { + switch v := v.(*ComputationRunReq); i { case 0: return &v.state case 1: @@ -1164,7 +1311,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[9].Exporter = func(v any, i int) any { - switch v := v.(*Dataset); i { + switch v := v.(*BackendInfoReq); i { case 0: return &v.state case 1: @@ -1176,7 +1323,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*Algorithm); i { + switch v := v.(*ResultConsumer); i { case 0: return &v.state case 1: @@ -1188,6 +1335,30 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[11].Exporter = func(v any, i int) any { + switch v := v.(*Dataset); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_manager_manager_proto_msgTypes[12].Exporter = func(v any, i int) any { + switch v := v.(*Algorithm); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_manager_manager_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*AgentConfig); i { case 0: return &v.state @@ -1200,15 +1371,17 @@ func file_manager_manager_proto_init() { } } } - file_manager_manager_proto_msgTypes[5].OneofWrappers = []any{ + file_manager_manager_proto_msgTypes[6].OneofWrappers = []any{ (*ClientStreamMessage_AgentLog)(nil), (*ClientStreamMessage_AgentEvent)(nil), (*ClientStreamMessage_RunRes)(nil), + (*ClientStreamMessage_BackendInfo)(nil), } - file_manager_manager_proto_msgTypes[6].OneofWrappers = []any{ + file_manager_manager_proto_msgTypes[7].OneofWrappers = []any{ (*ServerStreamMessage_RunReq)(nil), (*ServerStreamMessage_TerminateReq)(nil), (*ServerStreamMessage_StopComputation)(nil), + (*ServerStreamMessage_BackendInfoReq)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -1216,7 +1389,7 @@ func file_manager_manager_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_manager_manager_proto_rawDesc, NumEnums: 0, - NumMessages: 12, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/scripts/backend_info/backend_info.json b/scripts/backend_info/backend_info.json new file mode 100644 index 000000000..86973b474 --- /dev/null +++ b/scripts/backend_info/backend_info.json @@ -0,0 +1,101 @@ +{ + "snp_policy": { + "policy": 1966081, + "family_id": [ + 0 + ], + "image_id": [ + 0 + ], + "vmpl": { + "value": 0 + }, + "minimum_tcb": 15063977803600887811, + "minimum_launch_tcb": 15063977803600887811, + "require_author_key": false, + "measurement": [ + 0 + ], + "host_data": [ + 0 + ], + "report_id_ma": [ + 0 + ], + "chip_id": [ + 26, + 177, + 106, + 181, + 15, + 165, + 174, + 66, + 236, + 140, + 27, + 37, + 187, + 218, + 92, + 11, + 165, + 234, + 146, + 187, + 69, + 89, + 141, + 64, + 172, + 132, + 62, + 35, + 136, + 46, + 129, + 2, + 44, + 188, + 33, + 180, + 169, + 233, + 18, + 188, + 75, + 68, + 224, + 255, + 210, + 45, + 34, + 122, + 152, + 115, + 105, + 58, + 70, + 52, + 48, + 121, + 198, + 166, + 252, + 245, + 58, + 69, + 126, + 147 + ], + "minimum_build": 7, + "minimum_version": "1.55", + "permit_provisional_firmware": false, + "require_id_block": false + }, + "root_of_trust": { + "product": "Milan", + "check_crl": true, + "disallow_network": false + } +} diff --git a/test/computations/main.go b/test/computations/main.go index 05e9efaeb..bc9e74b7b 100644 --- a/test/computations/main.go +++ b/test/computations/main.go @@ -73,6 +73,16 @@ func (s *svc) Run(ipAdress string, reqChan chan *manager.ServerStreamMessage, au } algoHash := sha3.Sum256(algo) + + // Uncomment this to run tests on the manager service on a SEV enabled backend. + reqChan <- &manager.ServerStreamMessage{ + Message: &manager.ServerStreamMessage_BackendInfoReq{ + BackendInfoReq: &manager.BackendInfoReq{ + Id: "1", + }, + }, + } + reqChan <- &manager.ServerStreamMessage{ Message: &manager.ServerStreamMessage_RunReq{ RunReq: &manager.ComputationRunReq{ @@ -126,6 +136,8 @@ func main() { fmt.Println("received agent event") case *manager.ClientStreamMessage_AgentLog: fmt.Println("received agent log") + case *manager.ClientStreamMessage_BackendInfo: + fmt.Println("received backend info measurement request") } fmt.Println(incoming.Message) } diff --git a/test/manual/computation/computation.json b/test/manual/computation/computation.json deleted file mode 100644 index 1fa620ffd..000000000 --- a/test/manual/computation/computation.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "snp_policy": { - "minimum_guest_svn": 0, - "policy": 196608, - "minimum_tcb": 1, - "minimum_version": "1.0", - "minimum_launch_tcb": 1, - "measurement": [232, 141, 188, 114, 162, 221, 214, 6, 150, 248, 3, 173, 230, 39, 48, 120, 105, 243, 15, 242, 79, 67, 112, 128, 44, 119, 216, 226, 170, 255, 212, 154, 58, 68, 231, 30, 20, 235, 228, 42, 43, 1, 95, 191, 51, 113, 19, 72], - "minimum_build": 1 - }, - "root_of_trust": { - "product": "Milan", - "check_crl": true - } -} From 3c855e3b68d039b63b50a23d85a0dbe4b022594e Mon Sep 17 00:00:00 2001 From: Smith Jilks <41241359+smithjilks@users.noreply.github.com> Date: Tue, 6 Aug 2024 17:25:35 +0300 Subject: [PATCH 02/18] NOISSUE - Handle larger manifests exceeding the default grpc limit (#161) * Handle larger manifests exceeding the default grpc limit Signed-off-by: Jilks Smith * Update manager tests Signed-off-by: Jilks Smith * Update manager tests * Update manager client.go * Update manager client.go * Update manager client.go * Update manager grpc server.go * Update manager grpc server and client --------- Signed-off-by: Jilks Smith --- manager/api/grpc/client.go | 40 ++++- manager/api/grpc/server.go | 45 ++++- manager/manager.proto | 13 +- manager/manager_test.go | 264 ++++++----------------------- manager/service_test.go | 232 ++++++++++++++++++++++++++ manager/setup_test.go | 126 ++++++++++++++ pkg/manager/manager.pb.go | 334 +++++++++++++++++++++++-------------- 7 files changed, 698 insertions(+), 356 deletions(-) create mode 100644 manager/service_test.go create mode 100644 manager/setup_test.go diff --git a/manager/api/grpc/client.go b/manager/api/grpc/client.go index d2b1faf34..4dffe2c8f 100644 --- a/manager/api/grpc/client.go +++ b/manager/api/grpc/client.go @@ -3,15 +3,20 @@ package grpc import ( + "bytes" "context" - "errors" + "github.com/absmach/magistrala/pkg/errors" "github.com/ultravioletrs/cocos/manager" pkgmanager "github.com/ultravioletrs/cocos/pkg/manager" "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" ) -var errTerminationFromServer = errors.New("server requested client termination") +var ( + errTerminationFromServer = errors.New("server requested client termination") + errCorruptedManifest = errors.New("received manifest may be corrupted") +) type ManagerClient struct { stream pkgmanager.ManagerService_ProcessClient @@ -32,24 +37,41 @@ func (client ManagerClient) Process(ctx context.Context, cancel context.CancelFu eg, ctx := errgroup.WithContext(ctx) eg.Go(func() error { + var runReqBuffer bytes.Buffer for { req, err := client.stream.Recv() if err != nil { return err } + switch mes := req.Message.(type) { - case *pkgmanager.ServerStreamMessage_RunReq: - port, err := client.svc.Run(ctx, mes.RunReq) - if err != nil { - return err + case *pkgmanager.ServerStreamMessage_RunReqChunks: + if len(mes.RunReqChunks.Data) == 0 { + var runReq pkgmanager.ComputationRunReq + if err = proto.Unmarshal(runReqBuffer.Bytes(), &runReq); err != nil { + return errors.Wrap(err, errCorruptedManifest) + } + port, err := client.svc.Run(ctx, &runReq) + if err != nil { + return err + } + runRes := &pkgmanager.ClientStreamMessage_RunRes{ + RunRes: &pkgmanager.RunResponse{ + AgentPort: port, + ComputationId: runReq.Id, + }, + } + if err := client.stream.Send(&pkgmanager.ClientStreamMessage{Message: runRes}); err != nil { + return err + } } - runRes := &pkgmanager.ClientStreamMessage_RunRes{RunRes: &pkgmanager.RunResponse{AgentPort: port, ComputationId: mes.RunReq.Id}} - if err := client.stream.Send(&pkgmanager.ClientStreamMessage{Message: runRes}); err != nil { + if _, err := runReqBuffer.Write(mes.RunReqChunks.Data); err != nil { return err } + case *pkgmanager.ServerStreamMessage_TerminateReq: cancel() - return errors.Join(errTerminationFromServer, errors.New(mes.TerminateReq.Message)) + return errors.Wrap(errTerminationFromServer, errors.New(mes.TerminateReq.Message)) case *pkgmanager.ServerStreamMessage_StopComputation: if err := client.svc.Stop(ctx, mes.StopComputation.ComputationId); err != nil { return err diff --git a/manager/api/grpc/server.go b/manager/api/grpc/server.go index b922a7196..a5a7abb98 100644 --- a/manager/api/grpc/server.go +++ b/manager/api/grpc/server.go @@ -3,13 +3,23 @@ package grpc import ( + "bytes" + "errors" + "io" + "github.com/ultravioletrs/cocos/pkg/manager" "golang.org/x/sync/errgroup" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/protobuf/proto" +) + +var ( + _ manager.ManagerServiceServer = (*grpcServer)(nil) + ErrUnexpectedMsg = errors.New("unknown message type") ) -var _ manager.ManagerServiceServer = (*grpcServer)(nil) +const bufferSize = 1024 * 1024 // 1 MB type grpcServer struct { manager.UnimplementedManagerServiceServer @@ -54,8 +64,37 @@ func (s *grpcServer) Process(stream manager.ManagerService_ProcessServer) error case <-ctx.Done(): return nil case req := <-runReqChan: - if err := stream.Send(req); err != nil { - return err + switch msg := req.Message.(type) { + case *manager.ServerStreamMessage_RunReq: + data, err := proto.Marshal(msg.RunReq) + if err != nil { + return err + } + dataBuffer := bytes.NewBuffer(data) + buf := make([]byte, bufferSize) + for { + n, err := dataBuffer.Read(buf) + chunk := &manager.ServerStreamMessage{ + Message: &manager.ServerStreamMessage_RunReqChunks{ + RunReqChunks: &manager.RunReqChunks{ + Data: buf[:n], + }, + }, + } + + if err := stream.Send(chunk); err != nil { + return err + } + + if err == io.EOF { + break + } + } + + default: + if err := stream.Send(req); err != nil { + return err + } } } } diff --git a/manager/manager.proto b/manager/manager.proto index e4c367552..075a33832 100644 --- a/manager/manager.proto +++ b/manager/manager.proto @@ -58,13 +58,18 @@ message ClientStreamMessage { message ServerStreamMessage { oneof message { - ComputationRunReq runReq = 1; - Terminate terminateReq = 2; - StopComputation stopComputation = 3; - BackendInfoReq backendInfoReq = 4; + RunReqChunks runReqChunks = 1; + ComputationRunReq runReq = 2; + Terminate terminateReq = 3; + StopComputation stopComputation = 4; + BackendInfoReq backendInfoReq = 5; } } +message RunReqChunks { + bytes data = 1; +} + message ComputationRunReq { string id = 1; string name = 2; diff --git a/manager/manager_test.go b/manager/manager_test.go index 1babee22e..33265782e 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -1,232 +1,68 @@ // Copyright (c) Ultraviolet // SPDX-License-Identifier: Apache-2.0 -package manager +package manager_test import ( + "bytes" "context" - "encoding/json" - "log/slog" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/ultravioletrs/cocos/manager/qemu" - "github.com/ultravioletrs/cocos/manager/vm" - "github.com/ultravioletrs/cocos/manager/vm/mocks" "github.com/ultravioletrs/cocos/pkg/manager" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" ) -func TestNew(t *testing.T) { - cfg := qemu.Config{} - logger := slog.Default() - eventsChan := make(chan *manager.ClientStreamMessage) - vmf := new(mocks.Provider) - - service := New(cfg, "", logger, eventsChan, vmf.Execute) - - assert.NotNil(t, service) - assert.IsType(t, &managerService{}, service) -} - -func TestRun(t *testing.T) { - vmf := new(mocks.Provider) - vmMock := new(mocks.VM) - vmf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(vmMock) - tests := []struct { - name string - req *manager.ComputationRunReq - vmStartError error - expectedError error - }{ - { - name: "Successful run", - req: &manager.ComputationRunReq{ - Id: "test-computation", - Name: "Test Computation", - Algorithm: &manager.Algorithm{ - Hash: make([]byte, hashLength), - }, - AgentConfig: &manager.AgentConfig{}, - }, - vmStartError: nil, - expectedError: nil, - }, - { - name: "VM start failure", - req: &manager.ComputationRunReq{ - Id: "test-computation", - Name: "Test Computation", - Algorithm: &manager.Algorithm{ - Hash: make([]byte, hashLength), - }, - AgentConfig: &manager.AgentConfig{}, - }, - vmStartError: assert.AnError, - expectedError: assert.AnError, - }, +func TestProcess(t *testing.T) { + ctx := context.Background() + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) } + defer conn.Close() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.vmStartError == nil { - vmMock.On("Start").Return(nil).Once() - } else { - vmMock.On("Start").Return(tt.vmStartError).Once() - } - - vmMock.On("SendAgentConfig", mock.Anything).Return(nil) - - qemuCfg := qemu.Config{ - VSockConfig: qemu.VSockConfig{ - GuestCID: 3, - Vnc: 5900, - }, - } - logger := slog.Default() - eventsChan := make(chan *manager.ClientStreamMessage, 10) - - ms := &managerService{ - qemuCfg: qemuCfg, - logger: logger, - agents: make(map[int]string), - vms: make(map[string]vm.VM), - eventsChan: eventsChan, - vmFactory: vmf.Execute, - } - - ctx := context.Background() - - port, err := ms.Run(ctx, tt.req) - - if tt.expectedError != nil { - assert.Error(t, err) - assert.ErrorIs(t, err, tt.expectedError) - assert.Empty(t, port) - } else { - assert.NoError(t, err) - assert.NotEmpty(t, port) - assert.Len(t, ms.vms, 1) - assert.Len(t, ms.agents, 1) - } - - vmf.AssertExpectations(t) - - // Clear the events channel - for len(eventsChan) > 0 { - <-eventsChan - } - }) + client := manager.NewManagerServiceClient(conn) + stream, err := client.Process(ctx) + if err != nil { + t.Fatalf("Process failed: %v", err) } -} -func TestStop(t *testing.T) { - tests := []struct { - name string - computationID string - vmStopError error - expectedError error - initialVMCount int - }{ - { - name: "Successful stop", - computationID: "existing-computation", - vmStopError: nil, - expectedError: nil, - initialVMCount: 1, - }, - { - name: "Non-existent computation", - computationID: "non-existent-computation", - vmStopError: nil, - expectedError: ErrNotFound, - initialVMCount: 0, - }, - { - name: "VM stop error", - computationID: "error-computation", - vmStopError: assert.AnError, - expectedError: assert.AnError, - initialVMCount: 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ms := &managerService{ - vms: make(map[string]vm.VM), - } - vmMock := new(mocks.VM) - - if tt.vmStopError == nil { - vmMock.On("Stop").Return(nil).Once() - } else { - vmMock.On("Stop").Return(assert.AnError).Once() - } - - if tt.initialVMCount > 0 { - ms.vms[tt.computationID] = vmMock + var data bytes.Buffer + for { + msg, err := stream.Recv() + if err != nil { + t.Fatalf("Failed to receive ServerStreamMessage: %v", err) + } + + switch m := msg.Message.(type) { + case *manager.ServerStreamMessage_TerminateReq: + if m.TerminateReq.Message != "test terminate" { + t.Fatalf("Unexpected terminate message: %v", m.TerminateReq.Message) } - - err := ms.Stop(context.Background(), tt.computationID) - - if tt.expectedError != nil { - assert.Error(t, err) - assert.ErrorIs(t, err, tt.expectedError) - } else { - assert.NoError(t, err) - assert.Len(t, ms.vms, 0) - } - }) - } -} - -func TestGetFreePort(t *testing.T) { - port, err := getFreePort() - - assert.NoError(t, err) - assert.Greater(t, port, 0) -} - -func TestPublishEvent(t *testing.T) { - tests := []struct { - name string - event string - computationID string - status string - details json.RawMessage - }{ - { - name: "Standard event", - event: "test-event", - computationID: "test-computation", - status: "test-status", - details: nil, - }, - { - name: "Event with details", - event: "detailed-event", - computationID: "detailed-computation", - status: "detailed-status", - details: json.RawMessage(`{"key": "value"}`), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - eventsChan := make(chan *manager.ClientStreamMessage, 1) - ms := &managerService{ - eventsChan: eventsChan, + case *manager.ServerStreamMessage_RunReqChunks: + if len(m.RunReqChunks.Data) == 0 { + var runReq manager.ComputationRunReq + if err = proto.Unmarshal(data.Bytes(), &runReq); err != nil { + t.Fatalf("Failed to create run request: %v", err) + } + + runRes := &manager.ClientStreamMessage_AgentLog{ + AgentLog: &manager.AgentLog{ + Message: "test log", + ComputationId: "comp1", + Level: "DEBUG", + }, + } + if runReq.Id != "1" || runReq.Name != "sample computation" || runReq.Description != "sample description" { + t.Fatalf("Unexpected run request message: %v", &runReq) + } + if err := stream.Send(&manager.ClientStreamMessage{Message: runRes}); err != nil { + t.Fatalf("Failed to send ClientStreamMessage: %v", err) + } + return } - - ms.publishEvent(tt.event, tt.computationID, tt.status, tt.details) - - assert.Len(t, eventsChan, 1) - event := <-eventsChan - assert.Equal(t, tt.event, event.GetAgentEvent().EventType) - assert.Equal(t, tt.computationID, event.GetAgentEvent().ComputationId) - assert.Equal(t, tt.status, event.GetAgentEvent().Status) - assert.Equal(t, "manager", event.GetAgentEvent().Originator) - assert.Equal(t, tt.details, json.RawMessage(event.GetAgentEvent().Details)) - }) + data.Write(m.RunReqChunks.Data) + default: + t.Fatalf("Unexpected message type: %T", m) + } } } diff --git a/manager/service_test.go b/manager/service_test.go new file mode 100644 index 000000000..1babee22e --- /dev/null +++ b/manager/service_test.go @@ -0,0 +1,232 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 +package manager + +import ( + "context" + "encoding/json" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/ultravioletrs/cocos/manager/qemu" + "github.com/ultravioletrs/cocos/manager/vm" + "github.com/ultravioletrs/cocos/manager/vm/mocks" + "github.com/ultravioletrs/cocos/pkg/manager" +) + +func TestNew(t *testing.T) { + cfg := qemu.Config{} + logger := slog.Default() + eventsChan := make(chan *manager.ClientStreamMessage) + vmf := new(mocks.Provider) + + service := New(cfg, "", logger, eventsChan, vmf.Execute) + + assert.NotNil(t, service) + assert.IsType(t, &managerService{}, service) +} + +func TestRun(t *testing.T) { + vmf := new(mocks.Provider) + vmMock := new(mocks.VM) + vmf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(vmMock) + tests := []struct { + name string + req *manager.ComputationRunReq + vmStartError error + expectedError error + }{ + { + name: "Successful run", + req: &manager.ComputationRunReq{ + Id: "test-computation", + Name: "Test Computation", + Algorithm: &manager.Algorithm{ + Hash: make([]byte, hashLength), + }, + AgentConfig: &manager.AgentConfig{}, + }, + vmStartError: nil, + expectedError: nil, + }, + { + name: "VM start failure", + req: &manager.ComputationRunReq{ + Id: "test-computation", + Name: "Test Computation", + Algorithm: &manager.Algorithm{ + Hash: make([]byte, hashLength), + }, + AgentConfig: &manager.AgentConfig{}, + }, + vmStartError: assert.AnError, + expectedError: assert.AnError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.vmStartError == nil { + vmMock.On("Start").Return(nil).Once() + } else { + vmMock.On("Start").Return(tt.vmStartError).Once() + } + + vmMock.On("SendAgentConfig", mock.Anything).Return(nil) + + qemuCfg := qemu.Config{ + VSockConfig: qemu.VSockConfig{ + GuestCID: 3, + Vnc: 5900, + }, + } + logger := slog.Default() + eventsChan := make(chan *manager.ClientStreamMessage, 10) + + ms := &managerService{ + qemuCfg: qemuCfg, + logger: logger, + agents: make(map[int]string), + vms: make(map[string]vm.VM), + eventsChan: eventsChan, + vmFactory: vmf.Execute, + } + + ctx := context.Background() + + port, err := ms.Run(ctx, tt.req) + + if tt.expectedError != nil { + assert.Error(t, err) + assert.ErrorIs(t, err, tt.expectedError) + assert.Empty(t, port) + } else { + assert.NoError(t, err) + assert.NotEmpty(t, port) + assert.Len(t, ms.vms, 1) + assert.Len(t, ms.agents, 1) + } + + vmf.AssertExpectations(t) + + // Clear the events channel + for len(eventsChan) > 0 { + <-eventsChan + } + }) + } +} + +func TestStop(t *testing.T) { + tests := []struct { + name string + computationID string + vmStopError error + expectedError error + initialVMCount int + }{ + { + name: "Successful stop", + computationID: "existing-computation", + vmStopError: nil, + expectedError: nil, + initialVMCount: 1, + }, + { + name: "Non-existent computation", + computationID: "non-existent-computation", + vmStopError: nil, + expectedError: ErrNotFound, + initialVMCount: 0, + }, + { + name: "VM stop error", + computationID: "error-computation", + vmStopError: assert.AnError, + expectedError: assert.AnError, + initialVMCount: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ms := &managerService{ + vms: make(map[string]vm.VM), + } + vmMock := new(mocks.VM) + + if tt.vmStopError == nil { + vmMock.On("Stop").Return(nil).Once() + } else { + vmMock.On("Stop").Return(assert.AnError).Once() + } + + if tt.initialVMCount > 0 { + ms.vms[tt.computationID] = vmMock + } + + err := ms.Stop(context.Background(), tt.computationID) + + if tt.expectedError != nil { + assert.Error(t, err) + assert.ErrorIs(t, err, tt.expectedError) + } else { + assert.NoError(t, err) + assert.Len(t, ms.vms, 0) + } + }) + } +} + +func TestGetFreePort(t *testing.T) { + port, err := getFreePort() + + assert.NoError(t, err) + assert.Greater(t, port, 0) +} + +func TestPublishEvent(t *testing.T) { + tests := []struct { + name string + event string + computationID string + status string + details json.RawMessage + }{ + { + name: "Standard event", + event: "test-event", + computationID: "test-computation", + status: "test-status", + details: nil, + }, + { + name: "Event with details", + event: "detailed-event", + computationID: "detailed-computation", + status: "detailed-status", + details: json.RawMessage(`{"key": "value"}`), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eventsChan := make(chan *manager.ClientStreamMessage, 1) + ms := &managerService{ + eventsChan: eventsChan, + } + + ms.publishEvent(tt.event, tt.computationID, tt.status, tt.details) + + assert.Len(t, eventsChan, 1) + event := <-eventsChan + assert.Equal(t, tt.event, event.GetAgentEvent().EventType) + assert.Equal(t, tt.computationID, event.GetAgentEvent().ComputationId) + assert.Equal(t, tt.status, event.GetAgentEvent().Status) + assert.Equal(t, "manager", event.GetAgentEvent().Originator) + assert.Equal(t, tt.details, json.RawMessage(event.GetAgentEvent().Details)) + }) + } +} diff --git a/manager/setup_test.go b/manager/setup_test.go new file mode 100644 index 000000000..442bf4249 --- /dev/null +++ b/manager/setup_test.go @@ -0,0 +1,126 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 +package manager_test + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "log/slog" + "net" + "os" + "testing" + "time" + + mglog "github.com/absmach/magistrala/logger" + managergrpc "github.com/ultravioletrs/cocos/manager/api/grpc" + "github.com/ultravioletrs/cocos/pkg/manager" + "golang.org/x/crypto/sha3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/test/bufconn" +) + +const ( + bufSize = 1024 * 1024 + keyBitSize = 4096 +) + +var ( + lis *bufconn.Listener + algoPath = "../test/manual/algo/lin_reg.py" + dataPath = "../test/manual/data/iris.csv" + attestedTLS = false +) + +type svc struct { + logger *slog.Logger + t *testing.T +} + +func TestMain(m *testing.M) { + logger := mglog.NewMock() + + lis = bufconn.Listen(bufSize) + s := grpc.NewServer() + + manager.RegisterManagerServiceServer(s, managergrpc.NewServer(make(chan *manager.ClientStreamMessage, 1), &svc{logger: logger})) + go func() { + if err := s.Serve(lis); err != nil { + panic(err) + } + }() + + code := m.Run() + + s.Stop() + lis.Close() + os.Exit(code) +} + +func bufDialer(context.Context, string) (net.Conn, error) { + return lis.Dial() +} + +func (s *svc) Run(ipAddress string, runReqChan chan *manager.ServerStreamMessage, authInfo credentials.AuthInfo) { + privKey, err := rsa.GenerateKey(rand.Reader, keyBitSize) + if err != nil { + s.t.Fatalf("Error generating public key: %v", err) + } + + pubKey, err := x509.MarshalPKIXPublicKey(&privKey.PublicKey) + if err != nil { + s.t.Fatalf("Error marshalling public key: %v", err) + } + + pubPemBytes := pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Bytes: pubKey, + }) + + go func() { + time.Sleep(time.Millisecond * 100) + runReqChan <- &manager.ServerStreamMessage{ + Message: &manager.ServerStreamMessage_TerminateReq{ + TerminateReq: &manager.Terminate{Message: "test terminate"}, + }, + } + }() + + go func() { + time.Sleep(time.Millisecond * 100) + algo, err := os.ReadFile(algoPath) + if err != nil { + s.t.Fatalf("failed to read algorithm file: %s", err) + return + } + data, err := os.ReadFile(dataPath) + if err != nil { + s.t.Fatalf("failed to read data file: %s", err) + return + } + + pubPem, _ := pem.Decode(pubPemBytes) + algoHash := sha3.Sum256(algo) + dataHash := sha3.Sum256(data) + runReqChan <- &manager.ServerStreamMessage{ + Message: &manager.ServerStreamMessage_RunReq{ + RunReq: &manager.ComputationRunReq{ + Id: "1", + Name: "sample computation", + Description: "sample description", + Datasets: []*manager.Dataset{{Hash: dataHash[:], UserKey: pubPem.Bytes}}, + Algorithm: &manager.Algorithm{Hash: algoHash[:], UserKey: pubPem.Bytes}, + ResultConsumers: []*manager.ResultConsumer{{UserKey: pubPem.Bytes}}, + AgentConfig: &manager.AgentConfig{ + Port: "7002", + LogLevel: "debug", + AttestedTls: attestedTLS, + }, + }, + }, + } + }() +} diff --git a/pkg/manager/manager.pb.go b/pkg/manager/manager.pb.go index 3048e53ac..1c8a34780 100644 --- a/pkg/manager/manager.pb.go +++ b/pkg/manager/manager.pb.go @@ -502,6 +502,7 @@ type ServerStreamMessage struct { // Types that are assignable to Message: // + // *ServerStreamMessage_RunReqChunks // *ServerStreamMessage_RunReq // *ServerStreamMessage_TerminateReq // *ServerStreamMessage_StopComputation @@ -548,6 +549,13 @@ func (m *ServerStreamMessage) GetMessage() isServerStreamMessage_Message { return nil } +func (x *ServerStreamMessage) GetRunReqChunks() *RunReqChunks { + if x, ok := x.GetMessage().(*ServerStreamMessage_RunReqChunks); ok { + return x.RunReqChunks + } + return nil +} + func (x *ServerStreamMessage) GetRunReq() *ComputationRunReq { if x, ok := x.GetMessage().(*ServerStreamMessage_RunReq); ok { return x.RunReq @@ -580,22 +588,28 @@ type isServerStreamMessage_Message interface { isServerStreamMessage_Message() } +type ServerStreamMessage_RunReqChunks struct { + RunReqChunks *RunReqChunks `protobuf:"bytes,1,opt,name=runReqChunks,proto3,oneof"` +} + type ServerStreamMessage_RunReq struct { - RunReq *ComputationRunReq `protobuf:"bytes,1,opt,name=runReq,proto3,oneof"` + RunReq *ComputationRunReq `protobuf:"bytes,2,opt,name=runReq,proto3,oneof"` } type ServerStreamMessage_TerminateReq struct { - TerminateReq *Terminate `protobuf:"bytes,2,opt,name=terminateReq,proto3,oneof"` + TerminateReq *Terminate `protobuf:"bytes,3,opt,name=terminateReq,proto3,oneof"` } type ServerStreamMessage_StopComputation struct { - StopComputation *StopComputation `protobuf:"bytes,3,opt,name=stopComputation,proto3,oneof"` + StopComputation *StopComputation `protobuf:"bytes,4,opt,name=stopComputation,proto3,oneof"` } type ServerStreamMessage_BackendInfoReq struct { - BackendInfoReq *BackendInfoReq `protobuf:"bytes,4,opt,name=backendInfoReq,proto3,oneof"` + BackendInfoReq *BackendInfoReq `protobuf:"bytes,5,opt,name=backendInfoReq,proto3,oneof"` } +func (*ServerStreamMessage_RunReqChunks) isServerStreamMessage_Message() {} + func (*ServerStreamMessage_RunReq) isServerStreamMessage_Message() {} func (*ServerStreamMessage_TerminateReq) isServerStreamMessage_Message() {} @@ -604,6 +618,53 @@ func (*ServerStreamMessage_StopComputation) isServerStreamMessage_Message() {} func (*ServerStreamMessage_BackendInfoReq) isServerStreamMessage_Message() {} +type RunReqChunks struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *RunReqChunks) Reset() { + *x = RunReqChunks{} + if protoimpl.UnsafeEnabled { + mi := &file_manager_manager_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RunReqChunks) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunReqChunks) ProtoMessage() {} + +func (x *RunReqChunks) ProtoReflect() protoreflect.Message { + mi := &file_manager_manager_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunReqChunks.ProtoReflect.Descriptor instead. +func (*RunReqChunks) Descriptor() ([]byte, []int) { + return file_manager_manager_proto_rawDescGZIP(), []int{8} +} + +func (x *RunReqChunks) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + type ComputationRunReq struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -621,7 +682,7 @@ type ComputationRunReq struct { func (x *ComputationRunReq) Reset() { *x = ComputationRunReq{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[8] + mi := &file_manager_manager_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -634,7 +695,7 @@ func (x *ComputationRunReq) String() string { func (*ComputationRunReq) ProtoMessage() {} func (x *ComputationRunReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[8] + mi := &file_manager_manager_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -647,7 +708,7 @@ func (x *ComputationRunReq) ProtoReflect() protoreflect.Message { // Deprecated: Use ComputationRunReq.ProtoReflect.Descriptor instead. func (*ComputationRunReq) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{8} + return file_manager_manager_proto_rawDescGZIP(), []int{9} } func (x *ComputationRunReq) GetId() string { @@ -710,7 +771,7 @@ type BackendInfoReq struct { func (x *BackendInfoReq) Reset() { *x = BackendInfoReq{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[9] + mi := &file_manager_manager_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -723,7 +784,7 @@ func (x *BackendInfoReq) String() string { func (*BackendInfoReq) ProtoMessage() {} func (x *BackendInfoReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[9] + mi := &file_manager_manager_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -736,7 +797,7 @@ func (x *BackendInfoReq) ProtoReflect() protoreflect.Message { // Deprecated: Use BackendInfoReq.ProtoReflect.Descriptor instead. func (*BackendInfoReq) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{9} + return file_manager_manager_proto_rawDescGZIP(), []int{10} } func (x *BackendInfoReq) GetId() string { @@ -757,7 +818,7 @@ type ResultConsumer struct { func (x *ResultConsumer) Reset() { *x = ResultConsumer{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[10] + mi := &file_manager_manager_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -770,7 +831,7 @@ func (x *ResultConsumer) String() string { func (*ResultConsumer) ProtoMessage() {} func (x *ResultConsumer) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[10] + mi := &file_manager_manager_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -783,7 +844,7 @@ func (x *ResultConsumer) ProtoReflect() protoreflect.Message { // Deprecated: Use ResultConsumer.ProtoReflect.Descriptor instead. func (*ResultConsumer) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{10} + return file_manager_manager_proto_rawDescGZIP(), []int{11} } func (x *ResultConsumer) GetUserKey() []byte { @@ -805,7 +866,7 @@ type Dataset struct { func (x *Dataset) Reset() { *x = Dataset{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[11] + mi := &file_manager_manager_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -818,7 +879,7 @@ func (x *Dataset) String() string { func (*Dataset) ProtoMessage() {} func (x *Dataset) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[11] + mi := &file_manager_manager_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -831,7 +892,7 @@ func (x *Dataset) ProtoReflect() protoreflect.Message { // Deprecated: Use Dataset.ProtoReflect.Descriptor instead. func (*Dataset) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{11} + return file_manager_manager_proto_rawDescGZIP(), []int{12} } func (x *Dataset) GetHash() []byte { @@ -860,7 +921,7 @@ type Algorithm struct { func (x *Algorithm) Reset() { *x = Algorithm{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[12] + mi := &file_manager_manager_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -873,7 +934,7 @@ func (x *Algorithm) String() string { func (*Algorithm) ProtoMessage() {} func (x *Algorithm) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[12] + mi := &file_manager_manager_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -886,7 +947,7 @@ func (x *Algorithm) ProtoReflect() protoreflect.Message { // Deprecated: Use Algorithm.ProtoReflect.Descriptor instead. func (*Algorithm) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{12} + return file_manager_manager_proto_rawDescGZIP(), []int{13} } func (x *Algorithm) GetHash() []byte { @@ -921,7 +982,7 @@ type AgentConfig struct { func (x *AgentConfig) Reset() { *x = AgentConfig{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[13] + mi := &file_manager_manager_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -934,7 +995,7 @@ func (x *AgentConfig) String() string { func (*AgentConfig) ProtoMessage() {} func (x *AgentConfig) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[13] + mi := &file_manager_manager_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -947,7 +1008,7 @@ func (x *AgentConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentConfig.ProtoReflect.Descriptor instead. func (*AgentConfig) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{13} + return file_manager_manager_proto_rawDescGZIP(), []int{14} } func (x *AgentConfig) GetPort() string { @@ -1067,79 +1128,85 @@ var file_manager_manager_proto_rawDesc = []byte{ 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x22, 0x99, 0x02, 0x0a, 0x13, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x75, 0x6e, - 0x52, 0x65, 0x71, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x75, 0x6e, 0x52, 0x65, 0x71, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x12, - 0x38, 0x0a, 0x0c, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x74, 0x65, 0x72, - 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x44, 0x0a, 0x0f, 0x73, 0x74, 0x6f, - 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x6f, - 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0f, - 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x41, 0x0a, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, - 0x71, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, - 0x48, 0x00, 0x52, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x65, 0x71, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xb6, 0x02, - 0x0a, 0x11, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, - 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x08, 0x64, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x08, 0x64, - 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x73, 0x12, 0x30, 0x0a, 0x09, 0x61, 0x6c, 0x67, 0x6f, 0x72, - 0x69, 0x74, 0x68, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x52, 0x09, - 0x61, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x42, 0x0a, 0x10, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x52, 0x0f, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x37, 0x0a, - 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x20, 0x0a, 0x0e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, - 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2a, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, - 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, - 0x72, 0x4b, 0x65, 0x79, 0x22, 0x37, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x39, 0x0a, - 0x09, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, - 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x18, - 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0xf9, 0x01, 0x0a, 0x0b, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, - 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, 0x0a, - 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, - 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x61, - 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, - 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, - 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x6c, - 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, - 0x64, 0x54, 0x6c, 0x73, 0x32, 0x5d, 0x0a, 0x0e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4b, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, - 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, - 0x01, 0x30, 0x01, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x22, 0xd6, 0x02, 0x0a, 0x13, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3b, 0x0a, 0x0c, 0x72, 0x75, 0x6e, + 0x52, 0x65, 0x71, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, + 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x71, + 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x71, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x52, + 0x65, 0x71, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x12, 0x38, 0x0a, 0x0c, + 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x72, + 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x44, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, + 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6f, + 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0f, 0x73, 0x74, 0x6f, + 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x41, 0x0a, 0x0e, + 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x42, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x48, 0x00, 0x52, + 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x42, + 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x22, 0x0a, 0x0c, 0x52, 0x75, + 0x6e, 0x52, 0x65, 0x71, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xb6, + 0x02, 0x0a, 0x11, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, + 0x6e, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x08, 0x64, 0x61, + 0x74, 0x61, 0x73, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x08, + 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x73, 0x12, 0x30, 0x0a, 0x09, 0x61, 0x6c, 0x67, 0x6f, + 0x72, 0x69, 0x74, 0x68, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x52, + 0x09, 0x61, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x42, 0x0a, 0x10, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x52, 0x0f, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x37, + 0x0a, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x61, 0x67, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x20, 0x0a, 0x0e, 0x42, 0x61, 0x63, 0x6b, 0x65, + 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2a, 0x0a, 0x0e, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x75, + 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, + 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x37, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x68, 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x39, + 0x0a, 0x09, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x68, + 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, + 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0xf9, 0x01, 0x0a, 0x0b, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, + 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, + 0x24, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, + 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, + 0x65, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, + 0x6c, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, + 0x65, 0x64, 0x54, 0x6c, 0x73, 0x32, 0x5d, 0x0a, 0x0e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4b, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, + 0x73, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1154,7 +1221,7 @@ func file_manager_manager_proto_rawDescGZIP() []byte { return file_manager_manager_proto_rawDescData } -var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_manager_manager_proto_goTypes = []any{ (*Terminate)(nil), // 0: manager.Terminate (*StopComputation)(nil), // 1: manager.StopComputation @@ -1164,36 +1231,38 @@ var file_manager_manager_proto_goTypes = []any{ (*AgentLog)(nil), // 5: manager.AgentLog (*ClientStreamMessage)(nil), // 6: manager.ClientStreamMessage (*ServerStreamMessage)(nil), // 7: manager.ServerStreamMessage - (*ComputationRunReq)(nil), // 8: manager.ComputationRunReq - (*BackendInfoReq)(nil), // 9: manager.BackendInfoReq - (*ResultConsumer)(nil), // 10: manager.ResultConsumer - (*Dataset)(nil), // 11: manager.Dataset - (*Algorithm)(nil), // 12: manager.Algorithm - (*AgentConfig)(nil), // 13: manager.AgentConfig - (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp + (*RunReqChunks)(nil), // 8: manager.RunReqChunks + (*ComputationRunReq)(nil), // 9: manager.ComputationRunReq + (*BackendInfoReq)(nil), // 10: manager.BackendInfoReq + (*ResultConsumer)(nil), // 11: manager.ResultConsumer + (*Dataset)(nil), // 12: manager.Dataset + (*Algorithm)(nil), // 13: manager.Algorithm + (*AgentConfig)(nil), // 14: manager.AgentConfig + (*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp } var file_manager_manager_proto_depIdxs = []int32{ - 14, // 0: manager.AgentEvent.timestamp:type_name -> google.protobuf.Timestamp - 14, // 1: manager.AgentLog.timestamp:type_name -> google.protobuf.Timestamp + 15, // 0: manager.AgentEvent.timestamp:type_name -> google.protobuf.Timestamp + 15, // 1: manager.AgentLog.timestamp:type_name -> google.protobuf.Timestamp 5, // 2: manager.ClientStreamMessage.agent_log:type_name -> manager.AgentLog 4, // 3: manager.ClientStreamMessage.agent_event:type_name -> manager.AgentEvent 2, // 4: manager.ClientStreamMessage.run_res:type_name -> manager.RunResponse 3, // 5: manager.ClientStreamMessage.backendInfo:type_name -> manager.BackendInfo - 8, // 6: manager.ServerStreamMessage.runReq:type_name -> manager.ComputationRunReq - 0, // 7: manager.ServerStreamMessage.terminateReq:type_name -> manager.Terminate - 1, // 8: manager.ServerStreamMessage.stopComputation:type_name -> manager.StopComputation - 9, // 9: manager.ServerStreamMessage.backendInfoReq:type_name -> manager.BackendInfoReq - 11, // 10: manager.ComputationRunReq.datasets:type_name -> manager.Dataset - 12, // 11: manager.ComputationRunReq.algorithm:type_name -> manager.Algorithm - 10, // 12: manager.ComputationRunReq.result_consumers:type_name -> manager.ResultConsumer - 13, // 13: manager.ComputationRunReq.agent_config:type_name -> manager.AgentConfig - 6, // 14: manager.ManagerService.Process:input_type -> manager.ClientStreamMessage - 7, // 15: manager.ManagerService.Process:output_type -> manager.ServerStreamMessage - 15, // [15:16] is the sub-list for method output_type - 14, // [14:15] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 8, // 6: manager.ServerStreamMessage.runReqChunks:type_name -> manager.RunReqChunks + 9, // 7: manager.ServerStreamMessage.runReq:type_name -> manager.ComputationRunReq + 0, // 8: manager.ServerStreamMessage.terminateReq:type_name -> manager.Terminate + 1, // 9: manager.ServerStreamMessage.stopComputation:type_name -> manager.StopComputation + 10, // 10: manager.ServerStreamMessage.backendInfoReq:type_name -> manager.BackendInfoReq + 12, // 11: manager.ComputationRunReq.datasets:type_name -> manager.Dataset + 13, // 12: manager.ComputationRunReq.algorithm:type_name -> manager.Algorithm + 11, // 13: manager.ComputationRunReq.result_consumers:type_name -> manager.ResultConsumer + 14, // 14: manager.ComputationRunReq.agent_config:type_name -> manager.AgentConfig + 6, // 15: manager.ManagerService.Process:input_type -> manager.ClientStreamMessage + 7, // 16: manager.ManagerService.Process:output_type -> manager.ServerStreamMessage + 16, // [16:17] is the sub-list for method output_type + 15, // [15:16] is the sub-list for method input_type + 15, // [15:15] is the sub-list for extension type_name + 15, // [15:15] is the sub-list for extension extendee + 0, // [0:15] is the sub-list for field type_name } func init() { file_manager_manager_proto_init() } @@ -1299,7 +1368,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*ComputationRunReq); i { + switch v := v.(*RunReqChunks); i { case 0: return &v.state case 1: @@ -1311,7 +1380,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[9].Exporter = func(v any, i int) any { - switch v := v.(*BackendInfoReq); i { + switch v := v.(*ComputationRunReq); i { case 0: return &v.state case 1: @@ -1323,7 +1392,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*ResultConsumer); i { + switch v := v.(*BackendInfoReq); i { case 0: return &v.state case 1: @@ -1335,7 +1404,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[11].Exporter = func(v any, i int) any { - switch v := v.(*Dataset); i { + switch v := v.(*ResultConsumer); i { case 0: return &v.state case 1: @@ -1347,7 +1416,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[12].Exporter = func(v any, i int) any { - switch v := v.(*Algorithm); i { + switch v := v.(*Dataset); i { case 0: return &v.state case 1: @@ -1359,6 +1428,18 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[13].Exporter = func(v any, i int) any { + switch v := v.(*Algorithm); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_manager_manager_proto_msgTypes[14].Exporter = func(v any, i int) any { switch v := v.(*AgentConfig); i { case 0: return &v.state @@ -1378,6 +1459,7 @@ func file_manager_manager_proto_init() { (*ClientStreamMessage_BackendInfo)(nil), } file_manager_manager_proto_msgTypes[7].OneofWrappers = []any{ + (*ServerStreamMessage_RunReqChunks)(nil), (*ServerStreamMessage_RunReq)(nil), (*ServerStreamMessage_TerminateReq)(nil), (*ServerStreamMessage_StopComputation)(nil), @@ -1389,7 +1471,7 @@ func file_manager_manager_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_manager_manager_proto_rawDesc, NumEnums: 0, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 1, }, From afc306a85b7f3e5e363cf64d8f018d2e1871f19e Mon Sep 17 00:00:00 2001 From: b1ackd0t <28790446+rodneyosodo@users.noreply.github.com> Date: Tue, 6 Aug 2024 20:06:48 +0300 Subject: [PATCH 03/18] NOISSUE - Enable WASM Support and FileSystem Support (#189) * feat(algorithm): Add wasm as an algo type Signed-off-by: Rodney Osodo * feat(algorithm): Use filesystem to store results Move from unix socket for results storage to filesystem * test: test new filesystem changes Signed-off-by: Rodney Osodo * refactor(files): rename resultFile to resultsFilePath * feat(wasm-runtime): change from wasmtime to wasmedge Wasmedge enables easier directory mapping to get results Signed-off-by: Rodney Osodo * feat(algorithm): send results as zipped directory Create a new function to zip the results directory and send it back to the user * fix(wasm): runtime argument Fix the directory mapping for wasm runtime arguments Signed-off-by: Rodney Osodo * fix(errors): provide useful error message * chore(gitignore): add results zip to gitignore * feat(filesystem): Enable storing results on filesystem for python algos * refactor: revert to upstream cocos repo Signed-off-by: Rodney Osodo * fix: remove AddDataset from algorithm interface * fix: agent to handle results zipping * test: test zipping directories * refactor(agent): Handle file operations from agent * test: run test inside eos Signed-off-by: Rodney Osodo * refactor(test): Document and test algos are running Document steps on running the 2 python exampls and ensure they are running on eos Signed-off-by: Rodney Osodo * fix: remove witheDataset option * test: test without dataset argument Signed-off-by: Rodney Osodo --------- Signed-off-by: Rodney Osodo --- .github/workflows/checkproto.yaml | 2 +- .gitignore | 2 +- agent/agent.pb.go | 2 +- agent/agent_grpc.pb.go | 2 +- agent/algorithm/algorithm.go | 9 +- agent/algorithm/binary/binary.go | 49 ++------- agent/algorithm/python/python.go | 54 +++------- agent/algorithm/results.go | 59 +++++++++++ agent/algorithm/results_test.go | 81 +++++++++++++++ agent/algorithm/wasm/wasm.go | 50 ++++++++++ agent/service.go | 72 ++++++++++---- cli/result.go | 2 +- hal/linux/Config.in | 2 +- hal/linux/package/wasmedge/Config.in | 6 ++ hal/linux/package/wasmedge/wasmedge.mk | 8 ++ hal/linux/package/wasmtime/Config.in | 5 - hal/linux/package/wasmtime/wasmtime.mk | 11 --- pkg/manager/manager.pb.go | 2 +- pkg/manager/manager_grpc.pb.go | 2 +- test/manual/algo/README.md | 108 ++++++++++++++++++-- test/manual/algo/addition.py | 72 ++++++-------- test/manual/algo/lin_reg.py | 131 +++++++++++++++++++------ test/manual/algo/lin_reg_test.py | 51 ---------- 23 files changed, 517 insertions(+), 265 deletions(-) create mode 100644 agent/algorithm/results.go create mode 100644 agent/algorithm/results_test.go create mode 100644 agent/algorithm/wasm/wasm.go create mode 100644 hal/linux/package/wasmedge/Config.in create mode 100644 hal/linux/package/wasmedge/wasmedge.mk delete mode 100644 hal/linux/package/wasmtime/Config.in delete mode 100644 hal/linux/package/wasmtime/wasmtime.mk delete mode 100644 test/manual/algo/lin_reg_test.py diff --git a/.github/workflows/checkproto.yaml b/.github/workflows/checkproto.yaml index 798625d57..51be82135 100644 --- a/.github/workflows/checkproto.yaml +++ b/.github/workflows/checkproto.yaml @@ -33,7 +33,7 @@ jobs: - name: Set up protoc run: | - PROTOC_VERSION=25.3 + PROTOC_VERSION=27.2 PROTOC_GEN_VERSION=v1.34.2 PROTOC_GRPC_VERSION=v1.4.0 diff --git a/.gitignore b/.gitignore index f34a3e2e3..457ef1d81 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,5 @@ cmd/manager/tmp *.pem dist/ -result.bin +result.zip *.spec diff --git a/agent/agent.pb.go b/agent/agent.pb.go index 492a75d40..959991e8a 100644 --- a/agent/agent.pb.go +++ b/agent/agent.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v4.25.3 +// protoc v5.27.2 // source: agent/agent.proto package agent diff --git a/agent/agent_grpc.pb.go b/agent/agent_grpc.pb.go index 173589555..ac2bab2ee 100644 --- a/agent/agent_grpc.pb.go +++ b/agent/agent_grpc.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v4.25.3 +// - protoc v5.27.2 // source: agent/agent.proto package agent diff --git a/agent/algorithm/algorithm.go b/agent/algorithm/algorithm.go index 1e6bd9b0f..9c4b53194 100644 --- a/agent/algorithm/algorithm.go +++ b/agent/algorithm/algorithm.go @@ -13,7 +13,11 @@ type AlgorithType string const ( AlgoTypeBin AlgorithType = "bin" AlgoTypePython AlgorithType = "python" + AlgoTypeWasm AlgorithType = "wasm" AlgoTypeKey = "algo_type" + + ResultsDir = "results" + DatasetsDir = "datasets" ) func AlgorithmTypeToContext(ctx context.Context, algoType string) context.Context { @@ -27,8 +31,5 @@ func AlgorithmTypeFromContext(ctx context.Context) string { // Algorithm is an interface that specifies the API for an algorithm. type Algorithm interface { // Run executes the algorithm and returns the result. - Run() ([]byte, error) - - // Add dataset to algorithm. - AddDataset(dataset string) + Run() error } diff --git a/agent/algorithm/binary/binary.go b/agent/algorithm/binary/binary.go index d487dd60f..df6766c12 100644 --- a/agent/algorithm/binary/binary.go +++ b/agent/algorithm/binary/binary.go @@ -6,77 +6,40 @@ import ( "fmt" "io" "log/slog" - "os" "os/exec" "github.com/ultravioletrs/cocos/agent/algorithm" "github.com/ultravioletrs/cocos/agent/events" - "github.com/ultravioletrs/cocos/pkg/socket" ) -const socketPath = "unix_socket" - var _ algorithm.Algorithm = (*binary)(nil) type binary struct { algoFile string - datasets []string - logger *slog.Logger stderr io.Writer stdout io.Writer } -func New(logger *slog.Logger, eventsSvc events.Service, algoFile string) algorithm.Algorithm { +func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string) algorithm.Algorithm { return &binary{ algoFile: algoFile, - logger: logger, stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc}, stdout: &algorithm.Stdout{Logger: logger}, } } -func (b *binary) AddDataset(dataset string) { - b.datasets = append(b.datasets, dataset) -} - -func (b *binary) Run() ([]byte, error) { - defer os.Remove(b.algoFile) - defer func() { - for _, file := range b.datasets { - os.Remove(file) - } - }() - listener, err := socket.StartUnixSocketServer(socketPath) - if err != nil { - return nil, fmt.Errorf("error creating stdout pipe: %v", err) - } - defer listener.Close() - - // Create channels for received data and errors - dataChannel := make(chan []byte) - errorChannel := make(chan error) - - var result []byte - - go socket.AcceptConnection(listener, dataChannel, errorChannel) - - args := append([]string{socketPath}, b.datasets...) - cmd := exec.Command(b.algoFile, args...) +func (b *binary) Run() error { + cmd := exec.Command(b.algoFile) cmd.Stderr = b.stderr cmd.Stdout = b.stdout if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("error starting algorithm: %v", err) + return fmt.Errorf("error starting algorithm: %v", err) } if err := cmd.Wait(); err != nil { - return nil, fmt.Errorf("algorithm execution error: %v", err) + return fmt.Errorf("algorithm execution error: %v", err) } - select { - case result = <-dataChannel: - return result, nil - case err = <-errorChannel: - return nil, fmt.Errorf("error receiving data: %v", err) - } + return nil } diff --git a/agent/algorithm/python/python.go b/agent/algorithm/python/python.go index 0fd88313a..a2a2afdb5 100644 --- a/agent/algorithm/python/python.go +++ b/agent/algorithm/python/python.go @@ -13,12 +13,10 @@ import ( "github.com/ultravioletrs/cocos/agent/algorithm" "github.com/ultravioletrs/cocos/agent/events" - "github.com/ultravioletrs/cocos/pkg/socket" "google.golang.org/grpc/metadata" ) const ( - socketPath = "unix_socket" PyRuntime = "python3" pyRuntimeKey = "python_runtime" ) @@ -35,18 +33,15 @@ var _ algorithm.Algorithm = (*python)(nil) type python struct { algoFile string - datasets []string - logger *slog.Logger stderr io.Writer stdout io.Writer runtime string requirementsFile string } -func New(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFile, algoFile string) algorithm.Algorithm { +func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFile, algoFile string) algorithm.Algorithm { p := &python{ algoFile: algoFile, - logger: logger, stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc}, stdout: &algorithm.Stdout{Logger: logger}, requirementsFile: requirementsFile, @@ -59,17 +54,13 @@ func New(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFil return p } -func (p *python) AddDataset(dataset string) { - p.datasets = append(p.datasets, dataset) -} - -func (p *python) Run() ([]byte, error) { +func (p *python) Run() error { venvPath := "venv" createVenvCmd := exec.Command(p.runtime, "-m", "venv", venvPath) createVenvCmd.Stderr = p.stderr createVenvCmd.Stdout = p.stdout if err := createVenvCmd.Run(); err != nil { - return nil, fmt.Errorf("error creating virtual environment: %v", err) + return fmt.Errorf("error creating virtual environment: %v", err) } pythonPath := filepath.Join(venvPath, "bin", "python") @@ -79,48 +70,25 @@ func (p *python) Run() ([]byte, error) { rcmd.Stderr = p.stderr rcmd.Stdout = p.stdout if err := rcmd.Run(); err != nil { - return nil, fmt.Errorf("error installing requirements: %v", err) - } - } - - defer os.Remove(p.algoFile) - defer func() { - for _, file := range p.datasets { - os.Remove(file) + return fmt.Errorf("error installing requirements: %v", err) } - }() - defer os.RemoveAll(venvPath) - - listener, err := socket.StartUnixSocketServer(socketPath) - if err != nil { - return nil, fmt.Errorf("error creating stdout pipe: %v", err) } - defer listener.Close() - - dataChannel := make(chan []byte) - errorChannel := make(chan error) - - var result []byte - go socket.AcceptConnection(listener, dataChannel, errorChannel) - - args := append([]string{p.algoFile, socketPath}, p.datasets...) - cmd := exec.Command(pythonPath, args...) + cmd := exec.Command(pythonPath, p.algoFile) cmd.Stderr = p.stderr cmd.Stdout = p.stdout if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("error starting algorithm: %v", err) + return fmt.Errorf("error starting algorithm: %v", err) } if err := cmd.Wait(); err != nil { - return nil, fmt.Errorf("algorithm execution error: %v", err) + return fmt.Errorf("algorithm execution error: %v", err) } - select { - case result = <-dataChannel: - return result, nil - case err = <-errorChannel: - return nil, fmt.Errorf("error receiving data: %v", err) + if err := os.RemoveAll(venvPath); err != nil { + return fmt.Errorf("error removing virtual environment: %v", err) } + + return nil } diff --git a/agent/algorithm/results.go b/agent/algorithm/results.go new file mode 100644 index 000000000..114cfe058 --- /dev/null +++ b/agent/algorithm/results.go @@ -0,0 +1,59 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 +package algorithm + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "os" + "path/filepath" +) + +// ZipDirectory zips a directory and returns the zipped bytes. +func ZipDirectory() ([]byte, error) { + buf := new(bytes.Buffer) + zipWriter := zip.NewWriter(buf) + + err := filepath.Walk(ResultsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error walking the path %q: %v", path, err) + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(ResultsDir, path) + if err != nil { + return fmt.Errorf("error getting relative path for %q: %v", path, err) + } + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening file %q: %v", path, err) + } + defer file.Close() + + zipFile, err := zipWriter.Create(relPath) + if err != nil { + return fmt.Errorf("error creating zip file for %q: %v", path, err) + } + + if _, err = io.Copy(zipFile, file); err != nil { + return fmt.Errorf("error copying file %q to zip: %v", path, err) + } + + return err + }) + if err != nil { + return nil, err + } + + if err = zipWriter.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/agent/algorithm/results_test.go b/agent/algorithm/results_test.go new file mode 100644 index 000000000..c9745624f --- /dev/null +++ b/agent/algorithm/results_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 +package algorithm_test + +import ( + "os" + "testing" + + "github.com/ultravioletrs/cocos/agent/algorithm" +) + +func TestZipDirectory(t *testing.T) { + cases := []struct { + name string + directories []string + files []string + expected []string + }{ + { + name: "empty directory", + directories: []string{"testdata"}, + }, + { + name: "single file", + files: []string{"file1.txt"}, + }, + { + name: "directory with single file", + directories: []string{"testdata"}, + expected: []string{"testdata/file1.txt"}, + }, + { + name: "directory with multiple files", + directories: []string{"testdata"}, + expected: []string{ + "testdata/file1.txt", + "testdata/file2.txt", + "testdata/file3.txt", + }, + }, + { + name: "nested directories", + directories: []string{"testdata", "testdata/nested"}, + expected: []string{ + "testdata/nested/file1.txt", + "testdata/nested/file2.txt", + "testdata/nested/file3.txt", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if err := os.Mkdir(algorithm.ResultsDir, 0o755); err != nil { + t.Fatalf("error creating results directory: %s", err.Error()) + } + defer func() { + if err := os.RemoveAll(algorithm.ResultsDir); err != nil { + t.Fatalf("error removing results directory and its contents: %s", err.Error()) + } + }() + + for _, dir := range tc.directories { + if dir != "" { + if err := os.Mkdir(algorithm.ResultsDir+"/"+dir, 0o755); err != nil { + t.Fatalf("error creating test directory: %s", err.Error()) + } + } + } + for _, file := range tc.files { + if _, err := os.Create(algorithm.ResultsDir + "/" + file); err != nil { + t.Fatalf("error creating test file: %s", err.Error()) + } + } + + if _, err := algorithm.ZipDirectory(); err != nil { + t.Errorf("ZipDirectory() error = %v", err) + } + }) + } +} diff --git a/agent/algorithm/wasm/wasm.go b/agent/algorithm/wasm/wasm.go new file mode 100644 index 000000000..9d8bc721f --- /dev/null +++ b/agent/algorithm/wasm/wasm.go @@ -0,0 +1,50 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 +package wasm + +import ( + "fmt" + "io" + "log/slog" + "os/exec" + + "github.com/ultravioletrs/cocos/agent/algorithm" + "github.com/ultravioletrs/cocos/agent/events" +) + +const wasmRuntime = "wasmedge" + +var mapDirOption = []string{"--dir", ".:" + algorithm.ResultsDir} + +var _ algorithm.Algorithm = (*wasm)(nil) + +type wasm struct { + algoFile string + stderr io.Writer + stdout io.Writer +} + +func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string) algorithm.Algorithm { + return &wasm{ + algoFile: algoFile, + stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc}, + stdout: &algorithm.Stdout{Logger: logger}, + } +} + +func (w *wasm) Run() error { + args := append(mapDirOption, w.algoFile) + cmd := exec.Command(wasmRuntime, args...) + cmd.Stderr = w.stderr + cmd.Stdout = w.stdout + + if err := cmd.Start(); err != nil { + return fmt.Errorf("error starting algorithm: %v", err) + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("algorithm execution error: %v", err) + } + + return nil +} diff --git a/agent/service.go b/agent/service.go index 004f29d16..bc105ed6d 100644 --- a/agent/service.go +++ b/agent/service.go @@ -16,6 +16,7 @@ import ( "github.com/ultravioletrs/cocos/agent/algorithm" "github.com/ultravioletrs/cocos/agent/algorithm/binary" "github.com/ultravioletrs/cocos/agent/algorithm/python" + "github.com/ultravioletrs/cocos/agent/algorithm/wasm" "github.com/ultravioletrs/cocos/agent/events" "golang.org/x/crypto/sha3" ) @@ -89,6 +90,7 @@ func New(ctx context.Context, logger *slog.Logger, eventSvc events.Service, cmp svc.sm.StateFunctions[running] = svc.runComputation svc.computation = cmp + svc.sm.SendEvent(manifestReceived) return svc } @@ -131,21 +133,31 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error { switch algoType { case string(algorithm.AlgoTypeBin): - as.algorithm = binary.New(as.sm.logger, as.eventSvc, f.Name()) + as.algorithm = binary.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name()) case string(algorithm.AlgoTypePython): - fr, err := os.CreateTemp("", "requirements.txt") - if err != nil { - return fmt.Errorf("error creating requirments file: %v", err) - } - - if _, err := fr.Write(algo.Requirements); err != nil { - return fmt.Errorf("error writing requirements to file: %v", err) - } - if err := fr.Close(); err != nil { - return fmt.Errorf("error closing file: %v", err) + var requirementsFile string + if len(algo.Requirements) > 0 { + fr, err := os.CreateTemp("", "requirements.txt") + if err != nil { + return fmt.Errorf("error creating requirments file: %v", err) + } + + if _, err := fr.Write(algo.Requirements); err != nil { + return fmt.Errorf("error writing requirements to file: %v", err) + } + if err := fr.Close(); err != nil { + return fmt.Errorf("error closing file: %v", err) + } + requirementsFile = fr.Name() } runtime := python.PythonRunTimeFromContext(ctx) - as.algorithm = python.New(as.sm.logger, as.eventSvc, runtime, fr.Name(), f.Name()) + as.algorithm = python.NewAlgorithm(as.sm.logger, as.eventSvc, runtime, requirementsFile, f.Name()) + case string(algorithm.AlgoTypeWasm): + as.algorithm = wasm.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name()) + } + + if err := os.Mkdir(algorithm.DatasetsDir, 0o755); err != nil { + return fmt.Errorf("error creating datasets directory: %v", err) } if as.algorithm != nil { @@ -175,7 +187,7 @@ func (as *agentService) Data(ctx context.Context, dataset Dataset) error { } as.computation.Datasets = slices.Delete(as.computation.Datasets, index, index+1) - f, err := os.CreateTemp("", fmt.Sprintf("dataset-%d", index)) + f, err := os.Create(fmt.Sprintf("%s/dataset-%d", algorithm.DatasetsDir, index)) if err != nil { return fmt.Errorf("error creating dataset file: %v", err) } @@ -187,8 +199,6 @@ func (as *agentService) Data(ctx context.Context, dataset Dataset) error { return fmt.Errorf("error closing file: %v", err) } - as.algorithm.AddDataset(f.Name()) - if len(as.computation.Datasets) == 0 { as.sm.SendEvent(dataReceived) } @@ -233,16 +243,42 @@ func (as *agentService) runComputation() { as.publishEvent("starting", json.RawMessage{})() as.sm.logger.Debug("computation run started") defer as.sm.SendEvent(runComplete) + + if err := os.Mkdir(algorithm.ResultsDir, 0o755); err != nil { + as.runError = fmt.Errorf("error creating results directory: %s", err.Error()) + as.sm.logger.Warn(as.runError.Error()) + as.publishEvent("failed", json.RawMessage{})() + return + } + + defer func() { + if err := os.RemoveAll(algorithm.ResultsDir); err != nil { + as.sm.logger.Warn(fmt.Sprintf("error removing results directory and its contents: %s", err.Error())) + } + if err := os.RemoveAll(algorithm.DatasetsDir); err != nil { + as.sm.logger.Warn(fmt.Sprintf("error removing datasets directory and its contents: %s", err.Error())) + } + }() + as.publishEvent("in-progress", json.RawMessage{})() - result, err := as.algorithm.Run() + if err := as.algorithm.Run(); err != nil { + as.runError = err + as.sm.logger.Warn(fmt.Sprintf("failed to run computation: %s", err.Error())) + as.publishEvent("failed", json.RawMessage{})() + return + } + + results, err := algorithm.ZipDirectory() if err != nil { as.runError = err - as.sm.logger.Warn(fmt.Sprintf("computation failed with error: %s", err.Error())) + as.sm.logger.Warn(fmt.Sprintf("failed to zip results: %s", err.Error())) as.publishEvent("failed", json.RawMessage{})() return } + as.publishEvent("complete", json.RawMessage{})() - as.result = result + + as.result = results } func (as *agentService) publishEvent(status string, details json.RawMessage) func() { diff --git a/cli/result.go b/cli/result.go index a264c667c..73898cfb2 100644 --- a/cli/result.go +++ b/cli/result.go @@ -10,7 +10,7 @@ import ( "github.com/spf13/cobra" ) -const resultFilePath = "result.bin" +const resultFilePath = "result.zip" func (cli *CLI) NewResultsCmd() *cobra.Command { return &cobra.Command{ diff --git a/hal/linux/Config.in b/hal/linux/Config.in index cf84594f1..03271381a 100644 --- a/hal/linux/Config.in +++ b/hal/linux/Config.in @@ -1,2 +1,2 @@ source "$BR2_EXTERNAL_COCOS_PATH/package/agent/Config.in" -source "$BR2_EXTERNAL_COCOS_PATH/package/wasmtime/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/wasmedge/Config.in" diff --git a/hal/linux/package/wasmedge/Config.in b/hal/linux/package/wasmedge/Config.in new file mode 100644 index 000000000..c0a7be4e7 --- /dev/null +++ b/hal/linux/package/wasmedge/Config.in @@ -0,0 +1,6 @@ +config BR2_PACKAGE_WASMEDGE + bool "wasmedge" + default y + help + Wasmedge is a standalone runtime for WebAssembly. + https://wasmedge.org/docs/ diff --git a/hal/linux/package/wasmedge/wasmedge.mk b/hal/linux/package/wasmedge/wasmedge.mk new file mode 100644 index 000000000..9d62740d0 --- /dev/null +++ b/hal/linux/package/wasmedge/wasmedge.mk @@ -0,0 +1,8 @@ +WASMEDGE_DOWNLOAD_URL = https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh + +define WASMEDGE_INSTALL_TARGET_CMDS + curl -sSf $(WASMEDGE_DOWNLOAD_URL) | bash -s -- -p $(TARGET_DIR)/usr + echo "source /usr/env" >> $(TARGET_DIR)/etc/profile +endef + +$(eval $(generic-package)) diff --git a/hal/linux/package/wasmtime/Config.in b/hal/linux/package/wasmtime/Config.in deleted file mode 100644 index 06cbc61f6..000000000 --- a/hal/linux/package/wasmtime/Config.in +++ /dev/null @@ -1,5 +0,0 @@ -config BR2_PACKAGE_WASMTIME - bool "wasmtime" - help - Wasmtime is a standalone runtime for WebAssembly. - https://github.com/bytecodealliance/wasmtime diff --git a/hal/linux/package/wasmtime/wasmtime.mk b/hal/linux/package/wasmtime/wasmtime.mk deleted file mode 100644 index 1ec16882e..000000000 --- a/hal/linux/package/wasmtime/wasmtime.mk +++ /dev/null @@ -1,11 +0,0 @@ -WASMTIME_SITE = https://wasmtime.dev/install.sh - -define WASMTIME_BUILD_CMDS - curl $(WASMTIME_SITE) -sSf | bash -endef - -define WASMTIME_INSTALL_TARGET_CMDS - $(INSTALL) -D -m 0755 ~/.wasmtime/bin/wasmtime $(TARGET_DIR)/usr/bin/wasmtime -endef - -$(eval $(generic-package)) diff --git a/pkg/manager/manager.pb.go b/pkg/manager/manager.pb.go index 1c8a34780..1aa78cbdc 100644 --- a/pkg/manager/manager.pb.go +++ b/pkg/manager/manager.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v4.25.3 +// protoc v5.27.2 // source: manager/manager.proto package manager diff --git a/pkg/manager/manager_grpc.pb.go b/pkg/manager/manager_grpc.pb.go index 2aee96a58..ee6b7b52c 100644 --- a/pkg/manager/manager_grpc.pb.go +++ b/pkg/manager/manager_grpc.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v4.25.3 +// - protoc v5.27.2 // source: manager/manager.proto package manager diff --git a/test/manual/algo/README.md b/test/manual/algo/README.md index 60ee1efb7..a9cb79f6c 100644 --- a/test/manual/algo/README.md +++ b/test/manual/algo/README.md @@ -1,17 +1,105 @@ # Algorithm -Agent accepts binaries programs. To use the python program you need to bundle or compile it. -In this example we'll use [pyinstaller](https://pypi.org/project/pyinstaller/) +Agent accepts binaries programs, python scripts, and wasm files. It runs them in a sandboxed environment and returns the output. -```shell -pip install pandas scikit-learn -pip install -U pyinstaller -pyinstaller --onefile lin_reg.py +## Python Example + +To test this examples work on your local machine, you need to install the following dependencies: + +```bash +pip install -r requirements.txt +``` + +This can be done in a virtual environment. + +```bash +python -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +To run the example, you can use the following command: + +```bash +python3 test/manual/algo/addition.py +``` + +The addition example is a simple algorithm to demonstrate you can run an algorithm without any external dependencies and input arguments. It returns the sum of two numbers. + +```bash +python3 test/manual/algo/lin_reg.py ``` -Make the binary static: +The linear regression example is a more complex algorithm that requires external dependencies.It returns a linear regression model trained on the iris dataset found [here](../data/) for demonstration purposes. + +```bash +python3 test/manual/algo/lin_reg.py predict result.zip test/manual/data +``` + +This will make inference on the results of the linear regression model. + +To run the examples in the agent, you can use the following command: + +```bash +go run ./test/computations/main.go ./test/manual/algo/lin_reg.py public.pem false ./test/manual/data/iris.csv +``` + +This command is run from the root directory of the project. This will start the computation server. + +In another window, you can run the following command: + +```bash +sudo MANAGER_QEMU_SMP_MAXCPUS=4 MANAGER_GRPC_URL=localhost:7001 MANAGER_LOG_LEVEL=debug MANAGER_QEMU_USE_SUDO=false MANAGER_QEMU_ENABLE_SEV=false MANAGER_QEMU_SEV_CBITPOS=51 MANAGER_QEMU_ENABLE_SEV_SNP=false MANAGER_QEMU_OVMF_CODE_FILE=/usr/share/edk2/x64/OVMF_CODE.fd MANAGER_QEMU_OVMF_VARS_FILE=/usr/share/edk2/x64/OVMF_VARS.fd go run main.go +``` + +This command is run from the [manager main directory](../../../cmd/manager/). This will start the manager. Make sure you have already built the [qemu image](../../../hal/linux/README.md). + +In another window, you can run the following command: -```shell -pip install staticx -staticx +```bash +./build/cocos-cli algo ./test/manual/algo/lin_reg.py ./private.pem -a python -r ./test/manual/algo/requirements.txt ``` + +make sure you have built the cocos-cli. This will upload the algorithm and the requirements file. + +Next we need to upload the dataset + +```bash +./build/cocos-cli data ./test/manual/data/iris.csv ./private.pem +``` + +After some time when the results are ready, you can run the following command to get the results: + +```bash +./build/cocos-cli results ./private.pem +``` + +This will return the results of the algorithm. + +To make inference on the results, you can use the following command: + +```bash +python3 test/manual/algo/lin_reg.py predict result.zip test/manual/data +``` + +For addition example, you can use the following command: + +```bash +go run ./test/computations/main.go ./test/manual/algo/addition.py public.pem false +``` + +```bash +./build/cocos-cli algo ./test/manual/algo/addition.py ./private.pem -a python +``` + +```bash +./build/cocos-cli results ./private.pem +``` + +## Wasm Example + +More information on how to run wasm files can be found [here](https://github.com/ultravioletrs/ai/tree/main/burn-algorithms). + +## Binary Example + +More information on how to run binary files can be found [here](https://github.com/ultravioletrs/ai/tree/main/burn-algorithms). diff --git a/test/manual/algo/addition.py b/test/manual/algo/addition.py index 7dfc42ae3..9a4467b9b 100644 --- a/test/manual/algo/addition.py +++ b/test/manual/algo/addition.py @@ -1,9 +1,14 @@ -import sys, io -import joblib -import socket +import os +import sys +import zipfile + +RESULTS_DIR = "results" +RESULTS_FILE = "result.txt" + class Computation: result = 0 + def __init__(self): """ Initializes a new instance of the Computation class. @@ -16,45 +21,35 @@ def compute(self, a, b): """ self.result = a + b - def send_result(self, socket_path): + def save_result(self): """ - Sends the result to a socket. + Sends the result to a file. """ - buffer = io.BytesIO() - try: - joblib.dump(self.result, buffer) - except Exception as e: - print("Failed to dump the result to the buffer: ", e) - return + os.makedirs(RESULTS_DIR) + except FileExistsError: + pass - data = buffer.getvalue() + with open(RESULTS_DIR + os.sep + RESULTS_FILE, "w") as f: + f.write(str(self.result)) - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - try: - client.connect(socket_path) - except Exception as e: - print("Failed to connect to the socket: ", e) - return - try: - client.send(data) - except Exception as e: - print("Failed to send data to the socket: ", e) - return - finally: - client.close() - def read_results_from_file(self, results_file): """ Reads the results from a file. """ - try: - results = joblib.load(results_file) - print("Results: ", results) - except Exception as e: - print("Failed to load results from file: ", e) - return + if results_file.endswith(".zip"): + try: + os.makedirs(RESULTS_DIR) + except FileExistsError: + pass + with zipfile.ZipFile(results_file, "r") as zip_ref: + zip_ref.extractall(RESULTS_DIR) + with open(RESULTS_FILE, "r") as f: + print(f.read()) + else: + with open(results_file, "r") as f: + print(f.read()) + if __name__ == "__main__": a = 5 @@ -62,15 +57,10 @@ def read_results_from_file(self, results_file): computation = Computation() if len(sys.argv) == 1: - print("Please provide a socket path or a file path") - exit(1) - - if sys.argv[1] == "test" and len(sys.argv) == 3: - computation.read_results_from_file(sys.argv[2]) - elif len(sys.argv) == 2: computation.compute(a, b) - computation.send_result(sys.argv[1]) + computation.save_result() + elif len(sys.argv) == 3 and sys.argv[1] == "test": + computation.read_results_from_file(sys.argv[2]) else: print("Invalid arguments") exit(1) - diff --git a/test/manual/algo/lin_reg.py b/test/manual/algo/lin_reg.py index 191acaa49..701e0e183 100644 --- a/test/manual/algo/lin_reg.py +++ b/test/manual/algo/lin_reg.py @@ -1,47 +1,116 @@ -import sys, io +import os +import sys import joblib -import socket - import pandas as pd from sklearn.model_selection import train_test_split from sklearn.linear_model import LogisticRegression +import zipfile +from sklearn import metrics + +DATA_DIR = "datasets" +RESULTS_DIR = "results" +RESULTS_FILE = "model.bin" + + +class Computation: + model = None + + def __init__(self): + """ + Initializes a new instance of the Computation class. + """ + pass + + def _read_csv(self, data_path=""): + """ + Reads the CSV file. + """ + files = os.listdir(data_path) + if len(files) != 1: + print("No files found in the directory") + exit(1) + csv_file_path = data_path + os.sep + files[0] + return pd.read_csv(csv_file_path) + + def compute(self): + """ + Trains a logistic regression model. + """ + iris = self._read_csv(DATA_DIR) + + # Droping the Species since we only need the measurements + X = iris.drop(["Species"], axis=1) + + # converting into numpy array and assigning petal length and petal width + X = X.to_numpy()[:, (3, 4)] + y = iris["Species"] + + X_train, _, y_train, _ = train_test_split(X, y, test_size=0.5, random_state=42) -csv_file_path = sys.argv[2] -iris = pd.read_csv(csv_file_path) + log_reg = LogisticRegression() + log_reg.fit(X_train, y_train) + self.model = log_reg -# Droping the Species since we only need the measurements -X = iris.drop(['Species'], axis=1) + def save_result(self): + """ + Sends the result to a file. + """ + try: + os.makedirs(RESULTS_DIR) + except FileExistsError: + pass -# converting into numpy array and assigning petal length and petal width -X = X.to_numpy()[:, (3,4)] -y = iris['Species'] + results_file = RESULTS_DIR + os.sep + RESULTS_FILE + joblib.dump(self.model, results_file) -# Splitting into train and test -X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.5, random_state=42) + def read_results_from_file(self, results_file): + """ + Reads the results from a file. + """ + if results_file.endswith(".zip"): + try: + os.makedirs(RESULTS_DIR) + except FileExistsError: + pass + with zipfile.ZipFile(results_file, "r") as zip_ref: + zip_ref.extractall(RESULTS_DIR) + self.model = joblib.load(RESULTS_DIR + os.sep + RESULTS_FILE) + else: + self.model = joblib.load(results_file) -log_reg = LogisticRegression() -log_reg.fit(X_train,y_train) + def predict(self, data_path=""): + iris = self._read_csv(data_path) -# Serialize the trained model to a byte buffer -model_buffer = io.BytesIO() -joblib.dump(log_reg, model_buffer) + # Droping the Species since we only need the measurements + X = iris.drop(["Species"], axis=1) -# Get the serialized model as a bytes object -model_bytes = model_buffer.getvalue() + # converting into numpy array and assigning petal length and petal width + X = X.to_numpy()[:, (3, 4)] + y = iris["Species"] -# Define the path for the Unix domain socket -socket_path = sys.argv[1] + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.5, random_state=42 + ) -# Create a Unix domain socket client -client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + training_prediction = self.model.predict(X_train) + test_prediction = self.model.predict(X_test) -try: - # Connect to the server - client.connect(socket_path) + print("Precision, Recall, Confusion matrix, in training\n") + print(metrics.classification_report(y_train, training_prediction, digits=3)) + print(metrics.confusion_matrix(y_train, training_prediction)) + print("Precision, Recall, Confusion matrix, in testing\n") + print(metrics.classification_report(y_test, test_prediction, digits=3)) + print(metrics.confusion_matrix(y_test, test_prediction)) - # Send the serialized model over the socket - client.send(model_bytes) -finally: - # Close the socket - client.close() +if __name__ == "__main__": + computation = Computation() + if len(sys.argv) == 1: + computation.compute() + computation.save_result() + elif len(sys.argv) == 4 and sys.argv[1] == "predict": + computation.read_results_from_file(sys.argv[2]) + computation.predict(sys.argv[3]) + else: + print("Invalid arguments") + exit(1) diff --git a/test/manual/algo/lin_reg_test.py b/test/manual/algo/lin_reg_test.py deleted file mode 100644 index 3165873cd..000000000 --- a/test/manual/algo/lin_reg_test.py +++ /dev/null @@ -1,51 +0,0 @@ -import pandas as pd - -from sklearn.model_selection import train_test_split -from sklearn import metrics -import joblib - -import sys - -import warnings -warnings.filterwarnings("ignore", category=DeprecationWarning) -warnings.filterwarnings("ignore", category=UserWarning) - -csv_file_path = sys.argv[1] -model_filename = sys.argv[2] - -# Load the CSV file into a Pandas DataFrame -iris = pd.read_csv(csv_file_path) - -log_reg = joblib.load(model_filename) - -# Now you have the Iris dataset loaded into the iris_df DataFrame -print(iris.head()) # Display the first few rows of the DataFrame - -# Droping the Species since we only need the measurements -X = iris.drop(['Species'], axis=1) - -# converting into numpy array and assigning petal length and petal width -X = X.to_numpy()[:, (3,4)] -y = iris['Species'] - -# Splitting into train and test -X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.5, random_state=42) - -training_prediction = log_reg.predict(X_train) -test_prediction = log_reg.predict(X_test) - -print("Precision, Recall, Confusion matrix, in training\n") - -# Precision Recall scores -print(metrics.classification_report(y_train, training_prediction, digits=3)) - -# Confusion matrix -print(metrics.confusion_matrix(y_train, training_prediction)) - -print("Precision, Recall, Confusion matrix, in testing\n") - -# Precision Recall scores -print(metrics.classification_report(y_test, test_prediction, digits=3)) - -# Confusion matrix -print(metrics.confusion_matrix(y_test, test_prediction)) From 18cfa7619e63db5425d8bcb7ed2b764fd1640fde Mon Sep 17 00:00:00 2001 From: Washington Kigani Kamadi Date: Wed, 7 Aug 2024 16:58:25 +0300 Subject: [PATCH 04/18] PRISM-337: Add Filename to Dataset (#191) * add filename to dataset Signed-off-by: WashingtonKK * update protoc Signed-off-by: WashingtonKK --------- Signed-off-by: WashingtonKK --- manager/manager.proto | 1 + pkg/manager/manager.pb.go | 72 ++++++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/manager/manager.proto b/manager/manager.proto index 075a33832..b3654b71a 100644 --- a/manager/manager.proto +++ b/manager/manager.proto @@ -91,6 +91,7 @@ message ResultConsumer { message Dataset { bytes hash = 1; // should be sha3.Sum256, 32 byte length. bytes userKey = 2; + string filename = 3; } message Algorithm { diff --git a/pkg/manager/manager.pb.go b/pkg/manager/manager.pb.go index 1aa78cbdc..f3b8bafe4 100644 --- a/pkg/manager/manager.pb.go +++ b/pkg/manager/manager.pb.go @@ -859,8 +859,9 @@ type Dataset struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` // should be sha3.Sum256, 32 byte length. - UserKey []byte `protobuf:"bytes,2,opt,name=userKey,proto3" json:"userKey,omitempty"` + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` // should be sha3.Sum256, 32 byte length. + UserKey []byte `protobuf:"bytes,2,opt,name=userKey,proto3" json:"userKey,omitempty"` + Filename string `protobuf:"bytes,3,opt,name=filename,proto3" json:"filename,omitempty"` } func (x *Dataset) Reset() { @@ -909,6 +910,13 @@ func (x *Dataset) GetUserKey() []byte { return nil } +func (x *Dataset) GetFilename() string { + if x != nil { + return x.Filename + } + return "" +} + type Algorithm struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1176,37 +1184,39 @@ var file_manager_manager_proto_rawDesc = []byte{ 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2a, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, - 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x37, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, + 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x53, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0x39, - 0x0a, 0x09, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, - 0x18, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0xf9, 0x01, 0x0a, 0x0b, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, - 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, - 0x24, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, - 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, - 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, - 0x6c, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, - 0x65, 0x64, 0x54, 0x6c, 0x73, 0x32, 0x5d, 0x0a, 0x0e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4b, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, - 0x73, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, - 0x28, 0x01, 0x30, 0x01, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x1a, + 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x39, 0x0a, 0x09, 0x41, 0x6c, + 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x75, + 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x75, 0x73, + 0x65, 0x72, 0x4b, 0x65, 0x79, 0x22, 0xf9, 0x01, 0x0a, 0x0b, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x1b, 0x0a, + 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6b, 0x65, + 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, + 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x61, 0x46, 0x69, 0x6c, + 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x21, + 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x65, 0x64, 0x54, 0x6c, + 0x73, 0x32, 0x5d, 0x0a, 0x0e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x4b, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1c, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, + 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From 971e00aa68eaac9ec6ed064aa14e6851f210b7d5 Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:48:12 +0300 Subject: [PATCH 05/18] NOISSUE - Select from port range (#196) * select from port range Signed-off-by: Sammy Oina * fix failing test Signed-off-by: Sammy Oina * remove whitespace Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- cmd/manager/main.go | 25 +++++++----- manager/README.md | 66 ++++++++++++++++++++++++++------ manager/qemu/config.go | 3 ++ manager/service.go | 84 +++++++++++++++++++++++++++++++++-------- manager/service_test.go | 10 +++-- 5 files changed, 150 insertions(+), 38 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 69c338006..9489d478b 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -37,11 +37,11 @@ const ( ) type config struct { - LogLevel string `env:"MANAGER_LOG_LEVEL" envDefault:"info"` - JaegerURL url.URL `env:"COCOS_JAEGER_URL" envDefault:"http://localhost:4318"` - TraceRatio float64 `env:"MG_JAEGER_TRACE_RATIO" envDefault:"1.0"` - InstanceID string `env:"MANAGER_INSTANCE_ID" envDefault:""` - BackendMeasurementBinary string `env:"BACKEND_MEASUREMENT_BINARY" envDefault:"../../build"` + LogLevel string `env:"MANAGER_LOG_LEVEL" envDefault:"info"` + JaegerURL url.URL `env:"COCOS_JAEGER_URL" envDefault:"http://localhost:4318"` + TraceRatio float64 `env:"COCOS_JAEGER_TRACE_RATIO" envDefault:"1.0"` + InstanceID string `env:"MANAGER_INSTANCE_ID" envDefault:""` + BackendMeasurementBinary string `env:"MANAGER_BACKEND_MEASUREMENT_BINARY" envDefault:"../../build"` } func main() { @@ -104,7 +104,11 @@ func main() { } eventsChan := make(chan *pkgmanager.ClientStreamMessage) - svc := newService(logger, tracer, qemuCfg, eventsChan, cfg.BackendMeasurementBinary) + svc, err := newService(logger, tracer, qemuCfg, eventsChan, cfg.BackendMeasurementBinary) + if err != nil { + logger.Error(err.Error()) + return + } mc := managerapi.NewClient(pc, svc, eventsChan) @@ -121,13 +125,16 @@ func main() { } } -func newService(logger *slog.Logger, tracer trace.Tracer, qemuCfg qemu.Config, eventsChan chan *pkgmanager.ClientStreamMessage, backendMeasurementPath string) manager.Service { - svc := manager.New(qemuCfg, backendMeasurementPath, logger, eventsChan, qemu.NewVM) +func newService(logger *slog.Logger, tracer trace.Tracer, qemuCfg qemu.Config, eventsChan chan *pkgmanager.ClientStreamMessage, backendMeasurementPath string) (manager.Service, error) { + svc, err := manager.New(qemuCfg, backendMeasurementPath, logger, eventsChan, qemu.NewVM) + if err != nil { + return nil, err + } go svc.RetrieveAgentEventsLogs() svc = api.LoggingMiddleware(svc, logger) counter, latency := prometheus.MakeMetrics(svcName, "api") svc = api.MetricsMiddleware(svc, counter, latency) svc = tracing.New(svc, tracer) - return svc + return svc, nil } diff --git a/manager/README.md b/manager/README.md index f7237c7c1..9f412397b 100644 --- a/manager/README.md +++ b/manager/README.md @@ -6,17 +6,61 @@ Manager service provides a barebones gRPC API and Service interface implementati The service is configured using the environment variables from the following table. Note that any unset variables will be replaced with their default values. -| Variable | Description | Default | -| ----------------------------- | -------------------------------------------------------- | --------------------------------- | -| MANAGER_LOG_LEVEL | Log level for manager service (debug, info, warn, error) | info | -| MANAGER_GRPC_HOST | Manager service gRPC host | | -| MANAGER_GRPC_PORT | Manager service gRPC port | 7001 | -| MANAGER_GRPC_SERVER_CERT | Path to server certificate in pem format | | -| MANAGER_GRPC_SERVER_KEY | Path to server key in pem format | | -| MANAGER_GRPC_SERVER_CA_CERTS | Path to gRPC server CA certificate | | -| MANAGER_GRPC_CLIENT_CA_CERTS | Path to gRPC client CA certificate | | -| COCOS_JAEGER_URL | Jaeger server URL | http://localhost:4318 | -| MANAGER_INSTANCE_ID | Manager service instance ID | | +| Variable | Description | Default | +| ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| COCOS_JAEGER_URL | The URL for the Jaeger tracing endpoint. | http://localhost:4318 | +| COCOS_JAEGER_TRACE_RATIO | The ratio of traces to sample. | 1.0 | +| MANAGER_INSTANCE_ID | The instance ID for the manager service. | | +| MANAGER_BACKEND_MEASUREMENT_BINARY | The file path for the backend measurement binary. | ../../build | +| MANAGER_GRPC_CLIENT_CERT | The file path for the client certificate. | | +| MANAGER_GRPC_CLIENT_KEY | The file path for the client private key. | | +| MANAGER_GRPC_SERVER_CA_CERTS | The file path for the server CA certificate(s). | | +| MANAGER_GRPC_URL | The URL for the gRPC endpoint. | localhost:7001 | +| MANAGER_GRPC_TIMEOUT | The timeout for gRPC requests. | 60s | +| MANAGER_INSTANCE_ID | Manager service instance ID | | +| MANAGER_QEMU_MEMORY_SIZE | The total memory size for the virtual machine. Can be specified in a human-readable format like "2048M" or "4G". | 2048M | +| MANAGER_QEMU_MEMORY_SLOTS | The number of memory slots for the virtual machine. | 5 | +| MANAGER_QEMU_MAX_MEMORY | The maximum memory size for the virtual machine. Can be specified in a human-readable format like "30G". | 30G | +| MANAGER_QEMU_OVMF_CODE_IF | The interface type for the OVMF code. | pflash | +| MANAGER_QEMU_OVMF_CODE_FORMAT | The format of the OVMF code file. | raw | +| MANAGER_QEMU_OVMF_CODE_UNIT | The unit number for the OVMF code. | 0 | +| MANAGER_QEMU_OVMF_CODE_FILE | The file path for the OVMF code. | /usr/share/OVMF/OVMF_CODE.fd | +| MANAGER_QEMU_OVMF_CODE_READONLY | Whether the OVMF code should be read-only. | on | +| MANAGER_QEMU_OVMF_VARS_IF | The interface type for the OVMF variables. | pflash | +| MANAGER_QEMU_OVMF_VARS_FORMAT | The format of the OVMF variables file. | raw | +| MANAGER_QEMU_OVMF_VARS_UNIT | The unit number for the OVMF variables. | 1 | +| MANAGER_QEMU_OVMF_VARS_FILE | The file path for the OVMF variables. | /usr/share/OVMF/OVMF_VARS.fd | +| MANAGER_QEMU_NETDEV_ID | The ID for the network device. | vmnic | +| MANAGER_QEMU_HOST_FWD_AGENT | The port number for the host forward agent. | 7020 | +| MANAGER_QEMU_GUEST_FWD_AGENT | The port number for the guest forward agent. | 7002 | +| MANAGER_QEMU_VIRTIO_NET_PCI_DISABLE_LEGACY | Whether to disable the legacy PCI device. | on | +| MANAGER_QEMU_VIRTIO_NET_PCI_IOMMU_PLATFORM | Whether to enable the IOMMU platform for the virtio-net PCI device. | true | +| MANAGER_QEMU_VIRTIO_NET_PCI_ADDR | The PCI address for the virtio-net PCI device. | 0x2 | +| MANAGER_QEMU_VIRTIO_NET_PCI_ROMFILE | The file path for the ROM image for the virtio-net PCI device. | | +| MANAGER_QEMU_DISK_IMG_KERNEL_FILE | The file path for the kernel image. | img/bzImage | +| MANAGER_QEMU_DISK_IMG_ROOTFS_FILE | The file path for the root filesystem image. | img/rootfs.cpio.gz | +| MANAGER_QEMU_SEV_ID | The ID for the Secure Encrypted Virtualization (SEV) device. | sev0 | +| MANAGER_QEMU_SEV_CBITPOS | The position of the C-bit in the physical address. | 51 | +| MANAGER_QEMU_SEV_REDUCED_PHYS_BITS | The number of reduced physical address bits for SEV. | 1 | +| MANAGER_QEMU_HOST_DATA | Additional data for the SEV host. | | +| MANAGER_QEMU_VSOCK_ID | The ID for the virtual socket device. | vhost-vsock-pci0 | +| MANAGER_QEMU_VSOCK_GUEST_CID | The guest-side CID (Context ID) for the virtual socket device. | 3 | +| MANAGER_QEMU_VSOCK_VNC | Whether to enable the virtual socket device for VNC. | 0 | +| MANAGER_QEMU_BIN_PATH | The file path for the QEMU binary. | qemu-system-x86_64 | +| MANAGER_QEMU_TMP_FILE_LOC | The directory for temporary files. | tmp | +| MANAGER_QEMU_USE_SUDO | Whether to use sudo to run QEMU. | false | +| MANAGER_QEMU_ENABLE_SEV | Whether to enable Secure Encrypted Virtualization (SEV). | false | +| MANAGER_QEMU_ENABLE_SEV_SNP | Whether to enable Secure Nested Paging (SEV-SNP). | true | +| MANAGER_QEMU_ENABLE_KVM | Whether to enable the Kernel-based Virtual Machine (KVM) acceleration. | true | +| MANAGER_QEMU_MACHINE | The machine type for QEMU. | q35 | +| MANAGER_QEMU_CPU | The CPU model for QEMU. | EPYC | +| MANAGER_QEMU_SMP_COUNT | The number of virtual CPUs. | 4 | +| MANAGER_QEMU_SMP_MAXCPUS | The maximum number of virtual CPUs. | 64 | +| MANAGER_QEMU_MEM_ID | The ID for the memory device. | ram1 | +| MANAGER_QEMU_KERNEL_HASH | Whether to enable kernel hash verification. | false | +| MANAGER_QEMU_NO_GRAPHIC | Whether to disable the graphical display. | true | +| MANAGER_QEMU_MONITOR | The type of monitor to use. | pty | +| MANAGER_QEMU_HOST_FWD_RANGE | The range of host ports to forward. | 6000-6100 | ## Setup diff --git a/manager/qemu/config.go b/manager/qemu/config.go index fcd3bfede..a9df418a2 100644 --- a/manager/qemu/config.go +++ b/manager/qemu/config.go @@ -99,6 +99,9 @@ type Config struct { // display NoGraphic bool `env:"NO_GRAPHIC" envDefault:"true"` Monitor string `env:"MONITOR" envDefault:"pty"` + + // ports + HostFwdRange string `env:"HOST_FWD_RANGE" envDefault:"6000-6100"` } func (config Config) ConstructQemuArgs() []string { diff --git a/manager/service.go b/manager/service.go index 55526daa1..8e4c05486 100644 --- a/manager/service.go +++ b/manager/service.go @@ -11,7 +11,9 @@ import ( "net" "os" "os/exec" + "regexp" "strconv" + "sync" "github.com/absmach/magistrala/pkg/errors" "github.com/cenkalti/backoff/v4" @@ -67,12 +69,18 @@ type managerService struct { eventsChan chan *manager.ClientStreamMessage vms map[string]vm.VM vmFactory vm.Provider + portRangeMin int + portRangeMax int } var _ Service = (*managerService)(nil) // New instantiates the manager service implementation. -func New(cfg qemu.Config, backendMeasurementBinPath string, logger *slog.Logger, eventsChan chan *manager.ClientStreamMessage, vmFactory vm.Provider) Service { +func New(cfg qemu.Config, backendMeasurementBinPath string, logger *slog.Logger, eventsChan chan *manager.ClientStreamMessage, vmFactory vm.Provider) (Service, error) { + start, end, err := decodeRange(cfg.HostFwdRange) + if err != nil { + return nil, err + } ms := &managerService{ qemuCfg: cfg, logger: logger, @@ -81,8 +89,10 @@ func New(cfg qemu.Config, backendMeasurementBinPath string, logger *slog.Logger, eventsChan: eventsChan, vmFactory: vmFactory, backendMeasurementBinaryPath: backendMeasurementBinPath, + portRangeMin: start, + portRangeMax: end, } - return ms + return ms, nil } func (ms *managerService) Run(ctx context.Context, c *manager.ComputationRunReq) (string, error) { @@ -115,7 +125,7 @@ func (ms *managerService) Run(ctx context.Context, c *manager.ComputationRunReq) ac.ResultConsumers = append(ac.ResultConsumers, agent.ResultConsumer{UserKey: rc.UserKey}) } - agentPort, err := getFreePort() + agentPort, err := getFreePort(ms.portRangeMin, ms.portRangeMax) if err != nil { ms.publishEvent("vm-provision", c.Id, "failed", json.RawMessage{}) return "", errors.Wrap(ErrFailedToAllocatePort, err) @@ -182,21 +192,41 @@ func (ms *managerService) FetchBackendInfo() ([]byte, error) { return f, nil } -func getFreePort() (int, error) { - listener, err := net.Listen("tcp", "") - if err != nil { - return 0, err - } - defer listener.Close() - _, portStr, err := net.SplitHostPort(listener.Addr().String()) - if err != nil { - return 0, err +func getFreePort(minPort, maxPort int) (int, error) { + var wg sync.WaitGroup + portCh := make(chan int, maxPort-minPort+1) + errCh := make(chan error, 1) + + for port := minPort; port <= maxPort; port++ { + wg.Add(1) + go func(p int) { + defer wg.Done() + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", p)) + if err == nil { + defer listener.Close() + _, portStr, err := net.SplitHostPort(listener.Addr().String()) + if err == nil { + freePort, err := strconv.Atoi(portStr) + if err == nil { + portCh <- freePort + return + } + } + } + }(port) } - port, err := strconv.Atoi(portStr) - if err != nil { + + go func() { + wg.Wait() + close(portCh) + }() + + select { + case port := <-portCh: + return port, nil + case err := <-errCh: return 0, err } - return port, nil } func (ms *managerService) publishEvent(event, cmpID, status string, details json.RawMessage) { @@ -222,3 +252,27 @@ func computationHash(ac agent.Computation) ([32]byte, error) { return sha3.Sum256(jsonData), nil } + +func decodeRange(input string) (int, int, error) { + re := regexp.MustCompile(`(\d+)-(\d+)`) + matches := re.FindStringSubmatch(input) + if len(matches) != 3 { + return 0, 0, fmt.Errorf("invalid input format: %s", input) + } + + start, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, 0, err + } + + end, err := strconv.Atoi(matches[2]) + if err != nil { + return 0, 0, err + } + + if start > end { + return 0, 0, fmt.Errorf("invalid range: %d-%d", start, end) + } + + return start, end, nil +} diff --git a/manager/service_test.go b/manager/service_test.go index 1babee22e..87ce002da 100644 --- a/manager/service_test.go +++ b/manager/service_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/ultravioletrs/cocos/manager/qemu" "github.com/ultravioletrs/cocos/manager/vm" "github.com/ultravioletrs/cocos/manager/vm/mocks" @@ -17,12 +18,15 @@ import ( ) func TestNew(t *testing.T) { - cfg := qemu.Config{} + cfg := qemu.Config{ + HostFwdRange: "6000-6100", + } logger := slog.Default() eventsChan := make(chan *manager.ClientStreamMessage) vmf := new(mocks.Provider) - service := New(cfg, "", logger, eventsChan, vmf.Execute) + service, err := New(cfg, "", logger, eventsChan, vmf.Execute) + require.NoError(t, err) assert.NotNil(t, service) assert.IsType(t, &managerService{}, service) @@ -181,7 +185,7 @@ func TestStop(t *testing.T) { } func TestGetFreePort(t *testing.T) { - port, err := getFreePort() + port, err := getFreePort(6000, 6100) assert.NoError(t, err) assert.Greater(t, port, 0) From 31391a36a1a7da8739008fb55cbf08a654e1c655 Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:09:11 +0300 Subject: [PATCH 06/18] NOISSUE - Add optional Filename (#195) * add optional filename Signed-off-by: Sammy Oina * cover all bases Signed-off-by: Sammy Oina * maintain filename Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- agent/agent.pb.go | 70 +++++++++++++++++++--------------- agent/agent.proto | 1 + agent/api/grpc/endpoint.go | 2 +- agent/api/grpc/requests.go | 3 +- agent/api/grpc/server.go | 7 +++- agent/computations.go | 7 ++-- agent/mocks/agent.go | 2 +- agent/mocks/auth.go | 2 +- agent/service.go | 21 ++++++---- agent/state_string.go | 2 +- cli/datasets.go | 4 +- manager/service.go | 2 +- pkg/progressbar/progressbar.go | 4 +- pkg/sdk/agent.go | 2 +- test/computations/main.go | 9 ----- 15 files changed, 77 insertions(+), 61 deletions(-) diff --git a/agent/agent.pb.go b/agent/agent.pb.go index 959991e8a..e988b40bd 100644 --- a/agent/agent.pb.go +++ b/agent/agent.pb.go @@ -121,7 +121,8 @@ type DataRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Dataset []byte `protobuf:"bytes,1,opt,name=dataset,proto3" json:"dataset,omitempty"` + Dataset []byte `protobuf:"bytes,1,opt,name=dataset,proto3" json:"dataset,omitempty"` + Filename string `protobuf:"bytes,2,opt,name=filename,proto3" json:"filename,omitempty"` } func (x *DataRequest) Reset() { @@ -163,6 +164,13 @@ func (x *DataRequest) GetDataset() []byte { return nil } +func (x *DataRequest) GetFilename() string { + if x != nil { + return x.Filename + } + return "" +} + type DataResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -390,37 +398,39 @@ var file_agent_agent_proto_rawDesc = []byte{ 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x0e, 0x0a, 0x0c, 0x41, - 0x6c, 0x67, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x27, 0x0a, 0x0b, 0x44, + 0x6c, 0x67, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x43, 0x0a, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x64, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0f, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x35, 0x0a, 0x12, 0x41, - 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x44, 0x61, - 0x74, 0x61, 0x22, 0x29, 0x0a, 0x13, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x32, 0xfb, 0x01, - 0x0a, 0x0c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x33, - 0x0a, 0x04, 0x41, 0x6c, 0x67, 0x6f, 0x12, 0x12, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, - 0x6c, 0x67, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x2e, 0x41, 0x6c, 0x67, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x28, 0x01, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x2e, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x13, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x39, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x12, 0x14, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, - 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x30, 0x01, 0x12, 0x46, 0x0a, 0x0b, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x19, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x74, 0x74, 0x65, 0x73, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x09, 0x5a, 0x07, 0x2e, - 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x73, 0x65, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, + 0x22, 0x0e, 0x0a, 0x0c, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x0f, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x24, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x35, 0x0a, 0x12, 0x41, 0x74, 0x74, 0x65, 0x73, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, + 0x0b, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x44, 0x61, 0x74, 0x61, 0x22, 0x29, + 0x0a, 0x13, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x32, 0xfb, 0x01, 0x0a, 0x0c, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x41, 0x6c, + 0x67, 0x6f, 0x12, 0x12, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x6c, 0x67, 0x6f, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, + 0x6c, 0x67, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, + 0x33, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, + 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x28, 0x01, 0x12, 0x39, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, + 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, + 0x46, 0x0a, 0x0b, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x19, + 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x61, 0x67, 0x65, 0x6e, + 0x74, 0x2e, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/agent/agent.proto b/agent/agent.proto index 551dbc0be..d67ba246d 100644 --- a/agent/agent.proto +++ b/agent/agent.proto @@ -23,6 +23,7 @@ message AlgoResponse {} message DataRequest { bytes dataset = 1; + string filename = 2; } message DataResponse {} diff --git a/agent/api/grpc/endpoint.go b/agent/api/grpc/endpoint.go index 9d7fd1238..da41add6e 100644 --- a/agent/api/grpc/endpoint.go +++ b/agent/api/grpc/endpoint.go @@ -36,7 +36,7 @@ func dataEndpoint(svc agent.Service) endpoint.Endpoint { return dataRes{}, err } - dataset := agent.Dataset{Dataset: req.Dataset} + dataset := agent.Dataset{Dataset: req.Dataset, Filename: req.Filename} err := svc.Data(ctx, dataset) if err != nil { diff --git a/agent/api/grpc/requests.go b/agent/api/grpc/requests.go index eecc22a20..4d7d22061 100644 --- a/agent/api/grpc/requests.go +++ b/agent/api/grpc/requests.go @@ -19,7 +19,8 @@ func (req algoReq) validate() error { } type dataReq struct { - Dataset []byte `protobuf:"bytes,1,opt,name=dataset,proto3" json:"dataset,omitempty"` + Dataset []byte `protobuf:"bytes,1,opt,name=dataset,proto3" json:"dataset,omitempty"` + Filename string } func (req dataReq) validate() error { diff --git a/agent/api/grpc/server.go b/agent/api/grpc/server.go index f52c0bbb2..91d6b26a3 100644 --- a/agent/api/grpc/server.go +++ b/agent/api/grpc/server.go @@ -69,7 +69,8 @@ func decodeDataRequest(_ context.Context, grpcReq interface{}) (interface{}, err req := grpcReq.(*agent.DataRequest) return dataReq{ - Dataset: req.Dataset, + Dataset: req.Dataset, + Filename: req.Filename, }, nil } @@ -128,6 +129,7 @@ func (s *grpcServer) Algo(stream agent.AgentService_AlgoServer) error { // Data implements agent.AgentServiceServer. func (s *grpcServer) Data(stream agent.AgentService_DataServer) error { var dataFile []byte + var filename string for { dataChunk, err := stream.Recv() if err == io.EOF { @@ -137,8 +139,9 @@ func (s *grpcServer) Data(stream agent.AgentService_DataServer) error { return status.Error(codes.Internal, err.Error()) } dataFile = append(dataFile, dataChunk.Dataset...) + filename = dataChunk.Filename } - _, res, err := s.data.ServeGRPC(stream.Context(), &agent.DataRequest{Dataset: dataFile}) + _, res, err := s.data.ServeGRPC(stream.Context(), &agent.DataRequest{Dataset: dataFile, Filename: filename}) if err != nil { return err } diff --git a/agent/computations.go b/agent/computations.go index 22d966f58..79d30a8c3 100644 --- a/agent/computations.go +++ b/agent/computations.go @@ -44,9 +44,10 @@ func (d *Datasets) String() string { } type Dataset struct { - Dataset []byte `json:"-"` - Hash [32]byte `json:"hash,omitempty"` - UserKey []byte `json:"user_key,omitempty"` + Dataset []byte `json:"-"` + Hash [32]byte `json:"hash,omitempty"` + UserKey []byte `json:"user_key,omitempty"` + Filename string `json:"filename,omitempty"` } type Datasets []Dataset diff --git a/agent/mocks/agent.go b/agent/mocks/agent.go index 7f97ae969..64b2e9b08 100644 --- a/agent/mocks/agent.go +++ b/agent/mocks/agent.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.3. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. // Copyright (c) Ultraviolet // SPDX-License-Identifier: Apache-2.0 diff --git a/agent/mocks/auth.go b/agent/mocks/auth.go index 718c9c148..476d7ac79 100644 --- a/agent/mocks/auth.go +++ b/agent/mocks/auth.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.3. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. // Copyright (c) Ultraviolet // SPDX-License-Identifier: Apache-2.0 diff --git a/agent/service.go b/agent/service.go index bc105ed6d..de3d8d0db 100644 --- a/agent/service.go +++ b/agent/service.go @@ -36,18 +36,20 @@ var ( // ErrUnauthorizedAccess indicates missing or invalid credentials provided // when accessing a protected resource. ErrUnauthorizedAccess = errors.New("missing or invalid credentials provided") - // errUndeclaredAlgorithm indicates algorithm was not declared in computation manifest. + // ErrUndeclaredAlgorithm indicates algorithm was not declared in computation manifest. ErrUndeclaredDataset = errors.New("dataset not declared in computation manifest") - // errAllManifestItemsReceived indicates no new computation manifest items expected. + // ErrAllManifestItemsReceived indicates no new computation manifest items expected. ErrAllManifestItemsReceived = errors.New("all expected manifest Items have been received") - // errUndeclaredConsumer indicates the consumer requesting results in not declared in computation manifest. + // ErrUndeclaredConsumer indicates the consumer requesting results in not declared in computation manifest. ErrUndeclaredConsumer = errors.New("result consumer is undeclared in computation manifest") - // errResultsNotReady indicates the computation results are not ready. + // ErrResultsNotReady indicates the computation results are not ready. ErrResultsNotReady = errors.New("computation results are not yet ready") - // errStateNotReady agent received a request in the wrong state. + // ErrStateNotReady agent received a request in the wrong state. ErrStateNotReady = errors.New("agent not expecting this operation in the current state") - // errHashMismatch provided algorithm/dataset does not match hash in manifest. + // ErrHashMismatch provided algorithm/dataset does not match hash in manifest. ErrHashMismatch = errors.New("malformed data, hash does not match manifest") + // ErrFileNameMismatch provided dataset filename does not match filename in manifest. + ErrFileNameMismatch = errors.New("malformed data, filename does not match manifest") ) // Service specifies an API that must be fullfiled by the domain service @@ -185,9 +187,14 @@ func (as *agentService) Data(ctx context.Context, dataset Dataset) error { if hash != as.computation.Datasets[index].Hash { return ErrHashMismatch } + + if as.computation.Datasets[index].Filename != "" && as.computation.Datasets[index].Filename != dataset.Filename { + return ErrFileNameMismatch + } + as.computation.Datasets = slices.Delete(as.computation.Datasets, index, index+1) - f, err := os.Create(fmt.Sprintf("%s/dataset-%d", algorithm.DatasetsDir, index)) + f, err := os.Create(fmt.Sprintf("%s/%s", algorithm.DatasetsDir, dataset.Filename)) if err != nil { return fmt.Errorf("error creating dataset file: %v", err) } diff --git a/agent/state_string.go b/agent/state_string.go index 49bbea395..1694ded03 100644 --- a/agent/state_string.go +++ b/agent/state_string.go @@ -22,7 +22,7 @@ const _state_name = "idlereceivingManifestreceivingAlgorithmreceivingDatarunning var _state_index = [...]uint8{0, 4, 21, 39, 52, 59, 71, 79} func (i state) String() string { - if i < 0 || i >= state(len(_state_index)-1) { + if i >= state(len(_state_index)-1) { return "state(" + strconv.FormatInt(int64(i), 10) + ")" } return _state_name[_state_index[i]:_state_index[i+1]] diff --git a/cli/datasets.go b/cli/datasets.go index 4b60abc83..40debc968 100644 --- a/cli/datasets.go +++ b/cli/datasets.go @@ -7,6 +7,7 @@ import ( "encoding/pem" "log" "os" + "path" "github.com/spf13/cobra" "github.com/ultravioletrs/cocos/agent" @@ -29,7 +30,8 @@ func (cli *CLI) NewDatasetsCmd() *cobra.Command { } dataReq := agent.Dataset{ - Dataset: dataset, + Dataset: dataset, + Filename: path.Base(datasetFile), } privKeyFile, err := os.ReadFile(args[1]) diff --git a/manager/service.go b/manager/service.go index 8e4c05486..2b767d021 100644 --- a/manager/service.go +++ b/manager/service.go @@ -118,7 +118,7 @@ func (ms *managerService) Run(ctx context.Context, c *manager.ComputationRunReq) ms.publishEvent("vm-provision", c.Id, "failed", json.RawMessage{}) return "", errInvalidHashLength } - ac.Datasets = append(ac.Datasets, agent.Dataset{Hash: [hashLength]byte(data.Hash), UserKey: data.UserKey}) + ac.Datasets = append(ac.Datasets, agent.Dataset{Hash: [hashLength]byte(data.Hash), UserKey: data.UserKey, Filename: data.Filename}) } for _, rc := range c.ResultConsumers { diff --git a/pkg/progressbar/progressbar.go b/pkg/progressbar/progressbar.go index cd25e2c35..c113fe0d3 100644 --- a/pkg/progressbar/progressbar.go +++ b/pkg/progressbar/progressbar.go @@ -108,9 +108,9 @@ func (p *ProgressBar) SendAlgorithm(description string, algobuffer, reqBuffer *b return err } -func (p *ProgressBar) SendData(description string, buffer *bytes.Buffer, stream *agent.AgentService_DataClient) error { +func (p *ProgressBar) SendData(description, filename string, buffer *bytes.Buffer, stream *agent.AgentService_DataClient) error { return p.sendData(description, buffer, &dataClientWrapper{client: stream}, func(data []byte) interface{} { - return &agent.DataRequest{Dataset: data} + return &agent.DataRequest{Dataset: data, Filename: filename} }) } diff --git a/pkg/sdk/agent.go b/pkg/sdk/agent.go index fdc812ac7..fd2d75a82 100644 --- a/pkg/sdk/agent.go +++ b/pkg/sdk/agent.go @@ -91,7 +91,7 @@ func (sdk *agentSDK) Data(ctx context.Context, dataset agent.Dataset, privKey an dataBuffer := bytes.NewBuffer(dataset.Dataset) pb := progressbar.New() - if err := pb.SendData(dataProgressBarDescription, dataBuffer, &stream); err != nil { + if err := pb.SendData(dataProgressBarDescription, dataset.Filename, dataBuffer, &stream); err != nil { sdk.logger.Error("Failed to send Data") return err } diff --git a/test/computations/main.go b/test/computations/main.go index bc9e74b7b..e3006451d 100644 --- a/test/computations/main.go +++ b/test/computations/main.go @@ -74,15 +74,6 @@ func (s *svc) Run(ipAdress string, reqChan chan *manager.ServerStreamMessage, au algoHash := sha3.Sum256(algo) - // Uncomment this to run tests on the manager service on a SEV enabled backend. - reqChan <- &manager.ServerStreamMessage{ - Message: &manager.ServerStreamMessage_BackendInfoReq{ - BackendInfoReq: &manager.BackendInfoReq{ - Id: "1", - }, - }, - } - reqChan <- &manager.ServerStreamMessage{ Message: &manager.ServerStreamMessage_RunReq{ RunReq: &manager.ComputationRunReq{ From f596702e6ed3097590bb8dbcf33882d0a723d191 Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Wed, 21 Aug 2024 00:25:13 +0300 Subject: [PATCH 07/18] NOISSUE - Fix bug on same dataset provider for multiple datasets (#198) * fix bug on same dataset provider Signed-off-by: Sammy Oina * fix failing test Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- agent/auth/auth.go | 4 ++-- agent/auth/auth_test.go | 2 +- agent/service.go | 42 ++++++++++++++++++++++------------------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/agent/auth/auth.go b/agent/auth/auth.go index f94264d5c..0a1f59857 100644 --- a/agent/auth/auth.go +++ b/agent/auth/auth.go @@ -146,9 +146,9 @@ func (s *service) AuthenticateUser(ctx context.Context, role UserRole) (context. } } case DataProviderRole: - for i, dp := range s.datasetProviders { + for _, dp := range s.datasetProviders { if err := verifySignature(role, signature, dp); err == nil { - return agent.IndexToContext(ctx, i), nil + return ctx, nil } } case AlgorithmProviderRole: diff --git a/agent/auth/auth_test.go b/agent/auth/auth_test.go index e04bd18ba..dae13c37b 100644 --- a/agent/auth/auth_test.go +++ b/agent/auth/auth_test.go @@ -124,7 +124,7 @@ func TestAuthenticateUser(t *testing.T) { if err == nil { switch id, ok := agent.IndexFromContext(ctx); { - case tc.role == ConsumerRole, tc.role == DataProviderRole: + case tc.role == ConsumerRole: assert.True(t, ok, "expected index in context") assert.Equal(t, 0, id, "expected index 0 in context") default: diff --git a/agent/service.go b/agent/service.go index de3d8d0db..000128ff6 100644 --- a/agent/service.go +++ b/agent/service.go @@ -179,33 +179,37 @@ func (as *agentService) Data(ctx context.Context, dataset Dataset) error { hash := sha3.Sum256(dataset.Dataset) - index, ok := IndexFromContext(ctx) - if !ok { - return ErrUndeclaredDataset - } + matched := false + for i, d := range as.computation.Datasets { + if hash == d.Hash { + if d.Filename != "" && d.Filename != dataset.Filename { + return ErrFileNameMismatch + } - if hash != as.computation.Datasets[index].Hash { - return ErrHashMismatch - } + as.computation.Datasets = slices.Delete(as.computation.Datasets, i, i+1) - if as.computation.Datasets[index].Filename != "" && as.computation.Datasets[index].Filename != dataset.Filename { - return ErrFileNameMismatch - } + f, err := os.Create(fmt.Sprintf("%s/%s", algorithm.DatasetsDir, dataset.Filename)) + if err != nil { + return fmt.Errorf("error creating dataset file: %v", err) + } - as.computation.Datasets = slices.Delete(as.computation.Datasets, index, index+1) + if _, err := f.Write(dataset.Dataset); err != nil { + return fmt.Errorf("error writing dataset to file: %v", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("error closing file: %v", err) + } - f, err := os.Create(fmt.Sprintf("%s/%s", algorithm.DatasetsDir, dataset.Filename)) - if err != nil { - return fmt.Errorf("error creating dataset file: %v", err) + matched = true + break + } } - if _, err := f.Write(dataset.Dataset); err != nil { - return fmt.Errorf("error writing dataset to file: %v", err) - } - if err := f.Close(); err != nil { - return fmt.Errorf("error closing file: %v", err) + if !matched { + return ErrUndeclaredDataset } + // Check if all datasets have been received if len(as.computation.Datasets) == 0 { as.sm.SendEvent(dataReceived) } From 8f2cbd349dd963b380f7b100f0e7f1ad9172ca0a Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Wed, 21 Aug 2024 00:56:34 +0300 Subject: [PATCH 08/18] bump port range (#206) Signed-off-by: Sammy Oina --- manager/README.md | 2 +- manager/qemu/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manager/README.md b/manager/README.md index 9f412397b..049bcea50 100644 --- a/manager/README.md +++ b/manager/README.md @@ -60,7 +60,7 @@ The service is configured using the environment variables from the following tab | MANAGER_QEMU_KERNEL_HASH | Whether to enable kernel hash verification. | false | | MANAGER_QEMU_NO_GRAPHIC | Whether to disable the graphical display. | true | | MANAGER_QEMU_MONITOR | The type of monitor to use. | pty | -| MANAGER_QEMU_HOST_FWD_RANGE | The range of host ports to forward. | 6000-6100 | +| MANAGER_QEMU_HOST_FWD_RANGE | The range of host ports to forward. | 6100-6200 | ## Setup diff --git a/manager/qemu/config.go b/manager/qemu/config.go index a9df418a2..74cee5b26 100644 --- a/manager/qemu/config.go +++ b/manager/qemu/config.go @@ -101,7 +101,7 @@ type Config struct { Monitor string `env:"MONITOR" envDefault:"pty"` // ports - HostFwdRange string `env:"HOST_FWD_RANGE" envDefault:"6000-6100"` + HostFwdRange string `env:"HOST_FWD_RANGE" envDefault:"6100-6200"` } func (config Config) ConstructQemuArgs() []string { From 066dacd46ada44f3afcd2939cd849400f3dfb841 Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Wed, 21 Aug 2024 00:58:37 +0300 Subject: [PATCH 09/18] NOISSUE - Fix docs (#203) * fix docs Signed-off-by: Sammy Oina * fix typos Signed-off-by: Sammy Oina * cli Signed-off-by: Sammy Oina * add build instructions Signed-off-by: Sammy Oina * remove file Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- .gitignore | 2 +- cli/algorithms.go | 2 +- cli/datasets.go | 6 ++-- cli/result.go | 2 +- test/manual/README.md | 58 +++++++++++++++++++++++--------------- test/manual/algo/README.md | 10 +++---- 6 files changed, 47 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 457ef1d81..fa4d41fde 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,5 @@ cmd/manager/tmp *.pem dist/ -result.zip +results.zip *.spec diff --git a/cli/algorithms.go b/cli/algorithms.go index ec2626613..16922eae6 100644 --- a/cli/algorithms.go +++ b/cli/algorithms.go @@ -30,7 +30,7 @@ func (cli *CLI) NewAlgorithmCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { algorithmFile := args[0] - log.Println("Uploading algorithm binary:", algorithmFile) + log.Println("Uploading algorithm file:", algorithmFile) algorithm, err := os.ReadFile(algorithmFile) if err != nil { diff --git a/cli/datasets.go b/cli/datasets.go index 40debc968..1bbe4058c 100644 --- a/cli/datasets.go +++ b/cli/datasets.go @@ -16,13 +16,13 @@ import ( func (cli *CLI) NewDatasetsCmd() *cobra.Command { return &cobra.Command{ Use: "data", - Short: "Upload a dataset CSV file", - Example: "data ", + Short: "Upload a dataset", + Example: "data ", Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { datasetFile := args[0] - log.Println("Uploading dataset CSV:", datasetFile) + log.Println("Uploading dataset:", datasetFile) dataset, err := os.ReadFile(datasetFile) if err != nil { diff --git a/cli/result.go b/cli/result.go index 73898cfb2..5fb3ac7b5 100644 --- a/cli/result.go +++ b/cli/result.go @@ -10,7 +10,7 @@ import ( "github.com/spf13/cobra" ) -const resultFilePath = "result.zip" +const resultFilePath = "results.zip" func (cli *CLI) NewResultsCmd() *cobra.Command { return &cobra.Command{ diff --git a/test/manual/README.md b/test/manual/README.md index 4dda590ce..70ba29718 100644 --- a/test/manual/README.md +++ b/test/manual/README.md @@ -3,15 +3,17 @@ ## CLI Throughout the tests, we assume that our current working directory is the root of the `cocos` repository, both on the host machine and in the VM. +First, we will build cli by running in the root directory of `cocos`: +```bash +make cli +``` -### Algorithm requirements +The cli will be compiled to the build directory `./build/cocos-cli`. -Agent accepts the algorithm as a binary that take in two command line arguments. -```shell -algorithm-file -``` +### Algorithm requirements -The algorithm program should return the results to a socket and an example can be seen in this [file](./algo/lin_reg.py). +Agent accepts the algorithm as a binary or python or wasm file. +All assets/datasets the algorithm uses are stored in the `datasets` directory. The results from the algorithm run should be stored in the `results` directory. All these paths are relative to the algorithm working directory. ### Agent-CLI interaction @@ -32,7 +34,7 @@ sev-snp-measure --mode snp --vcpus 4 --vcpu-type EPYC-v4 --ovmf $OVMF_CODE --ker To speed up the verification process of attested TLS, download the ARK and ASK certificates using the CLI tool. The CLI tool will download the certificates under your home directory in the `.cocos` directory. ```bash -go run cmd/cli/main.go ca-bundle +./build/cocos-cli ca-bundle ``` In the following text, we can see an example of how the CLI tool is used. @@ -52,36 +54,35 @@ cd ../.. # The CLI should also be aware of the VM measurement. To add the measurement # to the .json file that contains the information about the platform, run CLI # with the measurement in base64 format and the path of the backend_info.json file.: -go run cmd/cli/main.go backend measurement '' '' +./build/cocos-cli backend measurement '' '' # If the VM is booted with the QEMU host data option, the CLI should also know # the host data information. To add the host data to the .json file that contains # the information about the platform, run CLI with the host data in base64 format # and the path of the backend_info.json file.: -go run cmd/cli/main.go backend measurement '' '' +./build/cocos-cli backend measurement '' '' # For attested TLS, also define the path to the backend_info.json that contains reference values for the fields of the attestation report export AGENT_GRPC_MANIFEST=./scripts/backend_info/backend_info.json export AGENT_GRPC_ATTESTED_TLS=true # Retieve Attestation -go run cmd/cli/main.go attestation get '' +./build/cocos-cli attestation get '' # Validate Attestation # Product name must be Milan or Genoa -go run cmd/cli/main.go attestation validate '' --report_data '' --product +./build/cocos-cli attestation validate '' --report_data '' --product # Run the CLI program with algorithm input -go run cmd/cli/main.go algo test/manual/algo/lin_reg.bin +./build/cocos-cli algo test/manual/algo/lin_reg.py -a python -r test/manual/algo/requirements.py # 2023/09/21 10:43:53 Uploading algorithm binary: test/manual/algo/lin_reg.bin # Run the CLI program with dataset input -go run cmd/cli/main.go data test/manual/data/iris.csv -go run cmd/cli/main.go data test/manual/data/iris.csv +./build/cocos-cli data test/manual/data/iris.csv # 2023/09/21 10:45:25 Uploading dataset CSV: test/manual/data/iris.csv # Run the CLI program to fetch computation result -go run cmd/cli/main.go result +./build/cocos-cli result # 2023/09/21 10:45:39 Retrieving computation result file # 2023/09/21 10:45:40 Computation result retrieved and saved successfully! ``` @@ -89,18 +90,12 @@ go run cmd/cli/main.go result Now there is a `result.bin` file in the current working directory. The file holds the trained logistic regression model. To test the model, run ```sh -python3 test/manual/algo/lin_reg_test.py test/manual/data/iris.csv result.bin +python ./test/manual/algo/lin_reg.py predict results.zip ./test/manual/data ``` You should get an output (truncated for the sake of brevity): ```sh - Id SepalLengthCm SepalWidthCm PetalLengthCm PetalWidthCm Species -0 1 5.1 3.5 1.4 0.2 Iris-setosa -1 2 4.9 3.0 1.4 0.2 Iris-setosa -2 3 4.7 3.2 1.3 0.2 Iris-setosa -3 4 4.6 3.1 1.5 0.2 Iris-setosa -4 5 5.0 3.6 1.4 0.2 Iris-setosa Precision, Recall, Confusion matrix, in training precision recall f1-score support @@ -112,4 +107,23 @@ Iris-versicolor 0.923 0.889 0.906 27 accuracy 0.933 75 macro avg 0.939 0.938 0.938 75 weighted avg 0.934 0.933 0.933 75 + +[[21 0 0] + [ 0 24 3] + [ 0 2 25]] +Precision, Recall, Confusion matrix, in testing + + precision recall f1-score support + + Iris-setosa 1.000 1.000 1.000 29 +Iris-versicolor 1.000 1.000 1.000 23 + Iris-virginica 1.000 1.000 1.000 23 + + accuracy 1.000 75 + macro avg 1.000 1.000 1.000 75 + weighted avg 1.000 1.000 1.000 75 + +[[29 0 0] + [ 0 23 0] + [ 0 0 23]] ``` diff --git a/test/manual/algo/README.md b/test/manual/algo/README.md index a9cb79f6c..3a5117e16 100644 --- a/test/manual/algo/README.md +++ b/test/manual/algo/README.md @@ -33,7 +33,7 @@ python3 test/manual/algo/lin_reg.py The linear regression example is a more complex algorithm that requires external dependencies.It returns a linear regression model trained on the iris dataset found [here](../data/) for demonstration purposes. ```bash -python3 test/manual/algo/lin_reg.py predict result.zip test/manual/data +python3 test/manual/algo/lin_reg.py predict results.zip test/manual/data ``` This will make inference on the results of the linear regression model. @@ -71,7 +71,7 @@ Next we need to upload the dataset After some time when the results are ready, you can run the following command to get the results: ```bash -./build/cocos-cli results ./private.pem +./build/cocos-cli result ./private.pem ``` This will return the results of the algorithm. @@ -79,13 +79,13 @@ This will return the results of the algorithm. To make inference on the results, you can use the following command: ```bash -python3 test/manual/algo/lin_reg.py predict result.zip test/manual/data +python3 test/manual/algo/lin_reg.py predict results.zip test/manual/data ``` For addition example, you can use the following command: ```bash -go run ./test/computations/main.go ./test/manual/algo/addition.py public.pem false +./build/cocos-cli ./test/manual/algo/addition.py public.pem false ``` ```bash @@ -93,7 +93,7 @@ go run ./test/computations/main.go ./test/manual/algo/addition.py public.pem fal ``` ```bash -./build/cocos-cli results ./private.pem +./build/cocos-cli result ./private.pem ``` ## Wasm Example From 6aec555f4e42864121c9f70a67d9122b906f6dec Mon Sep 17 00:00:00 2001 From: b1ackd0t <28790446+rodneyosodo@users.noreply.github.com> Date: Wed, 21 Aug 2024 00:59:28 +0300 Subject: [PATCH 10/18] NOISSUE - Start Port Search At Known Port (#207) * feat(manager): start port search at minimum port range When getting free port to be use we start at the minimum port range i.e 6000. This makes the agent url more deterministic since in most cases the agent port will be 6000 * fix(manager): Remove unused error channel The error channel is not used Signed-off-by: Rodney Osodo --------- Signed-off-by: Rodney Osodo --- manager/service.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/manager/service.go b/manager/service.go index 2b767d021..8dc6f9ee7 100644 --- a/manager/service.go +++ b/manager/service.go @@ -193,25 +193,20 @@ func (ms *managerService) FetchBackendInfo() ([]byte, error) { } func getFreePort(minPort, maxPort int) (int, error) { + if checkPortisFree(minPort) { + return minPort, nil + } + var wg sync.WaitGroup portCh := make(chan int, maxPort-minPort+1) - errCh := make(chan error, 1) for port := minPort; port <= maxPort; port++ { wg.Add(1) go func(p int) { defer wg.Done() - listener, err := net.Listen("tcp", fmt.Sprintf(":%d", p)) - if err == nil { - defer listener.Close() - _, portStr, err := net.SplitHostPort(listener.Addr().String()) - if err == nil { - freePort, err := strconv.Atoi(portStr) - if err == nil { - portCh <- freePort - return - } - } + + if checkPortisFree(p) { + portCh <- p } }(port) } @@ -224,9 +219,19 @@ func getFreePort(minPort, maxPort int) (int, error) { select { case port := <-portCh: return port, nil - case err := <-errCh: - return 0, err + default: + return 0, fmt.Errorf("failed to find free port in range %d-%d", minPort, maxPort) + } +} + +func checkPortisFree(port int) bool { + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return false } + defer listener.Close() + + return true } func (ms *managerService) publishEvent(event, cmpID, status string, details json.RawMessage) { From e4ef1aae3690e7955a89388eee17c54b9e4e424b Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Wed, 21 Aug 2024 01:02:09 +0300 Subject: [PATCH 11/18] COCOS-181 - Add failed state for algo execution (#197) * additional state failed Signed-off-by: Sammy Oina * generate state string Signed-off-by: Sammy Oina * fix logic Signed-off-by: Sammy Oina * fix typo Signed-off-by: Sammy Oina * conditional transition Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- agent/service.go | 21 +++++++++++++++------ agent/state.go | 3 +++ agent/state_string.go | 5 +++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/agent/service.go b/agent/service.go index 000128ff6..e84630338 100644 --- a/agent/service.go +++ b/agent/service.go @@ -50,6 +50,8 @@ var ( ErrHashMismatch = errors.New("malformed data, hash does not match manifest") // ErrFileNameMismatch provided dataset filename does not match filename in manifest. ErrFileNameMismatch = errors.New("malformed data, filename does not match manifest") + // ErrAllResultsConsumed indicates all results have been consumed. + ErrAllResultsConsumed = errors.New("all results have been consumed by declared consumers") ) // Service specifies an API that must be fullfiled by the domain service @@ -64,7 +66,7 @@ type Service interface { } type agentService struct { - computation Computation // Holds the current computation request details. + computation Computation // Holds the current computation manifest. algorithm algorithm.Algorithm // Filepath to the algorithm received for the computation. result []byte // Stores the result of the computation. sm *StateMachine // Manages the state transitions of the agent service. @@ -90,6 +92,7 @@ func New(ctx context.Context, logger *slog.Logger, eventSvc events.Service, cmp svc.sm.StateFunctions[resultsReady] = svc.publishEvent("in-progress", json.RawMessage{}) svc.sm.StateFunctions[complete] = svc.publishEvent("in-progress", json.RawMessage{}) svc.sm.StateFunctions[running] = svc.runComputation + svc.sm.StateFunctions[failed] = svc.publishEvent("failed", json.RawMessage{}) svc.computation = cmp @@ -218,11 +221,11 @@ func (as *agentService) Data(ctx context.Context, dataset Dataset) error { } func (as *agentService) Result(ctx context.Context) ([]byte, error) { - if as.sm.GetState() != resultsReady { + if as.sm.GetState() != resultsReady && as.sm.GetState() != failed { return []byte{}, ErrResultsNotReady } if len(as.computation.ResultConsumers) == 0 { - return []byte{}, ErrAllManifestItemsReceived + return []byte{}, ErrAllResultsConsumed } index, ok := IndexFromContext(ctx) if !ok { @@ -230,10 +233,10 @@ func (as *agentService) Result(ctx context.Context) ([]byte, error) { } as.computation.ResultConsumers = slices.Delete(as.computation.ResultConsumers, index, index+1) - if len(as.computation.ResultConsumers) == 0 { + if len(as.computation.ResultConsumers) == 0 && as.sm.GetState() == resultsReady { as.sm.SendEvent(resultsConsumed) } - // Return the result file or an error + return as.result, as.runError } @@ -253,7 +256,13 @@ func (as *agentService) Attestation(ctx context.Context, reportData [ReportDataS func (as *agentService) runComputation() { as.publishEvent("starting", json.RawMessage{})() as.sm.logger.Debug("computation run started") - defer as.sm.SendEvent(runComplete) + defer func() { + if as.runError != nil { + as.sm.SendEvent(runFailed) + } else { + as.sm.SendEvent(runComplete) + } + }() if err := os.Mkdir(algorithm.ResultsDir, 0o755); err != nil { as.runError = fmt.Errorf("error creating results directory: %s", err.Error()) diff --git a/agent/state.go b/agent/state.go index f61a978dc..61a388f74 100644 --- a/agent/state.go +++ b/agent/state.go @@ -20,6 +20,7 @@ const ( running resultsReady complete + failed ) type event uint8 @@ -31,6 +32,7 @@ const ( dataReceived runComplete resultsConsumed + runFailed ) // StateMachine represents the state machine. @@ -74,6 +76,7 @@ func NewStateMachine(logger *slog.Logger, cmp Computation) *StateMachine { sm.Transitions[running] = make(map[event]state) sm.Transitions[running][runComplete] = resultsReady + sm.Transitions[running][runFailed] = failed sm.Transitions[resultsReady] = make(map[event]state) sm.Transitions[resultsReady][resultsConsumed] = complete diff --git a/agent/state_string.go b/agent/state_string.go index 1694ded03..0a4a5a4eb 100644 --- a/agent/state_string.go +++ b/agent/state_string.go @@ -15,11 +15,12 @@ func _() { _ = x[running-4] _ = x[resultsReady-5] _ = x[complete-6] + _ = x[failed-7] } -const _state_name = "idlereceivingManifestreceivingAlgorithmreceivingDatarunningresultsReadycomplete" +const _state_name = "idlereceivingManifestreceivingAlgorithmreceivingDatarunningresultsReadycompletefailed" -var _state_index = [...]uint8{0, 4, 21, 39, 52, 59, 71, 79} +var _state_index = [...]uint8{0, 4, 21, 39, 52, 59, 71, 79, 85} func (i state) String() string { if i >= state(len(_state_index)-1) { From 899bfb0ec5c14f37432ef1fb4246c8126fd46612 Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:54:52 +0300 Subject: [PATCH 12/18] COCOS-151 - Add compression/decompression option for CLI/Agent (#200) * on the fly compression Signed-off-by: Sammy Oina * rename file-hash to checksum Signed-off-by: Sammy Oina * check error properly Signed-off-by: Sammy Oina * fix lint Signed-off-by: Sammy Oina * fix connection handling Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- agent/algorithm/results.go | 59 ----------------- agent/algorithm/results_test.go | 3 +- agent/computations.go | 17 +++++ agent/service.go | 30 +++++---- cli/README.md | 32 ++++++++-- cli/{file_hash.go => checksum.go} | 18 ++---- cli/datasets.go | 39 ++++++++++-- internal/file.go | 37 +++++++++++ internal/zip.go | 102 ++++++++++++++++++++++++++++++ pkg/sdk/agent.go | 5 +- test/computations/main.go | 19 +++--- test/manual/agent-config/main.go | 58 +++++++++-------- 12 files changed, 287 insertions(+), 132 deletions(-) delete mode 100644 agent/algorithm/results.go rename cli/{file_hash.go => checksum.go} (57%) create mode 100644 internal/zip.go diff --git a/agent/algorithm/results.go b/agent/algorithm/results.go deleted file mode 100644 index 114cfe058..000000000 --- a/agent/algorithm/results.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Ultraviolet -// SPDX-License-Identifier: Apache-2.0 -package algorithm - -import ( - "archive/zip" - "bytes" - "fmt" - "io" - "os" - "path/filepath" -) - -// ZipDirectory zips a directory and returns the zipped bytes. -func ZipDirectory() ([]byte, error) { - buf := new(bytes.Buffer) - zipWriter := zip.NewWriter(buf) - - err := filepath.Walk(ResultsDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("error walking the path %q: %v", path, err) - } - - if info.IsDir() { - return nil - } - - relPath, err := filepath.Rel(ResultsDir, path) - if err != nil { - return fmt.Errorf("error getting relative path for %q: %v", path, err) - } - - file, err := os.Open(path) - if err != nil { - return fmt.Errorf("error opening file %q: %v", path, err) - } - defer file.Close() - - zipFile, err := zipWriter.Create(relPath) - if err != nil { - return fmt.Errorf("error creating zip file for %q: %v", path, err) - } - - if _, err = io.Copy(zipFile, file); err != nil { - return fmt.Errorf("error copying file %q to zip: %v", path, err) - } - - return err - }) - if err != nil { - return nil, err - } - - if err = zipWriter.Close(); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} diff --git a/agent/algorithm/results_test.go b/agent/algorithm/results_test.go index c9745624f..594086ab5 100644 --- a/agent/algorithm/results_test.go +++ b/agent/algorithm/results_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/ultravioletrs/cocos/agent/algorithm" + "github.com/ultravioletrs/cocos/internal" ) func TestZipDirectory(t *testing.T) { @@ -73,7 +74,7 @@ func TestZipDirectory(t *testing.T) { } } - if _, err := algorithm.ZipDirectory(); err != nil { + if _, err := internal.ZipDirectoryToMemory(algorithm.ResultsDir); err != nil { t.Errorf("ZipDirectory() error = %v", err) } }) diff --git a/agent/computations.go b/agent/computations.go index 79d30a8c3..88f2daf0e 100644 --- a/agent/computations.go +++ b/agent/computations.go @@ -6,6 +6,8 @@ import ( "context" "encoding/json" "fmt" + + "google.golang.org/grpc/metadata" ) var _ fmt.Stringer = (*Datasets)(nil) @@ -69,3 +71,18 @@ func IndexFromContext(ctx context.Context) (int, bool) { index, ok := ctx.Value(ManifestIndexKey{}).(int) return index, ok } + +const DecompressKey = "decompress" + +func DecompressFromContext(ctx context.Context) bool { + vals := metadata.ValueFromIncomingContext(ctx, DecompressKey) + if len(vals) == 0 { + return false + } + + return vals[0] == "true" +} + +func DecompressToContext(ctx context.Context, decompress bool) context.Context { + return metadata.AppendToOutgoingContext(ctx, DecompressKey, fmt.Sprintf("%t", decompress)) +} diff --git a/agent/service.go b/agent/service.go index e84630338..0f1211ef7 100644 --- a/agent/service.go +++ b/agent/service.go @@ -18,6 +18,7 @@ import ( "github.com/ultravioletrs/cocos/agent/algorithm/python" "github.com/ultravioletrs/cocos/agent/algorithm/wasm" "github.com/ultravioletrs/cocos/agent/events" + "github.com/ultravioletrs/cocos/internal" "golang.org/x/crypto/sha3" ) @@ -191,16 +192,22 @@ func (as *agentService) Data(ctx context.Context, dataset Dataset) error { as.computation.Datasets = slices.Delete(as.computation.Datasets, i, i+1) - f, err := os.Create(fmt.Sprintf("%s/%s", algorithm.DatasetsDir, dataset.Filename)) - if err != nil { - return fmt.Errorf("error creating dataset file: %v", err) - } - - if _, err := f.Write(dataset.Dataset); err != nil { - return fmt.Errorf("error writing dataset to file: %v", err) - } - if err := f.Close(); err != nil { - return fmt.Errorf("error closing file: %v", err) + if DecompressFromContext(ctx) { + if err := internal.UnzipFromMemory(dataset.Dataset, algorithm.DatasetsDir); err != nil { + return fmt.Errorf("error decompressing dataset: %v", err) + } + } else { + f, err := os.Create(fmt.Sprintf("%s/%s", algorithm.DatasetsDir, dataset.Filename)) + if err != nil { + return fmt.Errorf("error creating dataset file: %v", err) + } + + if _, err := f.Write(dataset.Dataset); err != nil { + return fmt.Errorf("error writing dataset to file: %v", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("error closing file: %v", err) + } } matched = true @@ -212,7 +219,6 @@ func (as *agentService) Data(ctx context.Context, dataset Dataset) error { return ErrUndeclaredDataset } - // Check if all datasets have been received if len(as.computation.Datasets) == 0 { as.sm.SendEvent(dataReceived) } @@ -288,7 +294,7 @@ func (as *agentService) runComputation() { return } - results, err := algorithm.ZipDirectory() + results, err := internal.ZipDirectoryToMemory(algorithm.ResultsDir) if err != nil { as.runError = err as.sm.logger.Warn(fmt.Sprintf("failed to zip results: %s", err.Error())) diff --git a/cli/README.md b/cli/README.md index 2fb59f1ba..14d1d4193 100644 --- a/cli/README.md +++ b/cli/README.md @@ -16,14 +16,14 @@ make cli Retrieves attestation information from the SEV guest and saves it to a file. To retrieve attestation from agent, use the following command: ```bash -./build/cocos-cli agent attestation get '' +./build/cocos-cli attestation get '' ``` #### Validate attestation Validates the retrieved attestation information against a specified policy and checks its authenticity. To validate and verify attestation from agent, use the following command: ```bash -./build/cocos-cli agent attestation validate '' --report_data '' +./build/cocos-cli attestation validate '' --report_data '' ``` ##### Flags - --config: Path to a JSON file containing the validation configuration. This can be used to override individual flags. @@ -62,21 +62,41 @@ To validate and verify attestation from agent, use the following command: To upload an algorithm, use the following command: ```bash -./build/cocos-cli agent algo /path/to/algorithm +./build/cocos-cli algo /path/to/algorithm ``` +##### Flags +- -a, --algorithm string Algorithm type to run (default "bin") +- --python-runtime string Python runtime to use (default "python3") +- -r, --requirements string Python requirements file + + #### Upload Dataset To upload a dataset, use the following command: ```bash -./build/cocos-cli agent data /path/to/dataset.csv +./build/cocos-cli data /path/to/dataset.csv ``` +Users can also upload directories which will be compressed on transit. Once received by agent they will be stored as compressed files or decompressed if the user passed the decompression argument. + +##### Flags +- -d, --decompress Decompress the dataset on agent + + + #### Retrieve result To retrieve the computation result, use the following command: ```bash -./build/cocos-cli agent result -``` \ No newline at end of file +./build/cocos-cli result +``` + +#### Checksum +When defining the manifest dataset and algorithm checksums are required. This can be done as below: + +```bash +./build/cocos-cli checksum +``` diff --git a/cli/file_hash.go b/cli/checksum.go similarity index 57% rename from cli/file_hash.go rename to cli/checksum.go index e786c264a..c9b2cb0cb 100644 --- a/cli/file_hash.go +++ b/cli/checksum.go @@ -3,32 +3,26 @@ package cli import ( - "encoding/hex" "log" - "os" "github.com/spf13/cobra" - "golang.org/x/crypto/sha3" + "github.com/ultravioletrs/cocos/internal" ) func (cli *CLI) NewFileHashCmd() *cobra.Command { return &cobra.Command{ - Use: "file-hash", + Use: "checksum", Short: "Compute the sha3-256 hash of a file", - Example: "file-hash ", + Example: "checksum ", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - fileName := args[0] + path := args[0] - file, err := os.ReadFile(fileName) + hash, err := internal.ChecksumHex(path) if err != nil { - log.Fatalf("Error reading dataset file: %v", err) + log.Fatalf("Error computing hash: %v", err) } - hashBytes := sha3.Sum256(file) - - hash := hex.EncodeToString(hashBytes[:]) - log.Println("Hash of file:", hash) }, } diff --git a/cli/datasets.go b/cli/datasets.go index 1bbe4058c..875f1aa27 100644 --- a/cli/datasets.go +++ b/cli/datasets.go @@ -3,6 +3,7 @@ package cli import ( + "context" "crypto/x509" "encoding/pem" "log" @@ -11,27 +12,45 @@ import ( "github.com/spf13/cobra" "github.com/ultravioletrs/cocos/agent" + "github.com/ultravioletrs/cocos/internal" + "google.golang.org/grpc/metadata" ) +var decompressDataset bool + func (cli *CLI) NewDatasetsCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "data", Short: "Upload a dataset", Example: "data ", Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { - datasetFile := args[0] + datasetPath := args[0] - log.Println("Uploading dataset:", datasetFile) + log.Println("Uploading dataset:", datasetPath) - dataset, err := os.ReadFile(datasetFile) + f, err := os.Stat(datasetPath) if err != nil { log.Fatalf("Error reading dataset file: %v", err) } + var dataset []byte + + if f.IsDir() { + dataset, err = internal.ZipDirectoryToMemory(datasetPath) + if err != nil { + log.Fatalf("Error zipping dataset directory: %v", err) + } + } else { + dataset, err = os.ReadFile(datasetPath) + if err != nil { + log.Fatalf("Error reading dataset file: %v", err) + } + } + dataReq := agent.Dataset{ Dataset: dataset, - Filename: path.Base(datasetFile), + Filename: path.Base(datasetPath), } privKeyFile, err := os.ReadFile(args[1]) @@ -43,13 +62,17 @@ func (cli *CLI) NewDatasetsCmd() *cobra.Command { privKey := decodeKey(pemBlock) - if err := cli.agentSDK.Data(cmd.Context(), dataReq, privKey); err != nil { + ctx := metadata.NewOutgoingContext(cmd.Context(), metadata.New(make(map[string]string))) + if err := cli.agentSDK.Data(addDatasetMetadata(ctx), dataReq, privKey); err != nil { log.Fatalf("Error uploading dataset: %v", err) } log.Println("Successfully uploaded dataset") }, } + + cmd.Flags().BoolVarP(&decompressDataset, "decompress", "d", false, "Decompress the dataset on agent") + return cmd } func decodeKey(b *pem.Block) interface{} { @@ -74,3 +97,7 @@ func decodeKey(b *pem.Block) interface{} { return nil } } + +func addDatasetMetadata(ctx context.Context) context.Context { + return agent.DecompressToContext(ctx, decompressDataset) +} diff --git a/internal/file.go b/internal/file.go index 50416f519..100dfbb95 100644 --- a/internal/file.go +++ b/internal/file.go @@ -3,9 +3,12 @@ package internal import ( + "encoding/hex" "io" "os" "path/filepath" + + "golang.org/x/crypto/sha3" ) // CopyFile copies a file from srcPath to dstPath. @@ -46,3 +49,37 @@ func DeleteFilesInDir(dirPath string) error { return nil } + +// Checksum calculates the SHA3-256 checksum of the file or directory at path. +func Checksum(path string) ([]byte, error) { + file, err := os.Stat(path) + if err != nil { + return nil, err + } + + if file.IsDir() { + f, err := ZipDirectoryToMemory(path) + if err != nil { + return nil, err + } + sum := sha3.Sum256(f) + return sum[:], nil + } else { + f, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + sum := sha3.Sum256(f) + return sum[:], nil + } +} + +// ChecksumHex calculates the SHA3-256 checksum of the file or directory at path and returns it as a hex-encoded string. +func ChecksumHex(path string) (string, error) { + sum, err := Checksum(path) + if err != nil { + return "", err + } + return hex.EncodeToString(sum), nil +} diff --git a/internal/zip.go b/internal/zip.go new file mode 100644 index 000000000..25bce0548 --- /dev/null +++ b/internal/zip.go @@ -0,0 +1,102 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 +package internal + +import ( + "archive/zip" + "bytes" + "io" + "os" + "path/filepath" +) + +func ZipDirectoryToMemory(sourceDir string) ([]byte, error) { + buf := new(bytes.Buffer) + zipWriter := zip.NewWriter(buf) + + err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(sourceDir, path) + if err != nil { + return err + } + + zipHeader, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + zipHeader.Name = relPath + + zipWriterEntry, err := zipWriter.CreateHeader(zipHeader) + if err != nil { + return err + } + + fileToZip, err := os.Open(path) + if err != nil { + return err + } + defer fileToZip.Close() + + _, err = io.Copy(zipWriterEntry, fileToZip) + return err + }) + if err != nil { + zipWriter.Close() + return nil, err + } + + if err := zipWriter.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func UnzipFromMemory(zipData []byte, targetDir string) error { + reader := bytes.NewReader(zipData) + zipReader, err := zip.NewReader(reader, int64(len(zipData))) + if err != nil { + return err + } + + for _, file := range zipReader.File { + filePath := filepath.Join(targetDir, file.Name) + + if file.FileInfo().IsDir() { + if err := os.MkdirAll(filePath, os.ModePerm); err != nil { + return err + } + continue + } + + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + return err + } + + srcFile, err := file.Open() + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(filePath) + if err != nil { + return err + } + defer dstFile.Close() + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/sdk/agent.go b/pkg/sdk/agent.go index fd2d75a82..a1ad99de5 100644 --- a/pkg/sdk/agent.go +++ b/pkg/sdk/agent.go @@ -82,7 +82,10 @@ func (sdk *agentSDK) Data(ctx context.Context, dataset agent.Dataset, privKey an return err } - ctx = metadata.NewOutgoingContext(ctx, md) + for k, v := range md { + ctx = metadata.AppendToOutgoingContext(ctx, k, v[0]) + } + stream, err := sdk.client.Data(ctx) if err != nil { sdk.logger.Error("Failed to call Data RPC") diff --git a/test/computations/main.go b/test/computations/main.go index e3006451d..bcb422ee0 100644 --- a/test/computations/main.go +++ b/test/computations/main.go @@ -13,11 +13,11 @@ import ( mglog "github.com/absmach/magistrala/logger" "github.com/caarlos0/env/v11" + "github.com/ultravioletrs/cocos/internal" "github.com/ultravioletrs/cocos/internal/server" grpcserver "github.com/ultravioletrs/cocos/internal/server/grpc" managergrpc "github.com/ultravioletrs/cocos/manager/api/grpc" "github.com/ultravioletrs/cocos/pkg/manager" - "golang.org/x/crypto/sha3" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -44,11 +44,6 @@ type svc struct { func (s *svc) Run(ipAdress string, reqChan chan *manager.ServerStreamMessage, auth credentials.AuthInfo) { s.logger.Debug(fmt.Sprintf("received who am on ip address %s", ipAdress)) - algo, err := os.ReadFile(algoPath) - if err != nil { - s.logger.Error(fmt.Sprintf("failed to read algorithm file: %s", err)) - return - } pubKey, err := os.ReadFile(pubKeyFile) if err != nil { @@ -63,16 +58,20 @@ func (s *svc) Run(ipAdress string, reqChan chan *manager.ServerStreamMessage, au s.logger.Error(fmt.Sprintf("data file does not exist: %s", dataPath)) return } - data, err := os.ReadFile(dataPath) + dataHash, err := internal.Checksum(dataPath) if err != nil { - s.logger.Error(fmt.Sprintf("failed to read data file: %s", err)) + s.logger.Error(fmt.Sprintf("failed to calculate checksum: %s", err)) return } - dataHash := sha3.Sum256(data) + datasets = append(datasets, &manager.Dataset{Hash: dataHash[:], UserKey: pubPem.Bytes}) } - algoHash := sha3.Sum256(algo) + algoHash, err := internal.Checksum(algoPath) + if err != nil { + s.logger.Error(fmt.Sprintf("failed to calculate checksum: %s", err)) + return + } reqChan <- &manager.ServerStreamMessage{ Message: &manager.ServerStreamMessage_RunReq{ diff --git a/test/manual/agent-config/main.go b/test/manual/agent-config/main.go index 27ecd1feb..4a2868911 100644 --- a/test/manual/agent-config/main.go +++ b/test/manual/agent-config/main.go @@ -10,15 +10,16 @@ import ( "encoding/pem" "fmt" "log" + "net" "os" "strconv" "github.com/mdlayher/vsock" "github.com/ultravioletrs/cocos/agent" + "github.com/ultravioletrs/cocos/internal" "github.com/ultravioletrs/cocos/manager" "github.com/ultravioletrs/cocos/manager/qemu" pkgmanager "github.com/ultravioletrs/cocos/pkg/manager" - "golang.org/x/crypto/sha3" "google.golang.org/protobuf/proto" ) @@ -35,21 +36,19 @@ func main() { } attestedTLS := attestedTLSParam - algo, err := os.ReadFile(algoPath) + pubKey, err := os.ReadFile(pubKeyFile) if err != nil { - log.Fatalf(fmt.Sprintf("failed to read algorithm file: %s", err)) + log.Fatalf(fmt.Sprintf("failed to read public key file: %s", err)) } - data, err := os.ReadFile(dataPath) + pubPem, _ := pem.Decode(pubKey) + algoHash, err := internal.Checksum(algoPath) if err != nil { - log.Fatalf(fmt.Sprintf("failed to read data file: %s", err)) + log.Fatalf(fmt.Sprintf("failed to calculate checksum: %s", err)) } - pubKey, err := os.ReadFile(pubKeyFile) + dataHash, err := internal.Checksum(dataPath) if err != nil { - log.Fatalf(fmt.Sprintf("failed to read public key file: %s", err)) + log.Fatalf(fmt.Sprintf("failed to calculate checksum: %s", err)) } - pubPem, _ := pem.Decode(pubKey) - algoHash := sha3.Sum256(algo) - dataHash := sha3.Sum256(data) l, err := vsock.Listen(manager.ManagerVsockPort, nil) if err != nil { @@ -57,8 +56,8 @@ func main() { } ac := agent.Computation{ ID: "123", - Datasets: agent.Datasets{agent.Dataset{Hash: dataHash, UserKey: pubPem.Bytes}}, - Algorithm: agent.Algorithm{Hash: algoHash, UserKey: pubPem.Bytes}, + Datasets: agent.Datasets{agent.Dataset{Hash: [32]byte(dataHash), UserKey: pubPem.Bytes}}, + Algorithm: agent.Algorithm{Hash: [32]byte(algoHash), UserKey: pubPem.Bytes}, ResultConsumers: []agent.ResultConsumer{{UserKey: pubPem.Bytes}}, AgentConfig: agent.AgentConfig{ LogLevel: "debug", @@ -66,7 +65,9 @@ func main() { AttestedTls: attestedTLS, }, } - fmt.Println(SendAgentConfig(3, ac)) + if err := SendAgentConfig(3, ac); err != nil { + log.Fatal(err) + } for { conn, err := l.Accept() @@ -74,18 +75,7 @@ func main() { log.Println(err) continue } - b := make([]byte, 1024) - n, err := conn.Read(b) - if err != nil { - log.Println(err) - continue - } - conn.Close() - var mes pkgmanager.ClientStreamMessage - if err := proto.Unmarshal(b[:n], &mes); err != nil { - log.Println(err) - } - fmt.Println(mes.String()) + go handleConnections(conn) } } @@ -109,3 +99,21 @@ func SendAgentConfig(cid uint32, ac agent.Computation) error { } return nil } + +func handleConnections(conn net.Conn) { + defer conn.Close() + for { + b := make([]byte, 1024) + n, err := conn.Read(b) + if err != nil { + log.Println(err) + return + } + var message pkgmanager.ClientStreamMessage + if err := proto.Unmarshal(b[:n], &message); err != nil { + log.Println(err) + return + } + fmt.Println(message.String()) + } +} From f906593492d5ce1ac68370460209f421e7a6ea38 Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:55:36 +0300 Subject: [PATCH 13/18] remove tmp directory (#204) Signed-off-by: Sammy Oina --- .gitignore | 2 - cmd/manager/main.go | 5 - cmd/manager/script/launch-qemu.sh | 270 ------------------------------ manager/README.md | 3 +- manager/qemu/config.go | 1 - manager/qemu/vm.go | 29 ---- 6 files changed, 1 insertion(+), 309 deletions(-) delete mode 100755 cmd/manager/script/launch-qemu.sh diff --git a/.gitignore b/.gitignore index fa4d41fde..4c481b13d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ build build cmd/manager/img -cmd/manager/iso -cmd/manager/tmp .cov diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 9489d478b..64e19cc8a 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -17,7 +17,6 @@ import ( "github.com/absmach/magistrala/pkg/prometheus" "github.com/absmach/magistrala/pkg/uuid" "github.com/caarlos0/env/v11" - "github.com/ultravioletrs/cocos/internal" "github.com/ultravioletrs/cocos/manager" "github.com/ultravioletrs/cocos/manager/api" managerapi "github.com/ultravioletrs/cocos/manager/api/grpc" @@ -119,10 +118,6 @@ func main() { if err := g.Wait(); err != nil { logger.Error(fmt.Sprintf("%s service terminated: %s", svcName, err)) } - - if err = internal.DeleteFilesInDir(qemuCfg.TmpFileLoc); err != nil { - logger.Error(err.Error()) - } } func newService(logger *slog.Logger, tracer trace.Tracer, qemuCfg qemu.Config, eventsChan chan *pkgmanager.ClientStreamMessage, backendMeasurementPath string) (manager.Service, error) { diff --git a/cmd/manager/script/launch-qemu.sh b/cmd/manager/script/launch-qemu.sh deleted file mode 100755 index 06b83b8a3..000000000 --- a/cmd/manager/script/launch-qemu.sh +++ /dev/null @@ -1,270 +0,0 @@ -#!/bin/bash - -# -# user changeable parameters -# - -HDA_FILE="cmd/manager/img/focal-server-cloudimg-amd64.qcow2" -GUEST_SIZE_IN_MB="4096" -SEV_GUEST="1" -SMP_NCPUS="4" -CONSOLE="serial" -VNC_PORT="" -USE_VIRTIO="1" - -UEFI_BIOS_CODE="/usr/share/OVMF/OVMF_CODE.fd" -UEFI_BIOS_VARS_ORIG="/usr/share/OVMF/OVMF_VARS.fd" -UEFI_BIOS_VARS_COPY="cmd/manager/img/OVMF_VARS.fd" - -CBITPOS=51 -HOST_HTTP_PORT=9301 -GUEST_HTTP_PORT=9031 -HOST_GRPC_PORT=7020 -GUEST_GRPC_PORT=7002 - -ENABLE_FILE_LOG="0" -EXEC_QEMU_CMDLINE="0" - -usage() { - echo "$0 [options]" - echo "Available :" - echo " -hda hard disk ($HDA_FILE)" - echo " -nosev disable sev support" - echo " -mem guest memory" - echo " -smp number of cpus" - echo " -console display console to use (serial or gxl)" - echo " -vnc VNC port to use" - echo " -bios bios to use (default $UEFI_BIOS_CODE)" - echo " -kernel kernel to use" - echo " -initrd initrd to use" - echo " -cdrom CDROM image" - echo " -virtio use virtio devices" - echo " -cbitpos location of the C-bit" - echo " -hosthttp host http port" - echo " -guesthttp guest http port" - echo " -hostgrpc host grpc port" - echo " -guestgrpc guest grpc port" - echo " -origuefivars UEFI BIOS vars original file (default $UEFI_BIOS_VARS_ORIG)" - echo " -copyuefivars UEFI BIOS vars copy file (default $UEFI_BIOS_VARS_COPY)" - echo " -exec execute the QEMU command (default $EXEC_QEMU_CMDLINE)" - echo " -filelog enable/disable QEMU cmd line file log (default: $ENABLE_FILE_LOG)" - exit 1 -} - -while [[ $1 != "" ]]; do - case "$1" in - -hda) - HDA_FILE=${2} - shift - ;; - -nosev) - SEV_GUEST="0" - ;; - -mem) - GUEST_SIZE_IN_MB=${2} - shift - ;; - -console) - CONSOLE=${2} - shift - ;; - -smp) - SMP_NCPUS=$2 - shift - ;; - -vnc) - VNC_PORT=$2 - shift - ;; - -bios) - UEFI_BIOS_CODE=$2 - shift - ;; - -initrd) - INITRD_FILE=$2 - shift - ;; - -kernel) - KERNEL_FILE=$2 - shift - ;; - -cdrom) - CDROM_FILE=$2 - shift - ;; - -virtio) - USE_VIRTIO="1" - ;; - -cbitpos) - CBITPOS=$2 - shift - ;; - -hosthttp) - HOST_HTTP_PORT=$2 - shift - ;; - -guesthttp) - GUEST_HTTP_PORT=$2 - shift - ;; - -guestgrpc) - GUEST_GRPC_PORT=$2 - shift - ;; - -hostgrpc) - HOST_GRPC_PORT=$2 - shift - ;; - -origuefivars) - UEFI_BIOS_VARS_ORIG=$2 - shift - ;; - -copyuefivars) - UEFI_BIOS_VARS_COPY=$2 - shift - ;; - -exec) - EXEC_QEMU_CMDLINE="1" - ;; - -filelog) - ENABLE_FILE_LOG="1" - ;; - *) - usage;; - esac - shift -done - -# -# func definitions -# - -add_opts() { - echo -n "$* " >> ${QEMU_CMDLINE} -} - -run_cmd() { - if ! "$@"; then - echo "Command '$*' failed" - exit 1 - fi -} - -# copy BIOS variables to new dest for VM use without modifying the original ones -cp "$UEFI_BIOS_VARS_ORIG" "$UEFI_BIOS_VARS_COPY" - -# -# Qemu cmd line construction -# - -# we add all the qemu command line options into a file -QEMU_CMDLINE=/tmp/cmdline.$$ -rm -rf ${QEMU_CMDLINE} - -add_opts "$(which qemu-system-x86_64)" - -# Basic virtual machine property -add_opts "-enable-kvm -cpu EPYC -machine q35" - -# add number of VCPUs -[ -n "$SMP_NCPUS" ] && add_opts "-smp ${SMP_NCPUS},maxcpus=64" - -# define guest memory -add_opts "-m ${GUEST_SIZE_IN_MB}M,slots=5,maxmem=30G" - -# The OVMF binary, including the non-volatile variable store, appears as a -# "normal" qemu drive on the host side, and it is exposed to the guest as a -# persistent flash device. -add_opts "-drive if=pflash,format=raw,unit=0,file=${UEFI_BIOS_CODE},readonly=on" -add_opts "-drive if=pflash,format=raw,unit=1,file=${UEFI_BIOS_VARS_COPY}" - -# add CDROM if specified -[ -n "$CDROM_FILE" ] && add_opts "-drive file=${CDROM_FILE},media=cdrom -boot d" - -add_opts "-netdev user,id=vmnic,hostfwd=tcp::2222-:22,hostfwd=tcp::$HOST_HTTP_PORT-:$GUEST_HTTP_PORT,hostfwd=tcp::$HOST_GRPC_PORT-:$GUEST_GRPC_PORT" -add_opts "-device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic,romfile=" - -# If harddisk file is specified then add the HDD drive -if [ -n "$HDA_FILE" ]; then - if [ "$USE_VIRTIO" = "1" ]; then - if [[ ${HDA_FILE} = *"qcow2" ]]; then - add_opts "-drive file=${HDA_FILE},if=none,id=disk0,format=qcow2" - else - add_opts "-drive file=${HDA_FILE},if=none,id=disk0,format=raw" - fi - add_opts "-device virtio-scsi-pci,id=scsi,disable-legacy=on,iommu_platform=true" - add_opts "-device scsi-hd,drive=disk0" - else - if [[ ${HDA_FILE} = *"qcow2" ]]; then - add_opts "-drive file=${HDA_FILE},format=qcow2" - else - add_opts "-drive file=${HDA_FILE},format=raw" - fi - fi -fi - -# If this is SEV guest then add the encryption device objects to enable support -if [ ${SEV_GUEST} = "1" ]; then - add_opts "-object sev-guest,id=sev0,cbitpos=${CBITPOS},reduced-phys-bits=1" - add_opts "-machine memory-encryption=sev0" -fi - -# if console is serial then disable graphical interface -if [ "${CONSOLE}" = "serial" ]; then - add_opts "-nographic" -else - add_opts "-vga ${CONSOLE}" -fi - -# if -kernel arg is specified then use the kernel provided in command line for boot -if [ "${KERNEL_FILE}" != "" ]; then - add_opts "-kernel $KERNEL_FILE" - add_opts "-append \"console=ttyS0 earlyprintk=serial root=/dev/sda2\"" - [ -n "$INITRD_FILE" ] && add_opts "-initrd ${INITRD_FILE}" -fi - -# start vnc server -[ -n "$VNC_PORT" ] && add_opts "-vnc :${VNC_PORT}" && echo "Starting VNC on port ${VNC_PORT}" - -# start monitor on pty -add_opts "-monitor pty" - -# -# Qemu cmd line log -# - -# Set the log file path if ENABLE_FILE_LOG is 1 -if [ "$ENABLE_FILE_LOG" = "1" ]; then - LOG_FILE=$(pwd)/stdout.log - - # Save the command line args into log file - cat "$QEMU_CMDLINE" > "$LOG_FILE" - echo >> "$LOG_FILE" -fi - - # Log the command line to the console -cat "$QEMU_CMDLINE" - -# -# Qemu cmd line execution -# - -if [[ "${EXEC_QEMU_CMDLINE}" = "0" ]]; then - exit 0 -fi - -# map CTRL-C to CTRL ] -echo "Mapping CTRL-C to CTRL-]" -stty intr ^] - -echo "Launching VM ..." -if [ "$ENABLE_FILE_LOG" = "1" ]; then - bash ${QEMU_CMDLINE} 2>&1 | tee -a "${LOG_FILE}" -else - bash ${QEMU_CMDLINE} 2>&1 -fi - -# restore the mapping -stty intr ^c - -rm -rf ${QEMU_CMDLINE} diff --git a/manager/README.md b/manager/README.md index 049bcea50..74dc5c171 100644 --- a/manager/README.md +++ b/manager/README.md @@ -47,7 +47,6 @@ The service is configured using the environment variables from the following tab | MANAGER_QEMU_VSOCK_GUEST_CID | The guest-side CID (Context ID) for the virtual socket device. | 3 | | MANAGER_QEMU_VSOCK_VNC | Whether to enable the virtual socket device for VNC. | 0 | | MANAGER_QEMU_BIN_PATH | The file path for the QEMU binary. | qemu-system-x86_64 | -| MANAGER_QEMU_TMP_FILE_LOC | The directory for temporary files. | tmp | | MANAGER_QEMU_USE_SUDO | Whether to use sudo to run QEMU. | false | | MANAGER_QEMU_ENABLE_SEV | Whether to enable Secure Encrypted Virtualization (SEV). | false | | MANAGER_QEMU_ENABLE_SEV_SNP | Whether to enable Secure Nested Paging (SEV-SNP). | true | @@ -85,7 +84,7 @@ sudo apt update sudo apt install qemu-kvm ``` -Create `img` directory in `cmd/manager`. Create `tmp` directory in `cmd/manager`. +Create `img` directory in `cmd/manager`. #### Add Vsock The necessary kernel modules must be loaded on the hypervisor. To check if `vhost_vsock` is loaded run: diff --git a/manager/qemu/config.go b/manager/qemu/config.go index 74cee5b26..18a18bc8d 100644 --- a/manager/qemu/config.go +++ b/manager/qemu/config.go @@ -61,7 +61,6 @@ type VSockConfig struct { type Config struct { QemuBinPath string `env:"BIN_PATH" envDefault:"qemu-system-x86_64"` - TmpFileLoc string `env:"TMP_FILE_LOC" envDefault:"tmp"` UseSudo bool `env:"USE_SUDO" envDefault:"false"` EnableSEV bool `env:"ENABLE_SEV" envDefault:"false"` EnableSEVSNP bool `env:"ENABLE_SEV_SNP" envDefault:"true"` diff --git a/manager/qemu/vm.go b/manager/qemu/vm.go index 7365165e0..f0f94bb54 100644 --- a/manager/qemu/vm.go +++ b/manager/qemu/vm.go @@ -7,7 +7,6 @@ import ( "os/exec" "github.com/gofrs/uuid" - "github.com/ultravioletrs/cocos/internal" "github.com/ultravioletrs/cocos/manager/vm" "github.com/ultravioletrs/cocos/pkg/manager" ) @@ -43,34 +42,6 @@ func (v *qemuVM) Start() error { qemuCfg.NetDevConfig.ID = fmt.Sprintf("%s-%s", qemuCfg.NetDevConfig.ID, id) qemuCfg.SevConfig.ID = fmt.Sprintf("%s-%s", qemuCfg.SevConfig.ID, id) - if !v.config.KernelHash { - // Copy firmware vars file - srcFile := qemuCfg.OVMFVarsConfig.File - dstFile := fmt.Sprintf("%s/%s-%s.fd", v.config.TmpFileLoc, firmwareVars, id) - err = internal.CopyFile(srcFile, dstFile) - if err != nil { - return err - } - qemuCfg.OVMFVarsConfig.File = dstFile - } - - // Copy img files - srcFile := qemuCfg.DiskImgConfig.KernelFile - dstFile := fmt.Sprintf("%s/%s-%s", v.config.TmpFileLoc, KernelFile, id) - err = internal.CopyFile(srcFile, dstFile) - if err != nil { - return err - } - qemuCfg.DiskImgConfig.KernelFile = dstFile - - srcFile = qemuCfg.DiskImgConfig.RootFsFile - dstFile = fmt.Sprintf("%s/%s-%s.gz", v.config.TmpFileLoc, rootfsFile, id) - err = internal.CopyFile(srcFile, dstFile) - if err != nil { - return err - } - qemuCfg.DiskImgConfig.RootFsFile = dstFile - exe, args, err := v.executableAndArgs() if err != nil { return err From ee8370406cac9a17d88a340e2ac57b7a9ff49d22 Mon Sep 17 00:00:00 2001 From: Danko Miladinovic <72250944+danko-miladinovic@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:42:05 +0200 Subject: [PATCH 14/18] COCOS-165 - Add Docker support (#180) * add docker support * add copyright clause * rebase docker support * address blank lines * update manual tests to include docker * fix algo test * fix docker command * add docker doc * fix AddDataset method * fixed lin_reg.py * rebsed docker implementation * fix NewAlgorithm error * change docker README.md based on rebase * fix docker README * fix docker.go gofumpt * add option for datasets and results mount * edit README for docker * make docker container run command a part of docker image * remove unused code * make /cocos the default directory * updated documentation * removed docker dir * rebased docker --- agent/algorithm/algorithm.go | 6 +- agent/algorithm/docker/docker.go | 170 +++++++++++++++++++++++++++++ agent/service.go | 5 +- go.mod | 15 ++- go.sum | 57 ++++++++++ hal/linux/board/cocos/linux.config | 3 + init/systemd/agent_start_script.sh | 11 ++ init/systemd/cocos-agent.service | 2 + pkg/progressbar/progressbar.go | 20 ++-- pkg/sdk/agent_test.go | 2 +- test/manual/README.md | 8 +- test/manual/agent-config/main.go | 2 +- test/manual/algo/README.md | 76 ++++++++++++- 13 files changed, 357 insertions(+), 20 deletions(-) create mode 100644 agent/algorithm/docker/docker.go diff --git a/agent/algorithm/algorithm.go b/agent/algorithm/algorithm.go index 9c4b53194..040fda216 100644 --- a/agent/algorithm/algorithm.go +++ b/agent/algorithm/algorithm.go @@ -14,10 +14,12 @@ const ( AlgoTypeBin AlgorithType = "bin" AlgoTypePython AlgorithType = "python" AlgoTypeWasm AlgorithType = "wasm" + AlgoTypeDocker AlgorithType = "docker" AlgoTypeKey = "algo_type" - ResultsDir = "results" - DatasetsDir = "datasets" + ResultsDir = "results" + DatasetsDir = "datasets" + AlgoWorkingDir = "/cocos" ) func AlgorithmTypeToContext(ctx context.Context, algoType string) context.Context { diff --git a/agent/algorithm/docker/docker.go b/agent/algorithm/docker/docker.go new file mode 100644 index 000000000..0b467da9d --- /dev/null +++ b/agent/algorithm/docker/docker.go @@ -0,0 +1,170 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 +package docker + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "path" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/client" + "github.com/ultravioletrs/cocos/agent/algorithm" + "github.com/ultravioletrs/cocos/agent/events" +) + +const ( + containerName = "agent_container" + datasetsMountPath = "/cocos/datasets" + resultsMountPath = "/cocos/results" +) + +var _ algorithm.Algorithm = (*docker)(nil) + +type docker struct { + algoFile string + logger *slog.Logger + stderr io.Writer + stdout io.Writer +} + +func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string) algorithm.Algorithm { + d := &docker{ + algoFile: algoFile, + logger: logger, + stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc}, + stdout: &algorithm.Stdout{Logger: logger}, + } + + return d +} + +func (d *docker) Run() error { + ctx := context.Background() + + // Create a new Docker client. + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return fmt.Errorf("could not create a new Docker client: %v", err) + } + + // Open the Docker image tar file. + imageFile, err := os.Open(d.algoFile) + if err != nil { + return fmt.Errorf("could not open Docker image: %v", err) + } + defer imageFile.Close() + + // Load the Docker image from the tar file. + resp, err := cli.ImageLoad(ctx, imageFile, true) + if err != nil { + return fmt.Errorf("could not load Docker image from file: %v", err) + } + defer resp.Body.Close() + + // List the loaded images to get the image ID. + images, err := cli.ImageList(ctx, image.ListOptions{}) + if err != nil { + return fmt.Errorf("could not get the Docker image list: %v", err) + } + + var imageID string = "" + var dockerImageName string = "" + for _, image := range images { + for _, tag := range image.RepoTags { + imageID = image.ID + dockerImageName = tag + break + } + } + + if imageID == "" { + return fmt.Errorf("could not find image ID") + } + + // Create and start the container. + respContainer, err := cli.ContainerCreate(ctx, &container.Config{ + Image: dockerImageName, + }, &container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: path.Join(algorithm.AlgoWorkingDir, algorithm.DatasetsDir), + Target: datasetsMountPath, + }, + { + Type: mount.TypeBind, + Source: path.Join(algorithm.AlgoWorkingDir, algorithm.ResultsDir), + Target: resultsMountPath, + }, + }, + }, nil, nil, containerName) + if err != nil { + return fmt.Errorf("could not create a Docker container: %v", err) + } + + if err := cli.ContainerStart(ctx, respContainer.ID, container.StartOptions{}); err != nil { + return fmt.Errorf("could not start a Docker container: %v", err) + } + + statusCh, errCh := cli.ContainerWait(ctx, respContainer.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + return fmt.Errorf("could not wait for a Docker container: %v", err) + } + case <-statusCh: + } + + stdout, err := cli.ContainerLogs(ctx, respContainer.ID, container.LogsOptions{ShowStdout: true}) + if err != nil { + return fmt.Errorf("could not read stdout from the container: %v", err) + } + defer stdout.Close() + + err = writeToOut(stdout, d.stdout) + if err != nil { + d.logger.Warn(fmt.Sprintf("could not write to stdout: %v", err)) + } + + stderr, err := cli.ContainerLogs(ctx, respContainer.ID, container.LogsOptions{ShowStderr: true}) + if err != nil { + d.logger.Warn(fmt.Sprintf("could not read stderr from the container: %v", err)) + } + defer stderr.Close() + + err = writeToOut(stderr, d.stderr) + if err != nil { + d.logger.Warn(fmt.Sprintf("could not write to stderr: %v", err)) + } + + defer func() { + if err = cli.ContainerRemove(ctx, respContainer.ID, container.RemoveOptions{Force: true}); err != nil { + d.logger.Warn(fmt.Sprintf("error could not remove container: %v", err)) + } + + if _, err := cli.ImageRemove(ctx, imageID, image.RemoveOptions{Force: true}); err != nil { + d.logger.Warn(fmt.Sprintf("error could not remove image: %v", err)) + } + }() + + return nil +} + +func writeToOut(readCloser io.ReadCloser, ioWriter io.Writer) error { + content, err := io.ReadAll(readCloser) + if err != nil { + return fmt.Errorf("could not convert content from the container: %v", err) + } + + if _, err := ioWriter.Write(content); err != nil { + return fmt.Errorf("could not write to output: %v", err) + } + + return nil +} diff --git a/agent/service.go b/agent/service.go index 0f1211ef7..ae9ac2be9 100644 --- a/agent/service.go +++ b/agent/service.go @@ -15,6 +15,7 @@ import ( "github.com/google/go-sev-guest/client" "github.com/ultravioletrs/cocos/agent/algorithm" "github.com/ultravioletrs/cocos/agent/algorithm/binary" + "github.com/ultravioletrs/cocos/agent/algorithm/docker" "github.com/ultravioletrs/cocos/agent/algorithm/python" "github.com/ultravioletrs/cocos/agent/algorithm/wasm" "github.com/ultravioletrs/cocos/agent/events" @@ -115,7 +116,7 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error { return ErrHashMismatch } - f, err := os.CreateTemp("", "algorithm") + f, err := os.Create("algorithm") if err != nil { return fmt.Errorf("error creating algorithm file: %v", err) } @@ -160,6 +161,8 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error { as.algorithm = python.NewAlgorithm(as.sm.logger, as.eventSvc, runtime, requirementsFile, f.Name()) case string(algorithm.AlgoTypeWasm): as.algorithm = wasm.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name()) + case string(algorithm.AlgoTypeDocker): + as.algorithm = docker.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name()) } if err := os.Mkdir(algorithm.DatasetsDir, 0o755); err != nil { diff --git a/go.mod b/go.mod index 108dee3b2..26431fdf3 100644 --- a/go.mod +++ b/go.mod @@ -23,16 +23,29 @@ require ( ) require ( + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect go.opentelemetry.io/otel/sdk v1.28.0 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/tools v0.23.0 // indirect ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/docker v27.1.0+incompatible github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -55,7 +68,7 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index 3a7089ca8..c88c1bc30 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/absmach/magistrala v0.14.1-0.20240709113739-04c359462746 h1:Tj567KeGVygjTsSCxn4++skKiz9GkPugM1KMdIFxvfw= @@ -15,6 +17,16 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/digitalocean/go-libvirt v0.0.0-20240709142323-d8406205c752 h1:NI7XEcHzWVvBfVjSVK6Qk4wmrUfoyQxCNpBjrHelZFk= github.com/digitalocean/go-libvirt v0.0.0-20240709142323-d8406205c752/go.mod h1:/Ok8PA2qi/ve0Py38+oL+VxoYmlowigYRyLEODRYdgc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs= +github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -28,6 +40,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -45,6 +59,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -53,6 +69,12 @@ github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -78,8 +100,12 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= @@ -96,19 +122,50 @@ go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeX go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= diff --git a/hal/linux/board/cocos/linux.config b/hal/linux/board/cocos/linux.config index 5d49a7421..a9fc8f200 100644 --- a/hal/linux/board/cocos/linux.config +++ b/hal/linux/board/cocos/linux.config @@ -40,6 +40,9 @@ CONFIG_TMPFS_POSIX_ACL=y CONFIG_UNWINDER_FRAME_POINTER=y CONFIG_VSOCKETS=y CONFIG_VIRTIO_VSOCKETS=y +CONFIG_NF_TABLES=y +CONFIG_BPF_SYSCALL=y +CONFIG_CGROUP_BPF=y ### # AMD SEV-SNP diff --git a/init/systemd/agent_start_script.sh b/init/systemd/agent_start_script.sh index 0a0be0228..204023a35 100644 --- a/init/systemd/agent_start_script.sh +++ b/init/systemd/agent_start_script.sh @@ -1,5 +1,16 @@ #!/bin/sh +# Change the docker.service file to allow the Docker to run in RAM +mkdir -p /etc/systemd/system/docker.service.d + +# Create or overwrite the override.conf file with the new Environment variable +tee /etc/systemd/system/docker.service.d/override.conf > /dev/null <' '' +./build/cocos-cli backend hostdata '' '' # For attested TLS, also define the path to the backend_info.json that contains reference values for the fields of the attestation report export AGENT_GRPC_MANIFEST=./scripts/backend_info/backend_info.json @@ -77,6 +77,10 @@ export AGENT_GRPC_ATTESTED_TLS=true ./build/cocos-cli algo test/manual/algo/lin_reg.py -a python -r test/manual/algo/requirements.py # 2023/09/21 10:43:53 Uploading algorithm binary: test/manual/algo/lin_reg.bin +# In order to run the Docker image, run the CLI program with the algorithm docker option +go run ./cmd/cli/main.go algo -a docker +# 2023/09/21 10:43:53 Uploading algorithm binary: + # Run the CLI program with dataset input ./build/cocos-cli data test/manual/data/iris.csv # 2023/09/21 10:45:25 Uploading dataset CSV: test/manual/data/iris.csv @@ -87,7 +91,7 @@ export AGENT_GRPC_ATTESTED_TLS=true # 2023/09/21 10:45:40 Computation result retrieved and saved successfully! ``` -Now there is a `result.bin` file in the current working directory. The file holds the trained logistic regression model. To test the model, run +Now there is a `result.zip` file in the current working directory. The file holds the trained logistic regression model. To test the model, run ```sh python ./test/manual/algo/lin_reg.py predict results.zip ./test/manual/data diff --git a/test/manual/agent-config/main.go b/test/manual/agent-config/main.go index 4a2868911..075ac4919 100644 --- a/test/manual/agent-config/main.go +++ b/test/manual/agent-config/main.go @@ -32,7 +32,7 @@ func main() { pubKeyFile := os.Args[3] attestedTLSParam, err := strconv.ParseBool(os.Args[4]) if err != nil { - log.Fatalf("usage: %s , must be a bool value", os.Args[0]) + log.Fatalf("usage: %s , must be a bool value", os.Args[0]) } attestedTLS := attestedTLSParam diff --git a/test/manual/algo/README.md b/test/manual/algo/README.md index 3a5117e16..7ffb47fbf 100644 --- a/test/manual/algo/README.md +++ b/test/manual/algo/README.md @@ -1,6 +1,6 @@ # Algorithm -Agent accepts binaries programs, python scripts, and wasm files. It runs them in a sandboxed environment and returns the output. +Agent accepts binaries programs, python scripts, Docker images and wasm files. It runs them in a sandboxed environment and returns the output. ## Python Example @@ -38,7 +38,7 @@ python3 test/manual/algo/lin_reg.py predict results.zip test/manual/data This will make inference on the results of the linear regression model. -To run the examples in the agent, you can use the following command: +To run the examples in the secure VM (SVM) by the Agent, you can use the following command: ```bash go run ./test/computations/main.go ./test/manual/algo/lin_reg.py public.pem false ./test/manual/data/iris.csv @@ -96,6 +96,78 @@ For addition example, you can use the following command: ./build/cocos-cli result ./private.pem ``` +## Docker Example + +Here we will use the docker with the linear regression example (`lin_reg.py`). Throughout the example, we assume that our current working directory is the directory in which the `cocos` repository is cloned. For example: +```bash +# ls +cocos +``` + +The docker image must have a `cocos` directory containing the `datasets` and `results` directories. The Agent will run this image inside the SVM and will mount the datasets and results onto the `/cocos/datasets` and `/cocos/results` directories inside the image. The docker image must also contain the command that will be run when the docker container is run. + +The first step is to create a docker file. Use your favorite editor to create a file named `Dockerfile` in the current working directory and write in it the following code: + +```bash +FROM python:3.9-slim + +# set the working directory in the container +WORKDIR /cocos +RUN mkdir /cocos/results +RUN mkdir /cocos/datasets + +COPY ./cocos/test/manual/algo/requirements.txt /cocos/requirements.txt +COPY ./cocos/test/manual/algo/lin_reg.py /cocos/lin_reg.py + +# install dependencies +RUN pip install -r requirements.txt + +# command to be run when the docker container is started +CMD ["python3", "/cocos/lin_reg.py"] +``` + +Next, run the build command and then save the docker image as a `tar` file. +```bash +docker build -t linreg . +docker save linreg > linreg.tar +``` + +In another window, you can run the following command: + +```bash +sudo MANAGER_QEMU_SMP_MAXCPUS=4 MANAGER_GRPC_URL=localhost:7001 MANAGER_LOG_LEVEL=debug MANAGER_QEMU_USE_SUDO=false MANAGER_QEMU_ENABLE_SEV=false MANAGER_QEMU_SEV_CBITPOS=51 MANAGER_QEMU_ENABLE_SEV_SNP=false MANAGER_QEMU_OVMF_CODE_FILE=/usr/share/edk2/x64/OVMF_CODE.fd MANAGER_QEMU_OVMF_VARS_FILE=/usr/share/edk2/x64/OVMF_VARS.fd go run main.go +``` + +This command is run from the [manager main directory](../../../cmd/manager/). This will start the manager. Make sure you have already built the [qemu image](../../../hal/linux/README.md). + +In another window, specify what kind of algorithm you want the Agent to run (docker): + +```bash +./cocos/build/cocos-cli algo ./linreg.tar ./cocos/private.pem -a docker +``` + +make sure you have built the cocos-cli. This will upload the docker image. + +Next we need to upload the dataset + +```bash +./cocos/build/cocos-cli data ./cocos/test/manual/data/iris.csv ./cocos/private.pem +``` + +After some time when the results are ready, you can run the following command to get the results: + +```bash +./cocos/build/cocos-cli results ./cocos/private.pem +``` + +This will return the results of the algorithm. + +To make inference on the results, you can use the following command: + +```bash +python3 ./cocos/test/manual/algo/lin_reg.py predict result.zip ./cocos/test/manual/data +``` + ## Wasm Example More information on how to run wasm files can be found [here](https://github.com/ultravioletrs/ai/tree/main/burn-algorithms). From 4c80b574e3e7d47a68573f1bc21b68360e299f7e Mon Sep 17 00:00:00 2001 From: Danko Miladinovic <72250944+danko-miladinovic@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:30:04 +0200 Subject: [PATCH 15/18] add ramfs to HAL (#212) --- hal/linux/package/agent/agent.mk | 9 +++++---- init/systemd/{cocos_network_setup.sh => agent_setup.sh} | 8 ++++++++ init/systemd/cocos-agent.service | 4 ++-- manager/qemu/config.go | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) rename init/systemd/{cocos_network_setup.sh => agent_setup.sh} (81%) diff --git a/hal/linux/package/agent/agent.mk b/hal/linux/package/agent/agent.mk index 6257711d2..be1e096dd 100644 --- a/hal/linux/package/agent/agent.mk +++ b/hal/linux/package/agent/agent.mk @@ -12,15 +12,16 @@ define AGENT_BUILD_CMDS endef define AGENT_INSTALL_TARGET_CMDS - $(INSTALL) -D -m 0750 $(@D)/build/cocos-agent $(TARGET_DIR)/bin - mkdir -p $(TARGET_DIR)/var/log/cocos mkdir -p $(TARGET_DIR)/cocos/ + mkdir -p $(TARGET_DIR)/var/log/cocos + mkdir -p $(TARGET_DIR)/cocos_init/ + $(INSTALL) -D -m 0750 $(@D)/build/cocos-agent $(TARGET_DIR)/bin endef define AGENT_INSTALL_INIT_SYSTEMD $(INSTALL) -D -m 0640 $(@D)/init/systemd/cocos-agent.service $(TARGET_DIR)/usr/lib/systemd/system/cocos-agent.service - $(INSTALL) -D -m 0750 $(@D)/init/systemd/cocos_network_setup.sh $(TARGET_DIR)/cocos/cocos_network_setup.sh - $(INSTALL) -D -m 0750 $(@D)/init/systemd/agent_start_script.sh $(TARGET_DIR)/cocos/agent_start_script.sh + $(INSTALL) -D -m 0750 $(@D)/init/systemd/agent_setup.sh $(TARGET_DIR)/cocos_init/agent_setup.sh + $(INSTALL) -D -m 0750 $(@D)/init/systemd/agent_start_script.sh $(TARGET_DIR)/cocos_init/agent_start_script.sh endef $(eval $(golang-package)) diff --git a/init/systemd/cocos_network_setup.sh b/init/systemd/agent_setup.sh similarity index 81% rename from init/systemd/cocos_network_setup.sh rename to init/systemd/agent_setup.sh index 67ae39aeb..0f24b9e8c 100644 --- a/init/systemd/cocos_network_setup.sh +++ b/init/systemd/agent_setup.sh @@ -1,5 +1,7 @@ #!/bin/sh +MOUNT_POINT="/cocos" + # IFACES are all network interfaces excluding lo (LOOPBACK) and sit interfaces IFACES=$(ip link show | grep -vE 'LOOPBACK|sit*' | awk -F': ' '{print $2}') @@ -15,3 +17,9 @@ for IFACE in $IFACES; do dhclient $IFACE fi done + +if [ ! -d "$MOUNT_POINT" ]; then + mkdir -p $MOUNT_POINT +fi + +mount -t ramfs ramfs $MOUNT_POINT diff --git a/init/systemd/cocos-agent.service b/init/systemd/cocos-agent.service index 9e146b4e8..13c6c87f1 100644 --- a/init/systemd/cocos-agent.service +++ b/init/systemd/cocos-agent.service @@ -11,8 +11,8 @@ StandardError=file:/var/log/cocos/agent.stderr Environment=AGENT_GRPC_PORT=7002 Environment=AGENT_LOG_LEVEL=info -ExecStartPre=/cocos/cocos_network_setup.sh -ExecStart=/cocos/agent_start_script.sh +ExecStartPre=/cocos_init/agent_setup.sh +ExecStart=/cocos_init/agent_start_script.sh [Install] WantedBy=default.target diff --git a/manager/qemu/config.go b/manager/qemu/config.go index 18a18bc8d..f77b8f801 100644 --- a/manager/qemu/config.go +++ b/manager/qemu/config.go @@ -173,7 +173,7 @@ func (config Config) ConstructQemuArgs() []string { } args = append(args, "-kernel", config.DiskImgConfig.KernelFile) - args = append(args, "-append", strconv.Quote("earlyprintk=serial console=ttyS0")) + args = append(args, "-append", strconv.Quote("quiet console=null rootfstype=ramfs")) args = append(args, "-initrd", config.DiskImgConfig.RootFsFile) // SEV From 7155027440ea6a3bff5c4adc1050fbd44f231c72 Mon Sep 17 00:00:00 2001 From: Washington Kigani Kamadi Date: Fri, 23 Aug 2024 16:30:57 +0300 Subject: [PATCH 16/18] NOISSUE: Return Response on Computation Termination. (#211) * send response to manager on computation termination Signed-off-by: WashingtonKK * fix tests Signed-off-by: WashingtonKK * refactor: enhance stop computation Signed-off-by: WashingtonKK * remove comment and add event Signed-off-by: WashingtonKK --------- Signed-off-by: WashingtonKK --- manager/api/grpc/client.go | 6 + manager/manager.proto | 6 + manager/service.go | 3 + manager/service_test.go | 15 +- pkg/manager/manager.pb.go | 377 +++++++++++++++++++++++-------------- 5 files changed, 264 insertions(+), 143 deletions(-) diff --git a/manager/api/grpc/client.go b/manager/api/grpc/client.go index 4dffe2c8f..9e1c8306e 100644 --- a/manager/api/grpc/client.go +++ b/manager/api/grpc/client.go @@ -73,7 +73,13 @@ func (client ManagerClient) Process(ctx context.Context, cancel context.CancelFu cancel() return errors.Wrap(errTerminationFromServer, errors.New(mes.TerminateReq.Message)) case *pkgmanager.ServerStreamMessage_StopComputation: + msg := &pkgmanager.ClientStreamMessage_StopComputationRes{StopComputationRes: &pkgmanager.StopComputationResponse{ + ComputationId: mes.StopComputation.ComputationId, + }} if err := client.svc.Stop(ctx, mes.StopComputation.ComputationId); err != nil { + msg.StopComputationRes.Message = err.Error() + } + if err := client.stream.Send(&pkgmanager.ClientStreamMessage{Message: msg}); err != nil { return err } case *pkgmanager.ServerStreamMessage_BackendInfoReq: diff --git a/manager/manager.proto b/manager/manager.proto index b3654b71a..65aa316f1 100644 --- a/manager/manager.proto +++ b/manager/manager.proto @@ -21,6 +21,11 @@ message StopComputation { string computation_id = 1; } +message StopComputationResponse { + string computation_id = 1; + string message = 2; +} + message RunResponse{ string agent_port = 1; string computation_id = 2; @@ -53,6 +58,7 @@ message ClientStreamMessage { AgentEvent agent_event = 2; RunResponse run_res = 3; BackendInfo backendInfo = 4; + StopComputationResponse stopComputationRes = 5; } } diff --git a/manager/service.go b/manager/service.go index 8dc6f9ee7..bce53b659 100644 --- a/manager/service.go +++ b/manager/service.go @@ -167,12 +167,15 @@ func (ms *managerService) Run(ctx context.Context, c *manager.ComputationRunReq) func (ms *managerService) Stop(ctx context.Context, computationID string) error { cvm, ok := ms.vms[computationID] if !ok { + defer ms.publishEvent("stop-computation", computationID, "failed", json.RawMessage{}) return ErrNotFound } if err := cvm.Stop(); err != nil { + defer ms.publishEvent("stop-computation", computationID, "failed", json.RawMessage{}) return err } delete(ms.vms, computationID) + defer ms.publishEvent("stop-computation", computationID, "complete", json.RawMessage{}) return nil } diff --git a/manager/service_test.go b/manager/service_test.go index 87ce002da..e01ac1f7b 100644 --- a/manager/service_test.go +++ b/manager/service_test.go @@ -115,7 +115,6 @@ func TestRun(t *testing.T) { vmf.AssertExpectations(t) - // Clear the events channel for len(eventsChan) > 0 { <-eventsChan } @@ -124,6 +123,10 @@ func TestRun(t *testing.T) { } func TestStop(t *testing.T) { + vmf := new(mocks.Provider) + vmMock := new(mocks.VM) + vmf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(vmMock) + tests := []struct { name string computationID string @@ -156,8 +159,12 @@ func TestStop(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger := slog.Default() + eventsChan := make(chan *manager.ClientStreamMessage, 10) ms := &managerService{ - vms: make(map[string]vm.VM), + logger: logger, + vms: make(map[string]vm.VM), + eventsChan: eventsChan, } vmMock := new(mocks.VM) @@ -180,6 +187,10 @@ func TestStop(t *testing.T) { assert.NoError(t, err) assert.Len(t, ms.vms, 0) } + + for len(eventsChan) > 0 { + <-eventsChan + } }) } } diff --git a/pkg/manager/manager.pb.go b/pkg/manager/manager.pb.go index f3b8bafe4..80ccb028e 100644 --- a/pkg/manager/manager.pb.go +++ b/pkg/manager/manager.pb.go @@ -118,6 +118,61 @@ func (x *StopComputation) GetComputationId() string { return "" } +type StopComputationResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ComputationId string `protobuf:"bytes,1,opt,name=computation_id,json=computationId,proto3" json:"computation_id,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` +} + +func (x *StopComputationResponse) Reset() { + *x = StopComputationResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_manager_manager_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StopComputationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StopComputationResponse) ProtoMessage() {} + +func (x *StopComputationResponse) ProtoReflect() protoreflect.Message { + mi := &file_manager_manager_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StopComputationResponse.ProtoReflect.Descriptor instead. +func (*StopComputationResponse) Descriptor() ([]byte, []int) { + return file_manager_manager_proto_rawDescGZIP(), []int{2} +} + +func (x *StopComputationResponse) GetComputationId() string { + if x != nil { + return x.ComputationId + } + return "" +} + +func (x *StopComputationResponse) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + type RunResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -130,7 +185,7 @@ type RunResponse struct { func (x *RunResponse) Reset() { *x = RunResponse{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[2] + mi := &file_manager_manager_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -143,7 +198,7 @@ func (x *RunResponse) String() string { func (*RunResponse) ProtoMessage() {} func (x *RunResponse) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[2] + mi := &file_manager_manager_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -156,7 +211,7 @@ func (x *RunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunResponse.ProtoReflect.Descriptor instead. func (*RunResponse) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{2} + return file_manager_manager_proto_rawDescGZIP(), []int{3} } func (x *RunResponse) GetAgentPort() string { @@ -185,7 +240,7 @@ type BackendInfo struct { func (x *BackendInfo) Reset() { *x = BackendInfo{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[3] + mi := &file_manager_manager_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -198,7 +253,7 @@ func (x *BackendInfo) String() string { func (*BackendInfo) ProtoMessage() {} func (x *BackendInfo) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[3] + mi := &file_manager_manager_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -211,7 +266,7 @@ func (x *BackendInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use BackendInfo.ProtoReflect.Descriptor instead. func (*BackendInfo) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{3} + return file_manager_manager_proto_rawDescGZIP(), []int{4} } func (x *BackendInfo) GetInfo() []byte { @@ -244,7 +299,7 @@ type AgentEvent struct { func (x *AgentEvent) Reset() { *x = AgentEvent{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[4] + mi := &file_manager_manager_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -257,7 +312,7 @@ func (x *AgentEvent) String() string { func (*AgentEvent) ProtoMessage() {} func (x *AgentEvent) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[4] + mi := &file_manager_manager_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -270,7 +325,7 @@ func (x *AgentEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentEvent.ProtoReflect.Descriptor instead. func (*AgentEvent) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{4} + return file_manager_manager_proto_rawDescGZIP(), []int{5} } func (x *AgentEvent) GetEventType() string { @@ -329,7 +384,7 @@ type AgentLog struct { func (x *AgentLog) Reset() { *x = AgentLog{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[5] + mi := &file_manager_manager_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -342,7 +397,7 @@ func (x *AgentLog) String() string { func (*AgentLog) ProtoMessage() {} func (x *AgentLog) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[5] + mi := &file_manager_manager_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -355,7 +410,7 @@ func (x *AgentLog) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentLog.ProtoReflect.Descriptor instead. func (*AgentLog) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{5} + return file_manager_manager_proto_rawDescGZIP(), []int{6} } func (x *AgentLog) GetMessage() string { @@ -397,13 +452,14 @@ type ClientStreamMessage struct { // *ClientStreamMessage_AgentEvent // *ClientStreamMessage_RunRes // *ClientStreamMessage_BackendInfo + // *ClientStreamMessage_StopComputationRes Message isClientStreamMessage_Message `protobuf_oneof:"message"` } func (x *ClientStreamMessage) Reset() { *x = ClientStreamMessage{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[6] + mi := &file_manager_manager_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -416,7 +472,7 @@ func (x *ClientStreamMessage) String() string { func (*ClientStreamMessage) ProtoMessage() {} func (x *ClientStreamMessage) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[6] + mi := &file_manager_manager_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -429,7 +485,7 @@ func (x *ClientStreamMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ClientStreamMessage.ProtoReflect.Descriptor instead. func (*ClientStreamMessage) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{6} + return file_manager_manager_proto_rawDescGZIP(), []int{7} } func (m *ClientStreamMessage) GetMessage() isClientStreamMessage_Message { @@ -467,6 +523,13 @@ func (x *ClientStreamMessage) GetBackendInfo() *BackendInfo { return nil } +func (x *ClientStreamMessage) GetStopComputationRes() *StopComputationResponse { + if x, ok := x.GetMessage().(*ClientStreamMessage_StopComputationRes); ok { + return x.StopComputationRes + } + return nil +} + type isClientStreamMessage_Message interface { isClientStreamMessage_Message() } @@ -487,6 +550,10 @@ type ClientStreamMessage_BackendInfo struct { BackendInfo *BackendInfo `protobuf:"bytes,4,opt,name=backendInfo,proto3,oneof"` } +type ClientStreamMessage_StopComputationRes struct { + StopComputationRes *StopComputationResponse `protobuf:"bytes,5,opt,name=stopComputationRes,proto3,oneof"` +} + func (*ClientStreamMessage_AgentLog) isClientStreamMessage_Message() {} func (*ClientStreamMessage_AgentEvent) isClientStreamMessage_Message() {} @@ -495,6 +562,8 @@ func (*ClientStreamMessage_RunRes) isClientStreamMessage_Message() {} func (*ClientStreamMessage_BackendInfo) isClientStreamMessage_Message() {} +func (*ClientStreamMessage_StopComputationRes) isClientStreamMessage_Message() {} + type ServerStreamMessage struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -513,7 +582,7 @@ type ServerStreamMessage struct { func (x *ServerStreamMessage) Reset() { *x = ServerStreamMessage{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[7] + mi := &file_manager_manager_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -526,7 +595,7 @@ func (x *ServerStreamMessage) String() string { func (*ServerStreamMessage) ProtoMessage() {} func (x *ServerStreamMessage) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[7] + mi := &file_manager_manager_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -539,7 +608,7 @@ func (x *ServerStreamMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ServerStreamMessage.ProtoReflect.Descriptor instead. func (*ServerStreamMessage) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{7} + return file_manager_manager_proto_rawDescGZIP(), []int{8} } func (m *ServerStreamMessage) GetMessage() isServerStreamMessage_Message { @@ -629,7 +698,7 @@ type RunReqChunks struct { func (x *RunReqChunks) Reset() { *x = RunReqChunks{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[8] + mi := &file_manager_manager_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -642,7 +711,7 @@ func (x *RunReqChunks) String() string { func (*RunReqChunks) ProtoMessage() {} func (x *RunReqChunks) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[8] + mi := &file_manager_manager_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -655,7 +724,7 @@ func (x *RunReqChunks) ProtoReflect() protoreflect.Message { // Deprecated: Use RunReqChunks.ProtoReflect.Descriptor instead. func (*RunReqChunks) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{8} + return file_manager_manager_proto_rawDescGZIP(), []int{9} } func (x *RunReqChunks) GetData() []byte { @@ -682,7 +751,7 @@ type ComputationRunReq struct { func (x *ComputationRunReq) Reset() { *x = ComputationRunReq{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[9] + mi := &file_manager_manager_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -695,7 +764,7 @@ func (x *ComputationRunReq) String() string { func (*ComputationRunReq) ProtoMessage() {} func (x *ComputationRunReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[9] + mi := &file_manager_manager_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -708,7 +777,7 @@ func (x *ComputationRunReq) ProtoReflect() protoreflect.Message { // Deprecated: Use ComputationRunReq.ProtoReflect.Descriptor instead. func (*ComputationRunReq) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{9} + return file_manager_manager_proto_rawDescGZIP(), []int{10} } func (x *ComputationRunReq) GetId() string { @@ -771,7 +840,7 @@ type BackendInfoReq struct { func (x *BackendInfoReq) Reset() { *x = BackendInfoReq{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[10] + mi := &file_manager_manager_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -784,7 +853,7 @@ func (x *BackendInfoReq) String() string { func (*BackendInfoReq) ProtoMessage() {} func (x *BackendInfoReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[10] + mi := &file_manager_manager_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -797,7 +866,7 @@ func (x *BackendInfoReq) ProtoReflect() protoreflect.Message { // Deprecated: Use BackendInfoReq.ProtoReflect.Descriptor instead. func (*BackendInfoReq) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{10} + return file_manager_manager_proto_rawDescGZIP(), []int{11} } func (x *BackendInfoReq) GetId() string { @@ -818,7 +887,7 @@ type ResultConsumer struct { func (x *ResultConsumer) Reset() { *x = ResultConsumer{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[11] + mi := &file_manager_manager_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -831,7 +900,7 @@ func (x *ResultConsumer) String() string { func (*ResultConsumer) ProtoMessage() {} func (x *ResultConsumer) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[11] + mi := &file_manager_manager_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -844,7 +913,7 @@ func (x *ResultConsumer) ProtoReflect() protoreflect.Message { // Deprecated: Use ResultConsumer.ProtoReflect.Descriptor instead. func (*ResultConsumer) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{11} + return file_manager_manager_proto_rawDescGZIP(), []int{12} } func (x *ResultConsumer) GetUserKey() []byte { @@ -867,7 +936,7 @@ type Dataset struct { func (x *Dataset) Reset() { *x = Dataset{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[12] + mi := &file_manager_manager_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -880,7 +949,7 @@ func (x *Dataset) String() string { func (*Dataset) ProtoMessage() {} func (x *Dataset) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[12] + mi := &file_manager_manager_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -893,7 +962,7 @@ func (x *Dataset) ProtoReflect() protoreflect.Message { // Deprecated: Use Dataset.ProtoReflect.Descriptor instead. func (*Dataset) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{12} + return file_manager_manager_proto_rawDescGZIP(), []int{13} } func (x *Dataset) GetHash() []byte { @@ -929,7 +998,7 @@ type Algorithm struct { func (x *Algorithm) Reset() { *x = Algorithm{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[13] + mi := &file_manager_manager_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -942,7 +1011,7 @@ func (x *Algorithm) String() string { func (*Algorithm) ProtoMessage() {} func (x *Algorithm) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[13] + mi := &file_manager_manager_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -955,7 +1024,7 @@ func (x *Algorithm) ProtoReflect() protoreflect.Message { // Deprecated: Use Algorithm.ProtoReflect.Descriptor instead. func (*Algorithm) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{13} + return file_manager_manager_proto_rawDescGZIP(), []int{14} } func (x *Algorithm) GetHash() []byte { @@ -990,7 +1059,7 @@ type AgentConfig struct { func (x *AgentConfig) Reset() { *x = AgentConfig{} if protoimpl.UnsafeEnabled { - mi := &file_manager_manager_proto_msgTypes[14] + mi := &file_manager_manager_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1003,7 +1072,7 @@ func (x *AgentConfig) String() string { func (*AgentConfig) ProtoMessage() {} func (x *AgentConfig) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[14] + mi := &file_manager_manager_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1016,7 +1085,7 @@ func (x *AgentConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentConfig.ProtoReflect.Descriptor instead. func (*AgentConfig) Descriptor() ([]byte, []int) { - return file_manager_manager_proto_rawDescGZIP(), []int{14} + return file_manager_manager_proto_rawDescGZIP(), []int{15} } func (x *AgentConfig) GetPort() string { @@ -1088,54 +1157,65 @@ var file_manager_manager_proto_rawDesc = []byte{ 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x22, 0x53, 0x0a, 0x0b, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x31, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x65, - 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0xde, 0x01, 0x0a, 0x0a, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, - 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, - 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, - 0x69, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x6f, - 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, - 0x74, 0x6f, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x08, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, - 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, - 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xf5, 0x01, 0x0a, 0x13, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x30, 0x0a, 0x09, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, - 0x4c, 0x6f, 0x67, 0x12, 0x36, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, - 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x72, - 0x75, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x0b, - 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x42, 0x61, 0x63, 0x6b, - 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, - 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x49, 0x64, 0x22, 0x5a, 0x0a, 0x17, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, + 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x53, + 0x0a, 0x0b, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, + 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x25, 0x0a, 0x0e, + 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x64, 0x22, 0x31, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0xde, 0x01, 0x0a, 0x0a, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x25, + 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, + 0x1e, 0x0a, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x08, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x25, + 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x38, 0x0a, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xc9, 0x02, 0x0a, 0x13, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, + 0x09, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x4c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x12, + 0x36, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x72, 0x75, 0x6e, 0x5f, 0x72, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, + 0x52, 0x06, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x0b, 0x62, 0x61, 0x63, 0x6b, + 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, + 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x52, 0x0a, 0x12, 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, + 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x48, 0x00, 0x52, 0x12, 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xd6, 0x02, 0x0a, 0x13, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3b, 0x0a, 0x0c, 0x72, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, @@ -1231,48 +1311,50 @@ func file_manager_manager_proto_rawDescGZIP() []byte { return file_manager_manager_proto_rawDescData } -var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_manager_manager_proto_goTypes = []any{ - (*Terminate)(nil), // 0: manager.Terminate - (*StopComputation)(nil), // 1: manager.StopComputation - (*RunResponse)(nil), // 2: manager.RunResponse - (*BackendInfo)(nil), // 3: manager.BackendInfo - (*AgentEvent)(nil), // 4: manager.AgentEvent - (*AgentLog)(nil), // 5: manager.AgentLog - (*ClientStreamMessage)(nil), // 6: manager.ClientStreamMessage - (*ServerStreamMessage)(nil), // 7: manager.ServerStreamMessage - (*RunReqChunks)(nil), // 8: manager.RunReqChunks - (*ComputationRunReq)(nil), // 9: manager.ComputationRunReq - (*BackendInfoReq)(nil), // 10: manager.BackendInfoReq - (*ResultConsumer)(nil), // 11: manager.ResultConsumer - (*Dataset)(nil), // 12: manager.Dataset - (*Algorithm)(nil), // 13: manager.Algorithm - (*AgentConfig)(nil), // 14: manager.AgentConfig - (*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp + (*Terminate)(nil), // 0: manager.Terminate + (*StopComputation)(nil), // 1: manager.StopComputation + (*StopComputationResponse)(nil), // 2: manager.StopComputationResponse + (*RunResponse)(nil), // 3: manager.RunResponse + (*BackendInfo)(nil), // 4: manager.BackendInfo + (*AgentEvent)(nil), // 5: manager.AgentEvent + (*AgentLog)(nil), // 6: manager.AgentLog + (*ClientStreamMessage)(nil), // 7: manager.ClientStreamMessage + (*ServerStreamMessage)(nil), // 8: manager.ServerStreamMessage + (*RunReqChunks)(nil), // 9: manager.RunReqChunks + (*ComputationRunReq)(nil), // 10: manager.ComputationRunReq + (*BackendInfoReq)(nil), // 11: manager.BackendInfoReq + (*ResultConsumer)(nil), // 12: manager.ResultConsumer + (*Dataset)(nil), // 13: manager.Dataset + (*Algorithm)(nil), // 14: manager.Algorithm + (*AgentConfig)(nil), // 15: manager.AgentConfig + (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp } var file_manager_manager_proto_depIdxs = []int32{ - 15, // 0: manager.AgentEvent.timestamp:type_name -> google.protobuf.Timestamp - 15, // 1: manager.AgentLog.timestamp:type_name -> google.protobuf.Timestamp - 5, // 2: manager.ClientStreamMessage.agent_log:type_name -> manager.AgentLog - 4, // 3: manager.ClientStreamMessage.agent_event:type_name -> manager.AgentEvent - 2, // 4: manager.ClientStreamMessage.run_res:type_name -> manager.RunResponse - 3, // 5: manager.ClientStreamMessage.backendInfo:type_name -> manager.BackendInfo - 8, // 6: manager.ServerStreamMessage.runReqChunks:type_name -> manager.RunReqChunks - 9, // 7: manager.ServerStreamMessage.runReq:type_name -> manager.ComputationRunReq - 0, // 8: manager.ServerStreamMessage.terminateReq:type_name -> manager.Terminate - 1, // 9: manager.ServerStreamMessage.stopComputation:type_name -> manager.StopComputation - 10, // 10: manager.ServerStreamMessage.backendInfoReq:type_name -> manager.BackendInfoReq - 12, // 11: manager.ComputationRunReq.datasets:type_name -> manager.Dataset - 13, // 12: manager.ComputationRunReq.algorithm:type_name -> manager.Algorithm - 11, // 13: manager.ComputationRunReq.result_consumers:type_name -> manager.ResultConsumer - 14, // 14: manager.ComputationRunReq.agent_config:type_name -> manager.AgentConfig - 6, // 15: manager.ManagerService.Process:input_type -> manager.ClientStreamMessage - 7, // 16: manager.ManagerService.Process:output_type -> manager.ServerStreamMessage - 16, // [16:17] is the sub-list for method output_type - 15, // [15:16] is the sub-list for method input_type - 15, // [15:15] is the sub-list for extension type_name - 15, // [15:15] is the sub-list for extension extendee - 0, // [0:15] is the sub-list for field type_name + 16, // 0: manager.AgentEvent.timestamp:type_name -> google.protobuf.Timestamp + 16, // 1: manager.AgentLog.timestamp:type_name -> google.protobuf.Timestamp + 6, // 2: manager.ClientStreamMessage.agent_log:type_name -> manager.AgentLog + 5, // 3: manager.ClientStreamMessage.agent_event:type_name -> manager.AgentEvent + 3, // 4: manager.ClientStreamMessage.run_res:type_name -> manager.RunResponse + 4, // 5: manager.ClientStreamMessage.backendInfo:type_name -> manager.BackendInfo + 2, // 6: manager.ClientStreamMessage.stopComputationRes:type_name -> manager.StopComputationResponse + 9, // 7: manager.ServerStreamMessage.runReqChunks:type_name -> manager.RunReqChunks + 10, // 8: manager.ServerStreamMessage.runReq:type_name -> manager.ComputationRunReq + 0, // 9: manager.ServerStreamMessage.terminateReq:type_name -> manager.Terminate + 1, // 10: manager.ServerStreamMessage.stopComputation:type_name -> manager.StopComputation + 11, // 11: manager.ServerStreamMessage.backendInfoReq:type_name -> manager.BackendInfoReq + 13, // 12: manager.ComputationRunReq.datasets:type_name -> manager.Dataset + 14, // 13: manager.ComputationRunReq.algorithm:type_name -> manager.Algorithm + 12, // 14: manager.ComputationRunReq.result_consumers:type_name -> manager.ResultConsumer + 15, // 15: manager.ComputationRunReq.agent_config:type_name -> manager.AgentConfig + 7, // 16: manager.ManagerService.Process:input_type -> manager.ClientStreamMessage + 8, // 17: manager.ManagerService.Process:output_type -> manager.ServerStreamMessage + 17, // [17:18] is the sub-list for method output_type + 16, // [16:17] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name } func init() { file_manager_manager_proto_init() } @@ -1306,7 +1388,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[2].Exporter = func(v any, i int) any { - switch v := v.(*RunResponse); i { + switch v := v.(*StopComputationResponse); i { case 0: return &v.state case 1: @@ -1318,7 +1400,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[3].Exporter = func(v any, i int) any { - switch v := v.(*BackendInfo); i { + switch v := v.(*RunResponse); i { case 0: return &v.state case 1: @@ -1330,7 +1412,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[4].Exporter = func(v any, i int) any { - switch v := v.(*AgentEvent); i { + switch v := v.(*BackendInfo); i { case 0: return &v.state case 1: @@ -1342,7 +1424,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[5].Exporter = func(v any, i int) any { - switch v := v.(*AgentLog); i { + switch v := v.(*AgentEvent); i { case 0: return &v.state case 1: @@ -1354,7 +1436,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[6].Exporter = func(v any, i int) any { - switch v := v.(*ClientStreamMessage); i { + switch v := v.(*AgentLog); i { case 0: return &v.state case 1: @@ -1366,7 +1448,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[7].Exporter = func(v any, i int) any { - switch v := v.(*ServerStreamMessage); i { + switch v := v.(*ClientStreamMessage); i { case 0: return &v.state case 1: @@ -1378,7 +1460,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*RunReqChunks); i { + switch v := v.(*ServerStreamMessage); i { case 0: return &v.state case 1: @@ -1390,7 +1472,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[9].Exporter = func(v any, i int) any { - switch v := v.(*ComputationRunReq); i { + switch v := v.(*RunReqChunks); i { case 0: return &v.state case 1: @@ -1402,7 +1484,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*BackendInfoReq); i { + switch v := v.(*ComputationRunReq); i { case 0: return &v.state case 1: @@ -1414,7 +1496,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[11].Exporter = func(v any, i int) any { - switch v := v.(*ResultConsumer); i { + switch v := v.(*BackendInfoReq); i { case 0: return &v.state case 1: @@ -1426,7 +1508,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[12].Exporter = func(v any, i int) any { - switch v := v.(*Dataset); i { + switch v := v.(*ResultConsumer); i { case 0: return &v.state case 1: @@ -1438,7 +1520,7 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[13].Exporter = func(v any, i int) any { - switch v := v.(*Algorithm); i { + switch v := v.(*Dataset); i { case 0: return &v.state case 1: @@ -1450,6 +1532,18 @@ func file_manager_manager_proto_init() { } } file_manager_manager_proto_msgTypes[14].Exporter = func(v any, i int) any { + switch v := v.(*Algorithm); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_manager_manager_proto_msgTypes[15].Exporter = func(v any, i int) any { switch v := v.(*AgentConfig); i { case 0: return &v.state @@ -1462,13 +1556,14 @@ func file_manager_manager_proto_init() { } } } - file_manager_manager_proto_msgTypes[6].OneofWrappers = []any{ + file_manager_manager_proto_msgTypes[7].OneofWrappers = []any{ (*ClientStreamMessage_AgentLog)(nil), (*ClientStreamMessage_AgentEvent)(nil), (*ClientStreamMessage_RunRes)(nil), (*ClientStreamMessage_BackendInfo)(nil), + (*ClientStreamMessage_StopComputationRes)(nil), } - file_manager_manager_proto_msgTypes[7].OneofWrappers = []any{ + file_manager_manager_proto_msgTypes[8].OneofWrappers = []any{ (*ServerStreamMessage_RunReqChunks)(nil), (*ServerStreamMessage_RunReq)(nil), (*ServerStreamMessage_TerminateReq)(nil), @@ -1481,7 +1576,7 @@ func file_manager_manager_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_manager_manager_proto_rawDesc, NumEnums: 0, - NumMessages: 15, + NumMessages: 16, NumExtensions: 0, NumServices: 1, }, From c4022485158c654c7210179de9c2f6d075e5085f Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:51:35 +0300 Subject: [PATCH 17/18] COCOS-169 - Add support for algo arguments (#202) * custom args Signed-off-by: Sammy Oina * DEBUG Signed-off-by: Sammy Oina * args bug Signed-off-by: Sammy Oina * switch to slice Signed-off-by: Sammy Oina * add flags Signed-off-by: Sammy Oina * switch to string array Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- agent/algorithm/algorithm.go | 12 ++++++++++++ agent/algorithm/binary/binary.go | 6 ++++-- agent/algorithm/python/python.go | 7 +++++-- agent/algorithm/wasm/wasm.go | 5 ++++- agent/service.go | 8 +++++--- cli/README.md | 4 ++-- cli/algorithms.go | 3 +++ test/manual/algo/README.md | 2 +- test/manual/algo/addition.py | 26 ++++++++++++++++---------- 9 files changed, 52 insertions(+), 21 deletions(-) diff --git a/agent/algorithm/algorithm.go b/agent/algorithm/algorithm.go index 040fda216..56aa24f56 100644 --- a/agent/algorithm/algorithm.go +++ b/agent/algorithm/algorithm.go @@ -16,6 +16,7 @@ const ( AlgoTypeWasm AlgorithType = "wasm" AlgoTypeDocker AlgorithType = "docker" AlgoTypeKey = "algo_type" + AlgoArgsKey = "algo_args" ResultsDir = "results" DatasetsDir = "datasets" @@ -30,6 +31,17 @@ func AlgorithmTypeFromContext(ctx context.Context) string { return metadata.ValueFromIncomingContext(ctx, AlgoTypeKey)[0] } +func AlgorithmArgsToContext(ctx context.Context, algoArgs []string) context.Context { + for _, arg := range algoArgs { + ctx = metadata.AppendToOutgoingContext(ctx, AlgoArgsKey, arg) + } + return ctx +} + +func AlgorithmArgsFromContext(ctx context.Context) []string { + return metadata.ValueFromIncomingContext(ctx, AlgoArgsKey) +} + // Algorithm is an interface that specifies the API for an algorithm. type Algorithm interface { // Run executes the algorithm and returns the result. diff --git a/agent/algorithm/binary/binary.go b/agent/algorithm/binary/binary.go index df6766c12..21bdd0e2e 100644 --- a/agent/algorithm/binary/binary.go +++ b/agent/algorithm/binary/binary.go @@ -18,18 +18,20 @@ type binary struct { algoFile string stderr io.Writer stdout io.Writer + args []string } -func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string) algorithm.Algorithm { +func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string, args []string) algorithm.Algorithm { return &binary{ algoFile: algoFile, stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc}, stdout: &algorithm.Stdout{Logger: logger}, + args: args, } } func (b *binary) Run() error { - cmd := exec.Command(b.algoFile) + cmd := exec.Command(b.algoFile, b.args...) cmd.Stderr = b.stderr cmd.Stdout = b.stdout diff --git a/agent/algorithm/python/python.go b/agent/algorithm/python/python.go index a2a2afdb5..c739fcf4c 100644 --- a/agent/algorithm/python/python.go +++ b/agent/algorithm/python/python.go @@ -37,14 +37,16 @@ type python struct { stdout io.Writer runtime string requirementsFile string + args []string } -func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFile, algoFile string) algorithm.Algorithm { +func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFile, algoFile string, args []string) algorithm.Algorithm { p := &python{ algoFile: algoFile, stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc}, stdout: &algorithm.Stdout{Logger: logger}, requirementsFile: requirementsFile, + args: args, } if runtime != "" { p.runtime = runtime @@ -74,7 +76,8 @@ func (p *python) Run() error { } } - cmd := exec.Command(pythonPath, p.algoFile) + args := append([]string{p.algoFile}, p.args...) + cmd := exec.Command(pythonPath, args...) cmd.Stderr = p.stderr cmd.Stdout = p.stdout diff --git a/agent/algorithm/wasm/wasm.go b/agent/algorithm/wasm/wasm.go index 9d8bc721f..9bc2a2a3c 100644 --- a/agent/algorithm/wasm/wasm.go +++ b/agent/algorithm/wasm/wasm.go @@ -22,18 +22,21 @@ type wasm struct { algoFile string stderr io.Writer stdout io.Writer + args []string } -func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string) algorithm.Algorithm { +func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string, args []string) algorithm.Algorithm { return &wasm{ algoFile: algoFile, stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc}, stdout: &algorithm.Stdout{Logger: logger}, + args: args, } } func (w *wasm) Run() error { args := append(mapDirOption, w.algoFile) + args = append(args, w.args...) cmd := exec.Command(wasmRuntime, args...) cmd.Stderr = w.stderr cmd.Stdout = w.stdout diff --git a/agent/service.go b/agent/service.go index ae9ac2be9..23161d1dd 100644 --- a/agent/service.go +++ b/agent/service.go @@ -138,9 +138,11 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error { algoType = string(algorithm.AlgoTypeBin) } + args := algorithm.AlgorithmArgsFromContext(ctx) + switch algoType { case string(algorithm.AlgoTypeBin): - as.algorithm = binary.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name()) + as.algorithm = binary.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name(), args) case string(algorithm.AlgoTypePython): var requirementsFile string if len(algo.Requirements) > 0 { @@ -158,9 +160,9 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error { requirementsFile = fr.Name() } runtime := python.PythonRunTimeFromContext(ctx) - as.algorithm = python.NewAlgorithm(as.sm.logger, as.eventSvc, runtime, requirementsFile, f.Name()) + as.algorithm = python.NewAlgorithm(as.sm.logger, as.eventSvc, runtime, requirementsFile, f.Name(), args) case string(algorithm.AlgoTypeWasm): - as.algorithm = wasm.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name()) + as.algorithm = wasm.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name(), args) case string(algorithm.AlgoTypeDocker): as.algorithm = docker.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name()) } diff --git a/cli/README.md b/cli/README.md index 14d1d4193..2856050c6 100644 --- a/cli/README.md +++ b/cli/README.md @@ -67,10 +67,10 @@ To upload an algorithm, use the following command: ##### Flags - -a, --algorithm string Algorithm type to run (default "bin") -- --python-runtime string Python runtime to use (default "python3") +- --args stringArray Arguments to pass to the algorithm +- --python-runtime string Python runtime to use (default "python3") - -r, --requirements string Python requirements file - #### Upload Dataset To upload a dataset, use the following command: diff --git a/cli/algorithms.go b/cli/algorithms.go index 16922eae6..c9da27281 100644 --- a/cli/algorithms.go +++ b/cli/algorithms.go @@ -19,6 +19,7 @@ var ( pythonRuntime string algoType string requirementsFile string + algoArgs []string ) func (cli *CLI) NewAlgorithmCmd() *cobra.Command { @@ -72,12 +73,14 @@ func (cli *CLI) NewAlgorithmCmd() *cobra.Command { cmd.Flags().StringVarP(&algoType, "algorithm", "a", string(algorithm.AlgoTypeBin), "Algorithm type to run") cmd.Flags().StringVar(&pythonRuntime, "python-runtime", python.PyRuntime, "Python runtime to use") cmd.Flags().StringVarP(&requirementsFile, "requirements", "r", "", "Python requirements file") + cmd.Flags().StringArrayVar(&algoArgs, "args", []string{}, "Arguments to pass to the algorithm") return cmd } func addAlgoMetadata(ctx context.Context) context.Context { ctx = algorithm.AlgorithmTypeToContext(ctx, algoType) + ctx = algorithm.AlgorithmArgsToContext(ctx, algoArgs) ctx = python.PythonRunTimeToContext(ctx, pythonRuntime) return ctx } diff --git a/test/manual/algo/README.md b/test/manual/algo/README.md index 7ffb47fbf..e416556f4 100644 --- a/test/manual/algo/README.md +++ b/test/manual/algo/README.md @@ -89,7 +89,7 @@ For addition example, you can use the following command: ``` ```bash -./build/cocos-cli algo ./test/manual/algo/addition.py ./private.pem -a python +./build/cocos-cli algo ./test/manual/algo/addition.py ./private.pem -a python --args="--a" --args="100" --args="--b" --args="20" ``` ```bash diff --git a/test/manual/algo/addition.py b/test/manual/algo/addition.py index 9a4467b9b..79e1344b0 100644 --- a/test/manual/algo/addition.py +++ b/test/manual/algo/addition.py @@ -1,6 +1,6 @@ import os -import sys import zipfile +import argparse RESULTS_DIR = "results" RESULTS_FILE = "result.txt" @@ -52,15 +52,21 @@ def read_results_from_file(self, results_file): if __name__ == "__main__": - a = 5 - b = 10 + parser = argparse.ArgumentParser(description="Process some integers.") + parser.add_argument('--a', type=int, help="First number", default=5) + parser.add_argument('--b', type=int, help="Second number", default=10) + parser.add_argument('--test', type=str, help="Test with a results file", required=False) + + args = parser.parse_args() + computation = Computation() - if len(sys.argv) == 1: - computation.compute(a, b) - computation.save_result() - elif len(sys.argv) == 3 and sys.argv[1] == "test": - computation.read_results_from_file(sys.argv[2]) - else: - print("Invalid arguments") + try: + if args.test: + computation.read_results_from_file(args.test) + else: + computation.compute(args.a, args.b) + computation.save_result() + except Exception as e: + print(f"An error occurred: {e}") exit(1) From bdbeb4e976d2a8ed7b79a10bf13763cb478242ac Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:52:07 +0300 Subject: [PATCH 18/18] COCOS-199 - Enable testing of SEV features on any machine (#205) * make attestation embeddable Signed-off-by: Sammy Oina * mock backend info Signed-off-by: Sammy Oina * embed files Signed-off-by: Sammy Oina * finish up Signed-off-by: Sammy Oina --------- Signed-off-by: Sammy Oina --- Makefile | 12 +++++---- agent/quoteprovider/embed.go | 38 +++++++++++++++++++++++++++ agent/quoteprovider/sev.go | 13 +++++++++ agent/service.go | 26 +++++++++--------- attestation.bin | Bin 0 -> 1240 bytes attestation.go | 12 +++++++++ cmd/agent/main.go | 16 ++++++++--- hal/linux/configs/cocos_defconfig | 1 + hal/linux/package/agent/agent.mk | 2 +- internal/server/grpc/grpc.go | 14 +++++----- manager/backend_info.go | 29 ++++++++++++++++++++ manager/backend_info_embed.go | 13 +++++++++ manager/service.go | 18 ------------- scripts/backend_info/backend_info.go | 14 ++++++++++ 14 files changed, 159 insertions(+), 49 deletions(-) create mode 100644 agent/quoteprovider/embed.go create mode 100644 agent/quoteprovider/sev.go create mode 100644 attestation.bin create mode 100644 attestation.go create mode 100644 manager/backend_info.go create mode 100644 manager/backend_info_embed.go create mode 100644 scripts/backend_info/backend_info.go diff --git a/Makefile b/Makefile index 5db4989d8..7e81c2b81 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,12 @@ BUILD_DIR = build SERVICES = manager agent cli -PLATFORM_INFO = platform_info +BACKEND_INFO = backend_info CGO_ENABLED ?= 0 GOARCH ?= amd64 VERSION ?= $(shell git describe --abbrev=0 --tags --always) COMMIT ?= $(shell git rev-parse HEAD) TIME ?= $(shell date +%F_%T) +EMBED_ENABLED ?= 0 define compile_service CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \ @@ -13,18 +14,19 @@ define compile_service -X 'github.com/absmach/magistrala.BuildTime=$(TIME)' \ -X 'github.com/absmach/magistrala.Version=$(VERSION)' \ -X 'github.com/absmach/magistrala.Commit=$(COMMIT)'" \ + $(if $(filter 1,$(EMBED_ENABLED)),-tags "embed",) \ -o ${BUILD_DIR}/cocos-$(1) cmd/$(1)/main.go endef -.PHONY: all $(SERVICES) $(PLATFORM_INFO) +.PHONY: all $(SERVICES) $(BACKEND_INFO) all: $(SERVICES) $(SERVICES): - $(call compile_service,$(@)) + $(call compile_service,$@) -$(PLATFORM_INFO): - $(MAKE) -C ./scripts/platform_info +$(BACKEND_INFO): + $(MAKE) -C ./scripts/backend_info protoc: protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative agent/agent.proto diff --git a/agent/quoteprovider/embed.go b/agent/quoteprovider/embed.go new file mode 100644 index 000000000..8ad7a8b87 --- /dev/null +++ b/agent/quoteprovider/embed.go @@ -0,0 +1,38 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 + +//go:build embed +// +build embed + +package quoteprovider + +import ( + "github.com/google/go-sev-guest/client" + pb "github.com/google/go-sev-guest/proto/sevsnp" + cocosai "github.com/ultravioletrs/cocos" +) + +var _ client.QuoteProvider = (*embeddedQuoteProvider)(nil) + +type embeddedQuoteProvider struct { +} + +func GetQuoteProvider() (client.QuoteProvider, error) { + return &embeddedQuoteProvider{}, nil +} + +// GetQuote returns the SEV quote for the given report data. +func (e *embeddedQuoteProvider) GetRawQuote(reportData [64]byte) ([]byte, error) { + return cocosai.EmbeddedAttestation, nil +} + +// IsSupported returns true if the SEV platform is supported. +func (e *embeddedQuoteProvider) IsSupported() bool { + return true +} + +// Product returns the SEV product information. +// unimplemented since it is deprecated and not used. +func (e *embeddedQuoteProvider) Product() *pb.SevProduct { + panic("unimplemented") +} diff --git a/agent/quoteprovider/sev.go b/agent/quoteprovider/sev.go new file mode 100644 index 000000000..158c490e7 --- /dev/null +++ b/agent/quoteprovider/sev.go @@ -0,0 +1,13 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 + +//go:build !embed +// +build !embed + +package quoteprovider + +import "github.com/google/go-sev-guest/client" + +func GetQuoteProvider() (client.QuoteProvider, error) { + return client.GetQuoteProvider() +} diff --git a/agent/service.go b/agent/service.go index 23161d1dd..4c1b86ab5 100644 --- a/agent/service.go +++ b/agent/service.go @@ -68,21 +68,23 @@ type Service interface { } type agentService struct { - computation Computation // Holds the current computation manifest. - algorithm algorithm.Algorithm // Filepath to the algorithm received for the computation. - result []byte // Stores the result of the computation. - sm *StateMachine // Manages the state transitions of the agent service. - runError error // Stores any error encountered during the computation run. - eventSvc events.Service // Service for publishing events related to computation. + computation Computation // Holds the current computation request details. + algorithm algorithm.Algorithm // Filepath to the algorithm received for the computation. + result []byte // Stores the result of the computation. + sm *StateMachine // Manages the state transitions of the agent service. + runError error // Stores any error encountered during the computation run. + eventSvc events.Service // Service for publishing events related to computation. + quoteProvider client.QuoteProvider // Provider for generating attestation quotes. } var _ Service = (*agentService)(nil) // New instantiates the agent service implementation. -func New(ctx context.Context, logger *slog.Logger, eventSvc events.Service, cmp Computation) Service { +func New(ctx context.Context, logger *slog.Logger, eventSvc events.Service, cmp Computation, quoteProvider client.QuoteProvider) Service { svc := &agentService{ - sm: NewStateMachine(logger, cmp), - eventSvc: eventSvc, + sm: NewStateMachine(logger, cmp), + eventSvc: eventSvc, + quoteProvider: quoteProvider, } go svc.sm.Start(ctx) @@ -252,11 +254,7 @@ func (as *agentService) Result(ctx context.Context) ([]byte, error) { } func (as *agentService) Attestation(ctx context.Context, reportData [ReportDataSize]byte) ([]byte, error) { - provider, err := client.GetQuoteProvider() - if err != nil { - return []byte{}, err - } - rawQuote, err := provider.GetRawQuote(reportData) + rawQuote, err := as.quoteProvider.GetRawQuote(reportData) if err != nil { return []byte{}, err } diff --git a/attestation.bin b/attestation.bin new file mode 100644 index 0000000000000000000000000000000000000000..ab39d778875faf693e6a6584c27765fd737fae08 GIT binary patch literal 1240 zcmZQ#fB|M0gCNZaQ~^=VcM)B2c=Yd`oF%0kMfVhb#2uMz;U4XfJuUkB+b9*6DZl13 zEoZ2|zn*Ibm(Anb@`aq;ervjeS1c|NVYx1*y`YOXbk&rH?N^u07CbDYvXkxgM04?H z22+gYvcw&2mMXq@X{z? zn)8>xw7*lSQ6p#GraB+pebnZMf0Y)RTC!zX;v9!BDmNH4uPpHOVAE#&Ki%W-UF!|z z{Qn66T;T|Fr_{!*t^7;ZIlbwTR^5FohI{F&NxNMmdmYxa*eQ4DH8Sb!QQWffrO+O4 zmk0kZ=_*ytD9*HUGcl+^7itI=A>7=cX63BDo)=Jc(aa&e=3vn a#`TN19zD34y3GJ+JqI2qfaI9?8yEn)Y{+f^ literal 0 HcmV?d00001 diff --git a/attestation.go b/attestation.go new file mode 100644 index 000000000..3b2251efa --- /dev/null +++ b/attestation.go @@ -0,0 +1,12 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 + +//go:build embed +// +build embed + +package cocosai + +import _ "embed" + +//go:embed attestation.bin +var EmbeddedAttestation []byte diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 2bb22c24c..06b1e1b22 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -11,12 +11,14 @@ import ( "log/slog" "github.com/absmach/magistrala/pkg/prometheus" + "github.com/google/go-sev-guest/client" "github.com/mdlayher/vsock" "github.com/ultravioletrs/cocos/agent" "github.com/ultravioletrs/cocos/agent/api" agentgrpc "github.com/ultravioletrs/cocos/agent/api/grpc" "github.com/ultravioletrs/cocos/agent/auth" "github.com/ultravioletrs/cocos/agent/events" + "github.com/ultravioletrs/cocos/agent/quoteprovider" agentlogger "github.com/ultravioletrs/cocos/internal/logger" "github.com/ultravioletrs/cocos/internal/server" grpcserver "github.com/ultravioletrs/cocos/internal/server/grpc" @@ -62,7 +64,13 @@ func main() { } defer eventSvc.Close() - svc := newService(ctx, logger, eventSvc, cfg) + qp, err := quoteprovider.GetQuoteProvider() + if err != nil { + logger.Error(fmt.Sprintf("failed to create quote provider %s", err.Error())) + return + } + + svc := newService(ctx, logger, eventSvc, cfg, qp) grpcServerConfig := server.Config{ Port: cfg.AgentConfig.Port, @@ -85,7 +93,7 @@ func main() { return } - gs := grpcserver.New(ctx, cancel, svcName, grpcServerConfig, registerAgentServiceServer, logger, svc, authSvc) + gs := grpcserver.New(ctx, cancel, svcName, grpcServerConfig, registerAgentServiceServer, logger, qp, authSvc) g.Go(func() error { return gs.Start() @@ -100,8 +108,8 @@ func main() { } } -func newService(ctx context.Context, logger *slog.Logger, eventSvc events.Service, cmp agent.Computation) agent.Service { - svc := agent.New(ctx, logger, eventSvc, cmp) +func newService(ctx context.Context, logger *slog.Logger, eventSvc events.Service, cmp agent.Computation, qp client.QuoteProvider) agent.Service { + svc := agent.New(ctx, logger, eventSvc, cmp, qp) svc = api.LoggingMiddleware(svc, logger) counter, latency := prometheus.MakeMetrics(svcName, "api") diff --git a/hal/linux/configs/cocos_defconfig b/hal/linux/configs/cocos_defconfig index 10463a074..40f9ec06d 100644 --- a/hal/linux/configs/cocos_defconfig +++ b/hal/linux/configs/cocos_defconfig @@ -71,3 +71,4 @@ BR2_TOOLCHAIN_BUILDROOT_LIBSTDCPP=y BR2_PACKAGE_GCC=y BR2_PACKAGE_GCC_TARGET=y BR2_PACKAGE_LIBSTDCPP=y + diff --git a/hal/linux/package/agent/agent.mk b/hal/linux/package/agent/agent.mk index be1e096dd..4cbd301a0 100644 --- a/hal/linux/package/agent/agent.mk +++ b/hal/linux/package/agent/agent.mk @@ -8,7 +8,7 @@ AGENT_VERSION = main AGENT_SITE = $(call github,ultravioletrs,cocos,$(AGENT_VERSION)) define AGENT_BUILD_CMDS - $(MAKE) -C $(@D) agent + $(MAKE) -C $(@D) agent EMBED_ENABLED=$(AGENT_EMBED_ENABLED) endef define AGENT_INSTALL_TARGET_CMDS diff --git a/internal/server/grpc/grpc.go b/internal/server/grpc/grpc.go index 97eb22f88..4f9df0f36 100644 --- a/internal/server/grpc/grpc.go +++ b/internal/server/grpc/grpc.go @@ -20,7 +20,7 @@ import ( "os" "time" - "github.com/ultravioletrs/cocos/agent" + "github.com/google/go-sev-guest/client" agentgrpc "github.com/ultravioletrs/cocos/agent/api/grpc" "github.com/ultravioletrs/cocos/agent/auth" "github.com/ultravioletrs/cocos/internal/server" @@ -48,7 +48,7 @@ type Server struct { server.BaseServer server *grpc.Server registerService serviceRegister - agent agent.Service + quoteProvider client.QuoteProvider authSvc auth.Authenticator } @@ -56,7 +56,7 @@ type serviceRegister func(srv *grpc.Server) var _ server.Server = (*Server)(nil) -func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, registerService serviceRegister, logger *slog.Logger, agentSvc agent.Service, authSvc auth.Authenticator) server.Server { +func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, registerService serviceRegister, logger *slog.Logger, qp client.QuoteProvider, authSvc auth.Authenticator) server.Server { listenFullAddress := fmt.Sprintf("%s:%s", config.Host, config.Port) return &Server{ BaseServer: server.BaseServer{ @@ -68,7 +68,7 @@ func New(ctx context.Context, cancel context.CancelFunc, name string, config ser Logger: logger, }, registerService: registerService, - agent: agentSvc, + quoteProvider: qp, authSvc: authSvc, } } @@ -93,7 +93,7 @@ func (s *Server) Start() error { switch { case s.Config.AttestedTLS: - certificateBytes, privateKeyBytes, err := generateCertificatesForATLS(s.agent) + certificateBytes, privateKeyBytes, err := generateCertificatesForATLS(s.quoteProvider) if err != nil { return fmt.Errorf("failed to create certificate: %w", err) } @@ -228,7 +228,7 @@ func loadX509KeyPair(certfile, keyfile string) (tls.Certificate, error) { return tls.X509KeyPair(cert, key) } -func generateCertificatesForATLS(svc agent.Service) ([]byte, []byte, error) { +func generateCertificatesForATLS(qp client.QuoteProvider) ([]byte, []byte, error) { curve := elliptic.P256() privateKey, err := ecdsa.GenerateKey(curve, rand.Reader) if err != nil { @@ -241,7 +241,7 @@ func generateCertificatesForATLS(svc agent.Service) ([]byte, []byte, error) { } // The Attestation Report will be added as an X.509 certificate extension - attestationReport, err := svc.Attestation(context.Background(), sha3.Sum512(publicKeyBytes)) + attestationReport, err := qp.GetRawQuote(sha3.Sum512(publicKeyBytes)) if err != nil { return nil, nil, fmt.Errorf("failed to fetch the attestation report: %w", err) } diff --git a/manager/backend_info.go b/manager/backend_info.go new file mode 100644 index 000000000..528d8817e --- /dev/null +++ b/manager/backend_info.go @@ -0,0 +1,29 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 + +//go:build !embed +// +build !embed + +package manager + +import ( + "fmt" + "os" + "os/exec" +) + +func (ms *managerService) FetchBackendInfo() ([]byte, error) { + cmd := exec.Command("sudo", fmt.Sprintf("%s/backend_info", ms.backendMeasurementBinaryPath), "--policy", "1966081") + + _, err := cmd.Output() + if err != nil { + return nil, err + } + + f, err := os.ReadFile("./backend_info.json") + if err != nil { + return nil, err + } + + return f, nil +} diff --git a/manager/backend_info_embed.go b/manager/backend_info_embed.go new file mode 100644 index 000000000..5f9e6116d --- /dev/null +++ b/manager/backend_info_embed.go @@ -0,0 +1,13 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 + +//go:build embed +// +build embed + +package manager + +import backendinfo "github.com/ultravioletrs/cocos/scripts/backend_info" + +func (ms *managerService) FetchBackendInfo() ([]byte, error) { + return backendinfo.BackendInfo, nil +} diff --git a/manager/service.go b/manager/service.go index bce53b659..af9225cb3 100644 --- a/manager/service.go +++ b/manager/service.go @@ -9,8 +9,6 @@ import ( "fmt" "log/slog" "net" - "os" - "os/exec" "regexp" "strconv" "sync" @@ -179,22 +177,6 @@ func (ms *managerService) Stop(ctx context.Context, computationID string) error return nil } -func (ms *managerService) FetchBackendInfo() ([]byte, error) { - cmd := exec.Command("sudo", fmt.Sprintf("%s/backend_info", ms.backendMeasurementBinaryPath), "--policy", "1966081") - - _, err := cmd.Output() - if err != nil { - return nil, err - } - - f, err := os.ReadFile("./backend_info.json") - if err != nil { - return nil, err - } - - return f, nil -} - func getFreePort(minPort, maxPort int) (int, error) { if checkPortisFree(minPort) { return minPort, nil diff --git a/scripts/backend_info/backend_info.go b/scripts/backend_info/backend_info.go new file mode 100644 index 000000000..612dd9af1 --- /dev/null +++ b/scripts/backend_info/backend_info.go @@ -0,0 +1,14 @@ +// Copyright (c) Ultraviolet +// SPDX-License-Identifier: Apache-2.0 + +//go:build embed +// +build embed + +package backendinfo + +import ( + _ "embed" +) + +//go:embed backend_info.json +var BackendInfo []byte