From ee17eae9c3d1f30bece070c016cbcb24f45dd765 Mon Sep 17 00:00:00 2001 From: Lauri Peltonen Date: Mon, 4 May 2026 15:17:54 +0300 Subject: [PATCH 1/3] [runner] Sample cgroup counters and detect action OOM Add optional cgroup resource usage sampling to bb_runner. When assume_exclusive_cgroup is enabled, do best-effort validation that the runner is running in a dedicated cgroup (such as a container), and then sample memory.events, memory.peak, and PSI counters before and after each action, and attach the deltas to the runner response metadata. The following new fields are attached as CgroupResourceUsage metadata and logged: - memory_events_low - memory_events_high - memory_events_max - memory_events_oom - memory_events_oom_kill - memory_events_oom_group_kill // Linux >= 5.17 - memory_peak_bytes // resettable memory.peak, Linux >= 6.12 - psi_memory_some - psi_memory_full - psi_cpu_some - psi_cpu_full - psi_io_some - psi_io_full For more information about the meaning of each, refer to https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory-interface-files and https://www.kernel.org/doc/html/latest/accounting/psi.html The PSI counters are useful for determining what is the bottleneck of the action, is it IO-bound, memory-bound or CPU-bound. Finally, use the sampled memory.events counters in bb_worker to report OOM outcomes more accurately. If an action process was OOM-killed after the action cgroup hit its memory limit, surface information about the OOM in ExecuteResponse.Message and force a nonzero exit code even if the top-level command exited with zero. If an action process was OOM-killed without the action cgroup reaching its memory limit, treat it as a retryable infrastructure failure with codes.Unavailable, since that likely indicates system-level memory pressure such as node memory overcommitment. --- cmd/bb_runner/main.go | 7 + pkg/builder/local_build_executor.go | 28 ++ pkg/builder/local_build_executor_test.go | 146 ++++++ .../configuration/bb_runner/bb_runner.pb.go | 13 +- .../configuration/bb_runner/bb_runner.proto | 9 + pkg/proto/resourceusage/resourceusage.pb.go | 198 +++++++- pkg/proto/resourceusage/resourceusage.proto | 71 +++ pkg/runner/BUILD.bazel | 23 +- pkg/runner/cgroup_linux.go | 449 ++++++++++++++++++ pkg/runner/cgroup_linux_test.go | 192 ++++++++ pkg/runner/cgroup_other.go | 25 + .../cgroup_resource_usage_sampling_runner.go | 89 ++++ ...oup_resource_usage_sampling_runner_test.go | 174 +++++++ 13 files changed, 1397 insertions(+), 27 deletions(-) create mode 100644 pkg/runner/cgroup_linux.go create mode 100644 pkg/runner/cgroup_linux_test.go create mode 100644 pkg/runner/cgroup_other.go create mode 100644 pkg/runner/cgroup_resource_usage_sampling_runner.go create mode 100644 pkg/runner/cgroup_resource_usage_sampling_runner_test.go diff --git a/cmd/bb_runner/main.go b/cmd/bb_runner/main.go index 33457491..6141f0fb 100644 --- a/cmd/bb_runner/main.go +++ b/cmd/bb_runner/main.go @@ -70,6 +70,13 @@ func main() { commandCreator, configuration.SetTmpdirEnvironmentVariable, ) + if configuration.AssumeExclusiveCgroup { + var err error + r, err = runner.NewCgroupResourceUsageSamplingRunner(r) + if err != nil { + return util.StatusWrap(err, "Failed to enable cgroup resource usage sampling") + } + } // Let bb_runner replace temporary directories with symbolic // links pointing to the temporary directory set up by diff --git a/pkg/builder/local_build_executor.go b/pkg/builder/local_build_executor.go index a1b8ee07..a13f5083 100644 --- a/pkg/builder/local_build_executor.go +++ b/pkg/builder/local_build_executor.go @@ -11,6 +11,7 @@ import ( "github.com/buildbarn/bb-remote-execution/pkg/filesystem/access" "github.com/buildbarn/bb-remote-execution/pkg/filesystem/pool" "github.com/buildbarn/bb-remote-execution/pkg/proto/remoteworker" + "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/clock" @@ -36,6 +37,19 @@ var ( checkReadinessComponent = path.MustNewComponent("check_readiness") ) +func getCgroupResourceUsage(result *remoteexecution.ActionResult) *resourceusage.CgroupResourceUsage { + if result == nil || result.ExecutionMetadata == nil { + return nil + } + var cgroupUsage resourceusage.CgroupResourceUsage + for _, metadata := range result.ExecutionMetadata.AuxiliaryMetadata { + if metadata.UnmarshalTo(&cgroupUsage) == nil { + return &cgroupUsage + } + } + return nil +} + // capturingErrorLogger is an error logger that stores up to a single // error. When the error is stored, a context cancelation function is // invoked. This is used by localBuildExecutor to kill a build action in @@ -309,6 +323,20 @@ func (be *localBuildExecutor) Execute(ctx context.Context, filePool pool.FilePoo if runErr == nil { response.Result.ExitCode = int32(runResponse.ExitCode) response.Result.ExecutionMetadata.AuxiliaryMetadata = append(response.Result.ExecutionMetadata.AuxiliaryMetadata, runResponse.ResourceUsage...) + if cgroupUsage := getCgroupResourceUsage(response.Result); cgroupUsage != nil && cgroupUsage.MemoryEventsOomKill > 0 { + if cgroupUsage.MemoryEventsOom > 0 { + response.Message = "Action failed due to out of memory: cgroup memory limit was reached and a process was OOM-killed" + if response.Result.ExitCode == 0 { + response.Result.ExitCode = 1 + } + } else { + // The cgroup did not reach its memory limit, so the OOM + // kill likely came from system-level memory pressure, such + // as node memory overcommitment. Treat this as retryable + // infrastructure failure. + attachErrorToExecuteResponse(response, status.Error(codes.Unavailable, "An action process was OOM-killed without the action reaching its cgroup memory limit")) + } + } } else { attachErrorToExecuteResponse(response, util.StatusWrap(runErr, "Failed to run command")) } diff --git a/pkg/builder/local_build_executor_test.go b/pkg/builder/local_build_executor_test.go index 29255ea1..e46e25c1 100644 --- a/pkg/builder/local_build_executor_test.go +++ b/pkg/builder/local_build_executor_test.go @@ -12,6 +12,7 @@ import ( re_clock "github.com/buildbarn/bb-remote-execution/pkg/clock" "github.com/buildbarn/bb-remote-execution/pkg/filesystem/access" "github.com/buildbarn/bb-remote-execution/pkg/proto/remoteworker" + "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" @@ -794,6 +795,151 @@ func TestLocalBuildExecutorSuccess(t *testing.T) { }, executeResponse) } +func TestLocalBuildExecutorReportsCgroupOOMKill(t *testing.T) { + for _, testCase := range []struct { + name string + cgroupUsage *resourceusage.CgroupResourceUsage + wantExitCode int32 + wantMessage string + wantStatus error + }{ + { + name: "memory limit OOM kill with zero runner exit code", + cgroupUsage: &resourceusage.CgroupResourceUsage{ + MemoryEventsOom: 1, + MemoryEventsOomKill: 1, + }, + wantExitCode: 1, + wantMessage: "Action failed due to out of memory: cgroup memory limit was reached and a process was OOM-killed", + }, + { + name: "external OOM kill with zero runner exit code", + cgroupUsage: &resourceusage.CgroupResourceUsage{ + MemoryEventsOomKill: 1, + }, + wantStatus: status.Error(codes.Unavailable, "An action process was OOM-killed without the action reaching its cgroup memory limit"), + }, + } { + t.Run(testCase.name, func(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + resourceUsage, err := anypb.New(testCase.cgroupUsage) + require.NoError(t, err) + + contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage.EXPECT().Get( + gomock.Any(), + digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0000000000000000000000000000000000000000000000000000000000000002", 234), + ).Return(buffer.NewProtoBufferFromProto(&remoteexecution.Command{ + Arguments: []string{"clang"}, + }, buffer.UserProvided)) + + buildDirectory := mock.NewMockBuildDirectory(ctrl) + buildDirectoryCreator := mock.NewMockBuildDirectoryCreator(ctrl) + actionDigest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0000000000000000000000000000000000000000000000000000000000000001", 123) + buildDirectoryCreator.EXPECT().GetBuildDirectory(ctx, &actionDigest). + Return(buildDirectory, nil, nil) + filePool := mock.NewMockFilePool(ctrl) + monitor := mock.NewMockUnreadDirectoryMonitor(ctrl) + buildDirectory.EXPECT().InstallHooks(filePool, gomock.Any()) + buildDirectory.EXPECT().Mkdir(path.MustNewComponent("root"), os.FileMode(0o777)) + inputRootDirectory := mock.NewMockBuildDirectory(ctrl) + buildDirectory.EXPECT().EnterBuildDirectory(path.MustNewComponent("root")).Return(inputRootDirectory, nil) + inputRootDirectory.EXPECT().MergeDirectoryContents( + ctx, + gomock.Any(), + digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0000000000000000000000000000000000000000000000000000000000000003", 345), + monitor, + ).Return(nil) + buildDirectory.EXPECT().Mkdir(path.MustNewComponent("tmp"), os.FileMode(0o777)) + buildDirectory.EXPECT().Mkdir(path.MustNewComponent("server_logs"), os.FileMode(0o777)) + + runner := mock.NewMockRunnerClient(ctrl) + runner.EXPECT().Run(gomock.Any(), &runner_pb.RunRequest{ + Arguments: []string{"clang"}, + EnvironmentVariables: map[string]string{}, + WorkingDirectory: "", + StdoutPath: "stdout", + StderrPath: "stderr", + InputRootDirectory: "root", + TemporaryDirectory: "tmp", + ServerLogsDirectory: "server_logs", + }).Return(&runner_pb.RunResponse{ + ExitCode: 0, + ResourceUsage: []*anypb.Any{resourceUsage}, + }, nil) + inputRootDirectory.EXPECT().Close() + emptyDigest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 0) + buildDirectory.EXPECT().UploadFile(ctx, path.MustNewComponent("stdout"), gomock.Any(), gomock.Any()).Return(emptyDigest, nil) + buildDirectory.EXPECT().UploadFile(ctx, path.MustNewComponent("stderr"), gomock.Any(), gomock.Any()).Return(emptyDigest, nil) + serverLogsDirectory := mock.NewMockUploadableDirectory(ctrl) + buildDirectory.EXPECT().EnterUploadableDirectory(path.MustNewComponent("server_logs")).Return(serverLogsDirectory, nil) + serverLogsDirectory.EXPECT().ReadDir() + serverLogsDirectory.EXPECT().Close() + buildDirectory.EXPECT().Close() + + clock := mock.NewMockClock(ctrl) + clock.EXPECT().NewContextWithTimeout(gomock.Any(), time.Hour).DoAndReturn(func(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + return context.WithCancel(parent) + }) + clock.EXPECT().NewContextWithTimeout(gomock.Any(), 10*time.Second).DoAndReturn(func(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + return parent, func() {} + }) + localBuildExecutor := builder.NewLocalBuildExecutor( + contentAddressableStorage, + buildDirectoryCreator, + runner, + clock, + /* maximumWritableFileUploadDelay = */ 10*time.Second, + /* inputRootCharacterDevices = */ nil, + /* maximumMessageSizeBytes = */ 10000, + /* environmentVariables = */ map[string]string{}, + /* forceUploadTreesAndDirectories = */ false, + ) + + metadata := make(chan *remoteworker.CurrentState_Executing, 10) + executeResponse := localBuildExecutor.Execute( + ctx, + filePool, + monitor, + digest.MustNewFunction("ubuntu1804", remoteexecution.DigestFunction_SHA256), + &remoteworker.DesiredState_Executing{ + ActionDigest: &remoteexecution.Digest{ + Hash: "0000000000000000000000000000000000000000000000000000000000000001", + SizeBytes: 123, + }, + Action: &remoteexecution.Action{ + CommandDigest: &remoteexecution.Digest{ + Hash: "0000000000000000000000000000000000000000000000000000000000000002", + SizeBytes: 234, + }, + InputRootDigest: &remoteexecution.Digest{ + Hash: "0000000000000000000000000000000000000000000000000000000000000003", + SizeBytes: 345, + }, + Timeout: &durationpb.Duration{Seconds: 3600}, + }, + }, + metadata, + ) + + expectedResponse := &remoteexecution.ExecuteResponse{ + Result: &remoteexecution.ActionResult{ + ExitCode: testCase.wantExitCode, + ExecutionMetadata: &remoteexecution.ExecutedActionMetadata{ + AuxiliaryMetadata: []*anypb.Any{resourceUsage}, + }, + }, + Message: testCase.wantMessage, + } + if testCase.wantStatus != nil { + expectedResponse.Status = status.Convert(testCase.wantStatus).Proto() + } + testutil.RequireEqualProto(t, expectedResponse, executeResponse) + }) + } +} + func TestLocalBuildExecutorCachingInvalidTimeout(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/proto/configuration/bb_runner/bb_runner.pb.go b/pkg/proto/configuration/bb_runner/bb_runner.pb.go index 3d180dae..4ab2f091 100644 --- a/pkg/proto/configuration/bb_runner/bb_runner.pb.go +++ b/pkg/proto/configuration/bb_runner/bb_runner.pb.go @@ -39,6 +39,7 @@ type ApplicationConfiguration struct { SymlinkTemporaryDirectories []string `protobuf:"bytes,12,rep,name=symlink_temporary_directories,json=symlinkTemporaryDirectories,proto3" json:"symlink_temporary_directories,omitempty"` RunCommandCleaner []string `protobuf:"bytes,13,rep,name=run_command_cleaner,json=runCommandCleaner,proto3" json:"run_command_cleaner,omitempty"` AppleXcodeDeveloperDirectories map[string]string `protobuf:"bytes,14,rep,name=apple_xcode_developer_directories,json=appleXcodeDeveloperDirectories,proto3" json:"apple_xcode_developer_directories,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + AssumeExclusiveCgroup bool `protobuf:"varint,15,opt,name=assume_exclusive_cgroup,json=assumeExclusiveCgroup,proto3" json:"assume_exclusive_cgroup,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -164,11 +165,18 @@ func (x *ApplicationConfiguration) GetAppleXcodeDeveloperDirectories() map[strin return nil } +func (x *ApplicationConfiguration) GetAssumeExclusiveCgroup() bool { + if x != nil { + return x.AssumeExclusiveCgroup + } + return false +} + var File_github_com_buildbarn_bb_remote_execution_pkg_proto_configuration_bb_runner_bb_runner_proto protoreflect.FileDescriptor const file_github_com_buildbarn_bb_remote_execution_pkg_proto_configuration_bb_runner_bb_runner_proto_rawDesc = "" + "\n" + - "Zgithub.com/buildbarn/bb-remote-execution/pkg/proto/configuration/bb_runner/bb_runner.proto\x12!buildbarn.configuration.bb_runner\x1a^github.com/buildbarn/bb-remote-execution/pkg/proto/configuration/credentials/credentials.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\"\xf3\b\n" + + "Zgithub.com/buildbarn/bb-remote-execution/pkg/proto/configuration/bb_runner/bb_runner.proto\x12!buildbarn.configuration.bb_runner\x1a^github.com/buildbarn/bb-remote-execution/pkg/proto/configuration/credentials/credentials.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\"\xab\t\n" + "\x18ApplicationConfiguration\x120\n" + "\x14build_directory_path\x18\x01 \x01(\tR\x12buildDirectoryPath\x12T\n" + "\fgrpc_servers\x18\x02 \x03(\v21.buildbarn.configuration.grpc.ServerConfigurationR\vgrpcServers\x12>\n" + @@ -183,7 +191,8 @@ const file_github_com_buildbarn_bb_remote_execution_pkg_proto_configuration_bb_r "\x0frun_commands_as\x18\v \x01(\v2A.buildbarn.configuration.credentials.UNIXCredentialsConfigurationR\rrunCommandsAs\x12B\n" + "\x1dsymlink_temporary_directories\x18\f \x03(\tR\x1bsymlinkTemporaryDirectories\x12.\n" + "\x13run_command_cleaner\x18\r \x03(\tR\x11runCommandCleaner\x12\xaa\x01\n" + - "!apple_xcode_developer_directories\x18\x0e \x03(\v2_.buildbarn.configuration.bb_runner.ApplicationConfiguration.AppleXcodeDeveloperDirectoriesEntryR\x1eappleXcodeDeveloperDirectories\x1aQ\n" + + "!apple_xcode_developer_directories\x18\x0e \x03(\v2_.buildbarn.configuration.bb_runner.ApplicationConfiguration.AppleXcodeDeveloperDirectoriesEntryR\x1eappleXcodeDeveloperDirectories\x126\n" + + "\x17assume_exclusive_cgroup\x18\x0f \x01(\bR\x15assumeExclusiveCgroup\x1aQ\n" + "#AppleXcodeDeveloperDirectoriesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01J\x04\b\t\x10\n" + diff --git a/pkg/proto/configuration/bb_runner/bb_runner.proto b/pkg/proto/configuration/bb_runner/bb_runner.proto index d378faa8..48ca8608 100644 --- a/pkg/proto/configuration/bb_runner/bb_runner.proto +++ b/pkg/proto/configuration/bb_runner/bb_runner.proto @@ -135,4 +135,13 @@ message ApplicationConfiguration { // https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/exec/local/XcodeLocalEnvProvider.java // https://www.smileykeith.com/2021/03/08/locking-xcode-in-bazel/ map apple_xcode_developer_directories = 14; + + // Sample cgroup v2 resource usage counters around each action. + // + // This should only be enabled when the bb_runner process has an exclusive + // cgroup for action execution. If multiple runners or actions share the + // cgroup, the sampled counter deltas include unrelated work and are + // misleading. When this is enabled, bb_runner fails actions with + // codes.Internal if it detects multiple actions running concurrently. + bool assume_exclusive_cgroup = 15; } diff --git a/pkg/proto/resourceusage/resourceusage.pb.go b/pkg/proto/resourceusage/resourceusage.pb.go index 69a56188..50a85345 100644 --- a/pkg/proto/resourceusage/resourceusage.pb.go +++ b/pkg/proto/resourceusage/resourceusage.pb.go @@ -374,6 +374,146 @@ func (x *InputRootResourceUsage) GetFilesRead() uint64 { return 0 } +type CgroupResourceUsage struct { + state protoimpl.MessageState `protogen:"open.v1"` + MemoryEventsLow int64 `protobuf:"varint,1,opt,name=memory_events_low,json=memoryEventsLow,proto3" json:"memory_events_low,omitempty"` + MemoryEventsHigh int64 `protobuf:"varint,2,opt,name=memory_events_high,json=memoryEventsHigh,proto3" json:"memory_events_high,omitempty"` + MemoryEventsMax int64 `protobuf:"varint,3,opt,name=memory_events_max,json=memoryEventsMax,proto3" json:"memory_events_max,omitempty"` + MemoryEventsOom int64 `protobuf:"varint,4,opt,name=memory_events_oom,json=memoryEventsOom,proto3" json:"memory_events_oom,omitempty"` + MemoryEventsOomKill int64 `protobuf:"varint,5,opt,name=memory_events_oom_kill,json=memoryEventsOomKill,proto3" json:"memory_events_oom_kill,omitempty"` + MemoryEventsOomGroupKill int64 `protobuf:"varint,6,opt,name=memory_events_oom_group_kill,json=memoryEventsOomGroupKill,proto3" json:"memory_events_oom_group_kill,omitempty"` + MemoryPeakBytes int64 `protobuf:"varint,7,opt,name=memory_peak_bytes,json=memoryPeakBytes,proto3" json:"memory_peak_bytes,omitempty"` + PsiMemorySome *durationpb.Duration `protobuf:"bytes,8,opt,name=psi_memory_some,json=psiMemorySome,proto3" json:"psi_memory_some,omitempty"` + PsiMemoryFull *durationpb.Duration `protobuf:"bytes,9,opt,name=psi_memory_full,json=psiMemoryFull,proto3" json:"psi_memory_full,omitempty"` + PsiCpuSome *durationpb.Duration `protobuf:"bytes,10,opt,name=psi_cpu_some,json=psiCpuSome,proto3" json:"psi_cpu_some,omitempty"` + PsiCpuFull *durationpb.Duration `protobuf:"bytes,11,opt,name=psi_cpu_full,json=psiCpuFull,proto3" json:"psi_cpu_full,omitempty"` + PsiIoSome *durationpb.Duration `protobuf:"bytes,12,opt,name=psi_io_some,json=psiIoSome,proto3" json:"psi_io_some,omitempty"` + PsiIoFull *durationpb.Duration `protobuf:"bytes,13,opt,name=psi_io_full,json=psiIoFull,proto3" json:"psi_io_full,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CgroupResourceUsage) Reset() { + *x = CgroupResourceUsage{} + mi := &file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CgroupResourceUsage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CgroupResourceUsage) ProtoMessage() {} + +func (x *CgroupResourceUsage) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CgroupResourceUsage.ProtoReflect.Descriptor instead. +func (*CgroupResourceUsage) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_rawDescGZIP(), []int{4} +} + +func (x *CgroupResourceUsage) GetMemoryEventsLow() int64 { + if x != nil { + return x.MemoryEventsLow + } + return 0 +} + +func (x *CgroupResourceUsage) GetMemoryEventsHigh() int64 { + if x != nil { + return x.MemoryEventsHigh + } + return 0 +} + +func (x *CgroupResourceUsage) GetMemoryEventsMax() int64 { + if x != nil { + return x.MemoryEventsMax + } + return 0 +} + +func (x *CgroupResourceUsage) GetMemoryEventsOom() int64 { + if x != nil { + return x.MemoryEventsOom + } + return 0 +} + +func (x *CgroupResourceUsage) GetMemoryEventsOomKill() int64 { + if x != nil { + return x.MemoryEventsOomKill + } + return 0 +} + +func (x *CgroupResourceUsage) GetMemoryEventsOomGroupKill() int64 { + if x != nil { + return x.MemoryEventsOomGroupKill + } + return 0 +} + +func (x *CgroupResourceUsage) GetMemoryPeakBytes() int64 { + if x != nil { + return x.MemoryPeakBytes + } + return 0 +} + +func (x *CgroupResourceUsage) GetPsiMemorySome() *durationpb.Duration { + if x != nil { + return x.PsiMemorySome + } + return nil +} + +func (x *CgroupResourceUsage) GetPsiMemoryFull() *durationpb.Duration { + if x != nil { + return x.PsiMemoryFull + } + return nil +} + +func (x *CgroupResourceUsage) GetPsiCpuSome() *durationpb.Duration { + if x != nil { + return x.PsiCpuSome + } + return nil +} + +func (x *CgroupResourceUsage) GetPsiCpuFull() *durationpb.Duration { + if x != nil { + return x.PsiCpuFull + } + return nil +} + +func (x *CgroupResourceUsage) GetPsiIoSome() *durationpb.Duration { + if x != nil { + return x.PsiIoSome + } + return nil +} + +func (x *CgroupResourceUsage) GetPsiIoFull() *durationpb.Duration { + if x != nil { + return x.PsiIoFull + } + return nil +} + type MonetaryResourceUsage_Expense struct { state protoimpl.MessageState `protogen:"open.v1"` Currency string `protobuf:"bytes,1,opt,name=currency,proto3" json:"currency,omitempty"` @@ -384,7 +524,7 @@ type MonetaryResourceUsage_Expense struct { func (x *MonetaryResourceUsage_Expense) Reset() { *x = MonetaryResourceUsage_Expense{} - mi := &file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes[4] + mi := &file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -396,7 +536,7 @@ func (x *MonetaryResourceUsage_Expense) String() string { func (*MonetaryResourceUsage_Expense) ProtoMessage() {} func (x *MonetaryResourceUsage_Expense) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes[4] + mi := &file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -471,7 +611,24 @@ const file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_reso "\x14directories_resolved\x18\x01 \x01(\x04R\x13directoriesResolved\x12)\n" + "\x10directories_read\x18\x02 \x01(\x04R\x0fdirectoriesRead\x12\x1d\n" + "\n" + - "files_read\x18\x03 \x01(\x04R\tfilesReadBBZ@github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusageb\x06proto3" + "files_read\x18\x03 \x01(\x04R\tfilesRead\"\xde\x05\n" + + "\x13CgroupResourceUsage\x12*\n" + + "\x11memory_events_low\x18\x01 \x01(\x03R\x0fmemoryEventsLow\x12,\n" + + "\x12memory_events_high\x18\x02 \x01(\x03R\x10memoryEventsHigh\x12*\n" + + "\x11memory_events_max\x18\x03 \x01(\x03R\x0fmemoryEventsMax\x12*\n" + + "\x11memory_events_oom\x18\x04 \x01(\x03R\x0fmemoryEventsOom\x123\n" + + "\x16memory_events_oom_kill\x18\x05 \x01(\x03R\x13memoryEventsOomKill\x12>\n" + + "\x1cmemory_events_oom_group_kill\x18\x06 \x01(\x03R\x18memoryEventsOomGroupKill\x12*\n" + + "\x11memory_peak_bytes\x18\a \x01(\x03R\x0fmemoryPeakBytes\x12A\n" + + "\x0fpsi_memory_some\x18\b \x01(\v2\x19.google.protobuf.DurationR\rpsiMemorySome\x12A\n" + + "\x0fpsi_memory_full\x18\t \x01(\v2\x19.google.protobuf.DurationR\rpsiMemoryFull\x12;\n" + + "\fpsi_cpu_some\x18\n" + + " \x01(\v2\x19.google.protobuf.DurationR\n" + + "psiCpuSome\x12;\n" + + "\fpsi_cpu_full\x18\v \x01(\v2\x19.google.protobuf.DurationR\n" + + "psiCpuFull\x129\n" + + "\vpsi_io_some\x18\f \x01(\v2\x19.google.protobuf.DurationR\tpsiIoSome\x129\n" + + "\vpsi_io_full\x18\r \x01(\v2\x19.google.protobuf.DurationR\tpsiIoFullBBZ@github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusageb\x06proto3" var ( file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_rawDescOnce sync.Once @@ -485,26 +642,33 @@ func file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resou return file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_rawDescData } -var file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_goTypes = []any{ (*FilePoolResourceUsage)(nil), // 0: buildbarn.resourceusage.FilePoolResourceUsage (*POSIXResourceUsage)(nil), // 1: buildbarn.resourceusage.POSIXResourceUsage (*MonetaryResourceUsage)(nil), // 2: buildbarn.resourceusage.MonetaryResourceUsage (*InputRootResourceUsage)(nil), // 3: buildbarn.resourceusage.InputRootResourceUsage - (*MonetaryResourceUsage_Expense)(nil), // 4: buildbarn.resourceusage.MonetaryResourceUsage.Expense - nil, // 5: buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry - (*durationpb.Duration)(nil), // 6: google.protobuf.Duration + (*CgroupResourceUsage)(nil), // 4: buildbarn.resourceusage.CgroupResourceUsage + (*MonetaryResourceUsage_Expense)(nil), // 5: buildbarn.resourceusage.MonetaryResourceUsage.Expense + nil, // 6: buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry + (*durationpb.Duration)(nil), // 7: google.protobuf.Duration } var file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_depIdxs = []int32{ - 6, // 0: buildbarn.resourceusage.POSIXResourceUsage.user_time:type_name -> google.protobuf.Duration - 6, // 1: buildbarn.resourceusage.POSIXResourceUsage.system_time:type_name -> google.protobuf.Duration - 5, // 2: buildbarn.resourceusage.MonetaryResourceUsage.expenses:type_name -> buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry - 4, // 3: buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry.value:type_name -> buildbarn.resourceusage.MonetaryResourceUsage.Expense - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 7, // 0: buildbarn.resourceusage.POSIXResourceUsage.user_time:type_name -> google.protobuf.Duration + 7, // 1: buildbarn.resourceusage.POSIXResourceUsage.system_time:type_name -> google.protobuf.Duration + 6, // 2: buildbarn.resourceusage.MonetaryResourceUsage.expenses:type_name -> buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry + 7, // 3: buildbarn.resourceusage.CgroupResourceUsage.psi_memory_some:type_name -> google.protobuf.Duration + 7, // 4: buildbarn.resourceusage.CgroupResourceUsage.psi_memory_full:type_name -> google.protobuf.Duration + 7, // 5: buildbarn.resourceusage.CgroupResourceUsage.psi_cpu_some:type_name -> google.protobuf.Duration + 7, // 6: buildbarn.resourceusage.CgroupResourceUsage.psi_cpu_full:type_name -> google.protobuf.Duration + 7, // 7: buildbarn.resourceusage.CgroupResourceUsage.psi_io_some:type_name -> google.protobuf.Duration + 7, // 8: buildbarn.resourceusage.CgroupResourceUsage.psi_io_full:type_name -> google.protobuf.Duration + 5, // 9: buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry.value:type_name -> buildbarn.resourceusage.MonetaryResourceUsage.Expense + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { @@ -520,7 +684,7 @@ func file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resou GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_rawDesc), len(file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_rawDesc)), NumEnums: 0, - NumMessages: 6, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/resourceusage/resourceusage.proto b/pkg/proto/resourceusage/resourceusage.proto index 112b8f10..ec34c2fd 100644 --- a/pkg/proto/resourceusage/resourceusage.proto +++ b/pkg/proto/resourceusage/resourceusage.proto @@ -126,3 +126,74 @@ message InputRootResourceUsage { // Addressable Storage (CAS). uint64 files_read = 3; } + +// Resource usage metrics collected from cgroup v2 files by bb_runner. +// +// Fields represent the difference between samples taken before +// the action started and after it completed, unless noted otherwise. +// +// PSI total values are exported by Linux in microseconds, but stored +// here as durations. To express a PSI duration as a percentage, divide +// it by the action's execution duration from +// ActionResult.execution_metadata. For example: +// +// psi_cpu_some_percent = psi_cpu_some / ( +// execution_completed_timestamp - execution_start_timestamp) * 100 +// +// See: +// https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory-interface-files +// https://www.kernel.org/doc/html/latest/accounting/psi.html#psi +message CgroupResourceUsage { + // memory.events/low: Number of times the cgroup was reclaimed under + // high memory pressure even though it was below memory.low. + int64 memory_events_low = 1; + + // memory.events/high: Number of times cgroup processes were + // throttled and routed to direct reclaim after exceeding memory.high. + int64 memory_events_high = 2; + + // memory.events/max: Number of times the cgroup was about to exceed + // memory.max. + int64 memory_events_max = 3; + + // memory.events/oom: Number of times allocation was about to fail + // because the cgroup reached its memory limit. + int64 memory_events_oom = 4; + + // memory.events/oom_kill: Number of processes in the cgroup killed + // by any OOM killer. + int64 memory_events_oom_kill = 5; + + // memory.events/oom_group_kill: Number of group OOM kills. Zero if + // the running kernel does not expose this memory.events key. + int64 memory_events_oom_group_kill = 6; + + // memory.peak: Peak memory usage in bytes since resetting + // memory.peak at action start. Zero if memory.peak reset/read is + // unavailable. + int64 memory_peak_bytes = 7; + + // memory.pressure/some total: Memory PSI stall time with at least + // one task stalled on memory. + google.protobuf.Duration psi_memory_some = 8; + + // memory.pressure/full total: Memory PSI stall time with all + // non-idle tasks stalled on memory. + google.protobuf.Duration psi_memory_full = 9; + + // cpu.pressure/some total: CPU PSI stall time with at least one task + // waiting for CPU. + google.protobuf.Duration psi_cpu_some = 10; + + // cpu.pressure/full total: CPU PSI stall time with all non-idle + // tasks waiting for CPU. + google.protobuf.Duration psi_cpu_full = 11; + + // io.pressure/some total: I/O PSI stall time with at least one task + // stalled on I/O. + google.protobuf.Duration psi_io_some = 12; + + // io.pressure/full total: I/O PSI stall time with all non-idle + // tasks stalled on I/O. + google.protobuf.Duration psi_io_full = 13; +} diff --git a/pkg/runner/BUILD.bazel b/pkg/runner/BUILD.bazel index fb596676..2dd8241d 100644 --- a/pkg/runner/BUILD.bazel +++ b/pkg/runner/BUILD.bazel @@ -4,6 +4,9 @@ go_library( name = "runner", srcs = [ "apple_xcode_resolving_runner.go", + "cgroup_linux.go", + "cgroup_other.go", + "cgroup_resource_usage_sampling_runner.go", "clean_runner.go", "local_runner.go", "local_runner_darwin.go", @@ -19,6 +22,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/cleaner", + "//pkg/proto/resourceusage", "//pkg/proto/runner", "//pkg/proto/tmp_installer", "@com_github_buildbarn_bb_storage//pkg/filesystem", @@ -31,32 +35,26 @@ go_library( "@org_golang_google_protobuf//types/known/emptypb", ] + select({ "@rules_go//go/platform:android": [ - "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:darwin": [ - "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:freebsd": [ - "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:ios": [ - "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:linux": [ - "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:windows": [ - "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//windows", ], @@ -68,13 +66,22 @@ go_test( name = "runner_test", srcs = [ "apple_xcode_resolving_runner_test.go", + "cgroup_resource_usage_sampling_runner_test.go", "clean_runner_test.go", "local_runner_test.go", "path_existence_checking_runner_test.go", "temporary_directory_symlinking_runner_test.go", - ], + ] + select({ + "@rules_go//go/platform:android": [ + "cgroup_linux_test.go", + ], + "@rules_go//go/platform:linux": [ + "cgroup_linux_test.go", + ], + "//conditions:default": [], + }), + embed = [":runner"], deps = [ - ":runner", "//internal/mock", "//pkg/cleaner", "//pkg/proto/resourceusage", diff --git a/pkg/runner/cgroup_linux.go b/pkg/runner/cgroup_linux.go new file mode 100644 index 00000000..b17b643e --- /dev/null +++ b/pkg/runner/cgroup_linux.go @@ -0,0 +1,449 @@ +//go:build linux + +package runner + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" + "google.golang.org/protobuf/types/known/durationpb" +) + +var ( + currentCgroupPathOnce sync.Once + currentCgroupPath string + currentCgroupPathErr error +) + +// validateExclusiveCgroupResourceUsageSampling performs best-effort startup +// validation for cgroup resource usage sampling. It catches obvious +// misconfigurations, but cannot prove that no other process will enter the +// cgroup after startup. +func validateExclusiveCgroupResourceUsageSampling() error { + cgroupPath, err := getCurrentCgroupPath() + if err != nil { + return err + } + if _, err := readCgroupKeyValues(filepath.Join(cgroupPath, "memory.events")); err != nil { + return fmt.Errorf("failed to read cgroup v2 memory.events: %w", err) + } + if _, _, err := parsePSITotals(filepath.Join(cgroupPath, "memory.pressure")); err != nil { + return fmt.Errorf("failed to read cgroup v2 memory.pressure: %w", err) + } + if _, _, err := parsePSITotals(filepath.Join(cgroupPath, "cpu.pressure")); err != nil { + return fmt.Errorf("failed to read cgroup v2 cpu.pressure: %w", err) + } + if _, _, err := parsePSITotals(filepath.Join(cgroupPath, "io.pressure")); err != nil { + return fmt.Errorf("failed to read cgroup v2 io.pressure: %w", err) + } + processIDs, err := readCgroupProcessIDs(filepath.Join(cgroupPath, "cgroup.procs")) + if err != nil { + return fmt.Errorf("failed to read cgroup.procs: %w", err) + } + allowedProcessIDs := getAncestorProcessIDs(os.Getpid()) + for _, processID := range processIDs { + if _, ok := allowedProcessIDs[processID]; ok { + continue + } + return fmt.Errorf("cgroup %q is not exclusive; found extra process %d (%s)", cgroupPath, processID, getProcessName(processID)) + } + return nil +} + +type cgroupStatsReader struct { + cgroupPath string + + // memory.events counters + eventsLow int64 + eventsHigh int64 + eventsMax int64 + eventsOOM int64 + eventsOOMKill int64 + eventsOOMGroupKill int64 + + // PSI total values + psiMemorySomeUS int64 + psiMemoryFullUS int64 + psiCPUSomeUS int64 + psiCPUFullUS int64 + psiIOSomeUS int64 + psiIOFullUS int64 + + memoryPeakFile *os.File +} + +func newScopedCgroupStatsReader() (*cgroupStatsReader, error) { + cgroupPath, err := getCurrentCgroupPath() + if err != nil { + return nil, err + } + return newCgroupStatsReader(cgroupPath) +} + +func newCgroupStatsReader(cgroupPath string) (*cgroupStatsReader, error) { + events, err := readCgroupKeyValues(filepath.Join(cgroupPath, "memory.events")) + if err != nil { + return nil, fmt.Errorf("failed to read memory.events: %w", err) + } + + memorySome, memoryFull, err := parsePSITotals(filepath.Join(cgroupPath, "memory.pressure")) + if err != nil { + return nil, fmt.Errorf("failed to read memory.pressure: %w", err) + } + cpuSome, cpuFull, err := parsePSITotals(filepath.Join(cgroupPath, "cpu.pressure")) + if err != nil { + return nil, fmt.Errorf("failed to read cpu.pressure: %w", err) + } + ioSome, ioFull, err := parsePSITotals(filepath.Join(cgroupPath, "io.pressure")) + if err != nil { + return nil, fmt.Errorf("failed to read io.pressure: %w", err) + } + + // memory.peak is optional. If it is unavailable or cannot be reset/read, + // MemoryPeakBytes remains 0 to indicate that no peak data was collected. + memoryPeakFile := openAndResetCgroupMemoryPeak(filepath.Join(cgroupPath, "memory.peak")) + reader := &cgroupStatsReader{ + cgroupPath: cgroupPath, + + eventsLow: events["low"], + eventsHigh: events["high"], + eventsMax: events["max"], + eventsOOM: events["oom"], + eventsOOMKill: events["oom_kill"], + eventsOOMGroupKill: events["oom_group_kill"], + + psiMemorySomeUS: memorySome, + psiMemoryFullUS: memoryFull, + psiCPUSomeUS: cpuSome, + psiCPUFullUS: cpuFull, + psiIOSomeUS: ioSome, + psiIOFullUS: ioFull, + + memoryPeakFile: memoryPeakFile, + } + return reader, nil +} + +func (r *cgroupStatsReader) Close() error { + if r == nil || r.memoryPeakFile == nil { + return nil + } + return r.memoryPeakFile.Close() +} + +func (r *cgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { + events, err := readCgroupKeyValues(filepath.Join(r.cgroupPath, "memory.events")) + if err != nil { + return nil, fmt.Errorf("failed to read memory.events: %w", err) + } + + memorySome, memoryFull, err := parsePSITotals(filepath.Join(r.cgroupPath, "memory.pressure")) + if err != nil { + return nil, fmt.Errorf("failed to read memory.pressure: %w", err) + } + cpuSome, cpuFull, err := parsePSITotals(filepath.Join(r.cgroupPath, "cpu.pressure")) + if err != nil { + return nil, fmt.Errorf("failed to read cpu.pressure: %w", err) + } + ioSome, ioFull, err := parsePSITotals(filepath.Join(r.cgroupPath, "io.pressure")) + if err != nil { + return nil, fmt.Errorf("failed to read io.pressure: %w", err) + } + + memoryPeak := readCgroupMemoryPeak(r.memoryPeakFile) + + return &resourceusage.CgroupResourceUsage{ + MemoryEventsLow: events["low"] - r.eventsLow, + MemoryEventsHigh: events["high"] - r.eventsHigh, + MemoryEventsMax: events["max"] - r.eventsMax, + MemoryEventsOom: events["oom"] - r.eventsOOM, + MemoryEventsOomKill: events["oom_kill"] - r.eventsOOMKill, + MemoryEventsOomGroupKill: events["oom_group_kill"] - r.eventsOOMGroupKill, + + MemoryPeakBytes: memoryPeak, + + PsiMemorySome: microsecondsDuration(memorySome - r.psiMemorySomeUS), + PsiMemoryFull: microsecondsDuration(memoryFull - r.psiMemoryFullUS), + PsiCpuSome: microsecondsDuration(cpuSome - r.psiCPUSomeUS), + PsiCpuFull: microsecondsDuration(cpuFull - r.psiCPUFullUS), + PsiIoSome: microsecondsDuration(ioSome - r.psiIOSomeUS), + PsiIoFull: microsecondsDuration(ioFull - r.psiIOFullUS), + }, nil +} + +func microsecondsDuration(microseconds int64) *durationpb.Duration { + return durationpb.New(time.Duration(microseconds) * time.Microsecond) +} + +func openAndResetCgroupMemoryPeak(path string) *os.File { + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil + } + // Linux scopes memory.peak reset state to the file descriptor used for + // the write, so keep this descriptor open until the action finishes. + if _, err := f.Write([]byte("1")); err != nil { + f.Close() + return nil + } + return f +} + +func readCgroupMemoryPeak(f *os.File) int64 { + if f == nil { + return 0 + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return 0 + } + data, err := io.ReadAll(f) + if err != nil { + return 0 + } + value, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0 + } + return value +} + +// readCgroupKeyValues parses a cgroup file with key-value lines +// (e.g., memory.events, memory.stat). Each line has the form "key value". +func readCgroupKeyValues(path string) (map[string]int64, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + result := make(map[string]int64) + scanner := bufio.NewScanner(f) + for scanner.Scan() { + parts := strings.SplitN(scanner.Text(), " ", 2) + if len(parts) != 2 { + continue + } + v, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + result[parts[0]] = v + } + if err := scanner.Err(); err != nil { + return nil, err + } + return result, nil +} + +func readCgroupProcessIDs(path string) ([]int, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var processIDs []int + scanner := bufio.NewScanner(f) + for scanner.Scan() { + processID, err := strconv.Atoi(strings.TrimSpace(scanner.Text())) + if err != nil { + continue + } + processIDs = append(processIDs, processID) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return processIDs, nil +} + +// parsePSITotals parses a PSI pressure file and returns the total +// stall microseconds for the "some" and "full" lines. +// Format: some avg10=0.00 avg60=0.00 avg300=0.00 total=12345 +func parsePSITotals(path string) (someUS int64, fullUS int64, err error) { + f, err := os.Open(path) + if err != nil { + return 0, 0, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + prefix := fields[0] + var total int64 + foundTotal := false + for _, field := range fields[1:] { + if strings.HasPrefix(field, "total=") { + var err error + total, err = strconv.ParseInt(field[len("total="):], 10, 64) + if err != nil { + return 0, 0, err + } + foundTotal = true + break + } + } + if !foundTotal { + return 0, 0, fmt.Errorf("missing total field in %q", line) + } + switch prefix { + case "some": + someUS = total + case "full": + fullUS = total + } + } + if err := scanner.Err(); err != nil { + return 0, 0, err + } + return someUS, fullUS, nil +} + +func getCurrentCgroupPath() (string, error) { + currentCgroupPathOnce.Do(func() { + currentCgroupPath, currentCgroupPathErr = resolveCurrentCgroupPath("/proc/self/mountinfo", "/proc/self/cgroup") + }) + return currentCgroupPath, currentCgroupPathErr +} + +func resolveCurrentCgroupPath(mountInfoPath, cgroupPath string) (string, error) { + currentCgroupPath, err := readCurrentCgroupRelativePath(cgroupPath) + if err != nil { + return "", err + } + return resolveCgroupPathFromMountInfo(mountInfoPath, currentCgroupPath) +} + +func resolveCgroupPathFromMountInfo(path, currentCgroupPath string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read cgroup mount information: %w", err) + } + currentCgroupPath = cleanCgroupPath(currentCgroupPath) + var bestMountPoint, bestRoot string + for _, line := range strings.Split(string(data), "\n") { + separator := strings.Index(line, " - ") + if separator < 0 { + continue + } + mountFields := strings.Fields(line[:separator]) + filesystemFields := strings.Fields(line[separator+3:]) + if len(mountFields) < 5 || len(filesystemFields) < 1 || filesystemFields[0] != "cgroup2" { + continue + } + root := cleanCgroupPath(unescapeMountInfoField(mountFields[3])) + if !isCgroupPathPrefix(root, currentCgroupPath) { + continue + } + if len(root) <= len(bestRoot) { + continue + } + bestRoot = root + mountPoint := filepath.Clean(unescapeMountInfoField(mountFields[4])) + relativePath, err := filepath.Rel(root, currentCgroupPath) + if err != nil { + continue + } + bestMountPoint = filepath.Join(mountPoint, relativePath) + } + if bestMountPoint == "" { + return "", fmt.Errorf("cgroup v2 mount containing current cgroup %q not found in %s", currentCgroupPath, path) + } + return bestMountPoint, nil +} + +func cleanCgroupPath(path string) string { + path = filepath.Clean(path) + if filepath.IsAbs(path) { + return path + } + return filepath.Clean(string(filepath.Separator) + path) +} + +func isCgroupPathPrefix(root, path string) bool { + relativePath, err := filepath.Rel(root, path) + return err == nil && + relativePath != ".." && + !strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) && + !filepath.IsAbs(relativePath) +} + +// /proc/self/mountinfo encodes whitespace and backslash in path fields using +// octal escape sequences. +func unescapeMountInfoField(field string) string { + replacer := strings.NewReplacer( + `\011`, "\t", + `\012`, "\n", + `\040`, " ", + `\134`, `\`, + ) + return replacer.Replace(field) +} + +func readCurrentCgroupRelativePath(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read current cgroup: %w", err) + } + for _, line := range strings.Split(string(data), "\n") { + parts := strings.SplitN(line, ":", 3) + if len(parts) == 3 && parts[0] == "0" && parts[1] == "" { + return filepath.Clean(parts[2]), nil + } + } + return "", fmt.Errorf("cgroup v2 entry not found in %s", path) +} + +func getAncestorProcessIDs(processID int) map[int]struct{} { + processIDs := map[int]struct{}{} + for processID > 0 { + if _, ok := processIDs[processID]; ok { + break + } + processIDs[processID] = struct{}{} + parentProcessID, err := getParentProcessID(processID) + if err != nil { + break + } + processID = parentProcessID + } + return processIDs +} + +func getParentProcessID(processID int) (int, error) { + data, err := os.ReadFile("/proc/" + strconv.Itoa(processID) + "/status") + if err != nil { + return 0, err + } + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "PPid:") { + fields := strings.Fields(line) + if len(fields) != 2 { + return 0, fmt.Errorf("invalid PPid line for process %d", processID) + } + return strconv.Atoi(fields[1]) + } + } + return 0, fmt.Errorf("PPid line not found for process %d", processID) +} + +func getProcessName(processID int) string { + comm, err := os.ReadFile("/proc/" + strconv.Itoa(processID) + "/comm") + if err != nil { + return "" + } + return strings.TrimSpace(string(comm)) +} diff --git a/pkg/runner/cgroup_linux_test.go b/pkg/runner/cgroup_linux_test.go new file mode 100644 index 00000000..46b688d4 --- /dev/null +++ b/pkg/runner/cgroup_linux_test.go @@ -0,0 +1,192 @@ +//go:build linux + +package runner + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCgroupStatsReaderReportsDeltasFromCgroupFiles(t *testing.T) { + cgroupPath := t.TempDir() + + writeCgroupFile(t, cgroupPath, "memory.events", ` +low 1 +high 2 +max 3 +oom 4 +oom_kill 5 +oom_group_kill 6 +`) + writeCgroupFile(t, cgroupPath, "memory.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=100 +full avg10=0.00 avg60=0.00 avg300=0.00 total=200 +`) + writeCgroupFile(t, cgroupPath, "cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=300 +full avg10=0.00 avg60=0.00 avg300=0.00 total=310 +`) + writeCgroupFile(t, cgroupPath, "io.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=400 +full avg10=0.00 avg60=0.00 avg300=0.00 total=500 +`) + writeCgroupFile(t, cgroupPath, "memory.peak", "0\n") + + reader, err := newCgroupStatsReader(cgroupPath) + require.NoError(t, err) + defer func() { + require.NoError(t, reader.Close()) + }() + + writeCgroupFile(t, cgroupPath, "memory.events", ` +low 11 +high 22 +max 33 +oom 44 +oom_kill 55 +oom_group_kill 66 +`) + writeCgroupFile(t, cgroupPath, "memory.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=160 +full avg10=0.00 avg60=0.00 avg300=0.00 total=290 +`) + writeCgroupFile(t, cgroupPath, "cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=345 +full avg10=0.00 avg60=0.00 avg300=0.00 total=410 +`) + writeCgroupFile(t, cgroupPath, "io.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=480 +full avg10=0.00 avg60=0.00 avg300=0.00 total=610 +`) + writeCgroupFile(t, cgroupPath, "memory.peak", "4096\n") + + usage, err := reader.Read() + require.NoError(t, err) + + require.Equal(t, int64(10), usage.MemoryEventsLow) + require.Equal(t, int64(20), usage.MemoryEventsHigh) + require.Equal(t, int64(30), usage.MemoryEventsMax) + require.Equal(t, int64(40), usage.MemoryEventsOom) + require.Equal(t, int64(50), usage.MemoryEventsOomKill) + require.Equal(t, int64(60), usage.MemoryEventsOomGroupKill) + require.Equal(t, int64(4096), usage.MemoryPeakBytes) + require.Equal(t, 60*time.Microsecond, usage.GetPsiMemorySome().AsDuration()) + require.Equal(t, 90*time.Microsecond, usage.GetPsiMemoryFull().AsDuration()) + require.Equal(t, 45*time.Microsecond, usage.GetPsiCpuSome().AsDuration()) + require.Equal(t, 100*time.Microsecond, usage.GetPsiCpuFull().AsDuration()) + require.Equal(t, 80*time.Microsecond, usage.GetPsiIoSome().AsDuration()) + require.Equal(t, 110*time.Microsecond, usage.GetPsiIoFull().AsDuration()) +} + +func TestCgroupStatsReaderAllowsMissingOomGroupKillCounter(t *testing.T) { + cgroupPath := t.TempDir() + + writeCgroupFile(t, cgroupPath, "memory.events", ` +low 1 +high 2 +max 3 +oom 4 +oom_kill 5 +`) + writeCgroupFile(t, cgroupPath, "memory.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=100 +full avg10=0.00 avg60=0.00 avg300=0.00 total=200 +`) + writeCgroupFile(t, cgroupPath, "cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=300 +full avg10=0.00 avg60=0.00 avg300=0.00 total=310 +`) + writeCgroupFile(t, cgroupPath, "io.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=400 +full avg10=0.00 avg60=0.00 avg300=0.00 total=500 +`) + writeCgroupFile(t, cgroupPath, "memory.peak", "0\n") + + reader, err := newCgroupStatsReader(cgroupPath) + require.NoError(t, err) + defer func() { + require.NoError(t, reader.Close()) + }() + + writeCgroupFile(t, cgroupPath, "memory.events", ` +low 11 +high 22 +max 33 +oom 44 +oom_kill 55 +`) + + usage, err := reader.Read() + require.NoError(t, err) + + require.Equal(t, int64(50), usage.MemoryEventsOomKill) + require.Equal(t, int64(0), usage.MemoryEventsOomGroupKill) +} + +func TestResolveCurrentCgroupPathUsesMatchingCgroup2MountRoot(t *testing.T) { + for _, testCase := range []struct { + name string + mountInfo string + cgroup string + want string + }{ + { + name: "root mount", + mountInfo: ` +36 25 0:31 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice/runner.scope\n", + want: "/sys/fs/cgroup/worker.slice/runner.scope", + }, + { + name: "subtree mount", + mountInfo: ` +36 25 0:31 /unrelated /wrong rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +37 25 0:31 /worker.slice /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice/runner.scope\n", + want: "/sys/fs/cgroup/runner.scope", + }, + { + name: "most specific mount root", + mountInfo: ` +36 25 0:31 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +37 25 0:31 /worker.slice /run/worker-cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice/runner.scope\n", + want: "/run/worker-cgroup/runner.scope", + }, + { + name: "current cgroup is mount root", + mountInfo: ` +36 25 0:31 /worker.slice /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice\n", + want: "/sys/fs/cgroup", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + tempDir := t.TempDir() + mountInfoPath := filepath.Join(tempDir, "mountinfo") + cgroupPath := filepath.Join(tempDir, "cgroup") + writeFile(t, mountInfoPath, testCase.mountInfo) + writeFile(t, cgroupPath, testCase.cgroup) + + got, err := resolveCurrentCgroupPath(mountInfoPath, cgroupPath) + require.NoError(t, err) + require.Equal(t, testCase.want, got) + }) + } +} + +func writeCgroupFile(t *testing.T, cgroupPath, name, contents string) { + writeFile(t, filepath.Join(cgroupPath, name), contents) +} + +func writeFile(t *testing.T, path, contents string) { + require.NoError(t, os.WriteFile(path, []byte(contents), 0o666)) +} diff --git a/pkg/runner/cgroup_other.go b/pkg/runner/cgroup_other.go new file mode 100644 index 00000000..97f62475 --- /dev/null +++ b/pkg/runner/cgroup_other.go @@ -0,0 +1,25 @@ +//go:build !linux + +package runner + +import ( + "fmt" + + "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" +) + +type cgroupStatsReader struct{} + +func newScopedCgroupStatsReader() (*cgroupStatsReader, error) { + return &cgroupStatsReader{}, nil +} + +func (r *cgroupStatsReader) Close() error { return nil } + +func (r *cgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { + return nil, nil +} + +func validateExclusiveCgroupResourceUsageSampling() error { + return fmt.Errorf("cgroup resource usage sampling is only supported on Linux") +} diff --git a/pkg/runner/cgroup_resource_usage_sampling_runner.go b/pkg/runner/cgroup_resource_usage_sampling_runner.go new file mode 100644 index 00000000..ba74e568 --- /dev/null +++ b/pkg/runner/cgroup_resource_usage_sampling_runner.go @@ -0,0 +1,89 @@ +package runner + +import ( + "context" + "log" + "sync/atomic" + + "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" + runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) + +type cgroupResourceUsageSamplingRunner struct { + runner_pb.RunnerServer + newScopedCgroupStatsReader scopedCgroupStatsReaderFactory + activeRuns atomic.Int32 +} + +type scopedCgroupStatsReader interface { + Close() error + Read() (*resourceusage.CgroupResourceUsage, error) +} + +type scopedCgroupStatsReaderFactory func() (scopedCgroupStatsReader, error) + +func newDefaultScopedCgroupStatsReader() (scopedCgroupStatsReader, error) { + return newScopedCgroupStatsReader() +} + +// NewCgroupResourceUsageSamplingRunner creates a decorator for RunnerServer +// that samples cgroup v2 resource usage counters around actions and appends +// them to successful Run() responses. +// +// This decorator requires the runner's cgroup to be exclusive, so that +// sampled cgroup counters can be interpreted as per-action deltas. +func NewCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer) (runner_pb.RunnerServer, error) { + if err := validateExclusiveCgroupResourceUsageSampling(); err != nil { + return nil, err + } + return newCgroupResourceUsageSamplingRunner(base, newDefaultScopedCgroupStatsReader), nil +} + +func newCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer, newScopedCgroupStatsReader scopedCgroupStatsReaderFactory) runner_pb.RunnerServer { + return &cgroupResourceUsageSamplingRunner{ + RunnerServer: base, + newScopedCgroupStatsReader: newScopedCgroupStatsReader, + } +} + +func (r *cgroupResourceUsageSamplingRunner) Run(ctx context.Context, request *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + if r.activeRuns.Add(1) != 1 { + r.activeRuns.Add(-1) + return nil, status.Error(codes.Internal, "cgroup resource usage sampling requires an exclusive runner cgroup, but concurrent Run() calls were observed") + } + defer r.activeRuns.Add(-1) + + cgroupStatsReader, err := r.newScopedCgroupStatsReader() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to create scoped cgroup stats reader: %s", err) + } + defer func() { + _ = cgroupStatsReader.Close() + }() + + response, err := r.RunnerServer.Run(ctx, request) + if err != nil { + return response, err + } + + cgroupUsage, readErr := cgroupStatsReader.Read() + if readErr != nil { + log.Print("Failed to read scoped cgroup stats: ", readErr) + return response, err + } + if cgroupUsage == nil { + return response, err + } + cgroupAny, cgroupErr := anypb.New(cgroupUsage) + if cgroupErr != nil { + return response, err + } + if response != nil { + response.ResourceUsage = append(response.ResourceUsage, cgroupAny) + } + return response, nil +} diff --git a/pkg/runner/cgroup_resource_usage_sampling_runner_test.go b/pkg/runner/cgroup_resource_usage_sampling_runner_test.go new file mode 100644 index 00000000..7332d2c0 --- /dev/null +++ b/pkg/runner/cgroup_resource_usage_sampling_runner_test.go @@ -0,0 +1,174 @@ +package runner + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/buildbarn/bb-remote-execution/internal/mock" + "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" + runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" +) + +func TestCgroupResourceUsageSamplingRunnerAppendsResourceUsage(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupStatsReader := &fixedCgroupStatsReader{ + usage: &resourceusage.CgroupResourceUsage{ + MemoryEventsOomKill: 1, + MemoryPeakBytes: 4096, + PsiCpuSome: durationpb.New(123 * time.Microsecond), + PsiCpuFull: durationpb.New(45 * time.Microsecond), + }, + } + wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { + return cgroupStatsReader, nil + }) + + request := &runner_pb.RunRequest{} + baseRunner.EXPECT().Run(gomock.Any(), request).Return(&runner_pb.RunResponse{ + ExitCode: 7, + }, nil) + + response, err := wrappedRunner.Run(context.Background(), request) + require.NoError(t, err) + require.Equal(t, int64(7), response.ExitCode) + require.Len(t, response.ResourceUsage, 1) + var got resourceusage.CgroupResourceUsage + require.NoError(t, response.ResourceUsage[0].UnmarshalTo(&got)) + require.Equal(t, cgroupStatsReader.usage.GetMemoryEventsOomKill(), got.GetMemoryEventsOomKill()) + require.Equal(t, cgroupStatsReader.usage.GetMemoryPeakBytes(), got.GetMemoryPeakBytes()) + require.Equal(t, cgroupStatsReader.usage.GetPsiCpuSome().AsDuration(), got.GetPsiCpuSome().AsDuration()) + require.Equal(t, cgroupStatsReader.usage.GetPsiCpuFull().AsDuration(), got.GetPsiCpuFull().AsDuration()) + require.True(t, cgroupStatsReader.closed) +} + +func TestCgroupResourceUsageSamplingRunnerReadErrorPreservesRunResult(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupStatsReader := &fixedCgroupStatsReader{ + err: errors.New("read failed"), + } + wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { + return cgroupStatsReader, nil + }) + + request := &runner_pb.RunRequest{} + baseResponse := &runner_pb.RunResponse{ExitCode: 7} + baseRunner.EXPECT().Run(gomock.Any(), request).Return(baseResponse, nil) + + response, err := wrappedRunner.Run(context.Background(), request) + require.NoError(t, err) + require.Same(t, baseResponse, response) + require.Empty(t, response.ResourceUsage) + require.True(t, cgroupStatsReader.closed) +} + +func TestCgroupResourceUsageSamplingRunnerRunErrorDoesNotReadCgroupUsage(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupStatsReader := &fixedCgroupStatsReader{ + usage: &resourceusage.CgroupResourceUsage{ + MemoryPeakBytes: 4096, + }, + } + wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { + return cgroupStatsReader, nil + }) + + baseErr := status.Error(codes.FailedPrecondition, "failed") + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).Return(nil, baseErr) + + response, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + require.Nil(t, response) + require.Equal(t, baseErr, err) + require.False(t, cgroupStatsReader.read) + require.True(t, cgroupStatsReader.closed) +} + +func TestCgroupResourceUsageSamplingRunnerRejectsConcurrentRun(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { + return noopCgroupStatsReader{}, nil + }) + + started := make(chan struct{}) + release := make(chan struct{}) + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, request *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + close(started) + <-release + return &runner_pb.RunResponse{}, nil + }) + + firstRunErr := make(chan error, 1) + go func() { + _, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + firstRunErr <- err + }() + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for first Run() to enter wrapped runner") + } + + secondRunDone := make(chan struct{}) + var response *runner_pb.RunResponse + var err error + go func() { + response, err = wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + close(secondRunDone) + }() + select { + case <-secondRunDone: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for concurrent Run() rejection") + } + require.Nil(t, response) + require.Equal(t, codes.Internal, status.Code(err)) + require.Contains(t, status.Convert(err).Message(), "concurrent Run() calls") + + close(release) + select { + case err := <-firstRunErr: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for first Run() to finish") + } +} + +type fixedCgroupStatsReader struct { + usage *resourceusage.CgroupResourceUsage + err error + read bool + closed bool +} + +func (r *fixedCgroupStatsReader) Close() error { + r.closed = true + return nil +} + +func (r *fixedCgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { + r.read = true + return r.usage, r.err +} + +type noopCgroupStatsReader struct{} + +func (noopCgroupStatsReader) Close() error { + return nil +} + +func (noopCgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { + return nil, nil +} From f62693cffe01bc57136e440aa23ca3096702ae9b Mon Sep 17 00:00:00 2001 From: Lauri Peltonen Date: Wed, 10 Jun 2026 10:15:27 +0300 Subject: [PATCH 2/3] Address cgroup resource usage review comments - Rename CgroupResourceUsage fields to match their Linux names - Remove the best-effort startup validation for exclusive cgroups - Rename the configuration field from assume_exclusive_cgroup to sample_cgroup_resource_usage - Remove OOM classification from worker, keep "Unavailable" error code remapping on the runner side to keep failures caused by system-level memory pressure retryable - Refactor construction/injecting so that tests can now live in separate packages --- cmd/bb_runner/main.go | 8 +- pkg/builder/local_build_executor.go | 28 --- pkg/builder/local_build_executor_test.go | 146 ------------ .../configuration/bb_runner/bb_runner.pb.go | 12 +- .../configuration/bb_runner/bb_runner.proto | 15 +- pkg/proto/resourceusage/resourceusage.pb.go | 77 ++++--- pkg/proto/resourceusage/resourceusage.proto | 16 +- pkg/runner/BUILD.bazel | 5 +- pkg/runner/cgroup_linux.go | 161 +++---------- pkg/runner/cgroup_linux_test.go | 149 ++++++++----- pkg/runner/cgroup_other.go | 27 +-- .../cgroup_resource_usage_sampling_runner.go | 77 +++---- ...oup_resource_usage_sampling_runner_test.go | 211 ++++++++++++------ 13 files changed, 385 insertions(+), 547 deletions(-) diff --git a/cmd/bb_runner/main.go b/cmd/bb_runner/main.go index 6141f0fb..4078feb4 100644 --- a/cmd/bb_runner/main.go +++ b/cmd/bb_runner/main.go @@ -70,12 +70,12 @@ func main() { commandCreator, configuration.SetTmpdirEnvironmentVariable, ) - if configuration.AssumeExclusiveCgroup { - var err error - r, err = runner.NewCgroupResourceUsageSamplingRunner(r) + if configuration.SampleCgroupResourceUsage { + cgroupfsPath, err := runner.ResolveCurrentCgroupfsPath() if err != nil { - return util.StatusWrap(err, "Failed to enable cgroup resource usage sampling") + return util.StatusWrap(err, "Failed to resolve current cgroupfs path") } + r = runner.NewCgroupResourceUsageSamplingRunner(r, cgroupfsPath) } // Let bb_runner replace temporary directories with symbolic diff --git a/pkg/builder/local_build_executor.go b/pkg/builder/local_build_executor.go index a13f5083..a1b8ee07 100644 --- a/pkg/builder/local_build_executor.go +++ b/pkg/builder/local_build_executor.go @@ -11,7 +11,6 @@ import ( "github.com/buildbarn/bb-remote-execution/pkg/filesystem/access" "github.com/buildbarn/bb-remote-execution/pkg/filesystem/pool" "github.com/buildbarn/bb-remote-execution/pkg/proto/remoteworker" - "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/clock" @@ -37,19 +36,6 @@ var ( checkReadinessComponent = path.MustNewComponent("check_readiness") ) -func getCgroupResourceUsage(result *remoteexecution.ActionResult) *resourceusage.CgroupResourceUsage { - if result == nil || result.ExecutionMetadata == nil { - return nil - } - var cgroupUsage resourceusage.CgroupResourceUsage - for _, metadata := range result.ExecutionMetadata.AuxiliaryMetadata { - if metadata.UnmarshalTo(&cgroupUsage) == nil { - return &cgroupUsage - } - } - return nil -} - // capturingErrorLogger is an error logger that stores up to a single // error. When the error is stored, a context cancelation function is // invoked. This is used by localBuildExecutor to kill a build action in @@ -323,20 +309,6 @@ func (be *localBuildExecutor) Execute(ctx context.Context, filePool pool.FilePoo if runErr == nil { response.Result.ExitCode = int32(runResponse.ExitCode) response.Result.ExecutionMetadata.AuxiliaryMetadata = append(response.Result.ExecutionMetadata.AuxiliaryMetadata, runResponse.ResourceUsage...) - if cgroupUsage := getCgroupResourceUsage(response.Result); cgroupUsage != nil && cgroupUsage.MemoryEventsOomKill > 0 { - if cgroupUsage.MemoryEventsOom > 0 { - response.Message = "Action failed due to out of memory: cgroup memory limit was reached and a process was OOM-killed" - if response.Result.ExitCode == 0 { - response.Result.ExitCode = 1 - } - } else { - // The cgroup did not reach its memory limit, so the OOM - // kill likely came from system-level memory pressure, such - // as node memory overcommitment. Treat this as retryable - // infrastructure failure. - attachErrorToExecuteResponse(response, status.Error(codes.Unavailable, "An action process was OOM-killed without the action reaching its cgroup memory limit")) - } - } } else { attachErrorToExecuteResponse(response, util.StatusWrap(runErr, "Failed to run command")) } diff --git a/pkg/builder/local_build_executor_test.go b/pkg/builder/local_build_executor_test.go index e46e25c1..29255ea1 100644 --- a/pkg/builder/local_build_executor_test.go +++ b/pkg/builder/local_build_executor_test.go @@ -12,7 +12,6 @@ import ( re_clock "github.com/buildbarn/bb-remote-execution/pkg/clock" "github.com/buildbarn/bb-remote-execution/pkg/filesystem/access" "github.com/buildbarn/bb-remote-execution/pkg/proto/remoteworker" - "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" @@ -795,151 +794,6 @@ func TestLocalBuildExecutorSuccess(t *testing.T) { }, executeResponse) } -func TestLocalBuildExecutorReportsCgroupOOMKill(t *testing.T) { - for _, testCase := range []struct { - name string - cgroupUsage *resourceusage.CgroupResourceUsage - wantExitCode int32 - wantMessage string - wantStatus error - }{ - { - name: "memory limit OOM kill with zero runner exit code", - cgroupUsage: &resourceusage.CgroupResourceUsage{ - MemoryEventsOom: 1, - MemoryEventsOomKill: 1, - }, - wantExitCode: 1, - wantMessage: "Action failed due to out of memory: cgroup memory limit was reached and a process was OOM-killed", - }, - { - name: "external OOM kill with zero runner exit code", - cgroupUsage: &resourceusage.CgroupResourceUsage{ - MemoryEventsOomKill: 1, - }, - wantStatus: status.Error(codes.Unavailable, "An action process was OOM-killed without the action reaching its cgroup memory limit"), - }, - } { - t.Run(testCase.name, func(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - resourceUsage, err := anypb.New(testCase.cgroupUsage) - require.NoError(t, err) - - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0000000000000000000000000000000000000000000000000000000000000002", 234), - ).Return(buffer.NewProtoBufferFromProto(&remoteexecution.Command{ - Arguments: []string{"clang"}, - }, buffer.UserProvided)) - - buildDirectory := mock.NewMockBuildDirectory(ctrl) - buildDirectoryCreator := mock.NewMockBuildDirectoryCreator(ctrl) - actionDigest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0000000000000000000000000000000000000000000000000000000000000001", 123) - buildDirectoryCreator.EXPECT().GetBuildDirectory(ctx, &actionDigest). - Return(buildDirectory, nil, nil) - filePool := mock.NewMockFilePool(ctrl) - monitor := mock.NewMockUnreadDirectoryMonitor(ctrl) - buildDirectory.EXPECT().InstallHooks(filePool, gomock.Any()) - buildDirectory.EXPECT().Mkdir(path.MustNewComponent("root"), os.FileMode(0o777)) - inputRootDirectory := mock.NewMockBuildDirectory(ctrl) - buildDirectory.EXPECT().EnterBuildDirectory(path.MustNewComponent("root")).Return(inputRootDirectory, nil) - inputRootDirectory.EXPECT().MergeDirectoryContents( - ctx, - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0000000000000000000000000000000000000000000000000000000000000003", 345), - monitor, - ).Return(nil) - buildDirectory.EXPECT().Mkdir(path.MustNewComponent("tmp"), os.FileMode(0o777)) - buildDirectory.EXPECT().Mkdir(path.MustNewComponent("server_logs"), os.FileMode(0o777)) - - runner := mock.NewMockRunnerClient(ctrl) - runner.EXPECT().Run(gomock.Any(), &runner_pb.RunRequest{ - Arguments: []string{"clang"}, - EnvironmentVariables: map[string]string{}, - WorkingDirectory: "", - StdoutPath: "stdout", - StderrPath: "stderr", - InputRootDirectory: "root", - TemporaryDirectory: "tmp", - ServerLogsDirectory: "server_logs", - }).Return(&runner_pb.RunResponse{ - ExitCode: 0, - ResourceUsage: []*anypb.Any{resourceUsage}, - }, nil) - inputRootDirectory.EXPECT().Close() - emptyDigest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 0) - buildDirectory.EXPECT().UploadFile(ctx, path.MustNewComponent("stdout"), gomock.Any(), gomock.Any()).Return(emptyDigest, nil) - buildDirectory.EXPECT().UploadFile(ctx, path.MustNewComponent("stderr"), gomock.Any(), gomock.Any()).Return(emptyDigest, nil) - serverLogsDirectory := mock.NewMockUploadableDirectory(ctrl) - buildDirectory.EXPECT().EnterUploadableDirectory(path.MustNewComponent("server_logs")).Return(serverLogsDirectory, nil) - serverLogsDirectory.EXPECT().ReadDir() - serverLogsDirectory.EXPECT().Close() - buildDirectory.EXPECT().Close() - - clock := mock.NewMockClock(ctrl) - clock.EXPECT().NewContextWithTimeout(gomock.Any(), time.Hour).DoAndReturn(func(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { - return context.WithCancel(parent) - }) - clock.EXPECT().NewContextWithTimeout(gomock.Any(), 10*time.Second).DoAndReturn(func(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { - return parent, func() {} - }) - localBuildExecutor := builder.NewLocalBuildExecutor( - contentAddressableStorage, - buildDirectoryCreator, - runner, - clock, - /* maximumWritableFileUploadDelay = */ 10*time.Second, - /* inputRootCharacterDevices = */ nil, - /* maximumMessageSizeBytes = */ 10000, - /* environmentVariables = */ map[string]string{}, - /* forceUploadTreesAndDirectories = */ false, - ) - - metadata := make(chan *remoteworker.CurrentState_Executing, 10) - executeResponse := localBuildExecutor.Execute( - ctx, - filePool, - monitor, - digest.MustNewFunction("ubuntu1804", remoteexecution.DigestFunction_SHA256), - &remoteworker.DesiredState_Executing{ - ActionDigest: &remoteexecution.Digest{ - Hash: "0000000000000000000000000000000000000000000000000000000000000001", - SizeBytes: 123, - }, - Action: &remoteexecution.Action{ - CommandDigest: &remoteexecution.Digest{ - Hash: "0000000000000000000000000000000000000000000000000000000000000002", - SizeBytes: 234, - }, - InputRootDigest: &remoteexecution.Digest{ - Hash: "0000000000000000000000000000000000000000000000000000000000000003", - SizeBytes: 345, - }, - Timeout: &durationpb.Duration{Seconds: 3600}, - }, - }, - metadata, - ) - - expectedResponse := &remoteexecution.ExecuteResponse{ - Result: &remoteexecution.ActionResult{ - ExitCode: testCase.wantExitCode, - ExecutionMetadata: &remoteexecution.ExecutedActionMetadata{ - AuxiliaryMetadata: []*anypb.Any{resourceUsage}, - }, - }, - Message: testCase.wantMessage, - } - if testCase.wantStatus != nil { - expectedResponse.Status = status.Convert(testCase.wantStatus).Proto() - } - testutil.RequireEqualProto(t, expectedResponse, executeResponse) - }) - } -} - func TestLocalBuildExecutorCachingInvalidTimeout(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/proto/configuration/bb_runner/bb_runner.pb.go b/pkg/proto/configuration/bb_runner/bb_runner.pb.go index 4ab2f091..8021269a 100644 --- a/pkg/proto/configuration/bb_runner/bb_runner.pb.go +++ b/pkg/proto/configuration/bb_runner/bb_runner.pb.go @@ -39,7 +39,7 @@ type ApplicationConfiguration struct { SymlinkTemporaryDirectories []string `protobuf:"bytes,12,rep,name=symlink_temporary_directories,json=symlinkTemporaryDirectories,proto3" json:"symlink_temporary_directories,omitempty"` RunCommandCleaner []string `protobuf:"bytes,13,rep,name=run_command_cleaner,json=runCommandCleaner,proto3" json:"run_command_cleaner,omitempty"` AppleXcodeDeveloperDirectories map[string]string `protobuf:"bytes,14,rep,name=apple_xcode_developer_directories,json=appleXcodeDeveloperDirectories,proto3" json:"apple_xcode_developer_directories,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - AssumeExclusiveCgroup bool `protobuf:"varint,15,opt,name=assume_exclusive_cgroup,json=assumeExclusiveCgroup,proto3" json:"assume_exclusive_cgroup,omitempty"` + SampleCgroupResourceUsage bool `protobuf:"varint,15,opt,name=sample_cgroup_resource_usage,json=sampleCgroupResourceUsage,proto3" json:"sample_cgroup_resource_usage,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -165,9 +165,9 @@ func (x *ApplicationConfiguration) GetAppleXcodeDeveloperDirectories() map[strin return nil } -func (x *ApplicationConfiguration) GetAssumeExclusiveCgroup() bool { +func (x *ApplicationConfiguration) GetSampleCgroupResourceUsage() bool { if x != nil { - return x.AssumeExclusiveCgroup + return x.SampleCgroupResourceUsage } return false } @@ -176,7 +176,7 @@ var File_github_com_buildbarn_bb_remote_execution_pkg_proto_configuration_bb_run const file_github_com_buildbarn_bb_remote_execution_pkg_proto_configuration_bb_runner_bb_runner_proto_rawDesc = "" + "\n" + - "Zgithub.com/buildbarn/bb-remote-execution/pkg/proto/configuration/bb_runner/bb_runner.proto\x12!buildbarn.configuration.bb_runner\x1a^github.com/buildbarn/bb-remote-execution/pkg/proto/configuration/credentials/credentials.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\"\xab\t\n" + + "Zgithub.com/buildbarn/bb-remote-execution/pkg/proto/configuration/bb_runner/bb_runner.proto\x12!buildbarn.configuration.bb_runner\x1a^github.com/buildbarn/bb-remote-execution/pkg/proto/configuration/credentials/credentials.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\"\xb4\t\n" + "\x18ApplicationConfiguration\x120\n" + "\x14build_directory_path\x18\x01 \x01(\tR\x12buildDirectoryPath\x12T\n" + "\fgrpc_servers\x18\x02 \x03(\v21.buildbarn.configuration.grpc.ServerConfigurationR\vgrpcServers\x12>\n" + @@ -191,8 +191,8 @@ const file_github_com_buildbarn_bb_remote_execution_pkg_proto_configuration_bb_r "\x0frun_commands_as\x18\v \x01(\v2A.buildbarn.configuration.credentials.UNIXCredentialsConfigurationR\rrunCommandsAs\x12B\n" + "\x1dsymlink_temporary_directories\x18\f \x03(\tR\x1bsymlinkTemporaryDirectories\x12.\n" + "\x13run_command_cleaner\x18\r \x03(\tR\x11runCommandCleaner\x12\xaa\x01\n" + - "!apple_xcode_developer_directories\x18\x0e \x03(\v2_.buildbarn.configuration.bb_runner.ApplicationConfiguration.AppleXcodeDeveloperDirectoriesEntryR\x1eappleXcodeDeveloperDirectories\x126\n" + - "\x17assume_exclusive_cgroup\x18\x0f \x01(\bR\x15assumeExclusiveCgroup\x1aQ\n" + + "!apple_xcode_developer_directories\x18\x0e \x03(\v2_.buildbarn.configuration.bb_runner.ApplicationConfiguration.AppleXcodeDeveloperDirectoriesEntryR\x1eappleXcodeDeveloperDirectories\x12?\n" + + "\x1csample_cgroup_resource_usage\x18\x0f \x01(\bR\x19sampleCgroupResourceUsage\x1aQ\n" + "#AppleXcodeDeveloperDirectoriesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01J\x04\b\t\x10\n" + diff --git a/pkg/proto/configuration/bb_runner/bb_runner.proto b/pkg/proto/configuration/bb_runner/bb_runner.proto index 48ca8608..989b913c 100644 --- a/pkg/proto/configuration/bb_runner/bb_runner.proto +++ b/pkg/proto/configuration/bb_runner/bb_runner.proto @@ -138,10 +138,13 @@ message ApplicationConfiguration { // Sample cgroup v2 resource usage counters around each action. // - // This should only be enabled when the bb_runner process has an exclusive - // cgroup for action execution. If multiple runners or actions share the - // cgroup, the sampled counter deltas include unrelated work and are - // misleading. When this is enabled, bb_runner fails actions with - // codes.Internal if it detects multiple actions running concurrently. - bool assume_exclusive_cgroup = 15; + // This should only be enabled when bb_worker sends at most one action to + // this bb_runner at a time. In particular, setting + // RunnerConfiguration.concurrency > 1 for this runner is incompatible. + // + // The bb_runner process should also have a cgroup whose activity is + // acceptable to include in the reported usage. If multiple runners or + // actions share the cgroup, the sampled counter deltas include unrelated + // work and are misleading. + bool sample_cgroup_resource_usage = 15; } diff --git a/pkg/proto/resourceusage/resourceusage.pb.go b/pkg/proto/resourceusage/resourceusage.pb.go index 50a85345..8de68771 100644 --- a/pkg/proto/resourceusage/resourceusage.pb.go +++ b/pkg/proto/resourceusage/resourceusage.pb.go @@ -382,13 +382,13 @@ type CgroupResourceUsage struct { MemoryEventsOom int64 `protobuf:"varint,4,opt,name=memory_events_oom,json=memoryEventsOom,proto3" json:"memory_events_oom,omitempty"` MemoryEventsOomKill int64 `protobuf:"varint,5,opt,name=memory_events_oom_kill,json=memoryEventsOomKill,proto3" json:"memory_events_oom_kill,omitempty"` MemoryEventsOomGroupKill int64 `protobuf:"varint,6,opt,name=memory_events_oom_group_kill,json=memoryEventsOomGroupKill,proto3" json:"memory_events_oom_group_kill,omitempty"` - MemoryPeakBytes int64 `protobuf:"varint,7,opt,name=memory_peak_bytes,json=memoryPeakBytes,proto3" json:"memory_peak_bytes,omitempty"` - PsiMemorySome *durationpb.Duration `protobuf:"bytes,8,opt,name=psi_memory_some,json=psiMemorySome,proto3" json:"psi_memory_some,omitempty"` - PsiMemoryFull *durationpb.Duration `protobuf:"bytes,9,opt,name=psi_memory_full,json=psiMemoryFull,proto3" json:"psi_memory_full,omitempty"` - PsiCpuSome *durationpb.Duration `protobuf:"bytes,10,opt,name=psi_cpu_some,json=psiCpuSome,proto3" json:"psi_cpu_some,omitempty"` - PsiCpuFull *durationpb.Duration `protobuf:"bytes,11,opt,name=psi_cpu_full,json=psiCpuFull,proto3" json:"psi_cpu_full,omitempty"` - PsiIoSome *durationpb.Duration `protobuf:"bytes,12,opt,name=psi_io_some,json=psiIoSome,proto3" json:"psi_io_some,omitempty"` - PsiIoFull *durationpb.Duration `protobuf:"bytes,13,opt,name=psi_io_full,json=psiIoFull,proto3" json:"psi_io_full,omitempty"` + MemoryPeak int64 `protobuf:"varint,7,opt,name=memory_peak,json=memoryPeak,proto3" json:"memory_peak,omitempty"` + MemoryPressureSomeTotal *durationpb.Duration `protobuf:"bytes,8,opt,name=memory_pressure_some_total,json=memoryPressureSomeTotal,proto3" json:"memory_pressure_some_total,omitempty"` + MemoryPressureFullTotal *durationpb.Duration `protobuf:"bytes,9,opt,name=memory_pressure_full_total,json=memoryPressureFullTotal,proto3" json:"memory_pressure_full_total,omitempty"` + CpuPressureSomeTotal *durationpb.Duration `protobuf:"bytes,10,opt,name=cpu_pressure_some_total,json=cpuPressureSomeTotal,proto3" json:"cpu_pressure_some_total,omitempty"` + CpuPressureFullTotal *durationpb.Duration `protobuf:"bytes,11,opt,name=cpu_pressure_full_total,json=cpuPressureFullTotal,proto3" json:"cpu_pressure_full_total,omitempty"` + IoPressureSomeTotal *durationpb.Duration `protobuf:"bytes,12,opt,name=io_pressure_some_total,json=ioPressureSomeTotal,proto3" json:"io_pressure_some_total,omitempty"` + IoPressureFullTotal *durationpb.Duration `protobuf:"bytes,13,opt,name=io_pressure_full_total,json=ioPressureFullTotal,proto3" json:"io_pressure_full_total,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -465,51 +465,51 @@ func (x *CgroupResourceUsage) GetMemoryEventsOomGroupKill() int64 { return 0 } -func (x *CgroupResourceUsage) GetMemoryPeakBytes() int64 { +func (x *CgroupResourceUsage) GetMemoryPeak() int64 { if x != nil { - return x.MemoryPeakBytes + return x.MemoryPeak } return 0 } -func (x *CgroupResourceUsage) GetPsiMemorySome() *durationpb.Duration { +func (x *CgroupResourceUsage) GetMemoryPressureSomeTotal() *durationpb.Duration { if x != nil { - return x.PsiMemorySome + return x.MemoryPressureSomeTotal } return nil } -func (x *CgroupResourceUsage) GetPsiMemoryFull() *durationpb.Duration { +func (x *CgroupResourceUsage) GetMemoryPressureFullTotal() *durationpb.Duration { if x != nil { - return x.PsiMemoryFull + return x.MemoryPressureFullTotal } return nil } -func (x *CgroupResourceUsage) GetPsiCpuSome() *durationpb.Duration { +func (x *CgroupResourceUsage) GetCpuPressureSomeTotal() *durationpb.Duration { if x != nil { - return x.PsiCpuSome + return x.CpuPressureSomeTotal } return nil } -func (x *CgroupResourceUsage) GetPsiCpuFull() *durationpb.Duration { +func (x *CgroupResourceUsage) GetCpuPressureFullTotal() *durationpb.Duration { if x != nil { - return x.PsiCpuFull + return x.CpuPressureFullTotal } return nil } -func (x *CgroupResourceUsage) GetPsiIoSome() *durationpb.Duration { +func (x *CgroupResourceUsage) GetIoPressureSomeTotal() *durationpb.Duration { if x != nil { - return x.PsiIoSome + return x.IoPressureSomeTotal } return nil } -func (x *CgroupResourceUsage) GetPsiIoFull() *durationpb.Duration { +func (x *CgroupResourceUsage) GetIoPressureFullTotal() *durationpb.Duration { if x != nil { - return x.PsiIoFull + return x.IoPressureFullTotal } return nil } @@ -611,24 +611,23 @@ const file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_reso "\x14directories_resolved\x18\x01 \x01(\x04R\x13directoriesResolved\x12)\n" + "\x10directories_read\x18\x02 \x01(\x04R\x0fdirectoriesRead\x12\x1d\n" + "\n" + - "files_read\x18\x03 \x01(\x04R\tfilesRead\"\xde\x05\n" + + "files_read\x18\x03 \x01(\x04R\tfilesRead\"\xd1\x06\n" + "\x13CgroupResourceUsage\x12*\n" + "\x11memory_events_low\x18\x01 \x01(\x03R\x0fmemoryEventsLow\x12,\n" + "\x12memory_events_high\x18\x02 \x01(\x03R\x10memoryEventsHigh\x12*\n" + "\x11memory_events_max\x18\x03 \x01(\x03R\x0fmemoryEventsMax\x12*\n" + "\x11memory_events_oom\x18\x04 \x01(\x03R\x0fmemoryEventsOom\x123\n" + "\x16memory_events_oom_kill\x18\x05 \x01(\x03R\x13memoryEventsOomKill\x12>\n" + - "\x1cmemory_events_oom_group_kill\x18\x06 \x01(\x03R\x18memoryEventsOomGroupKill\x12*\n" + - "\x11memory_peak_bytes\x18\a \x01(\x03R\x0fmemoryPeakBytes\x12A\n" + - "\x0fpsi_memory_some\x18\b \x01(\v2\x19.google.protobuf.DurationR\rpsiMemorySome\x12A\n" + - "\x0fpsi_memory_full\x18\t \x01(\v2\x19.google.protobuf.DurationR\rpsiMemoryFull\x12;\n" + - "\fpsi_cpu_some\x18\n" + - " \x01(\v2\x19.google.protobuf.DurationR\n" + - "psiCpuSome\x12;\n" + - "\fpsi_cpu_full\x18\v \x01(\v2\x19.google.protobuf.DurationR\n" + - "psiCpuFull\x129\n" + - "\vpsi_io_some\x18\f \x01(\v2\x19.google.protobuf.DurationR\tpsiIoSome\x129\n" + - "\vpsi_io_full\x18\r \x01(\v2\x19.google.protobuf.DurationR\tpsiIoFullBBZ@github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusageb\x06proto3" + "\x1cmemory_events_oom_group_kill\x18\x06 \x01(\x03R\x18memoryEventsOomGroupKill\x12\x1f\n" + + "\vmemory_peak\x18\a \x01(\x03R\n" + + "memoryPeak\x12V\n" + + "\x1amemory_pressure_some_total\x18\b \x01(\v2\x19.google.protobuf.DurationR\x17memoryPressureSomeTotal\x12V\n" + + "\x1amemory_pressure_full_total\x18\t \x01(\v2\x19.google.protobuf.DurationR\x17memoryPressureFullTotal\x12P\n" + + "\x17cpu_pressure_some_total\x18\n" + + " \x01(\v2\x19.google.protobuf.DurationR\x14cpuPressureSomeTotal\x12P\n" + + "\x17cpu_pressure_full_total\x18\v \x01(\v2\x19.google.protobuf.DurationR\x14cpuPressureFullTotal\x12N\n" + + "\x16io_pressure_some_total\x18\f \x01(\v2\x19.google.protobuf.DurationR\x13ioPressureSomeTotal\x12N\n" + + "\x16io_pressure_full_total\x18\r \x01(\v2\x19.google.protobuf.DurationR\x13ioPressureFullTotalBBZ@github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusageb\x06proto3" var ( file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resourceusage_proto_rawDescOnce sync.Once @@ -657,12 +656,12 @@ var file_github_com_buildbarn_bb_remote_execution_pkg_proto_resourceusage_resour 7, // 0: buildbarn.resourceusage.POSIXResourceUsage.user_time:type_name -> google.protobuf.Duration 7, // 1: buildbarn.resourceusage.POSIXResourceUsage.system_time:type_name -> google.protobuf.Duration 6, // 2: buildbarn.resourceusage.MonetaryResourceUsage.expenses:type_name -> buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry - 7, // 3: buildbarn.resourceusage.CgroupResourceUsage.psi_memory_some:type_name -> google.protobuf.Duration - 7, // 4: buildbarn.resourceusage.CgroupResourceUsage.psi_memory_full:type_name -> google.protobuf.Duration - 7, // 5: buildbarn.resourceusage.CgroupResourceUsage.psi_cpu_some:type_name -> google.protobuf.Duration - 7, // 6: buildbarn.resourceusage.CgroupResourceUsage.psi_cpu_full:type_name -> google.protobuf.Duration - 7, // 7: buildbarn.resourceusage.CgroupResourceUsage.psi_io_some:type_name -> google.protobuf.Duration - 7, // 8: buildbarn.resourceusage.CgroupResourceUsage.psi_io_full:type_name -> google.protobuf.Duration + 7, // 3: buildbarn.resourceusage.CgroupResourceUsage.memory_pressure_some_total:type_name -> google.protobuf.Duration + 7, // 4: buildbarn.resourceusage.CgroupResourceUsage.memory_pressure_full_total:type_name -> google.protobuf.Duration + 7, // 5: buildbarn.resourceusage.CgroupResourceUsage.cpu_pressure_some_total:type_name -> google.protobuf.Duration + 7, // 6: buildbarn.resourceusage.CgroupResourceUsage.cpu_pressure_full_total:type_name -> google.protobuf.Duration + 7, // 7: buildbarn.resourceusage.CgroupResourceUsage.io_pressure_some_total:type_name -> google.protobuf.Duration + 7, // 8: buildbarn.resourceusage.CgroupResourceUsage.io_pressure_full_total:type_name -> google.protobuf.Duration 5, // 9: buildbarn.resourceusage.MonetaryResourceUsage.ExpensesEntry.value:type_name -> buildbarn.resourceusage.MonetaryResourceUsage.Expense 10, // [10:10] is the sub-list for method output_type 10, // [10:10] is the sub-list for method input_type diff --git a/pkg/proto/resourceusage/resourceusage.proto b/pkg/proto/resourceusage/resourceusage.proto index ec34c2fd..ee76d8ff 100644 --- a/pkg/proto/resourceusage/resourceusage.proto +++ b/pkg/proto/resourceusage/resourceusage.proto @@ -137,7 +137,7 @@ message InputRootResourceUsage { // it by the action's execution duration from // ActionResult.execution_metadata. For example: // -// psi_cpu_some_percent = psi_cpu_some / ( +// cpu_pressure_some_percent = cpu_pressure_some_total / ( // execution_completed_timestamp - execution_start_timestamp) * 100 // // See: @@ -171,29 +171,29 @@ message CgroupResourceUsage { // memory.peak: Peak memory usage in bytes since resetting // memory.peak at action start. Zero if memory.peak reset/read is // unavailable. - int64 memory_peak_bytes = 7; + int64 memory_peak = 7; // memory.pressure/some total: Memory PSI stall time with at least // one task stalled on memory. - google.protobuf.Duration psi_memory_some = 8; + google.protobuf.Duration memory_pressure_some_total = 8; // memory.pressure/full total: Memory PSI stall time with all // non-idle tasks stalled on memory. - google.protobuf.Duration psi_memory_full = 9; + google.protobuf.Duration memory_pressure_full_total = 9; // cpu.pressure/some total: CPU PSI stall time with at least one task // waiting for CPU. - google.protobuf.Duration psi_cpu_some = 10; + google.protobuf.Duration cpu_pressure_some_total = 10; // cpu.pressure/full total: CPU PSI stall time with all non-idle // tasks waiting for CPU. - google.protobuf.Duration psi_cpu_full = 11; + google.protobuf.Duration cpu_pressure_full_total = 11; // io.pressure/some total: I/O PSI stall time with at least one task // stalled on I/O. - google.protobuf.Duration psi_io_some = 12; + google.protobuf.Duration io_pressure_some_total = 12; // io.pressure/full total: I/O PSI stall time with all non-idle // tasks stalled on I/O. - google.protobuf.Duration psi_io_full = 13; + google.protobuf.Duration io_pressure_full_total = 13; } diff --git a/pkg/runner/BUILD.bazel b/pkg/runner/BUILD.bazel index 2dd8241d..12a5de33 100644 --- a/pkg/runner/BUILD.bazel +++ b/pkg/runner/BUILD.bazel @@ -66,7 +66,6 @@ go_test( name = "runner_test", srcs = [ "apple_xcode_resolving_runner_test.go", - "cgroup_resource_usage_sampling_runner_test.go", "clean_runner_test.go", "local_runner_test.go", "path_existence_checking_runner_test.go", @@ -74,14 +73,16 @@ go_test( ] + select({ "@rules_go//go/platform:android": [ "cgroup_linux_test.go", + "cgroup_resource_usage_sampling_runner_test.go", ], "@rules_go//go/platform:linux": [ "cgroup_linux_test.go", + "cgroup_resource_usage_sampling_runner_test.go", ], "//conditions:default": [], }), - embed = [":runner"], deps = [ + ":runner", "//internal/mock", "//pkg/cleaner", "//pkg/proto/resourceusage", diff --git a/pkg/runner/cgroup_linux.go b/pkg/runner/cgroup_linux.go index b17b643e..46c59269 100644 --- a/pkg/runner/cgroup_linux.go +++ b/pkg/runner/cgroup_linux.go @@ -10,55 +10,13 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" "google.golang.org/protobuf/types/known/durationpb" ) -var ( - currentCgroupPathOnce sync.Once - currentCgroupPath string - currentCgroupPathErr error -) - -// validateExclusiveCgroupResourceUsageSampling performs best-effort startup -// validation for cgroup resource usage sampling. It catches obvious -// misconfigurations, but cannot prove that no other process will enter the -// cgroup after startup. -func validateExclusiveCgroupResourceUsageSampling() error { - cgroupPath, err := getCurrentCgroupPath() - if err != nil { - return err - } - if _, err := readCgroupKeyValues(filepath.Join(cgroupPath, "memory.events")); err != nil { - return fmt.Errorf("failed to read cgroup v2 memory.events: %w", err) - } - if _, _, err := parsePSITotals(filepath.Join(cgroupPath, "memory.pressure")); err != nil { - return fmt.Errorf("failed to read cgroup v2 memory.pressure: %w", err) - } - if _, _, err := parsePSITotals(filepath.Join(cgroupPath, "cpu.pressure")); err != nil { - return fmt.Errorf("failed to read cgroup v2 cpu.pressure: %w", err) - } - if _, _, err := parsePSITotals(filepath.Join(cgroupPath, "io.pressure")); err != nil { - return fmt.Errorf("failed to read cgroup v2 io.pressure: %w", err) - } - processIDs, err := readCgroupProcessIDs(filepath.Join(cgroupPath, "cgroup.procs")) - if err != nil { - return fmt.Errorf("failed to read cgroup.procs: %w", err) - } - allowedProcessIDs := getAncestorProcessIDs(os.Getpid()) - for _, processID := range processIDs { - if _, ok := allowedProcessIDs[processID]; ok { - continue - } - return fmt.Errorf("cgroup %q is not exclusive; found extra process %d (%s)", cgroupPath, processID, getProcessName(processID)) - } - return nil -} - -type cgroupStatsReader struct { +type cgroupResourceUsageReader struct { cgroupPath string // memory.events counters @@ -80,15 +38,9 @@ type cgroupStatsReader struct { memoryPeakFile *os.File } -func newScopedCgroupStatsReader() (*cgroupStatsReader, error) { - cgroupPath, err := getCurrentCgroupPath() - if err != nil { - return nil, err - } - return newCgroupStatsReader(cgroupPath) -} - -func newCgroupStatsReader(cgroupPath string) (*cgroupStatsReader, error) { +// NewCgroupResourceUsageReaderFromPath creates a reader that samples resource +// usage counters from the cgroup v2 directory at cgroupPath. +func NewCgroupResourceUsageReaderFromPath(cgroupPath string) (CgroupResourceUsageReader, error) { events, err := readCgroupKeyValues(filepath.Join(cgroupPath, "memory.events")) if err != nil { return nil, fmt.Errorf("failed to read memory.events: %w", err) @@ -108,9 +60,9 @@ func newCgroupStatsReader(cgroupPath string) (*cgroupStatsReader, error) { } // memory.peak is optional. If it is unavailable or cannot be reset/read, - // MemoryPeakBytes remains 0 to indicate that no peak data was collected. + // MemoryPeak remains 0 to indicate that no peak data was collected. memoryPeakFile := openAndResetCgroupMemoryPeak(filepath.Join(cgroupPath, "memory.peak")) - reader := &cgroupStatsReader{ + reader := &cgroupResourceUsageReader{ cgroupPath: cgroupPath, eventsLow: events["low"], @@ -132,14 +84,14 @@ func newCgroupStatsReader(cgroupPath string) (*cgroupStatsReader, error) { return reader, nil } -func (r *cgroupStatsReader) Close() error { +func (r *cgroupResourceUsageReader) Close() error { if r == nil || r.memoryPeakFile == nil { return nil } return r.memoryPeakFile.Close() } -func (r *cgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { +func (r *cgroupResourceUsageReader) Read() (*resourceusage.CgroupResourceUsage, error) { events, err := readCgroupKeyValues(filepath.Join(r.cgroupPath, "memory.events")) if err != nil { return nil, fmt.Errorf("failed to read memory.events: %w", err) @@ -168,14 +120,14 @@ func (r *cgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { MemoryEventsOomKill: events["oom_kill"] - r.eventsOOMKill, MemoryEventsOomGroupKill: events["oom_group_kill"] - r.eventsOOMGroupKill, - MemoryPeakBytes: memoryPeak, + MemoryPeak: memoryPeak, - PsiMemorySome: microsecondsDuration(memorySome - r.psiMemorySomeUS), - PsiMemoryFull: microsecondsDuration(memoryFull - r.psiMemoryFullUS), - PsiCpuSome: microsecondsDuration(cpuSome - r.psiCPUSomeUS), - PsiCpuFull: microsecondsDuration(cpuFull - r.psiCPUFullUS), - PsiIoSome: microsecondsDuration(ioSome - r.psiIOSomeUS), - PsiIoFull: microsecondsDuration(ioFull - r.psiIOFullUS), + MemoryPressureSomeTotal: microsecondsDuration(memorySome - r.psiMemorySomeUS), + MemoryPressureFullTotal: microsecondsDuration(memoryFull - r.psiMemoryFullUS), + CpuPressureSomeTotal: microsecondsDuration(cpuSome - r.psiCPUSomeUS), + CpuPressureFullTotal: microsecondsDuration(cpuFull - r.psiCPUFullUS), + IoPressureSomeTotal: microsecondsDuration(ioSome - r.psiIOSomeUS), + IoPressureFullTotal: microsecondsDuration(ioFull - r.psiIOFullUS), }, nil } @@ -243,32 +195,10 @@ func readCgroupKeyValues(path string) (map[string]int64, error) { return result, nil } -func readCgroupProcessIDs(path string) ([]int, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - var processIDs []int - scanner := bufio.NewScanner(f) - for scanner.Scan() { - processID, err := strconv.Atoi(strings.TrimSpace(scanner.Text())) - if err != nil { - continue - } - processIDs = append(processIDs, processID) - } - if err := scanner.Err(); err != nil { - return nil, err - } - return processIDs, nil -} - // parsePSITotals parses a PSI pressure file and returns the total // stall microseconds for the "some" and "full" lines. // Format: some avg10=0.00 avg60=0.00 avg300=0.00 total=12345 -func parsePSITotals(path string) (someUS int64, fullUS int64, err error) { +func parsePSITotals(path string) (someUS, fullUS int64, err error) { f, err := os.Open(path) if err != nil { return 0, 0, err @@ -312,19 +242,21 @@ func parsePSITotals(path string) (someUS int64, fullUS int64, err error) { return someUS, fullUS, nil } -func getCurrentCgroupPath() (string, error) { - currentCgroupPathOnce.Do(func() { - currentCgroupPath, currentCgroupPathErr = resolveCurrentCgroupPath("/proc/self/mountinfo", "/proc/self/cgroup") - }) - return currentCgroupPath, currentCgroupPathErr +// ResolveCurrentCgroupfsPath resolves the cgroup v2 filesystem directory of +// the current process. +func ResolveCurrentCgroupfsPath() (string, error) { + return ResolveCurrentCgroupfsPathFromProcFiles("/proc/self/cgroup", "/proc/self/mountinfo") } -func resolveCurrentCgroupPath(mountInfoPath, cgroupPath string) (string, error) { - currentCgroupPath, err := readCurrentCgroupRelativePath(cgroupPath) +// ResolveCurrentCgroupfsPathFromProcFiles reads procCgroupPath and +// procMountInfoPath to resolve the cgroup v2 filesystem directory of the +// current process. +func ResolveCurrentCgroupfsPathFromProcFiles(procCgroupPath, procMountInfoPath string) (string, error) { + currentCgroupPath, err := readCurrentCgroupRelativePath(procCgroupPath) if err != nil { return "", err } - return resolveCgroupPathFromMountInfo(mountInfoPath, currentCgroupPath) + return resolveCgroupPathFromMountInfo(procMountInfoPath, currentCgroupPath) } func resolveCgroupPathFromMountInfo(path, currentCgroupPath string) (string, error) { @@ -406,44 +338,3 @@ func readCurrentCgroupRelativePath(path string) (string, error) { } return "", fmt.Errorf("cgroup v2 entry not found in %s", path) } - -func getAncestorProcessIDs(processID int) map[int]struct{} { - processIDs := map[int]struct{}{} - for processID > 0 { - if _, ok := processIDs[processID]; ok { - break - } - processIDs[processID] = struct{}{} - parentProcessID, err := getParentProcessID(processID) - if err != nil { - break - } - processID = parentProcessID - } - return processIDs -} - -func getParentProcessID(processID int) (int, error) { - data, err := os.ReadFile("/proc/" + strconv.Itoa(processID) + "/status") - if err != nil { - return 0, err - } - for _, line := range strings.Split(string(data), "\n") { - if strings.HasPrefix(line, "PPid:") { - fields := strings.Fields(line) - if len(fields) != 2 { - return 0, fmt.Errorf("invalid PPid line for process %d", processID) - } - return strconv.Atoi(fields[1]) - } - } - return 0, fmt.Errorf("PPid line not found for process %d", processID) -} - -func getProcessName(processID int) string { - comm, err := os.ReadFile("/proc/" + strconv.Itoa(processID) + "/comm") - if err != nil { - return "" - } - return strings.TrimSpace(string(comm)) -} diff --git a/pkg/runner/cgroup_linux_test.go b/pkg/runner/cgroup_linux_test.go index 46b688d4..78e6c76f 100644 --- a/pkg/runner/cgroup_linux_test.go +++ b/pkg/runner/cgroup_linux_test.go @@ -1,20 +1,22 @@ //go:build linux -package runner +package runner_test import ( + "fmt" "os" "path/filepath" "testing" "time" + "github.com/buildbarn/bb-remote-execution/pkg/runner" "github.com/stretchr/testify/require" ) -func TestCgroupStatsReaderReportsDeltasFromCgroupFiles(t *testing.T) { +func TestCgroupResourceUsageReaderReportsDeltasFromCgroupFiles(t *testing.T) { cgroupPath := t.TempDir() - writeCgroupFile(t, cgroupPath, "memory.events", ` + writeFile(t, cgroupPath, "memory.events", ` low 1 high 2 max 3 @@ -22,27 +24,27 @@ oom 4 oom_kill 5 oom_group_kill 6 `) - writeCgroupFile(t, cgroupPath, "memory.pressure", ` + writeFile(t, cgroupPath, "memory.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=100 full avg10=0.00 avg60=0.00 avg300=0.00 total=200 `) - writeCgroupFile(t, cgroupPath, "cpu.pressure", ` + writeFile(t, cgroupPath, "cpu.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=300 full avg10=0.00 avg60=0.00 avg300=0.00 total=310 `) - writeCgroupFile(t, cgroupPath, "io.pressure", ` + writeFile(t, cgroupPath, "io.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=400 full avg10=0.00 avg60=0.00 avg300=0.00 total=500 `) - writeCgroupFile(t, cgroupPath, "memory.peak", "0\n") + writeFile(t, cgroupPath, "memory.peak", "0\n") - reader, err := newCgroupStatsReader(cgroupPath) + reader, err := runner.NewCgroupResourceUsageReaderFromPath(cgroupPath) require.NoError(t, err) defer func() { require.NoError(t, reader.Close()) }() - writeCgroupFile(t, cgroupPath, "memory.events", ` + writeFile(t, cgroupPath, "memory.events", ` low 11 high 22 max 33 @@ -50,19 +52,19 @@ oom 44 oom_kill 55 oom_group_kill 66 `) - writeCgroupFile(t, cgroupPath, "memory.pressure", ` + writeFile(t, cgroupPath, "memory.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=160 full avg10=0.00 avg60=0.00 avg300=0.00 total=290 `) - writeCgroupFile(t, cgroupPath, "cpu.pressure", ` + writeFile(t, cgroupPath, "cpu.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=345 full avg10=0.00 avg60=0.00 avg300=0.00 total=410 `) - writeCgroupFile(t, cgroupPath, "io.pressure", ` + writeFile(t, cgroupPath, "io.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=480 full avg10=0.00 avg60=0.00 avg300=0.00 total=610 `) - writeCgroupFile(t, cgroupPath, "memory.peak", "4096\n") + writeFile(t, cgroupPath, "memory.peak", "4096\n") usage, err := reader.Read() require.NoError(t, err) @@ -73,46 +75,46 @@ full avg10=0.00 avg60=0.00 avg300=0.00 total=610 require.Equal(t, int64(40), usage.MemoryEventsOom) require.Equal(t, int64(50), usage.MemoryEventsOomKill) require.Equal(t, int64(60), usage.MemoryEventsOomGroupKill) - require.Equal(t, int64(4096), usage.MemoryPeakBytes) - require.Equal(t, 60*time.Microsecond, usage.GetPsiMemorySome().AsDuration()) - require.Equal(t, 90*time.Microsecond, usage.GetPsiMemoryFull().AsDuration()) - require.Equal(t, 45*time.Microsecond, usage.GetPsiCpuSome().AsDuration()) - require.Equal(t, 100*time.Microsecond, usage.GetPsiCpuFull().AsDuration()) - require.Equal(t, 80*time.Microsecond, usage.GetPsiIoSome().AsDuration()) - require.Equal(t, 110*time.Microsecond, usage.GetPsiIoFull().AsDuration()) + require.Equal(t, int64(4096), usage.MemoryPeak) + require.Equal(t, 60*time.Microsecond, usage.GetMemoryPressureSomeTotal().AsDuration()) + require.Equal(t, 90*time.Microsecond, usage.GetMemoryPressureFullTotal().AsDuration()) + require.Equal(t, 45*time.Microsecond, usage.GetCpuPressureSomeTotal().AsDuration()) + require.Equal(t, 100*time.Microsecond, usage.GetCpuPressureFullTotal().AsDuration()) + require.Equal(t, 80*time.Microsecond, usage.GetIoPressureSomeTotal().AsDuration()) + require.Equal(t, 110*time.Microsecond, usage.GetIoPressureFullTotal().AsDuration()) } -func TestCgroupStatsReaderAllowsMissingOomGroupKillCounter(t *testing.T) { +func TestCgroupResourceUsageReaderAllowsMissingOomGroupKillCounter(t *testing.T) { cgroupPath := t.TempDir() - writeCgroupFile(t, cgroupPath, "memory.events", ` + writeFile(t, cgroupPath, "memory.events", ` low 1 high 2 max 3 oom 4 oom_kill 5 `) - writeCgroupFile(t, cgroupPath, "memory.pressure", ` + writeFile(t, cgroupPath, "memory.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=100 full avg10=0.00 avg60=0.00 avg300=0.00 total=200 `) - writeCgroupFile(t, cgroupPath, "cpu.pressure", ` + writeFile(t, cgroupPath, "cpu.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=300 full avg10=0.00 avg60=0.00 avg300=0.00 total=310 `) - writeCgroupFile(t, cgroupPath, "io.pressure", ` + writeFile(t, cgroupPath, "io.pressure", ` some avg10=0.00 avg60=0.00 avg300=0.00 total=400 full avg10=0.00 avg60=0.00 avg300=0.00 total=500 `) - writeCgroupFile(t, cgroupPath, "memory.peak", "0\n") + writeFile(t, cgroupPath, "memory.peak", "0\n") - reader, err := newCgroupStatsReader(cgroupPath) + reader, err := runner.NewCgroupResourceUsageReaderFromPath(cgroupPath) require.NoError(t, err) defer func() { require.NoError(t, reader.Close()) }() - writeCgroupFile(t, cgroupPath, "memory.events", ` + writeFile(t, cgroupPath, "memory.events", ` low 11 high 22 max 33 @@ -127,66 +129,109 @@ oom_kill 55 require.Equal(t, int64(0), usage.MemoryEventsOomGroupKill) } -func TestResolveCurrentCgroupPathUsesMatchingCgroup2MountRoot(t *testing.T) { +func TestResolveCurrentCgroupfsPathUsesMatchingCgroup2MountRoot(t *testing.T) { for _, testCase := range []struct { name string mountInfo string cgroup string - want string + wantPath string }{ { name: "root mount", mountInfo: ` -36 25 0:31 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +36 25 0:31 / %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw `, - cgroup: "0::/worker.slice/runner.scope\n", - want: "/sys/fs/cgroup/worker.slice/runner.scope", + cgroup: "0::/worker.slice/runner.scope\n", + wantPath: "worker.slice/runner.scope", }, { name: "subtree mount", mountInfo: ` 36 25 0:31 /unrelated /wrong rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -37 25 0:31 /worker.slice /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +37 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw `, - cgroup: "0::/worker.slice/runner.scope\n", - want: "/sys/fs/cgroup/runner.scope", + cgroup: "0::/worker.slice/runner.scope\n", + wantPath: "runner.scope", }, { name: "most specific mount root", mountInfo: ` -36 25 0:31 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -37 25 0:31 /worker.slice /run/worker-cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +36 25 0:31 / %[1]s/unrelated rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +37 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw `, - cgroup: "0::/worker.slice/runner.scope\n", - want: "/run/worker-cgroup/runner.scope", + cgroup: "0::/worker.slice/runner.scope\n", + wantPath: "runner.scope", }, { name: "current cgroup is mount root", mountInfo: ` -36 25 0:31 /worker.slice /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +36 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw `, - cgroup: "0::/worker.slice\n", - want: "/sys/fs/cgroup", + cgroup: "0::/worker.slice\n", + wantPath: ".", }, } { t.Run(testCase.name, func(t *testing.T) { tempDir := t.TempDir() + cgroupfsPath := filepath.Join(tempDir, "cgroupfs") + resolvedCgroupPath := filepath.Join(cgroupfsPath, testCase.wantPath) + require.NoError(t, os.MkdirAll(resolvedCgroupPath, 0o777)) + writeInitialCgroupResourceUsageFiles(t, resolvedCgroupPath) + mountInfoPath := filepath.Join(tempDir, "mountinfo") - cgroupPath := filepath.Join(tempDir, "cgroup") - writeFile(t, mountInfoPath, testCase.mountInfo) - writeFile(t, cgroupPath, testCase.cgroup) + procCgroupPath := filepath.Join(tempDir, "cgroup") + writeFile(t, tempDir, "mountinfo", fmt.Sprintf(testCase.mountInfo, cgroupfsPath)) + writeFile(t, tempDir, "cgroup", testCase.cgroup) + + gotCgroupfsPath, err := runner.ResolveCurrentCgroupfsPathFromProcFiles(procCgroupPath, mountInfoPath) + require.NoError(t, err) + require.Equal(t, resolvedCgroupPath, gotCgroupfsPath) - got, err := resolveCurrentCgroupPath(mountInfoPath, cgroupPath) + reader, err := runner.NewCgroupResourceUsageReaderFromPath(gotCgroupfsPath) + require.NoError(t, err) + defer func() { + require.NoError(t, reader.Close()) + }() + + writeFile(t, resolvedCgroupPath, "memory.events", ` +low 11 +high 22 +max 33 +oom 44 +oom_kill 55 +oom_group_kill 66 +`) + usage, err := reader.Read() require.NoError(t, err) - require.Equal(t, testCase.want, got) + require.Equal(t, int64(10), usage.MemoryEventsLow) }) } } -func writeCgroupFile(t *testing.T, cgroupPath, name, contents string) { - writeFile(t, filepath.Join(cgroupPath, name), contents) +func writeInitialCgroupResourceUsageFiles(t *testing.T, cgroupPath string) { + writeFile(t, cgroupPath, "memory.events", ` +low 1 +high 2 +max 3 +oom 4 +oom_kill 5 +oom_group_kill 6 +`) + writeFile(t, cgroupPath, "memory.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=100 +full avg10=0.00 avg60=0.00 avg300=0.00 total=200 +`) + writeFile(t, cgroupPath, "cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=300 +full avg10=0.00 avg60=0.00 avg300=0.00 total=310 +`) + writeFile(t, cgroupPath, "io.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=400 +full avg10=0.00 avg60=0.00 avg300=0.00 total=500 +`) + writeFile(t, cgroupPath, "memory.peak", "0\n") } -func writeFile(t *testing.T, path, contents string) { - require.NoError(t, os.WriteFile(path, []byte(contents), 0o666)) +func writeFile(t *testing.T, dir, name, contents string) { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(contents), 0o666)) } diff --git a/pkg/runner/cgroup_other.go b/pkg/runner/cgroup_other.go index 97f62475..b953891e 100644 --- a/pkg/runner/cgroup_other.go +++ b/pkg/runner/cgroup_other.go @@ -3,23 +3,24 @@ package runner import ( - "fmt" - - "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -type cgroupStatsReader struct{} - -func newScopedCgroupStatsReader() (*cgroupStatsReader, error) { - return &cgroupStatsReader{}, nil +// ResolveCurrentCgroupfsPath returns an error, as cgroup resource usage +// sampling is only supported on Linux. +func ResolveCurrentCgroupfsPath() (string, error) { + return "", status.Error(codes.Unimplemented, "cgroup resource usage sampling is only supported on Linux") } -func (r *cgroupStatsReader) Close() error { return nil } - -func (r *cgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { - return nil, nil +// ResolveCurrentCgroupfsPathFromProcFiles returns an error, as cgroup resource +// usage sampling is only supported on Linux. +func ResolveCurrentCgroupfsPathFromProcFiles(procCgroupPath, procMountInfoPath string) (string, error) { + return "", status.Error(codes.Unimplemented, "cgroup resource usage sampling is only supported on Linux") } -func validateExclusiveCgroupResourceUsageSampling() error { - return fmt.Errorf("cgroup resource usage sampling is only supported on Linux") +// NewCgroupResourceUsageReaderFromPath returns an error, as cgroup resource +// usage sampling is only supported on Linux. +func NewCgroupResourceUsageReaderFromPath(cgroupPath string) (CgroupResourceUsageReader, error) { + return nil, status.Error(codes.Unimplemented, "cgroup resource usage sampling is only supported on Linux") } diff --git a/pkg/runner/cgroup_resource_usage_sampling_runner.go b/pkg/runner/cgroup_resource_usage_sampling_runner.go index ba74e568..242f955c 100644 --- a/pkg/runner/cgroup_resource_usage_sampling_runner.go +++ b/pkg/runner/cgroup_resource_usage_sampling_runner.go @@ -2,67 +2,63 @@ package runner import ( "context" - "log" "sync/atomic" "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" + "github.com/buildbarn/bb-storage/pkg/util" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/anypb" ) -type cgroupResourceUsageSamplingRunner struct { - runner_pb.RunnerServer - newScopedCgroupStatsReader scopedCgroupStatsReaderFactory - activeRuns atomic.Int32 -} - -type scopedCgroupStatsReader interface { +// CgroupResourceUsageReader reads cgroup resource usage deltas for a single +// action. +type CgroupResourceUsageReader interface { Close() error Read() (*resourceusage.CgroupResourceUsage, error) } -type scopedCgroupStatsReaderFactory func() (scopedCgroupStatsReader, error) - -func newDefaultScopedCgroupStatsReader() (scopedCgroupStatsReader, error) { - return newScopedCgroupStatsReader() +type cgroupResourceUsageSamplingRunner struct { + runner_pb.RunnerServer + newCgroupResourceUsageReader func() (CgroupResourceUsageReader, error) + activeRun atomic.Bool } // NewCgroupResourceUsageSamplingRunner creates a decorator for RunnerServer // that samples cgroup v2 resource usage counters around actions and appends // them to successful Run() responses. // -// This decorator requires the runner's cgroup to be exclusive, so that -// sampled cgroup counters can be interpreted as per-action deltas. -func NewCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer) (runner_pb.RunnerServer, error) { - if err := validateExclusiveCgroupResourceUsageSampling(); err != nil { - return nil, err - } - return newCgroupResourceUsageSamplingRunner(base, newDefaultScopedCgroupStatsReader), nil -} - -func newCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer, newScopedCgroupStatsReader scopedCgroupStatsReaderFactory) runner_pb.RunnerServer { +// Sampled cgroup counters are only meaningful as per-action deltas if +// bb_worker sends at most one action to the runner at a time +// (RunnerConfiguration.concurrency == 1), and if the runner is deployed in a +// cgroup whose other activity is acceptable to include in the reported usage. +// +// cgroupfsPath is the cgroup v2 filesystem directory whose counters should be +// sampled. A fresh reader is created for each Run() request to capture the +// baseline counters for that action. +func NewCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer, cgroupfsPath string) runner_pb.RunnerServer { return &cgroupResourceUsageSamplingRunner{ - RunnerServer: base, - newScopedCgroupStatsReader: newScopedCgroupStatsReader, + RunnerServer: base, + newCgroupResourceUsageReader: func() (CgroupResourceUsageReader, error) { + return NewCgroupResourceUsageReaderFromPath(cgroupfsPath) + }, } } func (r *cgroupResourceUsageSamplingRunner) Run(ctx context.Context, request *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { - if r.activeRuns.Add(1) != 1 { - r.activeRuns.Add(-1) + if !r.activeRun.CompareAndSwap(false, true) { return nil, status.Error(codes.Internal, "cgroup resource usage sampling requires an exclusive runner cgroup, but concurrent Run() calls were observed") } - defer r.activeRuns.Add(-1) + defer r.activeRun.Store(false) - cgroupStatsReader, err := r.newScopedCgroupStatsReader() + cgroupResourceUsageReader, err := r.newCgroupResourceUsageReader() if err != nil { - return nil, status.Errorf(codes.Internal, "failed to create scoped cgroup stats reader: %s", err) + return nil, util.StatusWrap(err, "Failed to create cgroup resource usage reader") } defer func() { - _ = cgroupStatsReader.Close() + _ = cgroupResourceUsageReader.Close() }() response, err := r.RunnerServer.Run(ctx, request) @@ -70,20 +66,25 @@ func (r *cgroupResourceUsageSamplingRunner) Run(ctx context.Context, request *ru return response, err } - cgroupUsage, readErr := cgroupStatsReader.Read() - if readErr != nil { - log.Print("Failed to read scoped cgroup stats: ", readErr) - return response, err + cgroupUsage, err := cgroupResourceUsageReader.Read() + if err != nil { + return response, util.StatusWrap(err, "Failed to read cgroup stats") } if cgroupUsage == nil { - return response, err + return response, nil } - cgroupAny, cgroupErr := anypb.New(cgroupUsage) - if cgroupErr != nil { - return response, err + cgroupAny, err := anypb.New(cgroupUsage) + if err != nil { + return response, util.StatusWrap(err, "Failed to marshal cgroup resource usage") } if response != nil { response.ResourceUsage = append(response.ResourceUsage, cgroupAny) } + if cgroupUsage.MemoryEventsOomKill > 0 && cgroupUsage.MemoryEventsOom == 0 { + // The cgroup did not reach its memory limit, so the OOM kill likely + // came from system-level memory pressure, such as node memory + // overcommitment. Treat this as retryable infrastructure failure. + return response, status.Error(codes.Unavailable, "An action process was OOM-killed without the action reaching its cgroup memory limit") + } return response, nil } diff --git a/pkg/runner/cgroup_resource_usage_sampling_runner_test.go b/pkg/runner/cgroup_resource_usage_sampling_runner_test.go index 7332d2c0..d6f1df1c 100644 --- a/pkg/runner/cgroup_resource_usage_sampling_runner_test.go +++ b/pkg/runner/cgroup_resource_usage_sampling_runner_test.go @@ -1,41 +1,52 @@ -package runner +//go:build linux + +package runner_test import ( "context" - "errors" + "os" + "path/filepath" "testing" "time" "github.com/buildbarn/bb-remote-execution/internal/mock" "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" + "github.com/buildbarn/bb-remote-execution/pkg/runner" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/durationpb" ) func TestCgroupResourceUsageSamplingRunnerAppendsResourceUsage(t *testing.T) { ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) - cgroupStatsReader := &fixedCgroupStatsReader{ - usage: &resourceusage.CgroupResourceUsage{ - MemoryEventsOomKill: 1, - MemoryPeakBytes: 4096, - PsiCpuSome: durationpb.New(123 * time.Microsecond), - PsiCpuFull: durationpb.New(45 * time.Microsecond), - }, - } - wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { - return cgroupStatsReader, nil - }) + cgroupPath := createTestCgroup(t) + wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) request := &runner_pb.RunRequest{} - baseRunner.EXPECT().Run(gomock.Any(), request).Return(&runner_pb.RunResponse{ - ExitCode: 7, - }, nil) + baseRunner.EXPECT().Run(gomock.Any(), request).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.events"), []byte(` +low 0 +high 1 +max 0 +oom 0 +oom_kill 0 +oom_group_kill 0 +`), 0o666)) + require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "cpu.pressure"), []byte(` +some avg10=0.00 avg60=0.00 avg300=0.00 total=123 +full avg10=0.00 avg60=0.00 avg300=0.00 total=45 +`), 0o666)) + require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.peak"), []byte("4096\n"), 0o666)) + return &runner_pb.RunResponse{ + ExitCode: 7, + }, nil + }, + ) response, err := wrappedRunner.Run(context.Background(), request) require.NoError(t, err) @@ -43,62 +54,115 @@ func TestCgroupResourceUsageSamplingRunnerAppendsResourceUsage(t *testing.T) { require.Len(t, response.ResourceUsage, 1) var got resourceusage.CgroupResourceUsage require.NoError(t, response.ResourceUsage[0].UnmarshalTo(&got)) - require.Equal(t, cgroupStatsReader.usage.GetMemoryEventsOomKill(), got.GetMemoryEventsOomKill()) - require.Equal(t, cgroupStatsReader.usage.GetMemoryPeakBytes(), got.GetMemoryPeakBytes()) - require.Equal(t, cgroupStatsReader.usage.GetPsiCpuSome().AsDuration(), got.GetPsiCpuSome().AsDuration()) - require.Equal(t, cgroupStatsReader.usage.GetPsiCpuFull().AsDuration(), got.GetPsiCpuFull().AsDuration()) - require.True(t, cgroupStatsReader.closed) + require.Equal(t, int64(1), got.GetMemoryEventsHigh()) + require.Equal(t, int64(4096), got.GetMemoryPeak()) + require.Equal(t, 123*time.Microsecond, got.GetCpuPressureSomeTotal().AsDuration()) + require.Equal(t, 45*time.Microsecond, got.GetCpuPressureFullTotal().AsDuration()) } -func TestCgroupResourceUsageSamplingRunnerReadErrorPreservesRunResult(t *testing.T) { +func TestCgroupResourceUsageSamplingRunnerReadErrorIsPropagated(t *testing.T) { ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) - cgroupStatsReader := &fixedCgroupStatsReader{ - err: errors.New("read failed"), - } - wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { - return cgroupStatsReader, nil - }) + cgroupPath := createTestCgroup(t) + wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) request := &runner_pb.RunRequest{} baseResponse := &runner_pb.RunResponse{ExitCode: 7} - baseRunner.EXPECT().Run(gomock.Any(), request).Return(baseResponse, nil) + baseRunner.EXPECT().Run(gomock.Any(), request).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + require.NoError(t, os.Remove(filepath.Join(cgroupPath, "memory.events"))) + return baseResponse, nil + }, + ) response, err := wrappedRunner.Run(context.Background(), request) - require.NoError(t, err) + require.Error(t, err) + require.Contains(t, status.Convert(err).Message(), "Failed to read cgroup stats") require.Same(t, baseResponse, response) require.Empty(t, response.ResourceUsage) - require.True(t, cgroupStatsReader.closed) } func TestCgroupResourceUsageSamplingRunnerRunErrorDoesNotReadCgroupUsage(t *testing.T) { ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) - cgroupStatsReader := &fixedCgroupStatsReader{ - usage: &resourceusage.CgroupResourceUsage{ - MemoryPeakBytes: 4096, - }, - } - wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { - return cgroupStatsReader, nil - }) + cgroupPath := createTestCgroup(t) + wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) baseErr := status.Error(codes.FailedPrecondition, "failed") - baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).Return(nil, baseErr) + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + require.NoError(t, os.Remove(filepath.Join(cgroupPath, "memory.events"))) + return nil, baseErr + }, + ) response, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) require.Nil(t, response) require.Equal(t, baseErr, err) - require.False(t, cgroupStatsReader.read) - require.True(t, cgroupStatsReader.closed) +} + +func TestCgroupResourceUsageSamplingRunnerSystemOOMKillIsUnavailable(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupPath := createTestCgroup(t) + wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.events"), []byte(` +low 0 +high 0 +max 0 +oom 0 +oom_kill 1 +oom_group_kill 0 +`), 0o666)) + return &runner_pb.RunResponse{ + ExitCode: 0, + }, nil + }, + ) + + response, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Equal(t, "An action process was OOM-killed without the action reaching its cgroup memory limit", status.Convert(err).Message()) + require.NotNil(t, response) + require.Len(t, response.ResourceUsage, 1) +} + +func TestCgroupResourceUsageSamplingRunnerCgroupOOMKillIsActionResult(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupPath := createTestCgroup(t) + wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.events"), []byte(` +low 0 +high 0 +max 0 +oom 1 +oom_kill 1 +oom_group_kill 0 +`), 0o666)) + return &runner_pb.RunResponse{ + ExitCode: 0, + }, nil + }, + ) + + response, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + require.NoError(t, err) + require.Equal(t, int64(0), response.ExitCode) + require.Len(t, response.ResourceUsage, 1) } func TestCgroupResourceUsageSamplingRunnerRejectsConcurrentRun(t *testing.T) { ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) - wrappedRunner := newCgroupResourceUsageSamplingRunner(baseRunner, func() (scopedCgroupStatsReader, error) { - return noopCgroupStatsReader{}, nil - }) + cgroupPath := createTestCgroup(t) + wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) started := make(chan struct{}) release := make(chan struct{}) @@ -107,7 +171,8 @@ func TestCgroupResourceUsageSamplingRunnerRejectsConcurrentRun(t *testing.T) { close(started) <-release return &runner_pb.RunResponse{}, nil - }) + }, + ) firstRunErr := make(chan error, 1) go func() { @@ -146,29 +211,35 @@ func TestCgroupResourceUsageSamplingRunnerRejectsConcurrentRun(t *testing.T) { } } -type fixedCgroupStatsReader struct { - usage *resourceusage.CgroupResourceUsage - err error - read bool - closed bool -} - -func (r *fixedCgroupStatsReader) Close() error { - r.closed = true - return nil -} - -func (r *fixedCgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { - r.read = true - return r.usage, r.err +func newTestCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer, cgroupPath string) runner_pb.RunnerServer { + return runner.NewCgroupResourceUsageSamplingRunner(base, cgroupPath) } -type noopCgroupStatsReader struct{} - -func (noopCgroupStatsReader) Close() error { - return nil -} - -func (noopCgroupStatsReader) Read() (*resourceusage.CgroupResourceUsage, error) { - return nil, nil +func createTestCgroup(t *testing.T) string { + cgroupPath := t.TempDir() + writeFile := func(name, contents string) { + require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, name), []byte(contents), 0o666)) + } + writeFile("memory.events", ` +low 0 +high 0 +max 0 +oom 0 +oom_kill 0 +oom_group_kill 0 +`) + writeFile("memory.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=0 +full avg10=0.00 avg60=0.00 avg300=0.00 total=0 +`) + writeFile("cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=0 +full avg10=0.00 avg60=0.00 avg300=0.00 total=0 +`) + writeFile("io.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=0 +full avg10=0.00 avg60=0.00 avg300=0.00 total=0 +`) + writeFile("memory.peak", "0\n") + return cgroupPath } From 8715eec282f8764a4303a3dede518282914e2bba Mon Sep 17 00:00:00 2001 From: Lauri Peltonen Date: Wed, 10 Jun 2026 12:55:16 +0300 Subject: [PATCH 3/3] Refactor the whole feature to be Linux-specific Trying to have a separation between generic CgroupResourceUsageSamplingRunner and Linux-specific Cgroup reader started to feel off, let's make the whole feature Linux-only. --- cmd/bb_runner/main.go | 5 +- pkg/runner/BUILD.bazel | 14 +- pkg/runner/cgroup_linux_test.go | 237 -------------- pkg/runner/cgroup_other.go | 26 -- .../cgroup_resource_usage_sampling_runner.go | 90 ------ ...p_resource_usage_sampling_runner_linux.go} | 131 ++++++-- ...up_resource_usage_sampling_runner_other.go | 17 + ...oup_resource_usage_sampling_runner_test.go | 294 ++++++++++++++---- 8 files changed, 366 insertions(+), 448 deletions(-) delete mode 100644 pkg/runner/cgroup_linux_test.go delete mode 100644 pkg/runner/cgroup_other.go delete mode 100644 pkg/runner/cgroup_resource_usage_sampling_runner.go rename pkg/runner/{cgroup_linux.go => cgroup_resource_usage_sampling_runner_linux.go} (66%) create mode 100644 pkg/runner/cgroup_resource_usage_sampling_runner_other.go diff --git a/cmd/bb_runner/main.go b/cmd/bb_runner/main.go index 4078feb4..de366add 100644 --- a/cmd/bb_runner/main.go +++ b/cmd/bb_runner/main.go @@ -71,11 +71,10 @@ func main() { configuration.SetTmpdirEnvironmentVariable, ) if configuration.SampleCgroupResourceUsage { - cgroupfsPath, err := runner.ResolveCurrentCgroupfsPath() + r, err = runner.NewCgroupResourceUsageSamplingRunner(r) if err != nil { - return util.StatusWrap(err, "Failed to resolve current cgroupfs path") + return util.StatusWrap(err, "Failed to create cgroup resource usage sampling runner") } - r = runner.NewCgroupResourceUsageSamplingRunner(r, cgroupfsPath) } // Let bb_runner replace temporary directories with symbolic diff --git a/pkg/runner/BUILD.bazel b/pkg/runner/BUILD.bazel index 12a5de33..4d786539 100644 --- a/pkg/runner/BUILD.bazel +++ b/pkg/runner/BUILD.bazel @@ -4,9 +4,8 @@ go_library( name = "runner", srcs = [ "apple_xcode_resolving_runner.go", - "cgroup_linux.go", - "cgroup_other.go", - "cgroup_resource_usage_sampling_runner.go", + "cgroup_resource_usage_sampling_runner_linux.go", + "cgroup_resource_usage_sampling_runner_other.go", "clean_runner.go", "local_runner.go", "local_runner_darwin.go", @@ -22,7 +21,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/cleaner", - "//pkg/proto/resourceusage", "//pkg/proto/runner", "//pkg/proto/tmp_installer", "@com_github_buildbarn_bb_storage//pkg/filesystem", @@ -35,26 +33,32 @@ go_library( "@org_golang_google_protobuf//types/known/emptypb", ] + select({ "@rules_go//go/platform:android": [ + "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:darwin": [ + "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:freebsd": [ + "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:ios": [ + "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:linux": [ + "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//unix", ], "@rules_go//go/platform:windows": [ + "//pkg/proto/resourceusage", "@org_golang_google_protobuf//types/known/durationpb", "@org_golang_x_sys//windows", ], @@ -72,11 +76,9 @@ go_test( "temporary_directory_symlinking_runner_test.go", ] + select({ "@rules_go//go/platform:android": [ - "cgroup_linux_test.go", "cgroup_resource_usage_sampling_runner_test.go", ], "@rules_go//go/platform:linux": [ - "cgroup_linux_test.go", "cgroup_resource_usage_sampling_runner_test.go", ], "//conditions:default": [], diff --git a/pkg/runner/cgroup_linux_test.go b/pkg/runner/cgroup_linux_test.go deleted file mode 100644 index 78e6c76f..00000000 --- a/pkg/runner/cgroup_linux_test.go +++ /dev/null @@ -1,237 +0,0 @@ -//go:build linux - -package runner_test - -import ( - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/buildbarn/bb-remote-execution/pkg/runner" - "github.com/stretchr/testify/require" -) - -func TestCgroupResourceUsageReaderReportsDeltasFromCgroupFiles(t *testing.T) { - cgroupPath := t.TempDir() - - writeFile(t, cgroupPath, "memory.events", ` -low 1 -high 2 -max 3 -oom 4 -oom_kill 5 -oom_group_kill 6 -`) - writeFile(t, cgroupPath, "memory.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=100 -full avg10=0.00 avg60=0.00 avg300=0.00 total=200 -`) - writeFile(t, cgroupPath, "cpu.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=300 -full avg10=0.00 avg60=0.00 avg300=0.00 total=310 -`) - writeFile(t, cgroupPath, "io.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=400 -full avg10=0.00 avg60=0.00 avg300=0.00 total=500 -`) - writeFile(t, cgroupPath, "memory.peak", "0\n") - - reader, err := runner.NewCgroupResourceUsageReaderFromPath(cgroupPath) - require.NoError(t, err) - defer func() { - require.NoError(t, reader.Close()) - }() - - writeFile(t, cgroupPath, "memory.events", ` -low 11 -high 22 -max 33 -oom 44 -oom_kill 55 -oom_group_kill 66 -`) - writeFile(t, cgroupPath, "memory.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=160 -full avg10=0.00 avg60=0.00 avg300=0.00 total=290 -`) - writeFile(t, cgroupPath, "cpu.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=345 -full avg10=0.00 avg60=0.00 avg300=0.00 total=410 -`) - writeFile(t, cgroupPath, "io.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=480 -full avg10=0.00 avg60=0.00 avg300=0.00 total=610 -`) - writeFile(t, cgroupPath, "memory.peak", "4096\n") - - usage, err := reader.Read() - require.NoError(t, err) - - require.Equal(t, int64(10), usage.MemoryEventsLow) - require.Equal(t, int64(20), usage.MemoryEventsHigh) - require.Equal(t, int64(30), usage.MemoryEventsMax) - require.Equal(t, int64(40), usage.MemoryEventsOom) - require.Equal(t, int64(50), usage.MemoryEventsOomKill) - require.Equal(t, int64(60), usage.MemoryEventsOomGroupKill) - require.Equal(t, int64(4096), usage.MemoryPeak) - require.Equal(t, 60*time.Microsecond, usage.GetMemoryPressureSomeTotal().AsDuration()) - require.Equal(t, 90*time.Microsecond, usage.GetMemoryPressureFullTotal().AsDuration()) - require.Equal(t, 45*time.Microsecond, usage.GetCpuPressureSomeTotal().AsDuration()) - require.Equal(t, 100*time.Microsecond, usage.GetCpuPressureFullTotal().AsDuration()) - require.Equal(t, 80*time.Microsecond, usage.GetIoPressureSomeTotal().AsDuration()) - require.Equal(t, 110*time.Microsecond, usage.GetIoPressureFullTotal().AsDuration()) -} - -func TestCgroupResourceUsageReaderAllowsMissingOomGroupKillCounter(t *testing.T) { - cgroupPath := t.TempDir() - - writeFile(t, cgroupPath, "memory.events", ` -low 1 -high 2 -max 3 -oom 4 -oom_kill 5 -`) - writeFile(t, cgroupPath, "memory.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=100 -full avg10=0.00 avg60=0.00 avg300=0.00 total=200 -`) - writeFile(t, cgroupPath, "cpu.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=300 -full avg10=0.00 avg60=0.00 avg300=0.00 total=310 -`) - writeFile(t, cgroupPath, "io.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=400 -full avg10=0.00 avg60=0.00 avg300=0.00 total=500 -`) - writeFile(t, cgroupPath, "memory.peak", "0\n") - - reader, err := runner.NewCgroupResourceUsageReaderFromPath(cgroupPath) - require.NoError(t, err) - defer func() { - require.NoError(t, reader.Close()) - }() - - writeFile(t, cgroupPath, "memory.events", ` -low 11 -high 22 -max 33 -oom 44 -oom_kill 55 -`) - - usage, err := reader.Read() - require.NoError(t, err) - - require.Equal(t, int64(50), usage.MemoryEventsOomKill) - require.Equal(t, int64(0), usage.MemoryEventsOomGroupKill) -} - -func TestResolveCurrentCgroupfsPathUsesMatchingCgroup2MountRoot(t *testing.T) { - for _, testCase := range []struct { - name string - mountInfo string - cgroup string - wantPath string - }{ - { - name: "root mount", - mountInfo: ` -36 25 0:31 / %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -`, - cgroup: "0::/worker.slice/runner.scope\n", - wantPath: "worker.slice/runner.scope", - }, - { - name: "subtree mount", - mountInfo: ` -36 25 0:31 /unrelated /wrong rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -37 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -`, - cgroup: "0::/worker.slice/runner.scope\n", - wantPath: "runner.scope", - }, - { - name: "most specific mount root", - mountInfo: ` -36 25 0:31 / %[1]s/unrelated rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -37 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -`, - cgroup: "0::/worker.slice/runner.scope\n", - wantPath: "runner.scope", - }, - { - name: "current cgroup is mount root", - mountInfo: ` -36 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw -`, - cgroup: "0::/worker.slice\n", - wantPath: ".", - }, - } { - t.Run(testCase.name, func(t *testing.T) { - tempDir := t.TempDir() - cgroupfsPath := filepath.Join(tempDir, "cgroupfs") - resolvedCgroupPath := filepath.Join(cgroupfsPath, testCase.wantPath) - require.NoError(t, os.MkdirAll(resolvedCgroupPath, 0o777)) - writeInitialCgroupResourceUsageFiles(t, resolvedCgroupPath) - - mountInfoPath := filepath.Join(tempDir, "mountinfo") - procCgroupPath := filepath.Join(tempDir, "cgroup") - writeFile(t, tempDir, "mountinfo", fmt.Sprintf(testCase.mountInfo, cgroupfsPath)) - writeFile(t, tempDir, "cgroup", testCase.cgroup) - - gotCgroupfsPath, err := runner.ResolveCurrentCgroupfsPathFromProcFiles(procCgroupPath, mountInfoPath) - require.NoError(t, err) - require.Equal(t, resolvedCgroupPath, gotCgroupfsPath) - - reader, err := runner.NewCgroupResourceUsageReaderFromPath(gotCgroupfsPath) - require.NoError(t, err) - defer func() { - require.NoError(t, reader.Close()) - }() - - writeFile(t, resolvedCgroupPath, "memory.events", ` -low 11 -high 22 -max 33 -oom 44 -oom_kill 55 -oom_group_kill 66 -`) - usage, err := reader.Read() - require.NoError(t, err) - require.Equal(t, int64(10), usage.MemoryEventsLow) - }) - } -} - -func writeInitialCgroupResourceUsageFiles(t *testing.T, cgroupPath string) { - writeFile(t, cgroupPath, "memory.events", ` -low 1 -high 2 -max 3 -oom 4 -oom_kill 5 -oom_group_kill 6 -`) - writeFile(t, cgroupPath, "memory.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=100 -full avg10=0.00 avg60=0.00 avg300=0.00 total=200 -`) - writeFile(t, cgroupPath, "cpu.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=300 -full avg10=0.00 avg60=0.00 avg300=0.00 total=310 -`) - writeFile(t, cgroupPath, "io.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=400 -full avg10=0.00 avg60=0.00 avg300=0.00 total=500 -`) - writeFile(t, cgroupPath, "memory.peak", "0\n") -} - -func writeFile(t *testing.T, dir, name, contents string) { - require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(contents), 0o666)) -} diff --git a/pkg/runner/cgroup_other.go b/pkg/runner/cgroup_other.go deleted file mode 100644 index b953891e..00000000 --- a/pkg/runner/cgroup_other.go +++ /dev/null @@ -1,26 +0,0 @@ -//go:build !linux - -package runner - -import ( - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -// ResolveCurrentCgroupfsPath returns an error, as cgroup resource usage -// sampling is only supported on Linux. -func ResolveCurrentCgroupfsPath() (string, error) { - return "", status.Error(codes.Unimplemented, "cgroup resource usage sampling is only supported on Linux") -} - -// ResolveCurrentCgroupfsPathFromProcFiles returns an error, as cgroup resource -// usage sampling is only supported on Linux. -func ResolveCurrentCgroupfsPathFromProcFiles(procCgroupPath, procMountInfoPath string) (string, error) { - return "", status.Error(codes.Unimplemented, "cgroup resource usage sampling is only supported on Linux") -} - -// NewCgroupResourceUsageReaderFromPath returns an error, as cgroup resource -// usage sampling is only supported on Linux. -func NewCgroupResourceUsageReaderFromPath(cgroupPath string) (CgroupResourceUsageReader, error) { - return nil, status.Error(codes.Unimplemented, "cgroup resource usage sampling is only supported on Linux") -} diff --git a/pkg/runner/cgroup_resource_usage_sampling_runner.go b/pkg/runner/cgroup_resource_usage_sampling_runner.go deleted file mode 100644 index 242f955c..00000000 --- a/pkg/runner/cgroup_resource_usage_sampling_runner.go +++ /dev/null @@ -1,90 +0,0 @@ -package runner - -import ( - "context" - "sync/atomic" - - "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" - runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" - "github.com/buildbarn/bb-storage/pkg/util" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/anypb" -) - -// CgroupResourceUsageReader reads cgroup resource usage deltas for a single -// action. -type CgroupResourceUsageReader interface { - Close() error - Read() (*resourceusage.CgroupResourceUsage, error) -} - -type cgroupResourceUsageSamplingRunner struct { - runner_pb.RunnerServer - newCgroupResourceUsageReader func() (CgroupResourceUsageReader, error) - activeRun atomic.Bool -} - -// NewCgroupResourceUsageSamplingRunner creates a decorator for RunnerServer -// that samples cgroup v2 resource usage counters around actions and appends -// them to successful Run() responses. -// -// Sampled cgroup counters are only meaningful as per-action deltas if -// bb_worker sends at most one action to the runner at a time -// (RunnerConfiguration.concurrency == 1), and if the runner is deployed in a -// cgroup whose other activity is acceptable to include in the reported usage. -// -// cgroupfsPath is the cgroup v2 filesystem directory whose counters should be -// sampled. A fresh reader is created for each Run() request to capture the -// baseline counters for that action. -func NewCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer, cgroupfsPath string) runner_pb.RunnerServer { - return &cgroupResourceUsageSamplingRunner{ - RunnerServer: base, - newCgroupResourceUsageReader: func() (CgroupResourceUsageReader, error) { - return NewCgroupResourceUsageReaderFromPath(cgroupfsPath) - }, - } -} - -func (r *cgroupResourceUsageSamplingRunner) Run(ctx context.Context, request *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { - if !r.activeRun.CompareAndSwap(false, true) { - return nil, status.Error(codes.Internal, "cgroup resource usage sampling requires an exclusive runner cgroup, but concurrent Run() calls were observed") - } - defer r.activeRun.Store(false) - - cgroupResourceUsageReader, err := r.newCgroupResourceUsageReader() - if err != nil { - return nil, util.StatusWrap(err, "Failed to create cgroup resource usage reader") - } - defer func() { - _ = cgroupResourceUsageReader.Close() - }() - - response, err := r.RunnerServer.Run(ctx, request) - if err != nil { - return response, err - } - - cgroupUsage, err := cgroupResourceUsageReader.Read() - if err != nil { - return response, util.StatusWrap(err, "Failed to read cgroup stats") - } - if cgroupUsage == nil { - return response, nil - } - cgroupAny, err := anypb.New(cgroupUsage) - if err != nil { - return response, util.StatusWrap(err, "Failed to marshal cgroup resource usage") - } - if response != nil { - response.ResourceUsage = append(response.ResourceUsage, cgroupAny) - } - if cgroupUsage.MemoryEventsOomKill > 0 && cgroupUsage.MemoryEventsOom == 0 { - // The cgroup did not reach its memory limit, so the OOM kill likely - // came from system-level memory pressure, such as node memory - // overcommitment. Treat this as retryable infrastructure failure. - return response, status.Error(codes.Unavailable, "An action process was OOM-killed without the action reaching its cgroup memory limit") - } - return response, nil -} diff --git a/pkg/runner/cgroup_linux.go b/pkg/runner/cgroup_resource_usage_sampling_runner_linux.go similarity index 66% rename from pkg/runner/cgroup_linux.go rename to pkg/runner/cgroup_resource_usage_sampling_runner_linux.go index 46c59269..ba2a2dc4 100644 --- a/pkg/runner/cgroup_linux.go +++ b/pkg/runner/cgroup_resource_usage_sampling_runner_linux.go @@ -1,25 +1,111 @@ //go:build linux +// +build linux package runner import ( "bufio" + "context" "fmt" "io" "os" "path/filepath" "strconv" "strings" + "sync/atomic" "time" "github.com/buildbarn/bb-remote-execution/pkg/proto/resourceusage" + runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/durationpb" ) +type cgroupResourceUsageSamplingRunner struct { + runner_pb.RunnerServer + cgroupfsPath string + activeRun atomic.Bool +} + +// NewCgroupResourceUsageSamplingRunner creates a decorator for RunnerServer +// that samples cgroup v2 resource usage counters around actions and appends +// them to successful Run() responses. +// +// Sampled cgroup counters are only meaningful as per-action deltas if +// bb_worker sends at most one action to the runner at a time +// (RunnerConfiguration.concurrency == 1), and if the runner is deployed in a +// cgroup whose other activity is acceptable to include in the reported usage. +func NewCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer) (runner_pb.RunnerServer, error) { + cgroupfsPath, err := ResolveCurrentCgroupfsPathFromProcFiles("/proc/self/cgroup", "/proc/self/mountinfo") + if err != nil { + return nil, err + } + return NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(base, cgroupfsPath), nil +} + +// NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath creates a decorator for +// RunnerServer that samples cgroup v2 resource usage counters from +// cgroupfsPath. +// +// cgroupfsPath is the cgroup v2 filesystem directory whose counters should be +// sampled. Counter deltas are measured over each Run() request. +func NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(base runner_pb.RunnerServer, cgroupfsPath string) runner_pb.RunnerServer { + return &cgroupResourceUsageSamplingRunner{ + RunnerServer: base, + cgroupfsPath: cgroupfsPath, + } +} + +func (r *cgroupResourceUsageSamplingRunner) Run(ctx context.Context, request *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + if !r.activeRun.CompareAndSwap(false, true) { + return nil, status.Error(codes.Internal, "cgroup resource usage sampling requires an exclusive runner cgroup, but concurrent Run() calls were observed") + } + defer r.activeRun.Store(false) + + cgroupResourceUsageReader, err := newCgroupResourceUsageReader(r.cgroupfsPath) + if err != nil { + return nil, util.StatusWrap(err, "Failed to create cgroup resource usage reader") + } + defer func() { + _ = cgroupResourceUsageReader.close() + }() + + response, err := r.RunnerServer.Run(ctx, request) + if err != nil { + return response, err + } + + cgroupUsage, err := cgroupResourceUsageReader.read() + if err != nil { + return response, util.StatusWrap(err, "Failed to read cgroup stats") + } + if cgroupUsage == nil { + return response, nil + } + cgroupAny, err := anypb.New(cgroupUsage) + if err != nil { + return response, util.StatusWrap(err, "Failed to marshal cgroup resource usage") + } + if response != nil { + response.ResourceUsage = append(response.ResourceUsage, cgroupAny) + } + if cgroupUsage.MemoryEventsOomKill > 0 && cgroupUsage.MemoryEventsOom == 0 { + // The cgroup did not reach its memory limit, so the OOM kill likely + // came from system-level memory pressure, such as node memory + // overcommitment. Treat this as retryable infrastructure failure. + return response, status.Error(codes.Unavailable, "An action process was OOM-killed without the action reaching its cgroup memory limit") + } + return response, nil +} + type cgroupResourceUsageReader struct { - cgroupPath string + cgroupfsPath string - // memory.events counters + // Initial memory.events counter values. eventsLow int64 eventsHigh int64 eventsMax int64 @@ -27,7 +113,7 @@ type cgroupResourceUsageReader struct { eventsOOMKill int64 eventsOOMGroupKill int64 - // PSI total values + // Initial PSI total stall durations, in microseconds. psiMemorySomeUS int64 psiMemoryFullUS int64 psiCPUSomeUS int64 @@ -38,32 +124,30 @@ type cgroupResourceUsageReader struct { memoryPeakFile *os.File } -// NewCgroupResourceUsageReaderFromPath creates a reader that samples resource -// usage counters from the cgroup v2 directory at cgroupPath. -func NewCgroupResourceUsageReaderFromPath(cgroupPath string) (CgroupResourceUsageReader, error) { - events, err := readCgroupKeyValues(filepath.Join(cgroupPath, "memory.events")) +func newCgroupResourceUsageReader(cgroupfsPath string) (*cgroupResourceUsageReader, error) { + events, err := readCgroupKeyValues(filepath.Join(cgroupfsPath, "memory.events")) if err != nil { return nil, fmt.Errorf("failed to read memory.events: %w", err) } - memorySome, memoryFull, err := parsePSITotals(filepath.Join(cgroupPath, "memory.pressure")) + memorySome, memoryFull, err := parsePSITotals(filepath.Join(cgroupfsPath, "memory.pressure")) if err != nil { return nil, fmt.Errorf("failed to read memory.pressure: %w", err) } - cpuSome, cpuFull, err := parsePSITotals(filepath.Join(cgroupPath, "cpu.pressure")) + cpuSome, cpuFull, err := parsePSITotals(filepath.Join(cgroupfsPath, "cpu.pressure")) if err != nil { return nil, fmt.Errorf("failed to read cpu.pressure: %w", err) } - ioSome, ioFull, err := parsePSITotals(filepath.Join(cgroupPath, "io.pressure")) + ioSome, ioFull, err := parsePSITotals(filepath.Join(cgroupfsPath, "io.pressure")) if err != nil { return nil, fmt.Errorf("failed to read io.pressure: %w", err) } // memory.peak is optional. If it is unavailable or cannot be reset/read, // MemoryPeak remains 0 to indicate that no peak data was collected. - memoryPeakFile := openAndResetCgroupMemoryPeak(filepath.Join(cgroupPath, "memory.peak")) + memoryPeakFile := openAndResetCgroupMemoryPeak(filepath.Join(cgroupfsPath, "memory.peak")) reader := &cgroupResourceUsageReader{ - cgroupPath: cgroupPath, + cgroupfsPath: cgroupfsPath, eventsLow: events["low"], eventsHigh: events["high"], @@ -84,28 +168,28 @@ func NewCgroupResourceUsageReaderFromPath(cgroupPath string) (CgroupResourceUsag return reader, nil } -func (r *cgroupResourceUsageReader) Close() error { +func (r *cgroupResourceUsageReader) close() error { if r == nil || r.memoryPeakFile == nil { return nil } return r.memoryPeakFile.Close() } -func (r *cgroupResourceUsageReader) Read() (*resourceusage.CgroupResourceUsage, error) { - events, err := readCgroupKeyValues(filepath.Join(r.cgroupPath, "memory.events")) +func (r *cgroupResourceUsageReader) read() (*resourceusage.CgroupResourceUsage, error) { + events, err := readCgroupKeyValues(filepath.Join(r.cgroupfsPath, "memory.events")) if err != nil { return nil, fmt.Errorf("failed to read memory.events: %w", err) } - memorySome, memoryFull, err := parsePSITotals(filepath.Join(r.cgroupPath, "memory.pressure")) + memorySome, memoryFull, err := parsePSITotals(filepath.Join(r.cgroupfsPath, "memory.pressure")) if err != nil { return nil, fmt.Errorf("failed to read memory.pressure: %w", err) } - cpuSome, cpuFull, err := parsePSITotals(filepath.Join(r.cgroupPath, "cpu.pressure")) + cpuSome, cpuFull, err := parsePSITotals(filepath.Join(r.cgroupfsPath, "cpu.pressure")) if err != nil { return nil, fmt.Errorf("failed to read cpu.pressure: %w", err) } - ioSome, ioFull, err := parsePSITotals(filepath.Join(r.cgroupPath, "io.pressure")) + ioSome, ioFull, err := parsePSITotals(filepath.Join(r.cgroupfsPath, "io.pressure")) if err != nil { return nil, fmt.Errorf("failed to read io.pressure: %w", err) } @@ -242,15 +326,8 @@ func parsePSITotals(path string) (someUS, fullUS int64, err error) { return someUS, fullUS, nil } -// ResolveCurrentCgroupfsPath resolves the cgroup v2 filesystem directory of -// the current process. -func ResolveCurrentCgroupfsPath() (string, error) { - return ResolveCurrentCgroupfsPathFromProcFiles("/proc/self/cgroup", "/proc/self/mountinfo") -} - -// ResolveCurrentCgroupfsPathFromProcFiles reads procCgroupPath and -// procMountInfoPath to resolve the cgroup v2 filesystem directory of the -// current process. +// ResolveCurrentCgroupfsPathFromProcFiles resolves the cgroup v2 filesystem +// directory for the process described by procCgroupPath and procMountInfoPath. func ResolveCurrentCgroupfsPathFromProcFiles(procCgroupPath, procMountInfoPath string) (string, error) { currentCgroupPath, err := readCurrentCgroupRelativePath(procCgroupPath) if err != nil { diff --git a/pkg/runner/cgroup_resource_usage_sampling_runner_other.go b/pkg/runner/cgroup_resource_usage_sampling_runner_other.go new file mode 100644 index 00000000..fe6651e3 --- /dev/null +++ b/pkg/runner/cgroup_resource_usage_sampling_runner_other.go @@ -0,0 +1,17 @@ +//go:build !linux +// +build !linux + +package runner + +import ( + runner_pb "github.com/buildbarn/bb-remote-execution/pkg/proto/runner" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// NewCgroupResourceUsageSamplingRunner returns an error, as cgroup resource +// usage sampling is only supported on Linux. +func NewCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer) (runner_pb.RunnerServer, error) { + return nil, status.Error(codes.Unimplemented, "cgroup resource usage sampling is only supported on Linux") +} diff --git a/pkg/runner/cgroup_resource_usage_sampling_runner_test.go b/pkg/runner/cgroup_resource_usage_sampling_runner_test.go index d6f1df1c..04ba1f3f 100644 --- a/pkg/runner/cgroup_resource_usage_sampling_runner_test.go +++ b/pkg/runner/cgroup_resource_usage_sampling_runner_test.go @@ -1,9 +1,11 @@ //go:build linux +// +build linux package runner_test import ( "context" + "fmt" "os" "path/filepath" "testing" @@ -20,28 +22,181 @@ import ( "google.golang.org/grpc/status" ) +func TestCgroupResourceUsageSamplingRunnerReportsDeltasFromCgroupFiles(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupPath := createTestCgroup(t) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) + + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + writeFile(t, cgroupPath, "memory.events", ` +low 11 +high 22 +max 33 +oom 44 +oom_kill 55 +oom_group_kill 66 +`) + writeFile(t, cgroupPath, "memory.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=160 +full avg10=0.00 avg60=0.00 avg300=0.00 total=290 +`) + writeFile(t, cgroupPath, "cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=345 +full avg10=0.00 avg60=0.00 avg300=0.00 total=410 +`) + writeFile(t, cgroupPath, "io.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=480 +full avg10=0.00 avg60=0.00 avg300=0.00 total=610 +`) + writeFile(t, cgroupPath, "memory.peak", "4096\n") + return &runner_pb.RunResponse{ + ExitCode: 7, + }, nil + }, + ) + + response, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + require.NoError(t, err) + require.Equal(t, int64(7), response.ExitCode) + require.Len(t, response.ResourceUsage, 1) + var usage resourceusage.CgroupResourceUsage + require.NoError(t, response.ResourceUsage[0].UnmarshalTo(&usage)) + + require.Equal(t, int64(10), usage.MemoryEventsLow) + require.Equal(t, int64(20), usage.MemoryEventsHigh) + require.Equal(t, int64(30), usage.MemoryEventsMax) + require.Equal(t, int64(40), usage.MemoryEventsOom) + require.Equal(t, int64(50), usage.MemoryEventsOomKill) + require.Equal(t, int64(60), usage.MemoryEventsOomGroupKill) + require.Equal(t, int64(4096), usage.MemoryPeak) + require.Equal(t, 60*time.Microsecond, usage.GetMemoryPressureSomeTotal().AsDuration()) + require.Equal(t, 90*time.Microsecond, usage.GetMemoryPressureFullTotal().AsDuration()) + require.Equal(t, 45*time.Microsecond, usage.GetCpuPressureSomeTotal().AsDuration()) + require.Equal(t, 100*time.Microsecond, usage.GetCpuPressureFullTotal().AsDuration()) + require.Equal(t, 80*time.Microsecond, usage.GetIoPressureSomeTotal().AsDuration()) + require.Equal(t, 110*time.Microsecond, usage.GetIoPressureFullTotal().AsDuration()) +} + +func TestCgroupResourceUsageSamplingRunnerAllowsMissingOomGroupKillCounter(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupPath := createTestCgroup(t) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) + + writeFile(t, cgroupPath, "memory.events", ` +low 1 +high 2 +max 3 +oom 4 +oom_kill 5 +`) + + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + writeFile(t, cgroupPath, "memory.events", ` +low 11 +high 22 +max 33 +oom 44 +oom_kill 55 +`) + return &runner_pb.RunResponse{}, nil + }, + ) + + response, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + require.NoError(t, err) + require.Len(t, response.ResourceUsage, 1) + var usage resourceusage.CgroupResourceUsage + require.NoError(t, response.ResourceUsage[0].UnmarshalTo(&usage)) + + require.Equal(t, int64(50), usage.MemoryEventsOomKill) + require.Equal(t, int64(0), usage.MemoryEventsOomGroupKill) +} + +func TestResolveCurrentCgroupfsPathUsesMatchingCgroup2MountRoot(t *testing.T) { + for _, testCase := range []struct { + name string + mountInfo string + cgroup string + wantPath string + }{ + { + name: "root mount", + mountInfo: ` +36 25 0:31 / %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice/runner.scope\n", + wantPath: "worker.slice/runner.scope", + }, + { + name: "subtree mount", + mountInfo: ` +36 25 0:31 /unrelated /wrong rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +37 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice/runner.scope\n", + wantPath: "runner.scope", + }, + { + name: "most specific mount root", + mountInfo: ` +36 25 0:31 / %[1]s/unrelated rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +37 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice/runner.scope\n", + wantPath: "runner.scope", + }, + { + name: "current cgroup is mount root", + mountInfo: ` +36 25 0:31 /worker.slice %[1]s rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +`, + cgroup: "0::/worker.slice\n", + wantPath: ".", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + tempDir := t.TempDir() + cgroupfsPath := filepath.Join(tempDir, "cgroupfs") + resolvedCgroupPath := filepath.Join(cgroupfsPath, testCase.wantPath) + + mountInfoPath := filepath.Join(tempDir, "mountinfo") + procCgroupPath := filepath.Join(tempDir, "cgroup") + writeFile(t, tempDir, "mountinfo", fmt.Sprintf(testCase.mountInfo, cgroupfsPath)) + writeFile(t, tempDir, "cgroup", testCase.cgroup) + + gotCgroupfsPath, err := runner.ResolveCurrentCgroupfsPathFromProcFiles(procCgroupPath, mountInfoPath) + require.NoError(t, err) + require.Equal(t, resolvedCgroupPath, gotCgroupfsPath) + }) + } +} + func TestCgroupResourceUsageSamplingRunnerAppendsResourceUsage(t *testing.T) { ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) cgroupPath := createTestCgroup(t) - wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) request := &runner_pb.RunRequest{} baseRunner.EXPECT().Run(gomock.Any(), request).DoAndReturn( func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { - require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.events"), []byte(` -low 0 -high 1 -max 0 -oom 0 -oom_kill 0 -oom_group_kill 0 -`), 0o666)) - require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "cpu.pressure"), []byte(` -some avg10=0.00 avg60=0.00 avg300=0.00 total=123 -full avg10=0.00 avg60=0.00 avg300=0.00 total=45 -`), 0o666)) - require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.peak"), []byte("4096\n"), 0o666)) + writeFile(t, cgroupPath, "memory.events", ` +low 1 +high 3 +max 3 +oom 4 +oom_kill 5 +oom_group_kill 6 +`) + writeFile(t, cgroupPath, "cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=423 +full avg10=0.00 avg60=0.00 avg300=0.00 total=355 +`) + writeFile(t, cgroupPath, "memory.peak", "4096\n") return &runner_pb.RunResponse{ ExitCode: 7, }, nil @@ -60,11 +215,35 @@ full avg10=0.00 avg60=0.00 avg300=0.00 total=45 require.Equal(t, 45*time.Microsecond, got.GetCpuPressureFullTotal().AsDuration()) } +func TestCgroupResourceUsageSamplingRunnerResetsMemoryPeakBeforeRun(t *testing.T) { + ctrl := gomock.NewController(t) + baseRunner := mock.NewMockRunnerServer(ctrl) + cgroupPath := createTestCgroup(t) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) + + baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { + memoryPeak, err := os.ReadFile(filepath.Join(cgroupPath, "memory.peak")) + require.NoError(t, err) + require.Equal(t, "1\n", string(memoryPeak)) + writeFile(t, cgroupPath, "memory.peak", "4096\n") + return &runner_pb.RunResponse{}, nil + }, + ) + + response, err := wrappedRunner.Run(context.Background(), &runner_pb.RunRequest{}) + require.NoError(t, err) + require.Len(t, response.ResourceUsage, 1) + var got resourceusage.CgroupResourceUsage + require.NoError(t, response.ResourceUsage[0].UnmarshalTo(&got)) + require.Equal(t, int64(4096), got.GetMemoryPeak()) +} + func TestCgroupResourceUsageSamplingRunnerReadErrorIsPropagated(t *testing.T) { ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) cgroupPath := createTestCgroup(t) - wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) request := &runner_pb.RunRequest{} baseResponse := &runner_pb.RunResponse{ExitCode: 7} @@ -86,7 +265,7 @@ func TestCgroupResourceUsageSamplingRunnerRunErrorDoesNotReadCgroupUsage(t *test ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) cgroupPath := createTestCgroup(t) - wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) baseErr := status.Error(codes.FailedPrecondition, "failed") baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( @@ -105,18 +284,18 @@ func TestCgroupResourceUsageSamplingRunnerSystemOOMKillIsUnavailable(t *testing. ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) cgroupPath := createTestCgroup(t) - wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { - require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.events"), []byte(` -low 0 -high 0 -max 0 -oom 0 -oom_kill 1 -oom_group_kill 0 -`), 0o666)) + writeFile(t, cgroupPath, "memory.events", ` +low 1 +high 2 +max 3 +oom 4 +oom_kill 6 +oom_group_kill 6 +`) return &runner_pb.RunResponse{ ExitCode: 0, }, nil @@ -134,18 +313,18 @@ func TestCgroupResourceUsageSamplingRunnerCgroupOOMKillIsActionResult(t *testing ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) cgroupPath := createTestCgroup(t) - wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) baseRunner.EXPECT().Run(gomock.Any(), gomock.Any()).DoAndReturn( func(context.Context, *runner_pb.RunRequest) (*runner_pb.RunResponse, error) { - require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, "memory.events"), []byte(` -low 0 -high 0 -max 0 -oom 1 -oom_kill 1 -oom_group_kill 0 -`), 0o666)) + writeFile(t, cgroupPath, "memory.events", ` +low 1 +high 2 +max 3 +oom 5 +oom_kill 6 +oom_group_kill 6 +`) return &runner_pb.RunResponse{ ExitCode: 0, }, nil @@ -162,7 +341,7 @@ func TestCgroupResourceUsageSamplingRunnerRejectsConcurrentRun(t *testing.T) { ctrl := gomock.NewController(t) baseRunner := mock.NewMockRunnerServer(ctrl) cgroupPath := createTestCgroup(t) - wrappedRunner := newTestCgroupResourceUsageSamplingRunner(baseRunner, cgroupPath) + wrappedRunner := runner.NewCgroupResourceUsageSamplingRunnerWithCgroupfsPath(baseRunner, cgroupPath) started := make(chan struct{}) release := make(chan struct{}) @@ -211,35 +390,32 @@ func TestCgroupResourceUsageSamplingRunnerRejectsConcurrentRun(t *testing.T) { } } -func newTestCgroupResourceUsageSamplingRunner(base runner_pb.RunnerServer, cgroupPath string) runner_pb.RunnerServer { - return runner.NewCgroupResourceUsageSamplingRunner(base, cgroupPath) -} - func createTestCgroup(t *testing.T) string { cgroupPath := t.TempDir() - writeFile := func(name, contents string) { - require.NoError(t, os.WriteFile(filepath.Join(cgroupPath, name), []byte(contents), 0o666)) - } - writeFile("memory.events", ` -low 0 -high 0 -max 0 -oom 0 -oom_kill 0 -oom_group_kill 0 + writeFile(t, cgroupPath, "memory.events", ` +low 1 +high 2 +max 3 +oom 4 +oom_kill 5 +oom_group_kill 6 `) - writeFile("memory.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=0 -full avg10=0.00 avg60=0.00 avg300=0.00 total=0 + writeFile(t, cgroupPath, "memory.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=100 +full avg10=0.00 avg60=0.00 avg300=0.00 total=200 `) - writeFile("cpu.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=0 -full avg10=0.00 avg60=0.00 avg300=0.00 total=0 + writeFile(t, cgroupPath, "cpu.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=300 +full avg10=0.00 avg60=0.00 avg300=0.00 total=310 `) - writeFile("io.pressure", ` -some avg10=0.00 avg60=0.00 avg300=0.00 total=0 -full avg10=0.00 avg60=0.00 avg300=0.00 total=0 + writeFile(t, cgroupPath, "io.pressure", ` +some avg10=0.00 avg60=0.00 avg300=0.00 total=400 +full avg10=0.00 avg60=0.00 avg300=0.00 total=500 `) - writeFile("memory.peak", "0\n") + writeFile(t, cgroupPath, "memory.peak", "0\n") return cgroupPath } + +func writeFile(t *testing.T, dir, name, contents string) { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(contents), 0o666)) +}