From 58d0da9669226dbb197ccec18c8711698bd3af72 Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Wed, 5 Aug 2026 17:03:29 -0700 Subject: [PATCH] Right-size actor sandboxes to declared ActorTemplate resource limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Actors previously ran in sandboxes sized to the whole node; there was no way to declare how much CPU/memory a given actor should get. This adds an explicit, immutable sizing knob on the ActorTemplate and plumbs it all the way into the sandbox's OCI spec, for both the gVisor and micro-VM runtimes. - API: ActorTemplate.spec.resources (*corev1.ResourceRequirements). The Limits size the sandbox and are baked into the immutable spec; the CRD and generated code are regenerated accordingly. - internal/sizing: new SandboxSize value (FromLimits / VCPUs / ApplyToOCISpec). ApplyToOCISpec writes CPU quota+period and the memory limit onto the OCI spec and is a no-op when neither dimension is set, so 0 means "unconstrained". - Plumbing: ateapi reads the template limits (actorResourceLimits → tmpl.Spec.Resources), supplies CpuMilli/MemoryBytes over the actor RPCs (ateapi → atelet → ateom), and ateom applies them — gVisor via the cgroup leaf (runsc --cpu-num-from-quota provisions the sentry vCPU count), micro-VM via the guest spec. Proto messages carry the two fields. - Scheduling: worker capacity is taken from the WorkerPool's per-worker limits and the scheduler only places an actor on a worker whose capacity >= the actor's declared limits; a missing worker or actor dimension is treated as unconstrained so placement is never blocked by absent data. - Tests: unit tests for sizing and scheduling, plus an e2e suite (internal/e2e/suites/sizing) that resumes an actor and asserts, via the probe fixture's new /resources endpoint, that the running sandbox observes the declared CPU/memory from the inside. - Docs & demos: document the model in api-guide.md; the counter, sandbox, and micro-VM demos declare actor limits and their WorkerPool comments now describe the real model (worker limits size the worker pod + advertise capacity; the sandbox is sized by ActorTemplate.spec.resources). --- .../internal/controlapi/actor_sizing_test.go | 138 ++++++++++++ cmd/ateapi/internal/controlapi/syncer.go | 48 ++++- .../internal/controlapi/workflow_resume.go | 31 +++ cmd/ateapi/internal/scheduling/scheduling.go | 20 ++ .../internal/scheduling/scheduling_test.go | 48 +++++ .../controllers/workerpool_apply_test.go | 12 +- cmd/atelet/main.go | 4 + cmd/ateom-gvisor/main.go | 4 + cmd/ateom-gvisor/runsc.go | 19 +- cmd/ateom-microvm/main.go | 23 +- cmd/ateom-microvm/restore.go | 4 +- cmd/ateom-microvm/run.go | 34 ++- cmd/ateom-microvm/spec.go | 9 +- demos/counter/counter-microvm.yaml.tmpl | 20 ++ demos/counter/counter.yaml.tmpl | 18 ++ demos/sandbox/sandbox.yaml.tmpl | 18 ++ docs/api-guide.md | 20 ++ internal/e2e/fixtures/probe/main.go | 65 ++++++ .../e2e/fixtures/probe/probe-sized.yaml.tmpl | 63 ++++++ internal/e2e/suites/sizing/sizing_test.go | 203 ++++++++++++++++++ internal/e2e/suites/sizing/testmain_test.go | 26 +++ internal/proto/ateletpb/atelet.pb.go | 56 ++++- internal/proto/ateletpb/atelet.proto | 13 ++ internal/proto/ateompb/ateom.pb.go | 60 +++++- internal/proto/ateompb/ateom.proto | 14 ++ internal/sizing/sizing.go | 105 +++++++++ internal/sizing/sizing_test.go | 103 +++++++++ .../generated/ate.dev_actortemplates.yaml | 68 ++++++ pkg/api/v1alpha1/actortemplate_types.go | 14 ++ pkg/api/v1alpha1/zz_generated.deepcopy.go | 5 + pkg/proto/ateapipb/ateapi.pb.go | 31 ++- pkg/proto/ateapipb/ateapi.proto | 8 + 32 files changed, 1256 insertions(+), 48 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/actor_sizing_test.go create mode 100644 internal/e2e/fixtures/probe/probe-sized.yaml.tmpl create mode 100644 internal/e2e/suites/sizing/sizing_test.go create mode 100644 internal/e2e/suites/sizing/testmain_test.go create mode 100644 internal/sizing/sizing.go create mode 100644 internal/sizing/sizing_test.go diff --git a/cmd/ateapi/internal/controlapi/actor_sizing_test.go b/cmd/ateapi/internal/controlapi/actor_sizing_test.go new file mode 100644 index 000000000..b8b0275c6 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/actor_sizing_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "testing" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// TestActorResourceLimits covers the actor-side extraction: the CPU/memory limits +// an ActorTemplate declares become the sandbox size and the scheduling floor. +func TestActorResourceLimits(t *testing.T) { + tests := []struct { + name string + res *corev1.ResourceRequirements + wantCPU int64 + wantMemory int64 + }{ + { + name: "nil resources yields zero", + res: nil, + wantCPU: 0, + wantMemory: 0, + }, + { + name: "cpu and memory limits are read", + res: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + }, + wantCPU: 2000, + wantMemory: 4 << 30, + }, + { + name: "millicpu is preserved", + res: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m")}, + }, + wantCPU: 1500, + wantMemory: 0, + }, + { + name: "requests are ignored; only limits size the actor", + res: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + wantCPU: 0, + wantMemory: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tmpl := &atev1alpha1.ActorTemplate{Spec: atev1alpha1.ActorTemplateSpec{Resources: tc.res}} + cpu, mem := actorResourceLimits(tmpl) + if cpu != tc.wantCPU || mem != tc.wantMemory { + t.Fatalf("actorResourceLimits() = (%d, %d), want (%d, %d)", cpu, mem, tc.wantCPU, tc.wantMemory) + } + }) + } +} + +// TestWorkerCapacity covers the worker-side extraction: capacity comes from the +// ateom container's limits, not the pod total, and other containers are ignored. +func TestWorkerCapacity(t *testing.T) { + pod := func(ctrs ...corev1.Container) *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{Containers: ctrs}} + } + limited := func(name, cpu, mem string) corev1.Container { + lim := corev1.ResourceList{} + if cpu != "" { + lim[corev1.ResourceCPU] = resource.MustParse(cpu) + } + if mem != "" { + lim[corev1.ResourceMemory] = resource.MustParse(mem) + } + return corev1.Container{Name: name, Resources: corev1.ResourceRequirements{Limits: lim}} + } + + tests := []struct { + name string + pod *corev1.Pod + wantCPU int64 + wantMemory int64 + }{ + { + name: "no ateom container yields zero", + pod: pod(limited("sidecar", "1", "1Gi")), + wantCPU: 0, + wantMemory: 0, + }, + { + name: "ateom container limits become capacity", + pod: pod(limited(ateomContainerName, "4", "8Gi")), + wantCPU: 4000, + wantMemory: 8 << 30, + }, + { + name: "only the ateom container counts, not the pod total", + pod: pod(limited("sidecar", "16", "64Gi"), limited(ateomContainerName, "2", "2Gi")), + wantCPU: 2000, + wantMemory: 2 << 30, + }, + { + name: "unset dimension reports zero", + pod: pod(limited(ateomContainerName, "2", "")), + wantCPU: 2000, + wantMemory: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cpu, mem := workerCapacity(tc.pod) + if cpu != tc.wantCPU || mem != tc.wantMemory { + t.Fatalf("workerCapacity() = (%d, %d), want (%d, %d)", cpu, mem, tc.wantCPU, tc.wantMemory) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/syncer.go b/cmd/ateapi/internal/controlapi/syncer.go index 77919b439..59711b40a 100644 --- a/cmd/ateapi/internal/controlapi/syncer.go +++ b/cmd/ateapi/internal/controlapi/syncer.go @@ -206,16 +206,19 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK return fmt.Errorf("getting worker from store: %w", err) } slog.InfoContext(ctx, "Syncer: creating worker in store", slog.String("worker", key.namespace+"/"+key.name)) + cpuCap, memCap := workerCapacity(pod) worker := &ateapipb.Worker{ - WorkerNamespace: pod.Namespace, - WorkerPool: key.pool, - WorkerPod: pod.Name, - Ip: pod.Status.PodIP, - WorkerPodUid: string(pod.UID), - NodeName: pod.Spec.NodeName, - SandboxClass: string(pool.Spec.SandboxClass), - Labels: pool.GetLabels(), - State: ateapipb.Worker_STATE_ACTIVE, + WorkerNamespace: pod.Namespace, + WorkerPool: key.pool, + WorkerPod: pod.Name, + Ip: pod.Status.PodIP, + WorkerPodUid: string(pod.UID), + NodeName: pod.Spec.NodeName, + SandboxClass: string(pool.Spec.SandboxClass), + Labels: pool.GetLabels(), + State: ateapipb.Worker_STATE_ACTIVE, + CpuMilliCapacity: cpuCap, + MemoryBytesCapacity: memCap, } // TODO(thockin): for now this is the only place Workers are // created. If/when this becomes a regular API, validation should @@ -273,6 +276,33 @@ func isWorkerEligible(pod *corev1.Pod) bool { return pod.Status.PodIP != "" } +// ateomContainerName is the name of the container in a worker pod that hosts the +// actor's sandbox; its resource limits bound what an actor placed here can use. +const ateomContainerName = "ateom" + +// workerCapacity returns the worker pod's CPU (millicores) and memory (bytes) +// capacity for hosting an actor, taken from the ateom container's resource +// limits. A dimension the pod does not limit reports 0, which the scheduler +// treats as "unknown" (unconstrained). The actor sandbox runs nested in the +// ateom container's cgroup, so that container's limits — not the pod total — +// are the relevant envelope. +func workerCapacity(pod *corev1.Pod) (cpuMilli, memBytes int64) { + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + if c.Name != ateomContainerName { + continue + } + if v := c.Resources.Limits.Cpu(); v != nil { + cpuMilli = v.MilliValue() + } + if v := c.Resources.Limits.Memory(); v != nil { + memBytes = v.Value() + } + break + } + return cpuMilli, memBytes +} + // markWorkerDraining transitions a worker to STATE_DRAINING so the scheduler // stops routing new actors to it while its pod is Terminating. If the worker is // already gone or already draining there is nothing more to do — the Pod diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index ef4004c62..d7f226f66 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -413,11 +413,32 @@ func workerAssignmentFrom(w *ateapipb.Worker) *ateapipb.WorkerAssignment { } } +// actorResourceLimits returns the actor's declared CPU (millicores) and memory +// (bytes) limits from its ActorTemplate, or 0 for a dimension the template did +// not set. These size the sandbox (supplied over the actor RPCs) and gate +// scheduling (a worker must have >= capacity). +func actorResourceLimits(tmpl *atev1alpha1.ActorTemplate) (cpuMilli, memBytes int64) { + res := tmpl.Spec.Resources + if res == nil { + return 0, 0 + } + if c := res.Limits.Cpu(); c != nil { + cpuMilli = c.MilliValue() + } + if m := res.Limits.Memory(); m != nil { + memBytes = m.Value() + } + return cpuMilli, memBytes +} + func schedulingConstraints(actor *ateapipb.Actor, tmpl *atev1alpha1.ActorTemplate) (scheduling.Constraints, error) { + cpuMilli, memBytes := actorResourceLimits(tmpl) c := scheduling.Constraints{ SandboxClass: string(tmpl.Spec.SandboxClass), ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: actor.GetLocalSnapshotInfo().GetNodeVmsWithLocalSnapshots(), + CPUMilli: cpuMilli, + MemoryBytes: memBytes, } if tmpl.Spec.WorkerSelector != nil { sel, err := metav1.LabelSelectorAsSelector(tmpl.Spec.WorkerSelector) @@ -549,6 +570,10 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, return err } + // The actor's declared limits ride the RPC down to the sandbox so it is sized + // to the actor (replacing the worker-pod downward-API approach). + cpuMilli, memBytes := actorResourceLimits(state.ActorTemplate) + if local := state.Actor.GetLocalSnapshotInfo(); local != nil { slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot") state.SnapshotKind = ateattr.SnapshotKindLocal @@ -561,6 +586,8 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, ActorTemplateName: state.Actor.GetActorTemplateName(), Spec: workloadSpec, ActorUid: state.Actor.GetMetadata().Uid, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, } req.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL req.Config = &ateletpb.RestoreRequest_LocalConfig{ @@ -613,6 +640,8 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, // Empty unless this is a Golden data resume. GoldenSnapshotUriPrefix: state.GoldenSnapshotLocation, ActorUid: state.Actor.GetMetadata().Uid, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, } _, err = client.Restore(ctx, req) return maybeCrashActor(ctx, s.store, input.ActorRef, err, "while restoring durable snapshot", ateattr.OperationResume) @@ -638,6 +667,8 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, SandboxAssets: sandboxAssets, Spec: workloadSpec, ActorUid: state.Actor.GetMetadata().Uid, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, } _, err = client.Run(ctx, req) return maybeCrashActor(ctx, s.store, input.ActorRef, err, "while creating workload from spec", ateattr.OperationResume) diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index a68b6d7c9..2484f42c5 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -40,6 +40,15 @@ type Constraints struct { // on one of these nodes. Used when the actor's latest snapshot is local // to specific node VMs. RequiredNodes []string + + // CPUMilli and MemoryBytes are the actor's declared resource limits, from + // the ActorTemplate. A worker is eligible only if its reported capacity is + // >= these. Zero means "unconstrained" for that dimension (the actor did not + // declare a limit), and a worker that reports zero capacity for a dimension + // is treated as unconstrained too, so placement is never blocked by missing + // data (matching the pre-capacity behaviour). + CPUMilli int64 + MemoryBytes int64 } // ErrNoCapacity is returned by Schedule when no free worker satisfies the @@ -122,5 +131,16 @@ func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bo return false } + // The worker must be able to contain the actor's declared limits. A zero + // constraint (actor declared no limit) or zero worker capacity (capacity + // unknown) is treated as unconstrained, so placement is never blocked by + // missing data. + if constraints.CPUMilli > 0 && worker.GetCpuMilliCapacity() > 0 && worker.GetCpuMilliCapacity() < constraints.CPUMilli { + return false + } + if constraints.MemoryBytes > 0 && worker.GetMemoryBytesCapacity() > 0 && worker.GetMemoryBytesCapacity() < constraints.MemoryBytes { + return false + } + return len(constraints.RequiredNodes) == 0 || slices.Contains(constraints.RequiredNodes, worker.GetNodeName()) } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 52062e47c..7f7ca416f 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -117,6 +117,47 @@ func TestSchedule(t *testing.T) { constraints: Constraints{SandboxClass: "gvisor"}, wantPod: "w-active", }, + { + name: "worker with too little cpu capacity is skipped", + fleet: fleet{ + worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 8<<30)), + worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30)), + }, + constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000}, + wantPod: "w-big", + }, + { + name: "worker with too little memory capacity is skipped", + fleet: fleet{ + worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(4000, 1<<30)), + worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 4<<30)), + }, + constraints: Constraints{SandboxClass: "gvisor", MemoryBytes: 2 << 30}, + wantPod: "w-big", + }, + { + name: "no worker with enough capacity yields ErrNoCapacity", + fleet: fleet{ + worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 1<<30)), + }, + constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000}, + }, + { + name: "zero worker capacity is treated as unconstrained", + fleet: fleet{ + worker("w-unknown", "gvisor", "node-a", tierTwo), + }, + constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000, MemoryBytes: 2 << 30}, + wantPod: "w-unknown", + }, + { + name: "zero constraint ignores worker capacity", + fleet: fleet{ + worker("w-tiny", "gvisor", "node-a", tierTwo, withCapacity(100, 1<<20)), + }, + constraints: Constraints{SandboxClass: "gvisor"}, + wantPod: "w-tiny", + }, { name: "empty fleet", fleet: fleet{}, @@ -265,5 +306,12 @@ func assigned(atespace, name string) func(*ateapipb.Worker) { } } +func withCapacity(cpuMilli, memBytes int64) func(*ateapipb.Worker) { + return func(w *ateapipb.Worker) { + w.CpuMilliCapacity = cpuMilli + w.MemoryBytesCapacity = memBytes + } +} + // firstIntn always picks the first candidate, making Schedule deterministic. func firstIntn(int) int { return 0 } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index f208a23f4..09de4873f 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -587,11 +587,13 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithAdd(ateomGvisorCapabilities...)). WithAppArmorProfile(corev1ac.AppArmorProfile(). WithType(corev1.AppArmorProfileTypeUnconfined))). - WithEnv(corev1ac.EnvVar(). - WithName("POD_UID"). - WithValueFrom(corev1ac.EnvVarSource(). - WithFieldRef(corev1ac.ObjectFieldSelector(). - WithFieldPath("metadata.uid")))). + WithEnv( + corev1ac.EnvVar(). + WithName("POD_UID"). + WithValueFrom(corev1ac.EnvVarSource(). + WithFieldRef(corev1ac.ObjectFieldSelector(). + WithFieldPath("metadata.uid"))), + ). WithVolumeMounts( corev1ac.VolumeMount(). WithName("run-ateom"). diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 95a4e16c4..bc555d9a4 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -345,6 +345,8 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * RuntimeAssetPaths: assetPaths, Spec: buildAteomWorkloadSpec(req.GetSpec()), ActorUid: actorUID, + CpuMilli: req.GetCpuMilli(), + MemoryBytes: req.GetMemoryBytes(), }); err != nil { return nil, fmt.Errorf("while calling ateom.RunWorkload: %w", err) } @@ -720,6 +722,8 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) Spec: buildAteomWorkloadSpec(req.GetSpec()), Scope: toAteomSnapshotScope(req.GetScope()), ActorUid: req.GetActorUid(), + CpuMilli: req.GetCpuMilli(), + MemoryBytes: req.GetMemoryBytes(), // Informational: for DATA_ON_GOLDEN the golden snapshot's files are // already staged into the restore dir by the combined download above; // ateom restores from the shared dir and never fetches this URI. diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 752acf2d3..48f1e8333 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -40,6 +40,7 @@ import ( "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/serverboot" + "github.com/agent-substrate/substrate/internal/sizing" "github.com/agent-substrate/substrate/internal/version" "github.com/hashicorp/go-reap" "github.com/spf13/pflag" @@ -300,6 +301,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } // Create and start pause container. The bundle rootfs is composed here — @@ -364,6 +366,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // * After we exit, atelet will upload checkpoint to GCS // * After we exit, atelet will tear down OCI bundles and reset the actor directory. + // Checkpoint only saves state; no sizing is applied, so size is left zero. rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), @@ -518,6 +521,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } checkpointDir := ateompath.RestoreStateDir(req.GetActorUid()) diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 003bd861a..ca3357de2 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -29,11 +29,15 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/sizing" ) type runsc struct { path string actorUID string + // size is the actor's declared limits, supplied on the RunWorkload / + // RestoreWorkload RPC; ensureContainerCgroupsPath writes it into the OCI spec. + size sizing.SandboxSize } // ensureContainerCgroupsPath sets the OCI spec's cgroupsPath so runsc creates a @@ -57,10 +61,13 @@ func (r *runsc) ensureContainerCgroupsPath(containerName string) error { if spec.Linux == nil { spec.Linux = &specs.Linux{} } - if spec.Linux.CgroupsPath != "" { - return nil + if spec.Linux.CgroupsPath == "" { + spec.Linux.CgroupsPath = "/" + containerName } - spec.Linux.CgroupsPath = "/" + containerName + // Right-size the per-container cgroup leaf to the actor's declared limits; + // runsc applies spec.Linux.Resources when it creates the leaf. Shared with the + // micro-VM runtime via internal/sizing. + r.size.ApplyToOCISpec(&spec) out, err := json.MarshalIndent(&spec, "", " ") if err != nil { return fmt.Errorf("marshaling %q: %w", specPath, err) @@ -90,6 +97,10 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri // "-log-packets", // "-strace", "-root", ateompath.RunSCStateDir(r.actorUID), + // Provision the sentry's vCPU count from the cgroup CPU quota written by + // sizing.ApplyToOCISpec, so the sandbox is sized to the pod's limit (runsc + // otherwise sizes to all host CPUs). Global flag: before the subcommand. + "--cpu-num-from-quota", "create", "-bundle", ateompath.OCIBundlePath(r.actorUID, containerName), "-pid-file", ateompath.PIDFilePath(r.actorUID, containerName), @@ -237,6 +248,8 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch // "-log-packets", // "-strace", "-root", ateompath.RunSCStateDir(r.actorUID), + // Match cmdCreate: size the restored sentry from the cgroup CPU quota. + "--cpu-num-from-quota", "restore", "-bundle", ateompath.OCIBundlePath(r.actorUID, containerName), "-image-path", checkpointPath, diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 057863ac8..61f2e3935 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -52,12 +52,13 @@ import ( ) var ( - podUID = flag.String("pod-uid", "", "The UID of the current pod") - chBinary = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") - kataConfig = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.") - kataDebug = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.") - showVersion = flag.Bool("version", false, "Print version and exit.") - logLevelFlag = flag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") + podUID = flag.String("pod-uid", "", "The UID of the current pod") + chBinary = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") + kataConfig = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.") + kataDebug = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.") + vmmMemReserve = flag.Int("vmm-mem-reserve-mib", vmmMemReserveMiB, "Guest RAM (MiB) held back from the pod's memory limit for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the pod cgroup alongside the guest RAM. Prevents the pod OOMing when the VM is sized to the pod's memory limit.") + showVersion = flag.Bool("version", false, "Print version and exit.") + logLevelFlag = flag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") atunnelListenAddress = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS") atunnelCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle for actor ingress HTTPS") @@ -211,7 +212,7 @@ func do(ctx context.Context) error { grpc.StatsHandler(otelgrpc.NewServerHandler()), grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), ) - ateompb.RegisterAteomServer(svr, NewService(*podUID, *chBinary, *kataConfig, *kataDebug, interiorNetNS, actorLogger, atunnelServer, atunnelEgress, atunnelEgressPort, *atunnelCredentialBundle, *atunnelEgressTrustBundle)) + ateompb.RegisterAteomServer(svr, NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelServer, atunnelEgress, atunnelEgressPort, *atunnelCredentialBundle, *atunnelEgressTrustBundle)) reflection.Register(svr) slog.InfoContext(ctx, "ateom-microvm serving", slog.String("socket", sockPath)) @@ -262,6 +263,11 @@ type AteomService struct { kataConfig string kataDebug bool + // memReserveMiB is guest RAM (MiB) held back from the pod's memory limit for + // the cloud-hypervisor VMM + virtiofsd (host processes sharing the pod cgroup + // with the guest RAM). Set from --vmm-mem-reserve-mib. + memReserveMiB int + // interiorNetNS hosts the per-activation actor veth peer (see net.go); // kata is pointed at it. interiorNetNS netns.NsHandle @@ -287,12 +293,13 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary, kataConfig string, kataDebug bool, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelServer *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, credentialBundle, egressTrustBundle string) *AteomService { +func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelServer *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, credentialBundle, egressTrustBundle string) *AteomService { return &AteomService{ podUID: podUID, chBinary: chBinary, kataConfig: kataConfig, kataDebug: kataDebug, + memReserveMiB: memReserveMiB, interiorNetNS: interiorNetNS, actorLogger: actorLogger, atunnel: atunnelServer, diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index a0c76affc..8f6944252 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -36,6 +36,7 @@ import ( "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/sizing" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -72,6 +73,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore actorVersion: req.GetActorVersion(), egressGatewayAddress: req.GetEgressGatewayAddress(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } restoreDir := ateompath.RestoreStateDir(p.actorUID) durableDir := ateompath.DurableDirVolumeMountsDir(p.actorUID) @@ -165,7 +167,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, if len(containers) > maxActorContainers { return status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } - ctrs, err := s.buildActorContainers(actorUID, containers) + ctrs, err := s.buildActorContainers(actorUID, containers, p.size) if err != nil { return err } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index ae691a2ca..4df6f5911 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -38,6 +38,7 @@ import ( "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/sizing" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" "google.golang.org/grpc/codes" @@ -105,6 +106,12 @@ const ( assetVirtiofsd = "virtiofsd" ) +// vmmMemReserveMiB is the DEFAULT guest RAM held back from the pod's memory limit +// for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the same +// pod cgroup as the guest RAM; without a margin the pod OOMs. Overridable per +// deployment via --vmm-mem-reserve-mib (see AteomService.memReserveMiB). +const vmmMemReserveMiB = 256 + // maxActorContainers is a sanity cap on containers per actor (all share the one // micro-VM + virtiofsd). 25 is far above any real pod. const maxActorContainers = 25 @@ -214,6 +221,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload actorVersion: req.GetActorVersion(), egressGatewayAddress: req.GetEgressGatewayAddress(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } s.actorLogger.EmitLifecycleLog("Actor starting", p.actorRef, p.actorUID, p.templateNS, p.templateName) @@ -241,6 +249,10 @@ type actorBootParams struct { // egressGatewayAddress is empty unless an egress gateway is configured, in // which case actor TCP egress is redirected to atunnel's local listener. egressGatewayAddress string + // size is the actor's declared limits (from the ActorTemplate), supplied on + // the RunWorkload / RestoreWorkload RPC. It sizes the VM (vCPUs, memory) and + // the guest container cgroup. Zero fields keep the kata defaults. + size sizing.SandboxSize } // coldBootAttempts is how many times a cold boot is tried when the micro-VM @@ -324,7 +336,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // Prepare each container's OCI spec + record its bundle rootfs (the overlay RO // lower). No host disk — the rootfs is overlay(virtio-fs lower + guest-tmpfs upper). - ctrs, err := s.buildActorContainers(actorUID, containers) + ctrs, err := s.buildActorContainers(actorUID, containers, p.size) if err != nil { return err } @@ -335,6 +347,22 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re return err } + // Right-size the VM to the actor's declared limits (see internal/sizing), + // keeping the kata-config values above as the fallback when a limit is unset. + // vCPUs round up; VM RAM reserves a fixed margin for the VMM + virtiofsd, which + // share the pod cgroup with the guest RAM. NB: a FULL-scope snapshot restore + // reuses the size baked into the snapshot (restoreFullScope), so resizing an + // existing actor takes effect on its next cold boot. + sz := p.size + if v := sz.VCPUs(); v > 0 { + vcpus = v + } + if sz.MemoryBytes > 0 { + if m := int(sz.MemoryBytes/(1024*1024)) - s.memReserveMiB; m > 0 { + memMiB = m + } + } + // Clean stale per-sandbox state + create the runtime dir for the sockets. kata.CleanupSandboxState(ctx, actorUID) if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil { @@ -481,13 +509,13 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // built — the rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper); the lowers // are bound into virtiofsd's shared dir in stageOverlayLowers after the sandbox state // is clean. Both RunWorkload and RestoreWorkload go through here. -func (s *AteomService) buildActorContainers(actorUID string, containers []*ateompb.Container) ([]actorContainer, error) { +func (s *AteomService) buildActorContainers(actorUID string, containers []*ateompb.Container, size sizing.SandboxSize) ([]actorContainer, error) { netnsPath := ateompath.AteomNetNSPath(s.podUID) ctrs := make([]actorContainer, len(containers)) for i, c := range containers { cn := c.GetName() bundle := ateompath.OCIBundlePath(actorUID, cn) - spec, err := ensureKataCompatibleSpec(bundle, actorUID, netnsPath) + spec, err := ensureKataCompatibleSpec(bundle, actorUID, netnsPath, size) if err != nil { return nil, fmt.Errorf("while preparing kata OCI spec for %q: %w", cn, err) } diff --git a/cmd/ateom-microvm/spec.go b/cmd/ateom-microvm/spec.go index 7962bc5aa..3e328b486 100644 --- a/cmd/ateom-microvm/spec.go +++ b/cmd/ateom-microvm/spec.go @@ -24,6 +24,8 @@ import ( "strings" specs "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/agent-substrate/substrate/internal/sizing" ) // ensureKataCompatibleSpec augments the bundle's config.json with the fields @@ -31,7 +33,7 @@ import ( // Without linux.resources, kata's ContainerConfig nil-derefs and the shim // crashes. This shaper is a bridge; a future atelet change should emit // runtime-appropriate specs so it can retire. -func ensureKataCompatibleSpec(bundle, id, netnsPath string) (*specs.Spec, error) { +func ensureKataCompatibleSpec(bundle, id, netnsPath string, size sizing.SandboxSize) (*specs.Spec, error) { specPath := filepath.Join(bundle, "config.json") b, err := os.ReadFile(specPath) if err != nil { @@ -51,6 +53,11 @@ func ensureKataCompatibleSpec(bundle, id, netnsPath string) (*specs.Spec, error) if spec.Linux.CgroupsPath == "" { spec.Linux.CgroupsPath = "/ateomchv/" + id } + // Right-size the guest container cgroup to the actor's declared limits; the + // kata-agent applies spec.Linux.Resources inside the VM. Shared with the gVisor + // runtime via internal/sizing; overlays the device allowlist + CPU shares set + // by defaultKataResources. + size.ApplyToOCISpec(&spec) // atelet's spec carries gVisor pause-model CRI annotations // (container-type=container, sandbox-id=pause). kata reads those and waits diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index e626c94ef..9dc5168d8 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -100,6 +100,18 @@ spec: sandboxClass: microvm sandboxConfigName: counter-microvm ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm + template: + # Per-worker resources: these size the worker POD and advertise its + # scheduling capacity (the per-actor ceiling). The micro-VM sandbox itself is + # sized by the ActorTemplate's spec.resources below, not by these; keep the + # actor memory below this limit so the VMM reserve fits (see internal/sizing). + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi --- @@ -127,6 +139,14 @@ spec: volumeMounts: - name: data mountPath: /home/counter + # Sandbox size: an actor occupies its whole worker. Size vCPUs + guest RAM at + # (or below) the pool's per-worker capacity above, leaving headroom under the + # worker memory limit for the VMM reserve. ateom applies these to the guest + # (see internal/sizing). + resources: + limits: + cpu: "2" + memory: 1536Mi workerSelector: matchLabels: workload: counter-microvm diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index fa8801842..f9ca8f7c7 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -29,6 +29,17 @@ metadata: spec: replicas: 5 ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + template: + # Per-worker resources: these size the worker POD and advertise its + # scheduling capacity (the per-actor ceiling). The sandbox itself is sized by + # the ActorTemplate's spec.resources below, not by these (see internal/sizing). + resources: + requests: + cpu: "250m" + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi --- @@ -53,6 +64,13 @@ ${VALIDATE_EXISTING_FILE_PATH_ARG} - name: data mountPath: /home/counter ${EXTERNAL_VOLUME_MOUNTS} + # Sandbox size: an actor occupies its whole worker, so size it at (or below) + # the pool's per-worker capacity above. ateom writes these to the OCI spec + # (cgroup CPU quota + memory limit) — see internal/sizing. + resources: + limits: + cpu: "1" + memory: 512Mi workerSelector: matchLabels: workload: counter diff --git a/demos/sandbox/sandbox.yaml.tmpl b/demos/sandbox/sandbox.yaml.tmpl index 0265eea62..f1d0ba8cd 100644 --- a/demos/sandbox/sandbox.yaml.tmpl +++ b/demos/sandbox/sandbox.yaml.tmpl @@ -27,6 +27,17 @@ metadata: spec: replicas: 2 ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + template: + # Per-worker resources: these size the worker POD and advertise its + # scheduling capacity (the per-actor ceiling). The sandbox itself is sized by + # the ActorTemplate's spec.resources below, not by these (see internal/sizing). + resources: + requests: + cpu: "500m" + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi --- apiVersion: ate.dev/v1alpha1 kind: ActorTemplate @@ -44,5 +55,12 @@ spec: env: - name: PORT value: "80" + # Sandbox size: an actor occupies its whole worker, so size it at (or below) + # the pool's per-worker capacity above. ateom writes these to the OCI spec + # (cgroup CPU quota + memory limit) — see internal/sizing. + resources: + limits: + cpu: "2" + memory: 1Gi snapshotsConfig: location: gs://${BUCKET_NAME}/ate-demo-sandbox/ diff --git a/docs/api-guide.md b/docs/api-guide.md index 982da52e7..48bd72a53 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -26,6 +26,13 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | `nodeAffinity` | `NodeAffinity` | `spec.affinity.nodeAffinity` | | `resources` | `ResourceRequirements` | `spec.containers[].resources` | +#### Worker Capacity (`spec.template.resources`) + +Setting `resources.limits` (CPU and Memory) on a `WorkerPool` establishes each worker pod's **capacity** — the envelope available to host an actor sandbox, taken from the `ateom` container's limits. The scheduler only places an actor on a worker whose capacity is `>=` the actor's declared resource limits (see [Sandbox Right-Sizing](#sandbox-right-sizing-specresources) on the `ActorTemplate`). + +- Size a pool's `limits` to the largest actor it should host. An actor occupies its whole worker, so worker capacity is the per-actor ceiling, not a shared budget. +- Capacity is advisory for placement only: a worker that declares no CPU/memory limit reports zero capacity for that dimension, which the scheduler treats as **unconstrained** (placement is never blocked by missing data). The actual sandbox size still comes from the `ActorTemplate`. + ### Example ```yaml @@ -101,11 +108,24 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `snapshotsConfig` | `SnapshotsConfig` | **Required.** GCS bucket and folder where memory snapshots are stored. | | `pauseImage` | `string` | **Required.** The image used for the sandbox root (e.g. `gcr.io/gke-release/pause`). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each either a `durableDir` or an `externalVolumeTemplate`. Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one, and `externalVolumeTemplate` is `gvisor`-only. | +| `resources` | `*ResourceRequirements` | Optional. Declares each actor's compute size via `limits` — see [Sandbox Right-Sizing](#sandbox-right-sizing-specresources). Immutable, like the rest of the spec. | The sandbox binaries (e.g. the gVisor `runsc` binary) are **no longer configured on the `ActorTemplate`**. They are resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-sandbox-binaries) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`. Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It defaults to `gvisor` and, like the rest of the spec, is immutable, so each template's class is fixed at creation. +### Sandbox Right-Sizing (`spec.resources`) + +Unlike a Pod, an actor is sized by its **`limits`** (CPU and Memory): the size is a property of the template, baked into snapshots, so it lives on the immutable `ActorTemplate` spec. Declared limits do three things: + +1. **Size the sandbox.** The limits are supplied to the sandbox over the actor RPCs (control plane → atelet → ateom) and applied to the container OCI spec: + - **gVisor (`ateom-gvisor`)** — `limits.cpu` sets the cgroup v2 CPU quota (`cpu.max`) and the Sentry vCPU count (`--cpu-num-from-quota`); `limits.memory` sets the cgroup v2 memory limit (`memory.max`) and bounds the virtual total memory the sandbox reports (so JVM/Go do not over-allocate from host RAM). + - **Micro-VM (`ateom-microvm`)** — `limits.cpu` sets Cloud Hypervisor `BootVcpus` / `MaxVcpus` (rounded up to whole vCPUs); `limits.memory` sets guest RAM, reserving a small configurable margin (default 256 MiB, `--vmm-mem-reserve-mib`) for the VMM and virtiofsd so the pod cgroup does not OOM. +2. **Gate scheduling.** An actor is only placed on a `WorkerPool` whose [worker capacity](#worker-capacity-spectemplateresources) is `>=` these limits. +3. **Fall back to runtime defaults.** A zero or absent limit leaves that dimension at the runtime default — unlimited for gVisor, the kata config for the micro-VM. + +`requests` are not consulted today (an actor occupies its whole worker). Because the size is baked into snapshots, a **micro-VM FULL-scope restore reuses the size in the snapshot**; changing an actor's limits takes effect on its next cold boot. + Container environment variables support literal `value` entries and `valueFrom.secretKeyRef`. Secret references are resolved by `ate-api-server` from the `ActorTemplate` namespace when a workload spec is materialized. For the golden actor, the resolved values are captured in the golden snapshot and future actors inherit those values until the golden snapshot is recreated. For an actor that bypasses the golden snapshot and boots from the current template spec, the resolved values are sent to atelet but are not serialized into the public Actor API. Other Kubernetes `valueFrom` sources are not supported yet. Secret changes do not automatically restart actors or invalidate snapshots; rotating a Secret requires an explicit actor or template lifecycle action. ### Workload Connectivity (Uniform DNS) diff --git a/internal/e2e/fixtures/probe/main.go b/internal/e2e/fixtures/probe/main.go index 6927a1d04..ec2192bea 100644 --- a/internal/e2e/fixtures/probe/main.go +++ b/internal/e2e/fixtures/probe/main.go @@ -21,10 +21,14 @@ package main import ( + "bufio" "encoding/json" "log" "net/http" "os" + "runtime" + "strconv" + "strings" ) // identityFile is the actor-id file inside the identity directory atelet @@ -48,6 +52,66 @@ func whoami(w http.ResponseWriter, _ *http.Request) { writeJSON(w, resp) } +// resources reports the compute envelope the actor observes from inside the +// sandbox, so the sizing e2e suite can assert the actor's declared limits +// actually shaped the runtime. +// +// - num_cpu is runtime.NumCPU(): for the gVisor runtime this is the sentry's +// vCPU count, provisioned from the CPU limit via runsc --cpu-num-from-quota, +// so it equals ceil(limits.cpu). +// - mem_total_bytes is MemTotal from /proc/meminfo: the memory the sandbox +// believes it has, bounded by limits.memory. +// - cpu_max / memory_max are the raw cgroup v2 files, reported best-effort for +// debugging; presence and format vary by runtime. +func resources(w http.ResponseWriter, _ *http.Request) { + resp := map[string]any{"num_cpu": runtime.NumCPU()} + + if v, err := memTotalBytes(); err == nil { + resp["mem_total_bytes"] = v + } else { + resp["mem_total_error"] = err.Error() + } + if b, err := os.ReadFile("/sys/fs/cgroup/cpu.max"); err == nil { + resp["cpu_max"] = strings.TrimSpace(string(b)) + } + if b, err := os.ReadFile("/sys/fs/cgroup/memory.max"); err == nil { + resp["memory_max"] = strings.TrimSpace(string(b)) + } + + writeJSON(w, resp) +} + +// memTotalBytes parses MemTotal (reported in kB) from /proc/meminfo. +func memTotalBytes() (int64, error) { + f, err := os.Open("/proc/meminfo") + if err != nil { + return 0, err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + rest, ok := strings.CutPrefix(line, "MemTotal:") + if !ok { + continue + } + fields := strings.Fields(rest) // e.g. "524288 kB" + if len(fields) == 0 { + break + } + kb, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return 0, err + } + return kb * 1024, nil + } + if err := sc.Err(); err != nil { + return 0, err + } + return 0, os.ErrNotExist +} + func writeJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(v); err != nil { @@ -58,6 +122,7 @@ func writeJSON(w http.ResponseWriter, v any) { func main() { mux := http.NewServeMux() mux.HandleFunc("/whoami", whoami) + mux.HandleFunc("/resources", resources) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) const addr = ":80" diff --git a/internal/e2e/fixtures/probe/probe-sized.yaml.tmpl b/internal/e2e/fixtures/probe/probe-sized.yaml.tmpl new file mode 100644 index 000000000..ebb9a4fd7 --- /dev/null +++ b/internal/e2e/fixtures/probe/probe-sized.yaml.tmpl @@ -0,0 +1,63 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sized variant of the probe fixture: the ActorTemplate declares +# spec.resources.limits, so the sizing e2e suite can assert the actor's +# sandbox is shaped to those limits. Reuses the probe image (it serves the +# /resources endpoint). Kept in its own namespace so it never collides with +# the plain probe fixture. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-e2e-sizing + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: probe-sized + namespace: ate-e2e-sizing + labels: + workload: probe-sized +spec: + replicas: 3 + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: probe-sized + namespace: ate-e2e-sizing +spec: + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + containers: + - name: probe + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + command: ["/ko-app/probe"] + # The feature under test: these limits size the sandbox (and, when the worker + # advertises capacity, gate scheduling). CPU=2 makes NumCPU() inside the + # gVisor sandbox a distinct, assertable value. + resources: + limits: + cpu: "2" + memory: 512Mi + workerSelector: + matchLabels: + workload: probe-sized + snapshotsConfig: + location: gs://${BUCKET_NAME}/ate-e2e-sizing/ diff --git a/internal/e2e/suites/sizing/sizing_test.go b/internal/e2e/suites/sizing/sizing_test.go new file mode 100644 index 000000000..6ff220331 --- /dev/null +++ b/internal/e2e/suites/sizing/sizing_test.go @@ -0,0 +1,203 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizing + +import ( + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + sizingNamespace = "ate-e2e-sizing" + sizingTemplate = "probe-sized" + + // The limits declared in probe-sized.yaml.tmpl. Keep these in sync with the + // manifest: the whole point of the suite is to assert the sandbox observes + // exactly what the ActorTemplate declared. + wantCPU = 2 + wantMemBytes = 512 * 1024 * 1024 // 512Mi +) + +// resourcesResponse mirrors the /resources endpoint of the probe fixture. +type resourcesResponse struct { + NumCPU int `json:"num_cpu"` + MemTotalBytes int64 `json:"mem_total_bytes"` + MemTotalError string `json:"mem_total_error"` + CPUMax string `json:"cpu_max"` + MemoryMax string `json:"memory_max"` +} + +// TestActorSizing_SandboxObservesDeclaredLimits is the end-to-end gate for the +// resource-limits redesign: an ActorTemplate that declares spec.resources.limits +// must produce a sandbox sized to those limits. The plumbing unit tests prove +// the limits reach the OCI spec; this proves the running sandbox actually +// honours them, by resuming an actor and asking it (via the probe /resources +// endpoint) what compute envelope it sees from the inside. +// +// gVisor is the default (and only) runtime in the macOS/colima kind +// environment, so the assertions target what runsc --cpu-num-from-quota and the +// cgroup memory limit produce inside the sentry. +func TestActorSizing_SandboxObservesDeclaredLimits(t *testing.T) { + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + clients := e2e.GetClients() + + deploySizedProbe(t, env["BUCKET_NAME"]) + waitForTemplateReady(t, ctx, clients) + + const id = "sized-actor" + createAndResumeActor(t, ctx, clients, id) + + rc, err := e2e.NewRouterClient(ctx) + if err != nil { + t.Fatalf("NewRouterClient: %v", err) + } + defer rc.Close() + + got := getResources(t, ctx, rc, id) + t.Logf("sandbox /resources: num_cpu=%d mem_total_bytes=%d cpu_max=%q memory_max=%q mem_total_error=%q", + got.NumCPU, got.MemTotalBytes, got.CPUMax, got.MemoryMax, got.MemTotalError) + + // CPU: runsc provisions the sentry's vCPU count from the CPU quota + // (--cpu-num-from-quota), so the sandbox must see exactly the declared limit. + if got.NumCPU != wantCPU { + t.Errorf("sandbox NumCPU = %d, want %d (declared limits.cpu=%d) — sandbox not sized to actor limits", got.NumCPU, wantCPU, wantCPU) + } + + // Memory: the sandbox must be bounded by the declared limit. gVisor may + // report slightly under the limit (reserved overhead) but must never see + // more; a value near the node's full RAM means the limit was not applied. + // Allow 10% headroom above the limit for accounting differences. + if got.MemTotalError != "" { + t.Errorf("probe could not read MemTotal: %s", got.MemTotalError) + } else if got.MemTotalBytes > wantMemBytes*11/10 { + t.Errorf("sandbox MemTotal = %d bytes, want <= ~%d (declared limits.memory=512Mi) — memory limit not applied", got.MemTotalBytes, wantMemBytes) + } else if got.MemTotalBytes < wantMemBytes/2 { + t.Errorf("sandbox MemTotal = %d bytes, unexpectedly far below the declared 512Mi limit", got.MemTotalBytes) + } +} + +func deploySizedProbe(t *testing.T, bucket string) { + t.Helper() + root, err := e2e.FindRepoRoot() + if err != nil { + t.Fatalf("FindRepoRoot: %v", err) + } + + // Render the manifest template to a file so both apply and delete can + // consume it without any shell involved (mirrors the identity suite). + tmpl, err := os.ReadFile(filepath.Join(root, "internal/e2e/fixtures/probe/probe-sized.yaml.tmpl")) + if err != nil { + t.Fatalf("reading sized probe manifest template: %v", err) + } + manifest := filepath.Join(t.TempDir(), "probe-sized.yaml") + rendered := strings.ReplaceAll(string(tmpl), "${BUCKET_NAME}", bucket) + if err := os.WriteFile(manifest, []byte(rendered), 0o644); err != nil { + t.Fatalf("writing rendered sized probe manifest: %v", err) + } + + // Build/push the probe image and apply through the repo's pinned ko. See the + // identity suite's deployProbe for why KO_CONFIG_PATH and the trailing + // `-- --context=...` are required. + applyArgs := []string{"ko", "apply", "-f", manifest} + if e2e.KubeContext != "" { + applyArgs = append(applyArgs, "--", "--context="+e2e.KubeContext) + } + e2e.RunCmdWithEnv(t, []string{"KO_CONFIG_PATH=" + root}, filepath.Join(root, "hack/run-tool.sh"), applyArgs...) + + t.Cleanup(func() { + delArgs := []string{"delete", "--ignore-not-found", "-f", manifest} + if e2e.KubeContext != "" { + delArgs = append([]string{"--context=" + e2e.KubeContext}, delArgs...) + } + e2e.RunCmd(t, "kubectl", delArgs...) + }) +} + +func waitForTemplateReady(t *testing.T, ctx context.Context, clients *e2e.Clients) { + t.Helper() + deadline := time.Now().Add(5 * time.Minute) + for time.Now().Before(deadline) { + at, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(sizingNamespace).Get(ctx, sizingTemplate, metav1.GetOptions{}) + if err == nil { + switch at.Status.Phase { + case v1alpha1.PhaseReady: + t.Logf("sized probe ActorTemplate ready, golden=%s", at.Status.GoldenActorID) + return + case v1alpha1.PhaseFailed: + t.Fatalf("sized probe ActorTemplate entered PhaseFailed") + } + } + time.Sleep(2 * time.Second) + } + t.Fatalf("timed out waiting for sized probe ActorTemplate to be Ready") +} + +func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Clients, id string) { + t.Helper() + // CreateActor requires the atespace to exist first. + _, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: sizingNamespace}}}) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: sizingNamespace, Name: id}, + ActorTemplateNamespace: sizingNamespace, + ActorTemplateName: sizingTemplate, + }}); err != nil { + t.Fatalf("CreateActor %q: %v", id, err) + } + t.Cleanup(func() { + // DeleteActor requires the actor to be suspended. + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: sizingNamespace, Name: id}}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: &ateapipb.ObjectRef{Atespace: sizingNamespace, Name: id}}) + }) + + // Resume from the golden snapshot (the restore path, not --boot). + if _, err := clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: sizingNamespace, Name: id}}); err != nil { + t.Fatalf("ResumeActor %q: %v", id, err) + } +} + +func getResources(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) resourcesResponse { + t.Helper() + resp, err := rc.Get(ctx, resources.ActorRef{Atespace: sizingNamespace, Name: id}, "/resources") + if err != nil { + t.Fatalf("GET /resources for %q: %v", id, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("GET /resources for %q: status %d, body %q", id, resp.StatusCode, body) + } + var out resourcesResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding /resources for %q: %v", id, err) + } + return out +} diff --git a/internal/e2e/suites/sizing/testmain_test.go b/internal/e2e/suites/sizing/testmain_test.go new file mode 100644 index 000000000..5ba57d803 --- /dev/null +++ b/internal/e2e/suites/sizing/testmain_test.go @@ -0,0 +1,26 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizing + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { + os.Exit(e2e.RunTestMain(m)) +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 5ed205914..bf2a1fc11 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -213,6 +213,11 @@ type RunRequest struct { // fetches the relevant assets and records them with the actor's on-node state // so a later Checkpoint can pin the same version into the snapshot manifest. SandboxAssets *SandboxAssets `protobuf:"bytes,8,opt,name=sandbox_assets,json=sandboxAssets,proto3" json:"sandbox_assets,omitempty"` + // The actor's declared size, from the ActorTemplate's resource limits. atelet + // passes these through to the sandbox so it is sized to the actor (not the + // whole host or worker pod). Zero means "unset": keep the runtime default. + CpuMilli int64 `protobuf:"varint,9,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,10,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -303,6 +308,20 @@ func (x *RunRequest) GetSandboxAssets() *SandboxAssets { return nil } +func (x *RunRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RunRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + // AssetFile is one content-addressed file atelet fetches for a sandbox runtime // (e.g. the gVisor release tarball). type AssetFile struct { @@ -1377,8 +1396,14 @@ type RestoreRequest struct { // of the `config` oneof: the actor's snapshot may be local (a pause // checkpoint) while the golden snapshot is always external. GoldenSnapshotUriPrefix string `protobuf:"bytes,12,opt,name=golden_snapshot_uri_prefix,json=goldenSnapshotUriPrefix,proto3" json:"golden_snapshot_uri_prefix,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The actor's declared size, from the ActorTemplate's resource limits. For + // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; + // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // means "unset": keep the runtime default. + CpuMilli int64 `protobuf:"varint,13,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,14,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreRequest) Reset() { @@ -1506,6 +1531,20 @@ func (x *RestoreRequest) GetGoldenSnapshotUriPrefix() string { return "" } +func (x *RestoreRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RestoreRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + type isRestoreRequest_Config interface { isRestoreRequest_Config() } @@ -1562,7 +1601,7 @@ var File_atelet_proto protoreflect.FileDescriptor const file_atelet_proto_rawDesc = "" + "\n" + - "\fatelet.proto\x12\x06atelet\"\xe0\x02\n" + + "\fatelet.proto\x12\x06atelet\"\xa0\x03\n" + "\n" + "RunRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + @@ -1573,7 +1612,10 @@ const file_atelet_proto_rawDesc = "" + "\x18actor_template_namespace\x18\x05 \x01(\tR\x16actorTemplateNamespace\x12.\n" + "\x13actor_template_name\x18\x06 \x01(\tR\x11actorTemplateName\x12(\n" + "\x04spec\x18\a \x01(\v2\x14.atelet.WorkloadSpecR\x04spec\x12<\n" + - "\x0esandbox_assets\x18\b \x01(\v2\x15.atelet.SandboxAssetsR\rsandboxAssets\"5\n" + + "\x0esandbox_assets\x18\b \x01(\v2\x15.atelet.SandboxAssetsR\rsandboxAssets\x12\x1b\n" + + "\tcpu_milli\x18\t \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\n" + + " \x01(\x03R\vmemoryBytes\"5\n" + "\tAssetFile\x12\x10\n" + "\x03url\x18\x01 \x01(\tR\x03url\x12\x16\n" + "\x06sha256\x18\x02 \x01(\tR\x06sha256\"\x8e\x01\n" + @@ -1650,7 +1692,7 @@ const file_atelet_proto_rawDesc = "" + " \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x12+\n" + "\x05scope\x18\v \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scopeB\b\n" + "\x06config\"\x14\n" + - "\x12CheckpointResponse\"\xe5\x04\n" + + "\x12CheckpointResponse\"\xa5\x05\n" + "\x0eRestoreRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + @@ -1665,7 +1707,9 @@ const file_atelet_proto_rawDesc = "" + "\x0fexternal_config\x18\n" + " \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x12+\n" + "\x05scope\x18\v \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scope\x12;\n" + - "\x1agolden_snapshot_uri_prefix\x18\f \x01(\tR\x17goldenSnapshotUriPrefixB\b\n" + + "\x1agolden_snapshot_uri_prefix\x18\f \x01(\tR\x17goldenSnapshotUriPrefix\x12\x1b\n" + + "\tcpu_milli\x18\r \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\x0e \x01(\x03R\vmemoryBytesB\b\n" + "\x06config\"\x11\n" + "\x0fRestoreResponse*`\n" + "\n" + diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 1aab6cc3b..97938b9f3 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -48,6 +48,12 @@ message RunRequest { // fetches the relevant assets and records them with the actor's on-node state // so a later Checkpoint can pin the same version into the snapshot manifest. SandboxAssets sandbox_assets = 8; + + // The actor's declared size, from the ActorTemplate's resource limits. atelet + // passes these through to the sandbox so it is sized to the actor (not the + // whole host or worker pod). Zero means "unset": keep the runtime default. + int64 cpu_milli = 9; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 10; // Memory limit in bytes. } // AssetFile is one content-addressed file atelet fetches for a sandbox runtime @@ -258,6 +264,13 @@ message RestoreRequest { // of the `config` oneof: the actor's snapshot may be local (a pause // checkpoint) while the golden snapshot is always external. string golden_snapshot_uri_prefix = 12; + + // The actor's declared size, from the ActorTemplate's resource limits. For + // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; + // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // means "unset": keep the runtime default. + int64 cpu_milli = 13; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 14; // Memory limit in bytes. } message RestoreResponse { diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index 7cd226315..7d575d0bf 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -117,8 +117,14 @@ type RunWorkloadRequest struct { // Remote egress gateway selected for this activation. When absent, actor // traffic uses direct egress instead of being redirected through atunnel. EgressGatewayAddress *string `protobuf:"bytes,10,opt,name=egress_gateway_address,json=egressGatewayAddress,proto3,oneof" json:"egress_gateway_address,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The actor's declared size, from the ActorTemplate's resource limits. ateom + // sizes the sandbox to these (cgroup caps via the OCI spec, and for the + // micro-VM the VM's vCPU count and memory). Zero means "unset": keep the + // runtime default (unlimited for gVisor, the kata config for the micro-VM). + CpuMilli int64 `protobuf:"varint,11,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,12,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RunWorkloadRequest) Reset() { @@ -221,6 +227,20 @@ func (x *RunWorkloadRequest) GetEgressGatewayAddress() string { return "" } +func (x *RunWorkloadRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RunWorkloadRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + // WorkloadSpec parallels Pod, but with far fewer configurable fields. type WorkloadSpec struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -730,8 +750,14 @@ type RestoreWorkloadRequest struct { // Set only when scope is SNAPSHOT_SCOPE_DATA_ON_GOLDEN. Mirrors the // snapshot_uri_prefix contract (field 8). GoldenSnapshotUriPrefix string `protobuf:"bytes,13,opt,name=golden_snapshot_uri_prefix,json=goldenSnapshotUriPrefix,proto3" json:"golden_snapshot_uri_prefix,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The actor's declared size, from the ActorTemplate's resource limits. Used to + // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a + // FULL micro-VM restore the size baked into the snapshot is authoritative and + // these are ignored. Zero means "unset": keep the runtime default. + CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreWorkloadRequest) Reset() { @@ -855,6 +881,20 @@ func (x *RestoreWorkloadRequest) GetGoldenSnapshotUriPrefix() string { return "" } +func (x *RestoreWorkloadRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RestoreWorkloadRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + type RestoreWorkloadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -895,7 +935,7 @@ var File_ateom_proto protoreflect.FileDescriptor const file_ateom_proto_rawDesc = "" + "\n" + - "\vateom.proto\x12\x05ateom\"\xc1\x04\n" + + "\vateom.proto\x12\x05ateom\"\x81\x05\n" + "\x12RunWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -909,7 +949,9 @@ const file_ateom_proto_rawDesc = "" + "\x04spec\x18\a \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12`\n" + "\x13runtime_asset_paths\x18\b \x03(\v20.ateom.RunWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x129\n" + "\x16egress_gateway_address\x18\n" + - " \x01(\tH\x00R\x14egressGatewayAddress\x88\x01\x01\x1aD\n" + + " \x01(\tH\x00R\x14egressGatewayAddress\x88\x01\x01\x12\x1b\n" + + "\tcpu_milli\x18\v \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\f \x01(\x03R\vmemoryBytes\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x19\n" + @@ -952,7 +994,7 @@ const file_ateom_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"C\n" + "\x1aCheckpointWorkloadResponse\x12%\n" + - "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\"\xe2\x05\n" + + "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\"\xa2\x06\n" + "\x16RestoreWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -969,7 +1011,9 @@ const file_ateom_proto_rawDesc = "" + "\x05scope\x18\n" + " \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x129\n" + "\x16egress_gateway_address\x18\f \x01(\tH\x00R\x14egressGatewayAddress\x88\x01\x01\x12;\n" + - "\x1agolden_snapshot_uri_prefix\x18\r \x01(\tR\x17goldenSnapshotUriPrefix\x1aD\n" + + "\x1agolden_snapshot_uri_prefix\x18\r \x01(\tR\x17goldenSnapshotUriPrefix\x12\x1b\n" + + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytes\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x19\n" + diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index c5f2293bd..f66ca5881 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -70,6 +70,13 @@ message RunWorkloadRequest { // Remote egress gateway selected for this activation. When absent, actor // traffic uses direct egress instead of being redirected through atunnel. optional string egress_gateway_address = 10; + + // The actor's declared size, from the ActorTemplate's resource limits. ateom + // sizes the sandbox to these (cgroup caps via the OCI spec, and for the + // micro-VM the VM's vCPU count and memory). Zero means "unset": keep the + // runtime default (unlimited for gVisor, the kata config for the micro-VM). + int64 cpu_milli = 11; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 12; // Memory limit in bytes. } // WorkloadSpec parallels Pod, but with far fewer configurable fields. @@ -207,6 +214,13 @@ message RestoreWorkloadRequest { // Set only when scope is SNAPSHOT_SCOPE_DATA_ON_GOLDEN. Mirrors the // snapshot_uri_prefix contract (field 8). string golden_snapshot_uri_prefix = 13; + + // The actor's declared size, from the ActorTemplate's resource limits. Used to + // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a + // FULL micro-VM restore the size baked into the snapshot is authoritative and + // these are ignored. Zero means "unset": keep the runtime default. + int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 15; // Memory limit in bytes. } message RestoreWorkloadResponse { diff --git a/internal/sizing/sizing.go b/internal/sizing/sizing.go new file mode 100644 index 000000000..ff1f46865 --- /dev/null +++ b/internal/sizing/sizing.go @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sizing right-sizes a sandbox to the actor's declared resource limits. +// The bulk of right-sizing is writing the correct cgroup values, which is +// identical for the gVisor and micro-VM runtimes, so both ateom binaries share +// this package: the actor's limits arrive over the ateom RPCs (RunWorkload / +// RestoreWorkload) and ApplyToOCISpec writes them into the container OCI spec. +// runsc then applies them to the host cgroup leaf (gVisor) and the kata-agent +// applies them to the guest cgroup (micro-VM). The micro-VM additionally sizes +// the VM itself from the same SandboxSize (see VCPUs). +package sizing + +import ( + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +const ( + // cpuQuotaPeriodMicros is the cgroup v2 cpu.max period (100ms, the kernel + // default) against which the CPU quota is expressed. + cpuQuotaPeriodMicros = 100000 +) + +// SandboxSize is the sandbox's target size, derived from the actor's declared +// resource limits. A zero field means "unset": the caller keeps its own default +// (the kata config for the micro-VM, unlimited for gVisor). +type SandboxSize struct { + // MilliCPU is the CPU limit in millicores (1000 = one core), or 0 if unset. + MilliCPU int64 + // MemoryBytes is the memory limit in bytes, or 0 if unset. + MemoryBytes int64 +} + +// FromLimits builds a SandboxSize from an actor's declared limits (millicores and +// bytes) as carried on the ateom RPCs. It is runtime-agnostic; both ateom-gvisor +// and ateom-microvm call it. Negative values are clamped to zero ("unset"). +func FromLimits(milliCPU, memoryBytes int64) SandboxSize { + if milliCPU < 0 { + milliCPU = 0 + } + if memoryBytes < 0 { + memoryBytes = 0 + } + return SandboxSize{MilliCPU: milliCPU, MemoryBytes: memoryBytes} +} + +// VCPUs converts the CPU limit to a whole vCPU count for the micro-VM, rounding +// up so a fractional limit still yields a usable core (minimum 1 when a limit is +// set). Returns 0 when the CPU limit is unset, letting the caller keep its +// default. +func (s SandboxSize) VCPUs() int { + if s.MilliCPU <= 0 { + return 0 + } + v := (s.MilliCPU + 999) / 1000 + if v < 1 { + v = 1 + } + return int(v) +} + +// ApplyToOCISpec writes the pod's CPU/memory limits into the container OCI spec's +// linux.resources so the sandbox cgroup is created with the right values. This is +// the piece shared by both runtimes: runsc applies it to the host cgroup leaf +// (gVisor) and the kata-agent applies it to the guest cgroup (micro-VM). Fields +// that are unset in SandboxSize are left untouched, preserving any existing values +// (e.g. the micro-VM's device allowlist and CPU shares). +func (s SandboxSize) ApplyToOCISpec(spec *specs.Spec) { + if s.MilliCPU <= 0 && s.MemoryBytes <= 0 { + return + } + if spec.Linux == nil { + spec.Linux = &specs.Linux{} + } + if spec.Linux.Resources == nil { + spec.Linux.Resources = &specs.LinuxResources{} + } + if s.MilliCPU > 0 { + if spec.Linux.Resources.CPU == nil { + spec.Linux.Resources.CPU = &specs.LinuxCPU{} + } + period := uint64(cpuQuotaPeriodMicros) + quota := s.MilliCPU * cpuQuotaPeriodMicros / 1000 + spec.Linux.Resources.CPU.Period = &period + spec.Linux.Resources.CPU.Quota = "a + } + if s.MemoryBytes > 0 { + if spec.Linux.Resources.Memory == nil { + spec.Linux.Resources.Memory = &specs.LinuxMemory{} + } + limit := s.MemoryBytes + spec.Linux.Resources.Memory.Limit = &limit + } +} diff --git a/internal/sizing/sizing_test.go b/internal/sizing/sizing_test.go new file mode 100644 index 000000000..1c9ac24fc --- /dev/null +++ b/internal/sizing/sizing_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizing + +import ( + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestFromLimits(t *testing.T) { + got := FromLimits(1500, 2147483648) + if got.MilliCPU != 1500 || got.MemoryBytes != 2147483648 { + t.Fatalf("FromLimits() = %+v", got) + } +} + +func TestFromLimitsClampsNegative(t *testing.T) { + got := FromLimits(-5, -1) + if got.MilliCPU != 0 || got.MemoryBytes != 0 { + t.Fatalf("FromLimits() = %+v, want zero", got) + } +} + +func TestVCPUs(t *testing.T) { + cases := []struct { + milli int64 + want int + }{ + {0, 0}, + {1, 1}, + {999, 1}, + {1000, 1}, + {1001, 2}, + {2500, 3}, + {4000, 4}, + } + for _, c := range cases { + if got := (SandboxSize{MilliCPU: c.milli}).VCPUs(); got != c.want { + t.Errorf("VCPUs(%d) = %d, want %d", c.milli, got, c.want) + } + } +} + +func TestApplyToOCISpec(t *testing.T) { + spec := &specs.Spec{} + (SandboxSize{MilliCPU: 2000, MemoryBytes: 1073741824}).ApplyToOCISpec(spec) + + if spec.Linux == nil || spec.Linux.Resources == nil { + t.Fatal("resources not set") + } + cpu := spec.Linux.Resources.CPU + if cpu == nil || cpu.Quota == nil || cpu.Period == nil { + t.Fatal("cpu not set") + } + if *cpu.Period != cpuQuotaPeriodMicros || *cpu.Quota != 2*cpuQuotaPeriodMicros { + t.Errorf("cpu = quota %d period %d", *cpu.Quota, *cpu.Period) + } + mem := spec.Linux.Resources.Memory + if mem == nil || mem.Limit == nil || *mem.Limit != 1073741824 { + t.Errorf("memory limit not set correctly: %+v", mem) + } +} + +func TestApplyToOCISpecPreservesExistingAndSkipsUnset(t *testing.T) { + shares := uint64(1024) + spec := &specs.Spec{Linux: &specs.Linux{Resources: &specs.LinuxResources{ + CPU: &specs.LinuxCPU{Shares: &shares}, + }}} + // Only memory set; CPU limit unset must not clobber existing shares and must + // not add a quota. + (SandboxSize{MemoryBytes: 512}).ApplyToOCISpec(spec) + + if spec.Linux.Resources.CPU.Shares == nil || *spec.Linux.Resources.CPU.Shares != 1024 { + t.Error("existing cpu shares clobbered") + } + if spec.Linux.Resources.CPU.Quota != nil { + t.Error("cpu quota set despite unset MilliCPU") + } + if spec.Linux.Resources.Memory == nil || *spec.Linux.Resources.Memory.Limit != 512 { + t.Error("memory limit not applied") + } +} + +func TestApplyToOCISpecNoopWhenEmpty(t *testing.T) { + spec := &specs.Spec{} + (SandboxSize{}).ApplyToOCISpec(spec) + if spec.Linux != nil { + t.Error("empty SandboxSize mutated spec") + } +} diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index e4fa9fca0..ef9be0fb9 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -277,6 +277,74 @@ spec: - message: All images must be pinned (changing the image invalidates snapshots) rule: self.contains('@') + resources: + description: |- + Resources declares the compute resources for each actor of this template. + Unlike a pod, an actor is sized by its Limits: the sandbox is built to the + CPU/memory limits (cgroup caps, and for the micro-VM the VM's vCPU count and + memory), the scheduler only places the actor on a worker whose capacity is + >= these limits, and the limits are supplied to the sandbox over the actor + RPCs. Because the size is baked into snapshots, it is part of the immutable + spec. Requests are not consulted today (an actor occupies its whole worker). + A zero or absent limit leaves the sandbox at the runtime default (unlimited + for gVisor, the kata config for the micro-VM). + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object sandboxClass: default: gvisor description: |- diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index fb18c267f..5995cf932 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -15,6 +15,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -415,6 +416,19 @@ type ActorTemplateSpec struct { // +optional // +kubebuilder:validation:MaxItems=32 Volumes []Volume `json:"volumes,omitempty"` + + // Resources declares the compute resources for each actor of this template. + // Unlike a pod, an actor is sized by its Limits: the sandbox is built to the + // CPU/memory limits (cgroup caps, and for the micro-VM the VM's vCPU count and + // memory), the scheduler only places the actor on a worker whose capacity is + // >= these limits, and the limits are supplied to the sandbox over the actor + // RPCs. Because the size is baked into snapshots, it is part of the immutable + // spec. Requests are not consulted today (an actor occupies its whole worker). + // A zero or absent limit leaves the sandbox at the runtime default (unlimited + // for gVisor, the kata config for the micro-VM). + // + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } // TODO: add validation diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 738367183..29baad253 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -106,6 +106,11 @@ func (in *ActorTemplateSpec) DeepCopyInto(out *ActorTemplateSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ActorTemplateSpec. diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index fdae6906f..4813b1086 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -2428,8 +2428,15 @@ type Worker struct { SandboxClass string `protobuf:"bytes,9,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` Labels map[string]string `protobuf:"bytes,10,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` State Worker_State `protobuf:"varint,11,opt,name=state,proto3,enum=ateapi.Worker_State" json:"state,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Capacity is the worker pod's compute capacity available to host an actor + // sandbox, taken from the ateom container's resource limits. The scheduler + // only places an actor on a worker whose capacity is >= the actor's declared + // resource limits. Zero means "unknown/unset": treated as unconstrained so + // placement is not blocked (matching the pre-capacity behaviour). + CpuMilliCapacity int64 `protobuf:"varint,12,opt,name=cpu_milli_capacity,json=cpuMilliCapacity,proto3" json:"cpu_milli_capacity,omitempty"` // CPU capacity in millicores. + MemoryBytesCapacity int64 `protobuf:"varint,13,opt,name=memory_bytes_capacity,json=memoryBytesCapacity,proto3" json:"memory_bytes_capacity,omitempty"` // Memory capacity in bytes. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Worker) Reset() { @@ -2539,6 +2546,20 @@ func (x *Worker) GetState() Worker_State { return Worker_STATE_UNSPECIFIED } +func (x *Worker) GetCpuMilliCapacity() int64 { + if x != nil { + return x.CpuMilliCapacity + } + return 0 +} + +func (x *Worker) GetMemoryBytesCapacity() int64 { + if x != nil { + return x.MemoryBytesCapacity + } + return 0 +} + type Assignment struct { state protoimpl.MessageState `protogen:"open.v1"` ActorTemplate *KubeNamespacedObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` @@ -3123,7 +3144,7 @@ const file_ateapi_proto_rawDesc = "" + "page_token\x18\x03 \x01(\tR\tpageToken\"c\n" + "\x12ListActorsResponse\x12%\n" + "\x06actors\x18\x01 \x03(\v2\r.ateapi.ActorR\x06actors\x12&\n" + - "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\x9a\x04\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xfc\x04\n" + "\x06Worker\x12)\n" + "\x10worker_namespace\x18\x01 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + "\vworker_pool\x18\x02 \x01(\tR\n" + @@ -3140,7 +3161,9 @@ const file_ateapi_proto_rawDesc = "" + "\rsandbox_class\x18\t \x01(\tR\fsandboxClass\x122\n" + "\x06labels\x18\n" + " \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12*\n" + - "\x05state\x18\v \x01(\x0e2\x14.ateapi.Worker.StateR\x05state\x1a9\n" + + "\x05state\x18\v \x01(\x0e2\x14.ateapi.Worker.StateR\x05state\x12,\n" + + "\x12cpu_milli_capacity\x18\f \x01(\x03R\x10cpuMilliCapacity\x122\n" + + "\x15memory_bytes_capacity\x18\r \x01(\x03R\x13memoryBytesCapacity\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"D\n" + diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index bc576cfc9..44c2fe3f4 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -456,6 +456,14 @@ message Worker { STATE_DRAINING = 2; } State state = 11; + + // Capacity is the worker pod's compute capacity available to host an actor + // sandbox, taken from the ateom container's resource limits. The scheduler + // only places an actor on a worker whose capacity is >= the actor's declared + // resource limits. Zero means "unknown/unset": treated as unconstrained so + // placement is not blocked (matching the pre-capacity behaviour). + int64 cpu_milli_capacity = 12; // CPU capacity in millicores. + int64 memory_bytes_capacity = 13; // Memory capacity in bytes. } message Assignment {