diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index e7dfb30e7d..f245ef4f09 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -38,6 +38,7 @@ require ( github.com/go-git/go-billy/v5 v5.9.0 github.com/go-openapi/runtime v0.29.2 github.com/go-openapi/strfmt v0.26.1 + github.com/gofrs/flock v0.13.0 github.com/google/go-containerregistry v0.21.7 github.com/google/nftables v0.3.0 github.com/google/uuid v1.6.0 diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 830e873b23..dedb79a8c8 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -617,6 +617,8 @@ github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= +github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= diff --git a/packages/orchestrator/pkg/factories/run.go b/packages/orchestrator/pkg/factories/run.go index 445afca748..945ee4ec36 100644 --- a/packages/orchestrator/pkg/factories/run.go +++ b/packages/orchestrator/pkg/factories/run.go @@ -13,10 +13,12 @@ import ( "os" "os/signal" "slices" + "strconv" "strings" "syscall" "time" + "github.com/gofrs/flock" "github.com/google/uuid" "github.com/soheilhy/cmux" "go.opentelemetry.io/otel/attribute" @@ -170,6 +172,61 @@ func ensureDirs(c cfg.Config) error { return nil } +func acquireOrchestratorLock(path string) (*flock.Flock, error) { + fileLock := flock.New(path, flock.SetPermissions(0o644)) + locked, err := fileLock.TryLock() + if err != nil { + return nil, fmt.Errorf("lock file: %w", err) + } + if !locked { + // flock(2) is released by the kernel on crash, so reaching here means a + // live process holds the lock. Surface the PID it recorded, if any. + if pid, perr := readLockHolderPID(path); perr == nil && pid > 0 { + return nil, fmt.Errorf("another instance is running with pid %d", pid) + } + + return nil, errors.New("another instance is running") + } + + // Record our PID so a future conflicting instance can report it. We hold the + // exclusive advisory lock, so no other process writes this file concurrently. + if err := writeLockHolderPID(path); err != nil { + _ = fileLock.Unlock() + + return nil, fmt.Errorf("write lock holder pid: %w", err) + } + + return fileLock, nil +} + +func writeLockHolderPID(path string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("open lock file: %w", err) + } + defer f.Close() + + if _, err := fmt.Fprintf(f, "%d\n", os.Getpid()); err != nil { + return fmt.Errorf("write pid: %w", err) + } + + return nil +} + +func readLockHolderPID(path string) (int, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, fmt.Errorf("read lock file: %w", err) + } + + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + return 0, fmt.Errorf("parse pid: %w", err) + } + + return pid, nil +} + func run(config cfg.Config, opts Options) (success bool) { success = true @@ -178,31 +235,27 @@ func run(config cfg.Config, opts Options) (success bool) { services := cfg.GetServices(config) - // Check if the orchestrator crashed and restarted - // Skip this check in development mode - // We don't want to lock if the service is running with force stop; the subsequent start would fail. - if !env.IsDevelopment() && !config.ForceStop && services.RunsOrchestrator() { - fileLockName := config.OrchestratorLockPath - info, err := os.Stat(fileLockName) - if err == nil { - log.Fatalf("Orchestrator was already started at %s, exiting", info.ModTime()) - } + usesSandboxRuntime := services.UsesSandboxRuntime() - f, err := os.Create(fileLockName) + // Enforce a single host-level sandbox runtime instance. + // Skip this check in development mode. + if !env.IsDevelopment() && usesSandboxRuntime { + f, err := acquireOrchestratorLock(config.OrchestratorLockPath) if err != nil { - log.Fatalf("Failed to create lock file %s: %v", fileLockName, err) + log.Fatalf("Failed to acquire orchestrator lock %s: %v", config.OrchestratorLockPath, err) } defer func() { fileErr := f.Close() if fileErr != nil { - log.Printf("Failed to close lock file %s: %v", fileLockName, fileErr) + log.Printf("Failed to close lock file %s: %v", config.OrchestratorLockPath, fileErr) } - - // Remove the lock file on graceful shutdown - if success == true { - if fileErr = os.Remove(fileLockName); fileErr != nil { - log.Printf("Failed to remove lock file %s: %v", fileLockName, fileErr) - } + // Remove the lock file on clean shutdown so a rollback to the older + // stat-based release can start: that guard exits whenever the lock + // file exists and cannot tell that this process is already gone. + // TODO: Remove this os.Remove once all hosts run a flock-based + // release and rollback to the stat-based guard is no longer possible. + if rmErr := os.Remove(config.OrchestratorLockPath); rmErr != nil && !os.IsNotExist(rmErr) { + log.Printf("Failed to remove lock file %s: %v", config.OrchestratorLockPath, rmErr) } }() } @@ -626,7 +679,7 @@ func run(config cfg.Config, opts Options) (success bool) { // Sandbox-runtime reclaim must run before newStorage below: reclaim deletes // leaked ns-* from /run/netns, and NewStorageLocal snapshots the remaining // namespaces as foreign at construction. - if services.UsesSandboxRuntime() && !config.DisableStartupReclaim { + if usesSandboxRuntime && !config.DisableStartupReclaim { startupreclaim.Run(ctx, startupreclaim.Config{ NetworkConfig: config.NetworkConfig, EgressProxy: egressSetup.Proxy,