diff --git a/cmd/entire/cli/agent/claudecode/hooks.go b/cmd/entire/cli/agent/claudecode/hooks.go index c72271d6f5..8f180165cd 100644 --- a/cmd/entire/cli/agent/claudecode/hooks.go +++ b/cmd/entire/cli/agent/claudecode/hooks.go @@ -111,6 +111,10 @@ func claudeHookConfig(ctx context.Context) (*agent.HookConfigFile, error) { return agent.OpenHookConfig(repoRoot, (&ClaudeCodeAgent{}).HookConfigRelPath()) //nolint:wrapcheck // agent.HookConfigFile already names the file in its error } +func claudeHookConfigForWorktreeRoot(worktreeRoot string) (*agent.HookConfigFile, error) { + return agent.OpenHookConfig(worktreeRoot, (&ClaudeCodeAgent{}).HookConfigRelPath()) //nolint:wrapcheck // agent.HookConfigFile already names the file in its error +} + // resolveInstallRepoRoot locates the repo root InstallHooks writes under, // falling back to CWD when not in a git repo (e.g. during tests). func resolveInstallRepoRoot(ctx context.Context) (string, error) { @@ -430,6 +434,10 @@ func loadClaudeSettings(ctx context.Context) (ClaudeSettings, error) { if err != nil { return ClaudeSettings{}, err } + return loadClaudeSettingsFromConfig(ctx, cfg) +} + +func loadClaudeSettingsFromConfig(ctx context.Context, cfg *agent.HookConfigFile) (ClaudeSettings, error) { data, err := cfg.Read() // No settings file means no hooks, which is an answer; anything else means we // could not read the answer. @@ -463,6 +471,21 @@ func (c *ClaudeCodeAgent) AreHooksInstalled(ctx context.Context) (bool, error) { return hasEntireHook(settings.Hooks.Stop), nil } +// AreProjectHooksInstalledInWorktree checks an explicit worktree's shared +// .claude/settings.json for Entire hooks. Claude user and local settings are +// outside this portability check because they do not travel with the checkout. +func AreProjectHooksInstalledInWorktree(ctx context.Context, worktreeRoot string) (bool, error) { + cfg, err := claudeHookConfigForWorktreeRoot(worktreeRoot) + if err != nil { + return false, err + } + settings, err := loadClaudeSettingsFromConfig(ctx, cfg) + if err != nil { + return false, err + } + return hasEntireHook(settings.Hooks.Stop), nil +} + // HookConfigState describes how Entire's Claude Code hooks compare to what // InstallHooks would write today. Aliased to the shared agent-package type so // `entire status` and `entire doctor` can treat every agent's drift check diff --git a/cmd/entire/cli/agent/claudecode/hooks_test.go b/cmd/entire/cli/agent/claudecode/hooks_test.go index 2b191a264e..02d4663372 100644 --- a/cmd/entire/cli/agent/claudecode/hooks_test.go +++ b/cmd/entire/cli/agent/claudecode/hooks_test.go @@ -267,6 +267,29 @@ func writeSettingsFile(t *testing.T, tempDir, content string) { } } +func TestAreProjectHooksInstalledInWorktree(t *testing.T) { + t.Parallel() + + configured := t.TempDir() + writeSettingsFile(t, configured, `{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"entire hooks claude-code stop"}]}]}}`) + + installed, err := AreProjectHooksInstalledInWorktree(t.Context(), configured) + if err != nil { + t.Fatalf("AreProjectHooksInstalledInWorktree() error = %v", err) + } + if !installed { + t.Error("AreProjectHooksInstalledInWorktree() = false, want true") + } + + installed, err = AreProjectHooksInstalledInWorktree(t.Context(), t.TempDir()) + if err != nil { + t.Fatalf("AreProjectHooksInstalledInWorktree() for empty worktree error = %v", err) + } + if installed { + t.Error("AreProjectHooksInstalledInWorktree() = true for empty worktree") + } +} + func containsRule(rules []string, rule string) bool { return slices.Contains(rules, rule) } diff --git a/cmd/entire/cli/checkpoint/fsstore/fsstore.go b/cmd/entire/cli/checkpoint/fsstore/fsstore.go index baa4c05084..5c22c9dc2a 100644 --- a/cmd/entire/cli/checkpoint/fsstore/fsstore.go +++ b/cmd/entire/cli/checkpoint/fsstore/fsstore.go @@ -329,7 +329,7 @@ func metadataFromWriteOptions(opts cp.WriteOptions) cp.Metadata { TranscriptLinesAtStart: opts.CheckpointTranscriptStart, //nolint:staticcheck // deliberate: git writes both so older CLIs can still read the metadata TokenUsage: opts.TokenUsage, SkillEvents: opts.SkillEvents, - PromptAttributions: opts.PromptAttributionsJSON, + PromptAttributions: checkpoint.CapPromptAttributions(context.Background(), opts.PromptAttributionsJSON, opts.SessionID), SessionMetrics: opts.SessionMetrics, Summary: checkpoint.RedactSummary(opts.Summary), Attribution: opts.Attribution, diff --git a/cmd/entire/cli/checkpoint/persistent.go b/cmd/entire/cli/checkpoint/persistent.go index e095d091a7..068b3d3fa6 100644 --- a/cmd/entire/cli/checkpoint/persistent.go +++ b/cmd/entire/cli/checkpoint/persistent.go @@ -734,7 +734,7 @@ func (s *treeWriter) writeSessionToSubdirectory(ctx context.Context, opts WriteO SkillEvents: opts.SkillEvents, SessionMetrics: opts.SessionMetrics, Attribution: opts.Attribution, - PromptAttributions: opts.PromptAttributionsJSON, + PromptAttributions: CapPromptAttributions(ctx, opts.PromptAttributionsJSON, opts.SessionID), Summary: RedactSummary(opts.Summary), CLIVersion: versioninfo.Version, Kind: opts.Kind, @@ -1789,6 +1789,10 @@ func (s *treeWriter) updateSessionMetadata(sessionDir string, entries map[string return fmt.Errorf("read session metadata: %w", err) } mutate(metadata) + // This path re-marshals metadata an earlier CLI wrote, so the write-time cap + // applies here too: otherwise a pre-v0.10.1 oversized prompt_attributions + // would be copied verbatim into every finalize or backfill rewrite. + metadata.PromptAttributions = CapPromptAttributions(context.Background(), metadata.PromptAttributions, metadata.SessionID) metadataJSON, err := jsonutil.MarshalIndentWithNewline(metadata, "", " ") if err != nil { diff --git a/cmd/entire/cli/checkpoint/prompt_attributions_cap.go b/cmd/entire/cli/checkpoint/prompt_attributions_cap.go new file mode 100644 index 0000000000..69a3425fd3 --- /dev/null +++ b/cmd/entire/cli/checkpoint/prompt_attributions_cap.go @@ -0,0 +1,38 @@ +package checkpoint + +import ( + "context" + "encoding/json" + "log/slog" + + "github.com/entireio/cli/cmd/entire/cli/logging" +) + +// MaxPromptAttributionsBytes bounds the prompt_attributions field a per-session +// metadata.json may carry. The field is a diagnostic record of the per-prompt +// line counts that fed the attribution summary: nothing in the CLI reads it back, +// and entire-api's ingest explicitly salvages a metadata.json whose tail it +// occupies. It is also the one field whose size scales with the working tree +// rather than with the session — CLI versions before v0.10.1 recorded every file +// of every nested git checkout on every prompt, and produced 70MB and 106MB +// metadata.json blobs that made entire/checkpoints/v1 unpushable to GitHub +// (100 MiB blob limit). A healthy session's record is tens of kilobytes, so +// 4 MiB is two orders of magnitude of headroom before the field is dropped. +const MaxPromptAttributionsBytes = 4 << 20 + +// CapPromptAttributions returns raw unchanged when it fits under +// MaxPromptAttributionsBytes, and nil — dropping the field from the written +// metadata — when it does not. The attribution summary itself is computed before +// this point and is unaffected; only the diagnostic input is withheld, and the +// drop is logged so the omission is explainable from .entire/logs. +func CapPromptAttributions(ctx context.Context, raw json.RawMessage, sessionID string) json.RawMessage { + if len(raw) <= MaxPromptAttributionsBytes { + return raw + } + logging.Warn(logging.WithComponent(ctx, "checkpoint"), + "dropping oversized prompt_attributions from session metadata", + slog.Int("bytes", len(raw)), + slog.Int("cap_bytes", MaxPromptAttributionsBytes), + slog.String("session_id", sessionID)) + return nil +} diff --git a/cmd/entire/cli/checkpoint/prompt_attributions_cap_test.go b/cmd/entire/cli/checkpoint/prompt_attributions_cap_test.go new file mode 100644 index 0000000000..b2812082e6 --- /dev/null +++ b/cmd/entire/cli/checkpoint/prompt_attributions_cap_test.go @@ -0,0 +1,147 @@ +package checkpoint + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" + "github.com/entireio/cli/cmd/entire/cli/gitrepo" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/testutil" + "github.com/entireio/cli/redact" +) + +func TestCapPromptAttributions_KeepsFittingPayload(t *testing.T) { + t.Parallel() + raw := json.RawMessage(`[{"checkpoint_number":1,"user_lines_added":3}]`) + assert.Equal(t, raw, CapPromptAttributions(context.Background(), raw, "s1")) + assert.Nil(t, CapPromptAttributions(context.Background(), nil, "s1")) +} + +func TestCapPromptAttributions_DropsOversizedPayload(t *testing.T) { + t.Parallel() + // One byte over the cap: the field is dropped rather than truncated, since a + // clipped JSON array is worse than an absent diagnostic. + raw := json.RawMessage(bytes.Repeat([]byte("x"), MaxPromptAttributionsBytes+1)) + assert.Nil(t, CapPromptAttributions(context.Background(), raw, "s1")) + + exact := json.RawMessage(bytes.Repeat([]byte("x"), MaxPromptAttributionsBytes)) + assert.Equal(t, exact, CapPromptAttributions(context.Background(), exact, "s1"), "the cap is inclusive") +} + +// oversizedPromptAttributions is a syntactically valid prompt_attributions +// payload just over the cap, the shape a pre-v0.10.1 nested-checkout walk left +// behind. +func oversizedPromptAttributions(t *testing.T) json.RawMessage { + t.Helper() + var b bytes.Buffer + b.WriteString(`[{"checkpoint_number":1,"user_added_per_file":{`) + for i := 0; b.Len() < MaxPromptAttributionsBytes+1; i++ { + if i > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `".claude/worktrees/agent/pkg/file%d.go":3`, i) + } + b.WriteString(`}}]`) + raw := json.RawMessage(b.Bytes()) + require.True(t, json.Valid(raw)) + return raw +} + +// sessionMetadataAt reads //metadata.json from the +// branch tip as a generic document, so a test can assert on key presence. +func sessionMetadataAt(t *testing.T, store *GitStore, cpID id.CheckpointID, sessionIndex int) map[string]json.RawMessage { + t.Helper() + ref, err := store.repo.Reference(store.refs.Primary, true) + require.NoError(t, err) + commit, err := store.repo.CommitObject(ref.Hash()) + require.NoError(t, err) + f, err := commit.File(fmt.Sprintf("%s/%d/%s", cpID.Path(), sessionIndex, paths.MetadataFileName)) + require.NoError(t, err) + content, err := f.Contents() + require.NoError(t, err) + var doc map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(content), &doc)) + return doc +} + +// The cap is applied at the writer boundary, not only in the helper: a +// session written with an oversized diagnostic lands without the field, and a +// session with a small one keeps it. +func TestGitStoreWrite_CapsPromptAttributionsInStoredMetadata(t *testing.T) { + t.Parallel() + dir := t.TempDir() + testutil.InitRepo(t, dir) + repo, err := gitrepo.OpenPath(dir) + require.NoError(t, err) + store := NewGitStore(repo, DefaultV1Refs()) + ctx := context.Background() + + write := func(cpID string, attrs json.RawMessage) id.CheckpointID { + checkpointID := id.MustCheckpointID(cpID) + require.NoError(t, store.Write(ctx, Session{ + CheckpointID: checkpointID, + SessionID: "session-" + cpID, + Strategy: "manual-commit", + Transcript: redact.AlreadyRedacted([]byte(`{"role":"user","content":"hi"}` + "\n")), + Prompts: []string{"hi"}, + AuthorName: "Test", + AuthorEmail: "test@test.com", + PromptAttributionsJSON: attrs, + })) + return checkpointID + } + + big := write("aaaaaaaaaaaa", oversizedPromptAttributions(t)) + doc := sessionMetadataAt(t, store, big, 0) + assert.NotContains(t, doc, "prompt_attributions", "an oversized diagnostic is dropped at the write boundary") + assert.Contains(t, doc, "session_id", "the rest of the metadata is intact") + + small := write("bbbbbbbbbbbb", json.RawMessage(`[{"checkpoint_number":1,"user_lines_added":2}]`)) + doc = sessionMetadataAt(t, store, small, 0) + assert.JSONEq(t, `[{"checkpoint_number":1,"user_lines_added":2}]`, string(doc["prompt_attributions"]), "a fitting diagnostic is kept verbatim") +} + +// The finalize/backfill path re-marshals metadata an older CLI wrote. Without +// the cap there, a pre-v0.10.1 oversized field would be copied into every later +// rewrite of that checkpoint. +func TestUpdateSessionMetadata_CapsPromptAttributionsFromOlderWriters(t *testing.T) { + t.Parallel() + dir := t.TempDir() + testutil.InitRepo(t, dir) + repo, err := gitrepo.OpenPath(dir) + require.NoError(t, err) + w := &treeWriter{repo: repo} + + existing, err := json.Marshal(map[string]any{ + "session_id": "legacy", + "prompt_attributions": oversizedPromptAttributions(t), + }) + require.NoError(t, err) + blob, err := CreateBlobFromContent(repo, existing) + require.NoError(t, err) + const sessionDir = "ab/cdef000001/0" + metadataPath := checkpointSubtreePath(sessionDir, paths.MetadataFileName) + entries := map[string]object.TreeEntry{ + metadataPath: {Name: metadataPath, Mode: filemode.Regular, Hash: blob}, + } + + require.NoError(t, w.updateSessionMetadata(sessionDir, entries, func(m *Metadata) { + m.CompactTranscriptStart = new(int) + })) + + assert.NotEqual(t, blob, entries[metadataPath].Hash, "the blob was rewritten") + updated, err := readJSONFromBlob[map[string]json.RawMessage](repo, entries[metadataPath].Hash) + require.NoError(t, err) + assert.NotContains(t, *updated, "prompt_attributions") + assert.Contains(t, *updated, "session_id") + assert.Contains(t, *updated, "compact_transcript_start", "the caller's mutation is applied") +} diff --git a/cmd/entire/cli/doctor.go b/cmd/entire/cli/doctor.go index bbbf009690..930fcd45cf 100644 --- a/cmd/entire/cli/doctor.go +++ b/cmd/entire/cli/doctor.go @@ -47,29 +47,43 @@ Checks performed: entire/checkpoints/v1 branches share no common ancestor (caused by a previous bug). Fixes by cherry-picking local checkpoints onto remote tip. - 2. Operational logs: warn when .entire/logs cannot be written. Every other + 2. Oversized checkpoint metadata: detects metadata.json blobs on + entire/checkpoints/v1 over 50 MiB (GitHub refuses blobs over 100 MiB, so + the branch cannot be pushed or mirrored there). Caused by CLI versions + before v0.10.1 recording nested git checkouts in the prompt_attributions + diagnostic field. Interactively offers to rewrite the branch without that + field and update the checkpoint sync remote (lease-guarded); --force does + NOT apply this one, because it rewrites history and pushes. Use + 'entire doctor shrink-checkpoint-metadata' to apply it non-interactively. + + 3. Linked-worktree portability: warn when Entire settings and the shared + Claude project hook config are present in another worktree, but this + worktree is missing either part. The check is read-only and explains how + to make the setup available to future worktrees and clones. + + 4. Operational logs: warn when .entire/logs cannot be written. Every other diagnostic is delivered by writing there, and that write is silent about its own failure, so an unwritable log directory looks exactly like a repo where nothing ran. When Codex hooks are installed: - 3. Codex hook trust: warn when hooks declared in .codex/hooks.json + 5. Codex hook trust: warn when hooks declared in .codex/hooks.json lack a trusted_hash entry in the user's Codex config (i.e. /hooks review hasn't run yet on this machine, or a newer entire release added a hook the user hasn't approved yet). For each installed agent that reports hook-config drift: - 4. Hook config: warn when the installed hooks no longer match what this + 6. Hook config: warn when the installed hooks no longer match what this CLI writes (e.g. an older release wrote Claude Code tool matchers that no longer fire, or a committed Pi/OpenCode extension has gone stale). Fix by re-running 'entire enable --force'. - 5. Summary provider: warn when summary_generation.provider names a registered + 7. Summary provider: warn when summary_generation.provider names a registered agent that cannot generate text (e.g. factoryai-droid), which makes 'entire checkpoint explain --generate', 'entire dispatch' and 'entire runner setup' fail. Reports the file to change; does not rewrite it. - 6. Stuck sessions: sessions stuck in ACTIVE or ENDED phase that need cleanup. + 8. Stuck sessions: sessions stuck in ACTIVE or ENDED phase that need cleanup. A session is considered stuck if: - It is in ACTIVE phase with no interaction for over 1 hour @@ -115,6 +129,7 @@ points at --force instead of prompting.`, cmd.AddCommand(newDoctorLogsCmd()) cmd.AddCommand(newDoctorBundleCmd()) cmd.AddCommand(newDoctorMigrateCheckpointsCmd()) + cmd.AddCommand(newDoctorShrinkCheckpointMetadataCmd()) return cmd } @@ -139,9 +154,20 @@ func runSessionsFix(cmd *cobra.Command, force bool) error { finalErr = NewSilentError(fmt.Errorf("metadata check failed: %w", metadataErr)) } + // Check 2: metadata.json blobs GitHub refuses. After the disconnection + // check, which may have advanced the local branch from the remote, so the + // scan sees the reconciled history. Deliberately not given `force`: the + // fix rewrites history and pushes, and only an interactive yes or the + // dedicated subcommand may trigger that. + if sizeErr := checkOversizedCheckpointMetadata(cmd); sizeErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Error: checkpoint metadata size check failed: %v\n", sizeErr) + finalErr = NewSilentError(fmt.Errorf("checkpoint metadata size check failed: %w", sizeErr)) + } + fmt.Fprintln(cmd.OutOrStdout()) ctx := cmd.Context() + setupIssue := inspectWorktreeSetup(ctx) // Ahead of checkGitHooks, which is the check a symlinked hooks directory // makes fail: the cause should be on screen before the failure it explains. @@ -150,10 +176,11 @@ func runSessionsFix(cmd *cobra.Command, force bool) error { // The git hook surface. Checked before the agent hook checks because it is // the more fundamental one: if git hooks are broken, commits are not captured // at all and agent-config drift is noise by comparison. - if hooksErr := checkGitHooks(cmd, force); hooksErr != nil { + if hooksErr := checkGitHooksWithWorktreeSetup(cmd, force, setupIssue); hooksErr != nil { fmt.Fprintf(cmd.ErrOrStderr(), "Error: git hook check failed: %v\n", hooksErr) finalErr = NewSilentError(fmt.Errorf("git hook check failed: %w", hooksErr)) } + writeWorktreeSetupIssue(cmd.OutOrStdout(), setupIssue) // Before checkLogSink, because a symlinked .entire/logs is one of the reasons // that check fires and this one names the cause. @@ -628,12 +655,17 @@ func confirmDoctorFix(ctx context.Context, w io.Writer, title string) (bool, err // writes exactly what the next turn-start would write anyway. Someone reaching // for doctor after a rejected push wants to be unblocked, not handed a second // command to run. -func checkGitHooks(cmd *cobra.Command, force bool) error { +func checkGitHooksWithWorktreeSetup(cmd *cobra.Command, force bool, setupIssue *worktreeSetupIssue) error { ctx := cmd.Context() w := cmd.OutOrStdout() switch strategy.CheckGitHookState(ctx) { case strategy.GitHooksCurrent: + if setupIssue != nil && setupIssue.CurrentCaptureInactive { + fmt.Fprintln(w, "Git hooks: INSTALLED BUT INACTIVE") + fmt.Fprintln(w, " This worktree has no Entire settings, so its hooks skip checkpoint capture.") + return nil + } fmt.Fprintln(w, "✓ Git hooks: OK") return nil diff --git a/cmd/entire/cli/doctor_metadata_size.go b/cmd/entire/cli/doctor_metadata_size.go new file mode 100644 index 0000000000..4a95a537b0 --- /dev/null +++ b/cmd/entire/cli/doctor_metadata_size.go @@ -0,0 +1,294 @@ +package cli + +import ( + "errors" + "fmt" + "io" + + git "github.com/go-git/go-git/v6" + "github.com/spf13/cobra" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/interactive" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/settings" + "github.com/entireio/cli/cmd/entire/cli/strategy" +) + +// oversizedMetadataListCap bounds how many oversized files doctor lists by +// name; the count line carries the rest. +const oversizedMetadataListCap = 5 + +// oversizedMetadataThreshold is the size doctor treats as oversized. A +// variable so tests can lower it instead of writing 50 MiB fixtures. +var oversizedMetadataThreshold = strategy.OversizedCheckpointMetadataThreshold + +// shrinkMetadataCommand is the explicit opt-in for the repair. It is a +// subcommand rather than part of `doctor --force` because this fix rewrites +// history and force-pushes a branch: `--force` is "apply every fix without +// asking", is run routinely by scripts and the e2e harness, and must not +// acquire a remote write as a side effect. An interactive `entire doctor` +// still offers it, because a human answering the prompt is the opt-in. +const shrinkMetadataCommand = "entire doctor shrink-checkpoint-metadata" + +// checkOversizedCheckpointMetadata reports metadata.json blobs on +// entire/checkpoints/v1 that GitHub will refuse. Interactively it offers the +// rewrite; otherwise it names the subcommand that applies it. It never acts +// under a bare `--force`. +// +// Only the git-branch backend is inspected — with git-refs as the primary store +// the v1 branch is no longer what gets pushed. +func checkOversizedCheckpointMetadata(cmd *cobra.Command) error { + ctx := cmd.Context() + w := cmd.OutOrStdout() + if cpCfg, _ := settings.LoadCheckpointsConfig(ctx); checkpoint.PrimaryIsRefs(cpCfg) { //nolint:errcheck // fail-soft: a bad checkpoints block already surfaces elsewhere; default to inspecting the branch + fmt.Fprintln(w, "✓ Checkpoint metadata size: skipped (git-refs store is primary)") + return nil + } + repo, err := openRepository(ctx) + if err != nil { + return err + } + defer repo.Close() + + remoteName, scan, err := scanCheckpointMetadataSize(cmd, repo) + if err != nil { + return err + } + if scan.Empty() { + if scan.RemoteErr != nil { + // The local branch is clean but the remote side is unknown, which + // is not the same as clean: say so instead of a bare OK. + fmt.Fprintf(w, "○ Checkpoint metadata size: local %s OK; %s could not be read, so its history was not checked\n", paths.MetadataBranchName, remoteName) + return nil + } + fmt.Fprintln(w, "✓ Checkpoint metadata size: OK") + return nil + } + reportOversizedCheckpointMetadata(w, scan, remoteName) + if scan.DedicatedCheckpointRemote { + return nil + } + + // Degrade to report-only when there is nobody to ask. Unlike the other + // checks this deliberately does not honor --force; see shrinkMetadataCommand. + if !interactive.CanPromptInteractively() { + fmt.Fprintf(w, " Run `%s` to apply it.\n", shrinkMetadataCommand) + return nil + } + proceed, promptErr := confirmDoctorFix(ctx, w, "Rewrite the checkpoint branch to drop the oversized field?") + if promptErr != nil { + return promptErr + } + if !proceed { + return nil + } + return applyCheckpointMetadataShrink(cmd, repo, scan, remoteName) +} + +// newDoctorShrinkCheckpointMetadataCmd is the non-interactive path to the +// repair that checkOversizedCheckpointMetadata reports. +func newDoctorShrinkCheckpointMetadataCmd() *cobra.Command { + var yes bool + cmd := &cobra.Command{ + Use: "shrink-checkpoint-metadata", + Short: "Rewrite entire/checkpoints/v1 without oversized prompt_attributions fields and force-push it", + Long: `Repair the checkpoint branch when a per-session metadata.json exceeds 50 MiB. + +GitHub refuses blobs over 100 MiB, so one such file anywhere in the history of +entire/checkpoints/v1 makes the branch unpushable — and unmirrorable — there. +CLI versions before v0.10.1 produced them by recording every file of nested git +checkouts (agent worktrees) in the prompt_attributions diagnostic field on every +prompt. + +The repair rewrites the branch dropping that field from the oversized files. +Nothing reads the field back: checkpoint IDs, transcripts, attribution summaries +and every other file are unchanged, and history before the first oversized blob +keeps its hashes. The rewritten branch is then force-pushed to the checkpoint +sync remote, guarded with --force-with-lease against the tip observed during the +scan, so a remote that moved in between is refused rather than overwritten. + +Run it in every clone that has a local entire/checkpoints/v1: another clone that +still holds the old history would replay it onto the remote on its next push. + +Interactively it asks before rewriting; pass --yes to skip the prompt.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + w := cmd.OutOrStdout() + if cpCfg, _ := settings.LoadCheckpointsConfig(ctx); checkpoint.PrimaryIsRefs(cpCfg) { //nolint:errcheck // same fail-soft default as the check + fmt.Fprintln(w, "The git-refs store is the primary checkpoint store; the v1 branch is not pushed, so there is nothing to repair.") + return nil + } + repo, err := openRepository(ctx) + if err != nil { + return err + } + defer repo.Close() + remoteName, scan, err := scanCheckpointMetadataSize(cmd, repo) + if err != nil { + return err + } + if scan.Empty() { + fmt.Fprintln(w, "No oversized checkpoint metadata found — nothing to repair.") + return nil + } + reportOversizedCheckpointMetadata(w, scan, remoteName) + if scan.DedicatedCheckpointRemote { + return NewSilentError(strategy.ErrDedicatedCheckpointRemote) + } + if !yes { + if !interactive.CanPromptInteractively() { + fmt.Fprintln(w, " Pass --yes to apply it.") + return nil + } + proceed, promptErr := confirmDoctorFix(ctx, w, "Rewrite the checkpoint branch to drop the oversized field?") + if promptErr != nil { + return promptErr + } + if !proceed { + return nil + } + } + return applyCheckpointMetadataShrink(cmd, repo, scan, remoteName) + }, + } + cmd.Flags().BoolVarP(&yes, "yes", "y", false, "Rewrite and push without prompting") + return cmd +} + +// scanCheckpointMetadataSize resolves the checkpoint sync remote and runs the +// read-only scan, warning (rather than failing) about a remote that cannot be +// read. An election error is reported as such: a checkpoint_push_remote naming +// a missing remote fails closed on purpose and must not read as "no remote". +func scanCheckpointMetadataSize(cmd *cobra.Command, repo *git.Repository) (string, *strategy.MetadataSizeScan, error) { + ctx := cmd.Context() + remoteName := "" + if elected, electErr := strategy.ResolveCheckpointSyncRemote(ctx); electErr == nil { + remoteName = elected.Name + } else { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: checkpoint sync remote could not be resolved (%v); the remote side is not inspected.\n", electErr) + } + scan, err := strategy.ScanOversizedCheckpointMetadata(ctx, repo, remoteName, oversizedMetadataThreshold) + if err != nil { + return "", nil, fmt.Errorf("scan checkpoint metadata sizes: %w", err) + } + if scan.RemoteErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: could not read %s's %s: %v\n", remoteName, paths.MetadataBranchName, scan.RemoteErr) + } + return remoteName, scan, nil +} + +// gitHubBlobHardLimit is the blob size GitHub refuses outright; the scan's +// threshold (50 MiB) is GitHub's warning line, well short of it. +const gitHubBlobHardLimit int64 = 100 << 20 + +func reportOversizedCheckpointMetadata(w io.Writer, scan *strategy.MetadataSizeScan, remoteName string) { + fmt.Fprintln(w, "Checkpoint metadata size: OVERSIZED") + all := scan.All() + overLimit := 0 + for _, b := range all { + if b.Size > gitHubBlobHardLimit { + overLimit++ + } + } + fmt.Fprintf(w, " %d file(s) on %s exceed %s, the size GitHub warns about.\n", len(all), paths.MetadataBranchName, humanBytes(scan.Threshold)) + if overLimit > 0 { + fmt.Fprintf(w, " %d of them exceed GitHub's %s hard limit: the branch cannot be pushed or mirrored\n", overLimit, humanBytes(gitHubBlobHardLimit)) + fmt.Fprintln(w, " there while any of those is in its history.") + } else { + fmt.Fprintf(w, " None exceeds GitHub's %s hard limit yet, so the branch still pushes; the field that\n", humanBytes(gitHubBlobHardLimit)) + fmt.Fprintln(w, " made them this large is diagnostic only and can be dropped.") + } + writeOversizedList(w, scan) + fmt.Fprintln(w, " Cause: CLI versions before v0.10.1 recorded every file of nested git checkouts") + fmt.Fprintln(w, " (agent worktrees) in the prompt_attributions diagnostic field on every prompt.") + if scan.DedicatedCheckpointRemote { + fmt.Fprintln(w, " This repository pushes the branch to a dedicated checkpoint remote (checkpoint_remote),") + fmt.Fprintln(w, " which the automatic repair does not handle yet: the branch has to be rewritten without") + fmt.Fprintln(w, " that field and force-pushed to the checkpoint remote by hand.") + return + } + fmt.Fprintf(w, " Fix: rewrite %s dropping prompt_attributions from those files (nothing reads it\n", paths.MetadataBranchName) + fmt.Fprintln(w, " back; checkpoint IDs, transcripts and attribution summaries are unchanged)") + switch { + case remoteName == "": + fmt.Fprintln(w, " and leave the branch local (no checkpoint sync remote).") + case scan.PushDisabled: + fmt.Fprintf(w, " and leave %s for you to push (checkpoint pushing is disabled in settings).\n", remoteName) + default: + fmt.Fprintf(w, " and force-push it to %s (lease-guarded against the tip observed just now).\n", remoteName) + } +} + +// applyCheckpointMetadataShrink runs the rewrite and reports the outcome, +// distinguishing a completed local rewrite whose push failed from a rewrite +// that did not happen. +func applyCheckpointMetadataShrink(cmd *cobra.Command, repo *git.Repository, scan *strategy.MetadataSizeScan, remoteName string) error { + w := cmd.OutOrStdout() + res, err := strategy.ShrinkOversizedCheckpointMetadata(cmd.Context(), repo, scan, w) + if err != nil { + var pushErr *strategy.PushFailedError + if errors.As(err, &pushErr) { + fmt.Fprintf(w, " Local %s was rewritten (%d commit(s), %d file(s) shrunk) but the push to %s failed.\n", paths.MetadataBranchName, res.CommitsRewritten, res.BlobsShrunk, pushErr.Remote) + fmt.Fprintf(w, " Re-run `%s` once the remote is reachable; it pushes the rewritten branch.\n", shrinkMetadataCommand) + } + return fmt.Errorf("shrink checkpoint metadata: %w", err) + } + switch { + case res.CommitsRewritten > 0 && res.RemoteCommitsRewritten > 0: + fmt.Fprintf(w, " ✓ Fixed: rewrote %d local and %d remote commit(s), shrank %d file(s)\n", res.CommitsRewritten, res.RemoteCommitsRewritten, res.BlobsShrunk) + case res.RemoteCommitsRewritten > 0: + fmt.Fprintf(w, " ✓ Fixed: rewrote %d commit(s) of %s's history, shrank %d file(s)\n", res.RemoteCommitsRewritten, remoteName, res.BlobsShrunk) + default: + fmt.Fprintf(w, " ✓ Fixed: rewrote %d commit(s), shrank %d file(s)\n", res.CommitsRewritten, res.BlobsShrunk) + } + switch { + case res.Pushed: + fmt.Fprintf(w, " ✓ Pushed %s to %s\n", paths.MetadataBranchName, remoteName) + case res.PushSkippedReason != "": + fmt.Fprintf(w, " Not pushed: %s\n", res.PushSkippedReason) + } + for _, b := range res.StillOversized { + fmt.Fprintf(w, " Warning: %s is still %s after the rewrite; it is large for another reason.\n", b.Path, humanBytes(b.Size)) + } + if remoteName != "" { + fmt.Fprintf(w, " Run `%s` in every other clone that has a local %s, or its next push replays the old history onto %s.\n", shrinkMetadataCommand, paths.MetadataBranchName, remoteName) + } + return nil +} + +func writeOversizedList(w io.Writer, scan *strategy.MetadataSizeScan) { + remoteOnly := make(map[string]struct{}, len(scan.Remote)) + for _, b := range scan.Remote { + remoteOnly[b.Hash.String()] = struct{}{} + } + all := scan.All() + for i, b := range all { + if i == oversizedMetadataListCap { + fmt.Fprintf(w, " ... and %d more\n", len(all)-oversizedMetadataListCap) + break + } + where := "" + if _, ok := remoteOnly[b.Hash.String()]; ok { + where = fmt.Sprintf(" (only on %s)", scan.RemoteName) + } + fmt.Fprintf(w, " %8s %s%s\n", humanBytes(b.Size), b.Path, where) + } +} + +// humanBytes renders a byte count in binary units, the ones GitHub's limits +// are stated in. +func humanBytes(n int64) string { + const unit = 1024 + switch { + case n >= unit*unit*unit: + return fmt.Sprintf("%.1f GiB", float64(n)/float64(unit*unit*unit)) + case n >= unit*unit: + return fmt.Sprintf("%.1f MiB", float64(n)/float64(unit*unit)) + case n >= unit: + return fmt.Sprintf("%.1f KiB", float64(n)/float64(unit)) + default: + return fmt.Sprintf("%d B", n) + } +} diff --git a/cmd/entire/cli/doctor_metadata_size_test.go b/cmd/entire/cli/doctor_metadata_size_test.go new file mode 100644 index 0000000000..60192b2499 --- /dev/null +++ b/cmd/entire/cli/doctor_metadata_size_test.go @@ -0,0 +1,170 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/entireio/cli/cmd/entire/cli/gitrepo" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/testutil" +) + +// withTinyOversizeThreshold lowers the doctor check's threshold so a few +// kilobytes count as oversized; the real threshold is 50 MiB. +func withTinyOversizeThreshold(t *testing.T) { + t.Helper() + old := oversizedMetadataThreshold + oversizedMetadataThreshold = 1024 + t.Cleanup(func() { oversizedMetadataThreshold = old }) +} + +const bloatedDoctorMetadataPath = "ab/cdef000001/0/" + paths.MetadataFileName + +// writeBloatedCheckpointBranch puts one checkpoint on entire/checkpoints/v1 +// whose session metadata.json carries a prompt_attributions record far over +// the test threshold, and returns the branch tip. +func writeBloatedCheckpointBranch(t *testing.T, repo *git.Repository) plumbing.Hash { + t.Helper() + perFile := make(map[string]int, 200) + for i := range 200 { + perFile[fmt.Sprintf(".claude/worktrees/agent/pkg/file%d.go", i)] = 2 + } + meta, err := json.Marshal(map[string]any{ + "session_id": "s1", + "attribution": map[string]any{"agent_lines": 5}, + "prompt_attributions": []map[string]any{{"checkpoint_number": 1, "user_added_per_file": perFile}}, + }) + require.NoError(t, err) + tip := testutil.CommitFiles(t, repo, nil, map[string][]byte{ + paths.MetadataFileName: []byte("{}\n"), + bloatedDoctorMetadataPath: meta, + "ab/cdef000001/0/" + paths.TranscriptFileName: []byte("{}\n"), + }, "Checkpoint: abcdef000001") + require.NoError(t, repo.Storer.SetReference(plumbing.NewHashReference( + plumbing.NewBranchReferenceName(paths.MetadataBranchName), tip))) + return tip +} + +func newDoctorTestCmd() (*cobra.Command, *bytes.Buffer) { + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + var stdout, stderr bytes.Buffer + cmd.SetOut(&stdout) + cmd.SetErr(&stderr) + return cmd, &stdout +} + +func v1Tip(t *testing.T, repo *git.Repository) plumbing.Hash { + t.Helper() + ref, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true) + require.NoError(t, err) + return ref.Hash() +} + +func TestCheckOversizedCheckpointMetadata_CleanRepoIsQuiet(t *testing.T) { + // Cannot use t.Parallel(): t.Chdir modifies process-global state. + dir := setupGitRepoForPhaseTest(t) + t.Chdir(dir) + cmd, stdout := newDoctorTestCmd() + + require.NoError(t, checkOversizedCheckpointMetadata(cmd)) + assert.Contains(t, stdout.String(), "✓ Checkpoint metadata size: OK") +} + +func TestCheckOversizedCheckpointMetadata_NonInteractive_ReportsAndNamesSubcommand(t *testing.T) { + // Cannot use t.Parallel(): t.Chdir and the threshold override are process-global. + withTinyOversizeThreshold(t) + dir := setupGitRepoForPhaseTest(t) + t.Chdir(dir) + repo, err := gitrepo.OpenPath(dir) + require.NoError(t, err) + tip := writeBloatedCheckpointBranch(t, repo) + cmd, stdout := newDoctorTestCmd() + + require.NoError(t, checkOversizedCheckpointMetadata(cmd)) + + out := stdout.String() + assert.Contains(t, out, "Checkpoint metadata size: OVERSIZED") + assert.Contains(t, out, bloatedDoctorMetadataPath) + assert.Contains(t, out, "Run `"+shrinkMetadataCommand+"` to apply it.") + assert.NotContains(t, out, "Fixed") + assert.Equal(t, tip, v1Tip(t, repo), "report-only: the branch is untouched") +} + +// `doctor --force` applies every other fix without asking; this one rewrites +// history and pushes, so it must stay report-only under --force. +func TestRunSessionsFix_Force_DoesNotRewriteCheckpointBranch(t *testing.T) { + // Cannot use t.Parallel(): t.Chdir and the threshold override are process-global. + withTinyOversizeThreshold(t) + dir := setupGitRepoForPhaseTest(t) + t.Chdir(dir) + repo, err := gitrepo.OpenPath(dir) + require.NoError(t, err) + tip := writeBloatedCheckpointBranch(t, repo) + cmd, stdout := newDoctorTestCmd() + + require.NoError(t, runSessionsFix(cmd, true)) + + out := stdout.String() + assert.Contains(t, out, "Checkpoint metadata size: OVERSIZED") + assert.Contains(t, out, shrinkMetadataCommand) + assert.NotContains(t, out, "Fixed: rewrote") + assert.Equal(t, tip, v1Tip(t, repo), "--force must not rewrite the checkpoint branch") +} + +func TestDoctorShrinkCheckpointMetadata_Yes_RewritesBranch(t *testing.T) { + // Cannot use t.Parallel(): t.Chdir and the threshold override are process-global. + withTinyOversizeThreshold(t) + dir := setupGitRepoForPhaseTest(t) + t.Chdir(dir) + repo, err := gitrepo.OpenPath(dir) + require.NoError(t, err) + tip := writeBloatedCheckpointBranch(t, repo) + + run := func(args ...string) string { + cmd := newDoctorShrinkCheckpointMetadataCmd() + cmd.SetContext(context.Background()) + var stdout, stderr bytes.Buffer + cmd.SetOut(&stdout) + cmd.SetErr(&stderr) + cmd.SetArgs(args) + require.NoError(t, cmd.Execute()) + return stdout.String() + } + + // Without --yes and without a terminal it reports and stops. + out := run() + assert.Contains(t, out, "Checkpoint metadata size: OVERSIZED") + assert.Contains(t, out, "Pass --yes to apply it.") + assert.Equal(t, tip, v1Tip(t, repo)) + + out = run("--yes") + assert.Contains(t, out, "✓ Fixed: rewrote 1 commit(s), shrank 1 file(s)") + assert.Contains(t, out, "Not pushed: no checkpoint sync remote") + newTip := v1Tip(t, repo) + assert.NotEqual(t, tip, newTip) + + c, err := repo.CommitObject(newTip) + require.NoError(t, err) + f, err := c.File(bloatedDoctorMetadataPath) + require.NoError(t, err) + content, err := f.Contents() + require.NoError(t, err) + assert.NotContains(t, content, "prompt_attributions") + assert.Contains(t, content, `"session_id"`) + + // A second run finds nothing. + assert.Contains(t, run("--yes"), "nothing to repair") + cmd, stdout := newDoctorTestCmd() + require.NoError(t, checkOversizedCheckpointMetadata(cmd)) + assert.Contains(t, stdout.String(), "✓ Checkpoint metadata size: OK") +} diff --git a/cmd/entire/cli/hooks_git_cmd.go b/cmd/entire/cli/hooks_git_cmd.go index 340d87cdad..ea97c210b3 100644 --- a/cmd/entire/cli/hooks_git_cmd.go +++ b/cmd/entire/cli/hooks_git_cmd.go @@ -232,7 +232,8 @@ func newHooksGitPrePushCmd() *cobra.Command { Short: "Handle pre-push git hook", Args: cobra.ExactArgs(1), // SilenceUsage/Errors so non-zero exits from privacy-critical - // failures (OPF rewrite errors) print only the error message, + // failures (OPF or checkpoint-metadata rewrite errors) print only + // the error message, // not cobra's usage banner. The error message itself already // includes user guidance (see ErrV1Diverged / ErrBootstrapTooLarge / // ErrV1RefMoved in strategy/manual_commit_opf_rewrite.go). @@ -254,7 +255,8 @@ func newHooksGitPrePushCmd() *cobra.Command { // Propagate the error so the hook script exits non-zero and // git push aborts the entire batch. PrePush itself only - // returns errors for privacy-critical failures (OPF rewrite — + // returns errors for privacy-critical failures (OPF or oversized + // checkpoint-metadata rewrite — // e.g., V1DivergedError, BootstrapTooLargeError, // V1RefMovedError, OPFRuntimeFailedError, // OPFNoCategoriesError); transient diff --git a/cmd/entire/cli/settings/settings.go b/cmd/entire/cli/settings/settings.go index fd8536f404..e867fb0066 100644 --- a/cmd/entire/cli/settings/settings.go +++ b/cmd/entire/cli/settings/settings.go @@ -690,7 +690,19 @@ func worktreeSettingsPaths(worktreeRoot string) (base, local string) { return filepath.Join(worktreeRoot, EntireSettingsFile), filepath.Join(worktreeRoot, EntireSettingsLocalFile) } +// LoadForWorktreeRoot is Load for an explicit worktree root instead of the +// process working directory. Code that already holds a repository — a doctor +// repair, a test fixture — must read that repository's settings, not whatever +// the current directory happens to be inside. +func LoadForWorktreeRoot(ctx context.Context, worktreeRoot string) (*EntireSettings, error) { + return loadForWorktreeRoot(ctx, worktreeRoot) +} + func loadForWorktreeRoot(ctx context.Context, worktreeRoot string) (*EntireSettings, error) { + return loadForWorktreeRootWithAgentPolicy(ctx, worktreeRoot, true) +} + +func loadForWorktreeRootWithAgentPolicy(ctx context.Context, worktreeRoot string, installAgentPolicy bool) (*EntireSettings, error) { settingsFileAbs, localSettingsFileAbs := worktreeSettingsPaths(worktreeRoot) preferencesFileAbs := "" if path, prefErr := clonePreferencesPathForWorktreeRoot(ctx, worktreeRoot); prefErr == nil { @@ -699,7 +711,7 @@ func loadForWorktreeRoot(ctx context.Context, worktreeRoot string) (*EntireSetti logging.Debug(ctx, "clone preferences path unresolved; skipping preferences layer", slog.String("error", prefErr.Error())) } - return loadMergedSettings(ctx, settingsFileAbs, preferencesFileAbs, localSettingsFileAbs) + return loadMergedSettingsWithAgentPolicy(ctx, settingsFileAbs, preferencesFileAbs, localSettingsFileAbs, installAgentPolicy) } func clonePreferencesPathForWorktreeRoot(ctx context.Context, worktreeRoot string) (string, error) { @@ -729,6 +741,14 @@ func worktreeRootOfSettingsFile(settingsFileAbs string) string { } func loadMergedSettings(ctx context.Context, settingsFileAbs, preferencesFileAbs, localSettingsFileAbs string) (*EntireSettings, error) { + return loadMergedSettingsWithAgentPolicy(ctx, settingsFileAbs, preferencesFileAbs, localSettingsFileAbs, true) +} + +func loadMergedSettingsWithAgentPolicy( + ctx context.Context, + settingsFileAbs, preferencesFileAbs, localSettingsFileAbs string, + installAgentPolicy bool, +) (*EntireSettings, error) { // Load base settings settings, err := loadFromFile(settingsFileAbs) if err != nil { @@ -777,7 +797,9 @@ func loadMergedSettings(ctx context.Context, settingsFileAbs, preferencesFileAbs // config, so it gets the same gate, and then installs the surviving set as // the process-wide policy. enforceSymlinkedAgentDirsTrust(ctx, settings, localSettingsFileAbs, localData) - applyVouchedAgentDirs(settings, worktreeRootOfSettingsFile(settingsFileAbs)) + if installAgentPolicy { + applyVouchedAgentDirs(settings, worktreeRootOfSettingsFile(settingsFileAbs)) + } // Re-validate after merge. Individual files are validated by loadFromFile, // but mergeJSON patches fields independently and can produce combinations @@ -1768,6 +1790,17 @@ func IsSetUp(ctx context.Context) bool { // job is to say why it cannot see something. func FilesPresent(ctx context.Context) (project, local bool, err error) { root, err := entiredir.OpenForRead(ctx) + return filesPresent(root, err) +} + +// FilesPresentForWorktreeRoot is FilesPresent for an explicit worktree root. +func FilesPresentForWorktreeRoot(worktreeRoot string) (project, local bool, err error) { + root, err := entiredir.OpenAtForRead(worktreeRoot) + return filesPresent(root, err) +} + +func filesPresent(root *os.Root, openErr error) (project, local bool, err error) { + err = openErr if err != nil { if errors.Is(err, fs.ErrNotExist) { return false, false, nil @@ -1844,6 +1877,34 @@ func IsSetUpAndEnabled(ctx context.Context) bool { return s.Enabled } +// IsSetUpAndEnabledForWorktreeRoot is a read-only IsSetUpAndEnabled for an +// explicit worktree root. It does not install that worktree's agent-directory +// policy into the process. +func IsSetUpAndEnabledForWorktreeRoot(ctx context.Context, worktreeRoot string) bool { + project, local, err := FilesPresentForWorktreeRoot(worktreeRoot) + if err != nil || (!project && !local) { + return false + } + // This is an inspection of another worktree. Loading its effective settings + // must not replace the current worktree's process-wide agent-directory policy. + s, err := loadForWorktreeRootWithAgentPolicy(ctx, worktreeRoot, false) + if err != nil { + return false + } + return s.Enabled +} + +// ProjectSettingsEnabledForWorktreeRoot reports whether an explicit worktree +// has an enabled project settings layer, without consulting local overrides. +func ProjectSettingsEnabledForWorktreeRoot(worktreeRoot string) bool { + project, _, err := FilesPresentForWorktreeRoot(worktreeRoot) + if err != nil || !project { + return false + } + s, err := loadFromFile(filepath.Join(worktreeRoot, EntireSettingsFile)) + return err == nil && s.Enabled +} + // IsFilteredFetchesEnabled checks if filtered fetches should be used. // When enabled, filtered fetches always resolve remote names to URLs first so // git does not persist promisor settings onto named remotes in local config. diff --git a/cmd/entire/cli/settings/settings_test.go b/cmd/entire/cli/settings/settings_test.go index f2a8792247..75591a3580 100644 --- a/cmd/entire/cli/settings/settings_test.go +++ b/cmd/entire/cli/settings/settings_test.go @@ -1536,6 +1536,44 @@ func TestIsSetUpAndEnabled_FalseOnInvalidScannerConfig(t *testing.T) { } } +func TestIsSetUpAndEnabledForWorktreeRoot(t *testing.T) { + t.Parallel() + + enabledRoot := t.TempDir() + testutil.InitRepo(t, enabledRoot) + testutil.WriteFile(t, enabledRoot, EntireSettingsFile, `{"enabled":true}`) + + disabledRoot := t.TempDir() + testutil.InitRepo(t, disabledRoot) + testutil.WriteFile(t, disabledRoot, EntireSettingsLocalFile, `{"enabled":false}`) + + if !IsSetUpAndEnabledForWorktreeRoot(t.Context(), enabledRoot) { + t.Error("enabled explicit worktree reported inactive") + } + if IsSetUpAndEnabledForWorktreeRoot(t.Context(), disabledRoot) { + t.Error("disabled explicit worktree reported active") + } + if IsSetUpAndEnabledForWorktreeRoot(t.Context(), t.TempDir()) { + t.Error("unconfigured explicit worktree reported active") + } +} + +func TestProjectSettingsEnabledForWorktreeRoot_IgnoresLocalOverride(t *testing.T) { + t.Parallel() + + root := t.TempDir() + testutil.InitRepo(t, root) + testutil.WriteFile(t, root, EntireSettingsFile, `{"enabled":false}`) + testutil.WriteFile(t, root, EntireSettingsLocalFile, `{"enabled":true}`) + + if ProjectSettingsEnabledForWorktreeRoot(root) { + t.Error("disabled project settings reported enabled through local override") + } + if !IsSetUpAndEnabledForWorktreeRoot(t.Context(), root) { + t.Error("effective worktree settings reported disabled despite local override") + } +} + func TestGetCheckpointPushRemote(t *testing.T) { t.Parallel() tests := []struct { diff --git a/cmd/entire/cli/status.go b/cmd/entire/cli/status.go index 88c4fce3f1..cbf37de186 100644 --- a/cmd/entire/cli/status.go +++ b/cmd/entire/cli/status.go @@ -79,16 +79,18 @@ func runStatus(ctx context.Context, w io.Writer, detailed, jsonOutput bool) erro if err != nil { return err //nolint:wrapcheck // already contextual; a bare %w only changes the concrete type } + setupIssue := inspectWorktreeSetup(ctx) if !projectExists && !localExists { fmt.Fprintln(w, "○ not set up (run `entire enable` to get started)") + writeWorktreeSetupIssue(w, setupIssue) return nil } sty := newStatusStyles(w) if detailed { - return runStatusDetailed(ctx, w, sty, settingsPath, localSettingsPath, projectExists, localExists) + return runStatusDetailed(ctx, w, sty, settingsPath, localSettingsPath, projectExists, localExists, setupIssue) } // Short output: just show the effective/merged state @@ -98,6 +100,7 @@ func runStatus(ctx context.Context, w io.Writer, detailed, jsonOutput bool) erro } fmt.Fprintln(w, formatSettingsStatusShort(ctx, s, sty)) + writeWorktreeSetupIssue(w, setupIssue) if s.Enabled { writeActiveSessions(ctx, w, sty) } @@ -121,13 +124,14 @@ func writeAgentHelpHint(w io.Writer, sty statusStyles) { } // runStatusDetailed shows the effective status plus detailed status for each settings file. -func runStatusDetailed(ctx context.Context, w io.Writer, sty statusStyles, settingsPath, localSettingsPath string, projectExists, localExists bool) error { +func runStatusDetailed(ctx context.Context, w io.Writer, sty statusStyles, settingsPath, localSettingsPath string, projectExists, localExists bool, setupIssue *worktreeSetupIssue) error { // First show the effective/merged status effectiveSettings, err := LoadEntireSettings(ctx) if err != nil { return fmt.Errorf("failed to load settings: %w", err) } fmt.Fprintln(w, formatSettingsStatusShort(ctx, effectiveSettings, sty)) + writeWorktreeSetupIssue(w, setupIssue) fmt.Fprintln(w) // blank line // Show project settings if it exists @@ -1038,6 +1042,9 @@ type statusJSON struct { // CodexHooks reports effective discovery/trust warnings separately from // current-checkout installation and freshness semantics. CodexHooks *codexHooksStatusJSON `json:"codex_hooks,omitempty"` + // WorktreeSetup reports missing Entire settings or shared Claude project + // hook config only when a configured sibling proves the portable setup. + WorktreeSetup *worktreeSetupStatusJSON `json:"worktree_setup,omitempty"` // CheckpointPushDisabled is emitted only when Entire is enabled and the // effective push_sessions setting is false. Its absence does not guarantee // that a push can succeed. @@ -1080,6 +1087,25 @@ type statusJSON struct { Error string `json:"error,omitempty"` } +type worktreeSetupStatusJSON struct { + State string `json:"state"` + Agent string `json:"agent"` + Missing []string `json:"missing"` + ConfiguredWorktree string `json:"configured_worktree"` +} + +func worktreeSetupStatusFromIssue(issue *worktreeSetupIssue) *worktreeSetupStatusJSON { + if issue == nil { + return nil + } + return &worktreeSetupStatusJSON{ + State: "incomplete", + Agent: claudeCodeAgentName, + Missing: issue.missingJSONFields(), + ConfiguredWorktree: issue.ConfiguredWorktree, + } +} + type codexHooksStatusJSON struct { State string `json:"state"` WorktreePath string `json:"worktree_path,omitempty"` @@ -1134,9 +1160,10 @@ func runStatusJSON(ctx context.Context, w io.Writer) error { if presenceErr != nil { return writeJSON(statusJSON{Error: presenceErr.Error()}) } + setupIssue := inspectWorktreeSetup(ctx) if !projectExists && !localExists { - return writeJSON(statusJSON{Error: "not set up"}) + return writeJSON(statusJSON{Error: "not set up", WorktreeSetup: worktreeSetupStatusFromIssue(setupIssue)}) } s, err := LoadEntireSettings(ctx) @@ -1149,6 +1176,7 @@ func runStatusJSON(ctx context.Context, w io.Writer) error { Agents: []string{}, ActiveSessions: []sessionBriefJSON{}, AgentHelp: agentHelpCommand, + WorktreeSetup: worktreeSetupStatusFromIssue(setupIssue), } if s.Enabled { diff --git a/cmd/entire/cli/status_test.go b/cmd/entire/cli/status_test.go index 644708b81b..1341d6fcbd 100644 --- a/cmd/entire/cli/status_test.go +++ b/cmd/entire/cli/status_test.go @@ -3276,7 +3276,7 @@ func TestRunStatusDetailed_ReportsRejectedExternalAgents(t *testing.T) { //nolin var out bytes.Buffer sty := statusStyles{colorEnabled: false, width: 80} if err := runStatusDetailed(t.Context(), &out, sty, projectPath, - filepath.Join(entireDir, "settings.local.json"), true, false); err != nil { + filepath.Join(entireDir, "settings.local.json"), true, false, nil); err != nil { t.Fatalf("runStatusDetailed: %v", err) } diff --git a/cmd/entire/cli/strategy/checkpoint_remote.go b/cmd/entire/cli/strategy/checkpoint_remote.go index 97de4bc33c..d6df643574 100644 --- a/cmd/entire/cli/strategy/checkpoint_remote.go +++ b/cmd/entire/cli/strategy/checkpoint_remote.go @@ -3,6 +3,8 @@ package strategy import ( "bytes" "context" + "crypto/rand" + "encoding/hex" "errors" "fmt" "log/slog" @@ -21,6 +23,18 @@ import ( // push hot path, where the user's own `git push` is blocked for the duration. const checkpointRemoteFetchTimeout = 30 * time.Second +func newFetchTmpRef(purpose string) (plumbing.ReferenceName, error) { + var suffix [12]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generate temporary fetch ref: %w", err) + } + ref := plumbing.ReferenceName(FetchTmpRefPrefix + purpose + "/" + hex.EncodeToString(suffix[:])) + if err := ref.Validate(); err != nil { + return "", fmt.Errorf("build temporary fetch ref: %w", err) + } + return ref, nil +} + // checkpointRemoteForegroundFetchTimeout bounds checkpoint-remote fetches made // by user-initiated foreground commands (enable, resume, explain). It matches // the origin-side metadata fetch budget in the cli package. @@ -146,17 +160,22 @@ func fetchMetadataBranchWithin(ctx context.Context, remoteURL string, timeout ti return fmt.Errorf("primary metadata ref %s is not a branch", refs.Primary) } branchName := refs.Primary.Short() - tmpRef := FetchTmpRefPrefix + branchName - srcRef := refs.Primary.String() - - if err := fetchURLIntoTmpRef(ctx, "", remoteURL, srcRef, tmpRef, "metadata branch", true, timeout); err != nil { + tmpRef, err := newFetchTmpRef("metadata-branch") + if err != nil { return err } - if err := PromoteTmpRefSafely(ctx, plumbing.ReferenceName(tmpRef), refs.Primary, branchName); err != nil { - return err + repo, err := OpenRepository(ctx) + if err != nil { + return fmt.Errorf("failed to open repository for %s fetch: %w", branchName, err) } + defer repo.Close() + defer func() { _ = repo.Storer.RemoveReference(tmpRef) }() //nolint:errcheck // cleanup is best-effort + srcRef := refs.Primary.String() - return nil + if err := fetchURLIntoTmpRef(ctx, "", remoteURL, srcRef, tmpRef.String(), "metadata branch", true, timeout); err != nil { + return err + } + return promoteTmpRefSafely(ctx, repo, tmpRef, refs.Primary, branchName) } // fetchURLIntoTmpRef runs `git fetch +:` via the diff --git a/cmd/entire/cli/strategy/checkpoint_remote_test.go b/cmd/entire/cli/strategy/checkpoint_remote_test.go index 8283c96ffe..99c4154b89 100644 --- a/cmd/entire/cli/strategy/checkpoint_remote_test.go +++ b/cmd/entire/cli/strategy/checkpoint_remote_test.go @@ -14,6 +14,7 @@ import ( "github.com/entireio/cli/cmd/entire/cli/testutil" "github.com/entireio/cli/cmd/entire/cli/vercelconfig" + git "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/plumbing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -570,8 +571,10 @@ func TestFetchMetadataBranch_FetchesAndCreatesLocalBranch(t *testing.T) { // Branch should now exist assert.True(t, testutil.BranchExists(t, localDir, "entire/checkpoints/v1")) - // Temp ref should be cleaned up - assert.False(t, testutil.BranchExists(t, localDir, "refs/entire-fetch-tmp/entire/checkpoints/v1")) + repo, err := git.PlainOpen(localDir) + require.NoError(t, err) + t.Cleanup(func() { _ = repo.Close() }) + assertNoFetchTmpRefsWithPurpose(t, repo, "metadata-branch") } // Not parallel: uses t.Chdir() @@ -875,6 +878,7 @@ func TestEnsurePrimaryRef_FetchesFromCheckpointRemoteInsteadOfOrphan(t *testing. files := checkpointRemoteMetadataFiles(t, localDir) assert.Contains(t, files, "aa/aaaaaaaaaa/"+paths.MetadataFileName, "the bootstrapped branch should contain the checkpoint committed on the remote") + assertNoFetchTmpRefsWithPurpose(t, repo, "metadata-bootstrap") } // TestEnsurePrimaryRef_ReplacesExistingEmptyOrphanFromCheckpointRemote verifies @@ -948,6 +952,7 @@ func TestEnsurePrimaryRef_ReplacesExistingEmptyOrphanFromCheckpointRemote(t *tes files := checkpointRemoteMetadataFiles(t, localDir) assert.Contains(t, files, "aa/aaaaaaaaaa/"+paths.MetadataFileName, "the healed branch should contain the checkpoint committed on the remote") + assertNoFetchTmpRefsWithPurpose(t, repo, "metadata-heal") } // TestEnsurePrimaryRef_SkipsCheckpointRemoteBootstrapOutsideEnableFlow verifies diff --git a/cmd/entire/cli/strategy/common.go b/cmd/entire/cli/strategy/common.go index df708383a3..8c21845827 100644 --- a/cmd/entire/cli/strategy/common.go +++ b/cmd/entire/cli/strategy/common.go @@ -142,7 +142,10 @@ func PromoteTmpRefSafely(ctx context.Context, tmpRefName, destRefName plumbing.R } defer repo.Close() defer func() { _ = repo.Storer.RemoveReference(tmpRefName) }() //nolint:errcheck // cleanup is best-effort + return promoteTmpRefSafely(ctx, repo, tmpRefName, destRefName, label) +} +func promoteTmpRefSafely(ctx context.Context, repo *git.Repository, tmpRefName, destRefName plumbing.ReferenceName, label string) error { tmpRef, err := repo.Reference(tmpRefName, true) if err != nil { return fmt.Errorf("%s not found after fetch (tmp ref %s missing): %w", label, tmpRefName, err) @@ -930,8 +933,14 @@ func bootstrapPrimaryFromCheckpointRemote(ctx context.Context, repo *git.Reposit return false } - branchName := primary.Short() - tmpRefName := plumbing.ReferenceName(FetchTmpRefPrefix + branchName) + tmpRefName, err := newFetchTmpRef("metadata-bootstrap") + if err != nil { + logging.Warn(ctx, "checkpoint-remote: cannot allocate bootstrap fetch ref", + slog.String("error", err.Error()), + ) + return false + } + defer func() { _ = repo.Storer.RemoveReference(tmpRefName) }() //nolint:errcheck // cleanup is best-effort // Unfiltered. The bootstrap itself only needs the ref, but it is only ever // reached under the git-branch primary (git-refs returns above), and there // the branch it lands IS the repo's checkpoint store — refs.Read and @@ -953,8 +962,6 @@ func bootstrapPrimaryFromCheckpointRemote(ctx context.Context, repo *git.Reposit fmt.Fprintln(os.Stderr, " Continuing — they will be fetched on demand when a command needs them.") return false } - defer func() { _ = repo.Storer.RemoveReference(tmpRefName) }() //nolint:errcheck // cleanup is best-effort - tmpRef, err := repo.Reference(tmpRefName, true) if err != nil { logging.Debug(ctx, "checkpoint-remote: fetched metadata ref missing after enable bootstrap", @@ -1042,7 +1049,10 @@ func healEmptyOrphanFromCheckpointRemote(ctx context.Context, repo *git.Reposito return false, nil } - tmpRefName := plumbing.ReferenceName(FetchTmpRefPrefix + primary.Short()) + tmpRefName, err := newFetchTmpRef("metadata-heal") + if err != nil { + return false, fmt.Errorf("allocate metadata heal fetch ref: %w", err) + } defer func() { _ = repo.Storer.RemoveReference(tmpRefName) }() //nolint:errcheck // cleanup is best-effort // Unfiltered, for the same reason as the bootstrap fetch above: this heal diff --git a/cmd/entire/cli/strategy/manual_commit_opf_rewrite.go b/cmd/entire/cli/strategy/manual_commit_opf_rewrite.go index e5cc6e9f3c..188074a02b 100644 --- a/cmd/entire/cli/strategy/manual_commit_opf_rewrite.go +++ b/cmd/entire/cli/strategy/manual_commit_opf_rewrite.go @@ -75,7 +75,7 @@ type V1RefMovedError struct { } func (e *V1RefMovedError) Error() string { - return fmt.Sprintf("entire/checkpoints/v1 moved during OPF rewrite "+ + return fmt.Sprintf("entire/checkpoints/v1 moved during checkpoint rewrite "+ "(expected %s, found %s); another local worktree advanced the ref "+ "mid-rewrite — re-run `git push` (no fetch needed; the move was local)", e.Expected.String()[:7], e.Actual.String()[:7]) @@ -258,6 +258,10 @@ const rawByteCapMultiplier = 100 // privacy-critical failures — the pre-push hook propagates these so // git push aborts. func RewriteUnpushedV1WithOPF(ctx context.Context, repo *git.Repository, target string) (plumbing.Hash, error) { + return rewriteUnpushedV1WithOPF(ctx, repo, target, OversizedCheckpointMetadataThreshold) +} + +func rewriteUnpushedV1WithOPF(ctx context.Context, repo *git.Repository, target string, metadataThreshold int64) (plumbing.Hash, error) { localTip, err := readV1Tip(repo, plumbing.NewBranchReferenceName(paths.MetadataBranchName)) if err != nil { return plumbing.ZeroHash, fmt.Errorf("read local v1: %w", err) @@ -270,6 +274,16 @@ func RewriteUnpushedV1WithOPF(ctx context.Context, repo *git.Repository, target return plumbing.ZeroHash, fmt.Errorf("read remote v1: %w", err) } + reconciledTip, handled, reconcileErr := reconcileOversizedV1ForPush( + ctx, repo, localTip, remoteTip, metadataThreshold, + ) + if reconcileErr != nil { + return plumbing.ZeroHash, fmt.Errorf("reconcile oversized checkpoint metadata: %w", reconcileErr) + } + if handled { + localTip = reconciledTip + } + if !remoteTip.IsZero() { mergeBase, mbErr := computeMergeBase(repo, localTip, remoteTip) if mbErr != nil { @@ -421,10 +435,7 @@ func readV1Tip(repo *git.Repository, refName plumbing.ReferenceName) (plumbing.H return ref.Hash(), nil } -// opfRewriteFetchTmpRef is the temp ref used to stage the URL-fetched -// remote v1 tip during OPF rewrite. Cleaned up at the end of each -// resolveRemoteV1Tip call so the tracking is invisible to the user. -const opfRewriteFetchTmpRef = FetchTmpRefPrefix + "opf-rewrite-v1" +const opfRewriteFetchPurpose = "opf-rewrite-v1" // resolveRemoteV1Tip returns the hash of the remote's // entire/checkpoints/v1 tip. @@ -446,7 +457,18 @@ func resolveRemoteV1Tip(ctx context.Context, repo *git.Repository, target string if wt, wtErr := repo.Worktree(); wtErr == nil { worktreeRoot = wt.Filesystem().Root() } - if err := fetchURLIntoTmpRef(ctx, worktreeRoot, target, srcRef, opfRewriteFetchTmpRef, "v1 for OPF rewrite", true, checkpointRemoteFetchTimeout); err != nil { + tmpRef, err := newFetchTmpRef(opfRewriteFetchPurpose) + if err != nil { + return plumbing.ZeroHash, err + } + defer func() { + if err := repo.Storer.RemoveReference(tmpRef); err != nil { + logging.Debug(ctx, "OPF rewrite: failed to clean up temp ref", + slog.String("error", err.Error()), + ) + } + }() + if err := fetchURLIntoTmpRef(ctx, worktreeRoot, target, srcRef, tmpRef.String(), "v1 for OPF rewrite", true, checkpointRemoteFetchTimeout); err != nil { if !remote.IsURL(target) { logging.Warn(ctx, "OPF rewrite: failed to fetch remote v1; using local remote-tracking ref", slog.String("remote", target), @@ -459,14 +481,7 @@ func resolveRemoteV1Tip(ctx context.Context, repo *git.Repository, target string ) return plumbing.ZeroHash, nil } - defer func() { - if err := repo.Storer.RemoveReference(plumbing.ReferenceName(opfRewriteFetchTmpRef)); err != nil { - logging.Debug(ctx, "OPF rewrite: failed to clean up temp ref", - slog.String("error", err.Error()), - ) - } - }() - ref, err := repo.Reference(plumbing.ReferenceName(opfRewriteFetchTmpRef), true) + ref, err := repo.Reference(tmpRef, true) if err != nil { if errors.Is(err, plumbing.ErrReferenceNotFound) { return plumbing.ZeroHash, nil diff --git a/cmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go b/cmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go index 66ca818a07..f3343d6d33 100644 --- a/cmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go +++ b/cmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go @@ -504,6 +504,7 @@ func TestResolveRemoteV1Tip_NamedRemoteFetchesLatestTip(t *testing.T) { got, err := resolveRemoteV1Tip(context.Background(), localRepo, "origin") require.NoError(t, err) require.Equal(t, latestRemoteTip, got) + assertNoFetchTmpRefsWithPurpose(t, localRepo, opfRewriteFetchPurpose) } // Bootstrap cap: a single table-driven test covers both the over-limit diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 1185702f38..9cb2235c68 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -54,8 +54,17 @@ func (s *ManualCommitStrategy) PrePushFromGitHook(ctx context.Context, remote st } func (s *ManualCommitStrategy) prePush(ctx context.Context, remote string, protectFirstUserBranch bool) error { + return s.prePushWithMetadataThreshold(ctx, remote, protectFirstUserBranch, OversizedCheckpointMetadataThreshold) +} + +func (s *ManualCommitStrategy) prePushWithMetadataThreshold( + ctx context.Context, + remote string, + protectFirstUserBranch bool, + metadataThreshold int64, +) error { // This runs inside the user's `git push` pre-push hook. Every checkpoint - // git subprocess spawned here (metadata fetch, policy sync, checkpoint + // git subprocess spawned here (metadata fetch and cleanup, checkpoint // push and its recovery fetch) must fail fast rather than block on an // interactive SSH passphrase prompt — there is no way to answer it here and // it would hang the user's push. Foreground commands do not set this. @@ -121,6 +130,23 @@ func (s *ManualCommitStrategy) prePush(ctx context.Context, remote string, prote // default. Defer publication until the user's own branch exists there. deferAutomaticCheckpointPush := protectFirstUserBranch && deferCheckpointPushOnEmptyRemote(ctx, ps) + repo, repoErr := OpenRepository(ctx) + if repoErr != nil { + logging.Warn(ctx, "checkpoint metadata cleanup: failed to open repository; aborting push", + slog.String("error", repoErr.Error()), + ) + return repoErr + } + defer repo.Close() + if err := prepareOversizedV1ForPush( + ctx, repo, ps.pushTarget(), metadataThreshold, + ); err != nil { + logging.Warn(ctx, "checkpoint metadata cleanup failed; aborting push", + slog.String("error", err.Error()), + ) + return err + } + // OPF pre-push rewrite: if OPF is configured, resolve the user's // decision (env > settings > prompt > non-TTY auto-run), then // re-redact unpushed v1 commits with OPF (producing the OPF-applied, diff --git a/cmd/entire/cli/strategy/metadata_push_reconcile.go b/cmd/entire/cli/strategy/metadata_push_reconcile.go new file mode 100644 index 0000000000..62f19e009d --- /dev/null +++ b/cmd/entire/cli/strategy/metadata_push_reconcile.go @@ -0,0 +1,514 @@ +package strategy + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "os/exec" + "path" + "strconv" + "strings" + "time" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" + "github.com/entireio/cli/cmd/entire/cli/paths" + git "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" +) + +const metadataCleanupFetchPurpose = "metadata-cleanup-v1" + +type shallowMetadataRepairError struct { + Boundary plumbing.Hash + Path string +} + +func (e *shallowMetadataRepairError) Error() string { + return fmt.Sprintf( + "checkpoint metadata at %s is oversized at shallow boundary %s; fetch the missing checkpoint history before retrying", + e.Path, e.Boundary, + ) +} + +func prepareOversizedV1ForPush(ctx context.Context, repo *git.Repository, target string, threshold int64) error { + localTip, err := readV1Tip(repo, plumbing.NewBranchReferenceName(paths.MetadataBranchName)) + if err != nil { + return fmt.Errorf("read local checkpoint history: %w", err) + } + if localTip.IsZero() { + return nil + } + hasOversizedHistory, err := hasOversizedMetadataIntroducedSince( + ctx, repo, localTip, plumbing.ZeroHash, threshold, + ) + if err != nil { + return err + } + if !hasOversizedHistory { + return nil + } + remoteTip, err := fetchV1TipForMetadataCleanup(ctx, repo, target) + if err != nil { + return err + } + _, _, err = reconcileOversizedV1ForPush(ctx, repo, localTip, remoteTip, threshold) + return err +} + +func fetchV1TipForMetadataCleanup(ctx context.Context, repo *git.Repository, target string) (plumbing.Hash, error) { + worktree, err := repo.Worktree() + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("resolve worktree for checkpoint metadata probe: %w", err) + } + ref := plumbing.NewBranchReferenceName(paths.MetadataBranchName) + probeCtx, cancel := context.WithTimeout(ctx, checkpointRemoteFetchTimeout) + defer cancel() + out, err := remote.LsRemoteInDir(probeCtx, worktree.Filesystem().Root(), target, ref.String()) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("probe %s on %s: %w", ref, remote.RedactURLOrPath(target), err) + } + if len(bytes.TrimSpace(out)) == 0 { + return plumbing.ZeroHash, nil + } + + tmpRef, err := newFetchTmpRef(metadataCleanupFetchPurpose) + if err != nil { + return plumbing.ZeroHash, err + } + defer func() { + _ = repo.Storer.RemoveReference(tmpRef) //nolint:errcheck // cleanup is best-effort + }() + if err := fetchURLIntoTmpRef( + ctx, + worktree.Filesystem().Root(), + target, + ref.String(), + tmpRef.String(), + "v1 for metadata cleanup", + true, + checkpointRemoteFetchTimeout, + ); err != nil { + return plumbing.ZeroHash, err + } + fetched, err := repo.Reference(tmpRef, true) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("read fetched checkpoint history: %w", err) + } + return fetched.Hash(), nil +} + +// reconcileOversizedV1ForPush rewrites local oversized history. An existing +// remote must prove where its shared checkpoint content ends before replay. +func reconcileOversizedV1ForPush( + ctx context.Context, + repo *git.Repository, + observedLocalTip, remoteTip plumbing.Hash, + threshold int64, +) (plumbing.Hash, bool, error) { + if observedLocalTip.IsZero() { + return observedLocalTip, false, nil + } + repoPath, err := getRepoPath(repo) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("get repository path: %w", err) + } + + mergeBase := plumbing.ZeroHash + if !remoteTip.IsZero() { + mergeBase, err = computeMergeBaseWithGit(ctx, repoPath, observedLocalTip, remoteTip) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("compute original merge-base: %w", err) + } + remoteHasOversizedHistory, scanErr := hasOversizedMetadataIntroducedSince( + ctx, repo, remoteTip, plumbing.ZeroHash, threshold, + ) + if scanErr != nil { + return plumbing.ZeroHash, false, fmt.Errorf("scan remote checkpoint metadata: %w", scanErr) + } + if remoteHasOversizedHistory { + return observedLocalTip, false, nil + } + } + hasOversizedLocalHistory, err := hasOversizedMetadataIntroducedSince( + ctx, repo, observedLocalTip, mergeBase, threshold, + ) + if err != nil { + return plumbing.ZeroHash, false, err + } + if !hasOversizedLocalHistory { + return observedLocalTip, false, nil + } + + shallow, err := loadShallowHashes(ctx, repoPath) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("load shallow boundaries: %w", err) + } + rewriter := newMetadataRewriter(ctx, repo, threshold) + rewriter.shallow = shallow + rewrittenTip, err := rewriter.rewriteHistory(observedLocalTip) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("rewrite oversized local metadata: %w", err) + } + if rewriter.commitsRewritten == 0 { + return observedLocalTip, false, nil + } + if remoteTip.IsZero() { + if err := atomicSetV1Ref(ctx, repo, observedLocalTip, rewrittenTip); err != nil { + return plumbing.ZeroHash, false, err + } + return rewrittenTip, true, nil + } + + rewrittenBoundary, originalBoundary, found, err := findSharedRewrittenTree( + ctx, repo, repoPath, rewrittenTip, remoteTip, rewriter.commits, + ) + if err != nil { + return plumbing.ZeroHash, false, err + } + if !found { + return observedLocalTip, false, nil + } + + if !mergeBase.IsZero() { + safe, err := boundaryAtOrAfterMergeBase(ctx, repoPath, mergeBase, originalBoundary) + if err != nil { + return plumbing.ZeroHash, false, err + } + if !safe { + return observedLocalTip, false, nil + } + } + + localOnly, err := collectFirstParentCommitsSince(ctx, repo, repoPath, rewrittenTip, rewrittenBoundary) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("collect local checkpoints after repaired boundary: %w", err) + } + + newTip := remoteTip + if len(localOnly) > 0 { + newTip, err = cherryPickOnto(ctx, repo, remoteTip, localOnly, shallow) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("replay local checkpoints onto repaired remote: %w", err) + } + } + + if err := atomicSetV1Ref(ctx, repo, observedLocalTip, newTip); err != nil { + return plumbing.ZeroHash, false, err + } + return newTip, true, nil +} + +func oversizedMetadataPathInTree(repo *git.Repository, root plumbing.Hash, threshold int64) (string, error) { + type pendingTree struct { + hash plumbing.Hash + prefix string + } + stack := []pendingTree{{hash: root}} + seen := make(map[plumbing.Hash]struct{}) + for len(stack) > 0 { + last := len(stack) - 1 + pending := stack[last] + stack = stack[:last] + if _, ok := seen[pending.hash]; ok { + continue + } + seen[pending.hash] = struct{}{} + + tree, err := repo.TreeObject(pending.hash) + if err != nil { + return "", fmt.Errorf("load tree %s: %w", pending.hash, err) + } + for _, entry := range tree.Entries { + entryPath := entry.Name + if pending.prefix != "" { + entryPath = pending.prefix + "/" + entry.Name + } + if entry.Mode == filemode.Dir { + stack = append(stack, pendingTree{hash: entry.Hash, prefix: entryPath}) + continue + } + if path.Base(entryPath) != paths.MetadataFileName || !isFileMode(entry.Mode) { + continue + } + blob, err := repo.BlobObject(entry.Hash) + if err != nil { + return "", fmt.Errorf("load metadata blob %s at %s: %w", entry.Hash, entryPath, err) + } + if blob.Size > threshold { + return entryPath, nil + } + } + } + return "", nil +} + +func hasOversizedMetadataIntroducedSince( + ctx context.Context, + repo *git.Repository, + tip, boundary plumbing.Hash, + threshold int64, +) (bool, error) { + repoPath, err := getRepoPath(repo) + if err != nil { + return false, fmt.Errorf("get repository path for metadata scan: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + rangeSpec := tip.String() + if !boundary.IsZero() { + rangeSpec = boundary.String() + ".." + tip.String() + } + // Raw path records keep metadata identity when one blob is also stored under + // an unrelated path; full merge diffs include side-parent and resolution data. + objectsCmd := exec.CommandContext( + ctx, + "git", "log", "--root", "--full-history", "-m", "--raw", "--no-renames", + "--diff-filter=AMT", "--format=", "--no-abbrev", "-z", rangeSpec, + "--", ":(glob)**/metadata.json", + ) + objectsCmd.Dir = repoPath + stdout, err := objectsCmd.StdoutPipe() + if err != nil { + return false, fmt.Errorf("open checkpoint metadata scan: %w", err) + } + var stderr bytes.Buffer + objectsCmd.Stderr = &stderr + if err := objectsCmd.Start(); err != nil { + return false, fmt.Errorf("start checkpoint metadata scan: %w", err) + } + waited := false + defer func() { + if !waited { + _ = objectsCmd.Process.Kill() //nolint:errcheck // best-effort after a parse/read failure + _ = objectsCmd.Wait() //nolint:errcheck // reap after best-effort termination + } + }() + + seen := make(map[plumbing.Hash]struct{}) + found := false + reader := bufio.NewReader(stdout) + for { + rawRecord, readErr := reader.ReadString(0) + if errors.Is(readErr, io.EOF) && rawRecord == "" { + break + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return false, fmt.Errorf("read checkpoint metadata scan: %w", readErr) + } + record := strings.TrimSpace(strings.TrimSuffix(rawRecord, "\x00")) + if record == "" { + if errors.Is(readErr, io.EOF) { + break + } + continue + } + if !strings.HasPrefix(record, ":") { + return false, fmt.Errorf("parse checkpoint metadata scan record %q", record) + } + pathField, pathErr := reader.ReadString(0) + if pathErr != nil && !errors.Is(pathErr, io.EOF) { + return false, fmt.Errorf("read checkpoint metadata path: %w", pathErr) + } + metadataPath := strings.TrimSuffix(pathField, "\x00") + if path.Base(metadataPath) != paths.MetadataFileName { + return false, fmt.Errorf("unexpected checkpoint metadata path %q", metadataPath) + } + + if err := ctx.Err(); err != nil { + return false, fmt.Errorf("scan local checkpoint metadata: %w", err) + } + fields := strings.Fields(record) + if len(fields) < 5 { + return false, fmt.Errorf("parse checkpoint metadata scan record %q", record) + } + mode, err := strconv.ParseUint(fields[1], 8, 32) + if err != nil { + return false, fmt.Errorf("parse checkpoint metadata mode %q: %w", fields[1], err) + } + if !isFileMode(filemode.FileMode(mode)) { + continue + } + hash := plumbing.NewHash(fields[3]) + if _, exists := seen[hash]; exists { + continue + } + seen[hash] = struct{}{} + blob, err := repo.BlobObject(hash) + if err != nil { + return false, fmt.Errorf("load metadata blob %s at %s: %w", hash, metadataPath, err) + } + if blob.Size > threshold { + found = true + if _, drainErr := io.Copy(io.Discard, reader); drainErr != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return false, fmt.Errorf("scan local checkpoint metadata: %w", ctxErr) + } + return false, fmt.Errorf("drain checkpoint metadata scan: %w", drainErr) + } + break + } + if errors.Is(readErr, io.EOF) { + break + } + } + if err := objectsCmd.Wait(); err != nil { + waited = true + return false, fmt.Errorf("list local checkpoint metadata objects: %s: %w", strings.TrimSpace(stderr.String()), err) + } + waited = true + return found, nil +} + +func findSharedRewrittenTree( + ctx context.Context, + repo *git.Repository, + repoPath string, + rewrittenLocalTip, remoteTip plumbing.Hash, + originalToRewritten map[plumbing.Hash]plumbing.Hash, +) (rewritten, original plumbing.Hash, found bool, err error) { + remoteTrees := make(map[plumbing.Hash]struct{}) + remoteHistory, err := reachableHistory(ctx, repo, repoPath, remoteTip) + if err != nil { + return plumbing.ZeroHash, plumbing.ZeroHash, false, fmt.Errorf("log remote checkpoint history: %w", err) + } + for _, commit := range remoteHistory { + if err := ctx.Err(); err != nil { + return plumbing.ZeroHash, plumbing.ZeroHash, false, fmt.Errorf("scan remote checkpoint trees: %w", err) + } + remoteTrees[commit.TreeHash] = struct{}{} + } + + rewrittenToOriginal := make(map[plumbing.Hash]plumbing.Hash, len(originalToRewritten)) + for oldHash, newHash := range originalToRewritten { + rewrittenToOriginal[newHash] = oldHash + } + localHistory, err := firstParentHistory(ctx, repo, repoPath, rewrittenLocalTip) + if err != nil { + return plumbing.ZeroHash, plumbing.ZeroHash, false, fmt.Errorf("log rewritten local checkpoint history: %w", err) + } + for _, commit := range localHistory { + if err := ctx.Err(); err != nil { + return plumbing.ZeroHash, plumbing.ZeroHash, false, fmt.Errorf("scan rewritten local checkpoint trees: %w", err) + } + if _, exists := remoteTrees[commit.TreeHash]; !exists { + continue + } + oldHash, exists := rewrittenToOriginal[commit.Hash] + if !exists { + return plumbing.ZeroHash, plumbing.ZeroHash, false, fmt.Errorf("rewritten commit %s has no original", commit.Hash) + } + return commit.Hash, oldHash, true, nil + } + return plumbing.ZeroHash, plumbing.ZeroHash, false, nil +} + +func reachableHistory( + ctx context.Context, + repo *git.Repository, + repoPath string, + tip plumbing.Hash, +) ([]*object.Commit, error) { + return historyFromRevList(ctx, repo, repoPath, tip.String()) +} + +func firstParentHistory( + ctx context.Context, + repo *git.Repository, + repoPath string, + tip plumbing.Hash, +) ([]*object.Commit, error) { + return historyFromRevList(ctx, repo, repoPath, "--first-parent", tip.String()) +} + +func historyFromRevList( + ctx context.Context, + repo *git.Repository, + repoPath string, + args ...string, +) ([]*object.Commit, error) { + cmd := exec.CommandContext(ctx, "git", append([]string{"rev-list"}, args...)...) + cmd.Dir = repoPath + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git rev-list failed: %w", err) + } + hashes := strings.Fields(string(output)) + commits := make([]*object.Commit, 0, len(hashes)) + for _, value := range hashes { + commit, err := repo.CommitObject(plumbing.NewHash(value)) + if err != nil { + return nil, fmt.Errorf("load history commit %s: %w", value, err) + } + commits = append(commits, commit) + } + return commits, nil +} + +func collectFirstParentCommitsSince( + ctx context.Context, + repo *git.Repository, + repoPath string, + tip, boundary plumbing.Hash, +) ([]*object.Commit, error) { + commits, err := historyFromRevList( + ctx, repo, repoPath, "--first-parent", "--reverse", boundary.String()+".."+tip.String(), + ) + if err != nil { + return nil, err + } + if len(commits) > MaxCommitTraversalDepth { + return nil, fmt.Errorf("commit chain exceeded %d commits; aborting rebase", MaxCommitTraversalDepth) + } + return commits, nil +} + +func computeMergeBaseWithGit( + ctx context.Context, + repoPath string, + local, remote plumbing.Hash, +) (plumbing.Hash, error) { + cmd := exec.CommandContext(ctx, "git", "merge-base", "--all", local.String(), remote.String()) + cmd.Dir = repoPath + output, err := cmd.Output() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return plumbing.ZeroHash, nil + } + return plumbing.ZeroHash, fmt.Errorf("git merge-base failed: %w", err) + } + bases := strings.Fields(string(output)) + if len(bases) > 1 { + return plumbing.ZeroHash, errors.New("multiple merge bases prevent safe checkpoint reconciliation") + } + if len(bases) == 0 { + return plumbing.ZeroHash, nil + } + return plumbing.NewHash(bases[0]), nil +} + +func boundaryAtOrAfterMergeBase( + ctx context.Context, + repoPath string, + mergeBase, boundary plumbing.Hash, +) (bool, error) { + if mergeBase.Equal(boundary) { + return true, nil + } + cmd := exec.CommandContext(ctx, "git", "merge-base", "--is-ancestor", mergeBase.String(), boundary.String()) + cmd.Dir = repoPath + if err := cmd.Run(); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return false, nil + } + return false, fmt.Errorf("compare repaired boundary with original merge-base: %w", err) + } + return true, nil +} diff --git a/cmd/entire/cli/strategy/metadata_push_reconcile_test.go b/cmd/entire/cli/strategy/metadata_push_reconcile_test.go new file mode 100644 index 0000000000..54d54a8579 --- /dev/null +++ b/cmd/entire/cli/strategy/metadata_push_reconcile_test.go @@ -0,0 +1,937 @@ +package strategy + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + git "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/gitrepo" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/testutil" + "github.com/entireio/cli/cmd/entire/cli/trailers" +) + +func TestFetchAndRebase_RepairedRemoteWithRemoteOnlySuffixReplaysLocalSuffix(t *testing.T) { + dir, repo, cleanRoot, _, sharedTip := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + + localPath := "22/3333333333/0/metadata.json" + localTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: 223333333333", map[string][]byte{ + localPath: []byte(`{"checkpoint_id":"223333333333","source":"local"}` + "\n"), + }) + repairedSharedTip, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(sharedTip) + require.NoError(t, err) + repairedC3 := readCommit(t, repo, repairedSharedTip) + repairedC2 := readCommit(t, repo, repairedC3.ParentHashes[0]) + independentC2 := makeOrphanCommit(t, repo, repairedC2.TreeHash, []plumbing.Hash{cleanRoot}, repairedC2.Message) + independentC3 := makeOrphanCommit(t, repo, repairedC3.TreeHash, []plumbing.Hash{independentC2}, repairedC3.Message) + require.NotEqual(t, repairedSharedTip, independentC3, + "an independently signed repair has different commits with the same checkpoint trees") + remotePath := "44/5555555555/0/metadata.json" + remoteTip := appendCheckpointFiles(t, repo, independentC3, "Checkpoint: 445555555555", map[string][]byte{ + remotePath: []byte(`{"checkpoint_id":"445555555555","source":"remote"}` + "\n"), + }) + + bare := pushV1ToBare(t, dir, remoteTip) + setV1Tip(t, repo, localTip) + ref := plumbing.NewBranchReferenceName(paths.MetadataBranchName) + delivered, err := pushRefIfNeededWithMetadataThreshold( + ctx, + "file://"+bare, + ref, + shrinkTestThreshold, + ) + require.NoError(t, err) + require.True(t, delivered) + + got := readTip(t, repo) + assert.Equal(t, got.String(), bareV1(t, bare), "the retry must deliver the reconciled tip") + gotCommit := readCommit(t, repo, got) + require.Equal(t, []plumbing.Hash{remoteTip}, gotCommit.ParentHashes, + "the local-only checkpoint must be replayed onto the actual remote tip") + assert.Equal(t, []string{ + "Checkpoint: 223333333333", + "Checkpoint: 445555555555", + "Checkpoint: eeffffffffff", + "Checkpoint: ccdddddddddd", + "Checkpoint: aabbbbbbbbbb", + }, commitMessages(t, repo, got), "shared repaired checkpoints must not be duplicated") + assertCommitHasFile(t, repo, got, localPath) + assertCommitHasFile(t, repo, got, remotePath) + + oversized, err := findOversizedMetadataBlobs(ctx, repo, got, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized) + assertNoFetchTmpRefsWithPurpose(t, repo, "push-recovery") + + again, handled, err := reconcileOversizedV1ForPush(ctx, repo, got, remoteTip, shrinkTestThreshold) + require.NoError(t, err) + assert.False(t, handled) + assert.Equal(t, got, again, "already-clean history is idempotent") + delivered, err = pushRefIfNeededWithMetadataThreshold(ctx, "file://"+bare, ref, shrinkTestThreshold) + require.NoError(t, err) + assert.True(t, delivered) + assert.Equal(t, got.String(), bareV1(t, bare), "a repeated push is a no-op") +} + +func TestPrePush_RepairedRemoteConvergesWithoutHistoricalBloat(t *testing.T) { + dir, repo, _, _, sharedTip := bloatedV1Fixture(t) + t.Chdir(dir) + paths.ClearWorktreeRootCache() + ctx := context.Background() + localTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: local-only", map[string][]byte{ + "22/3333333333/0/metadata.json": []byte(`{"checkpoint_id":"223333333333"}` + "\n"), + }) + repairedSharedTip, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(sharedTip) + require.NoError(t, err) + remoteTip := appendCheckpointFiles(t, repo, repairedSharedTip, "Checkpoint: remote-only", map[string][]byte{ + "44/5555555555/0/metadata.json": []byte(`{"checkpoint_id":"445555555555"}` + "\n"), + }) + bare := pushV1ToBare(t, dir, remoteTip) + setV1Tip(t, repo, localTip) + testutil.WriteFile(t, dir, ".entire/settings.json", `{"enabled":true}`+"\n") + + require.NoError(t, NewManualCommitStrategy().prePushWithMetadataThreshold( + ctx, "origin", false, shrinkTestThreshold, + )) + got := readTip(t, repo) + assert.Equal(t, got.String(), bareV1(t, bare)) + oversized, err := findOversizedMetadataBlobs(ctx, repo, got, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized) + assert.Equal(t, []string{ + "Checkpoint: local-only", + "Checkpoint: remote-only", + "Checkpoint: eeffffffffff", + "Checkpoint: ccdddddddddd", + "Checkpoint: aabbbbbbbbbb", + }, commitMessages(t, repo, got)) + require.NoError(t, NewManualCommitStrategy().prePushWithMetadataThreshold( + ctx, "origin", false, shrinkTestThreshold, + )) + assert.Equal(t, got, readTip(t, repo), "a repeated pre-push must not create another checkpoint commit") +} + +func TestPrepareOversizedV1ForPush_RemoteBranchAbsent(t *testing.T) { + for _, targetKind := range []string{"named", "url"} { + t.Run(targetKind, func(t *testing.T) { + dir, repo, _, _, oldTip := bloatedV1Fixture(t) + t.Chdir(dir) + bare := t.TempDir() + testutil.RunGit(t, dir, "init", "--bare", bare) + target := "file://" + bare + if targetKind == "named" { + testutil.RunGit(t, dir, "remote", "add", "origin", bare) + target = "origin" + } + + require.NoError(t, prepareOversizedV1ForPush( + context.Background(), repo, target, shrinkTestThreshold, + )) + newTip := readTip(t, repo) + require.NotEqual(t, oldTip, newTip) + oversized, err := findOversizedMetadataBlobs(context.Background(), repo, newTip, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized) + + delivered, err := pushRefIfNeededWithMetadataThreshold( + context.Background(), target, plumbing.NewBranchReferenceName(paths.MetadataBranchName), shrinkTestThreshold, + ) + require.NoError(t, err) + assert.True(t, delivered) + assert.Equal(t, newTip.String(), bareV1(t, bare)) + }) + } +} + +func TestNewFetchTmpRefIsUniqueAndValid(t *testing.T) { + t.Parallel() + + seen := make(map[plumbing.ReferenceName]struct{}) + for range 100 { + ref, err := newFetchTmpRef("metadata-cleanup-v1") + require.NoError(t, err) + require.NoError(t, ref.Validate()) + assert.Contains(t, ref.String(), FetchTmpRefPrefix+"metadata-cleanup-v1/") + if _, exists := seen[ref]; exists { + t.Fatalf("newFetchTmpRef() returned duplicate %s", ref) + } + seen[ref] = struct{}{} + } +} + +func TestFetchV1TipForMetadataCleanup_IsolatesConcurrentTargets(t *testing.T) { + t.Parallel() + dir, repo, firstTip := setupV1RepoInDir(t) + secondTip := appendCheckpointFiles(t, repo, firstTip, "Checkpoint: second remote", map[string][]byte{ + "22/2222222222/0/metadata.json": []byte(`{"checkpoint_id":"222222222222"}` + "\n"), + }) + targets := []struct { + path string + tip plumbing.Hash + }{ + {path: t.TempDir(), tip: firstTip}, + {path: t.TempDir(), tip: secondTip}, + } + for _, target := range targets { + testutil.RunGit(t, dir, "init", "--bare", target.path) + testutil.RunGit(t, dir, "push", target.path, target.tip.String()+":refs/heads/"+paths.MetadataBranchName) + } + + var wg sync.WaitGroup + errs := make([]error, len(targets)) + got := make([]plumbing.Hash, len(targets)) + for i, target := range targets { + wg.Add(1) + go func() { + defer wg.Done() + got[i], errs[i] = fetchV1TipForMetadataCleanup(t.Context(), repo, "file://"+target.path) + }() + } + wg.Wait() + for i, target := range targets { + require.NoError(t, errs[i]) + assert.Equal(t, target.tip, got[i]) + } + assertNoFetchTmpRefsWithPurpose(t, repo, metadataCleanupFetchPurpose) +} + +func TestPrepareOversizedV1ForPush_RemovesHistoricalBlobMissingFromTip(t *testing.T) { + dir, repo, _, _, tipWithBloat := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + tipTree := readCommit(t, repo, tipWithBloat).TreeHash + withoutBloatedMetadata, err := checkpoint.ApplyTreeChanges(ctx, repo, tipTree, []checkpoint.TreeChange{{ + Path: bloatedMetadataPath, + }}) + require.NoError(t, err) + oldTip := makeOrphanCommit(t, repo, withoutBloatedMetadata, []plumbing.Hash{tipWithBloat}, "metadata replaced") + setV1Tip(t, repo, oldTip) + _, err = readCommit(t, repo, oldTip).File(bloatedMetadataPath) + require.Error(t, err, "the oversized blob must be historical rather than present in the tip tree") + + bare := t.TempDir() + testutil.RunGit(t, dir, "init", "--bare", bare) + testutil.RunGit(t, dir, "remote", "add", "origin", bare) + require.NoError(t, prepareOversizedV1ForPush(ctx, repo, "origin", shrinkTestThreshold)) + + newTip := readTip(t, repo) + require.NotEqual(t, oldTip, newTip) + oversized, err := findOversizedMetadataBlobs(ctx, repo, newTip, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized, "the blob absent from the tip tree must also be removed from rewritten history") + delivered, err := pushRefIfNeededWithMetadataThreshold( + ctx, "origin", plumbing.NewBranchReferenceName(paths.MetadataBranchName), shrinkTestThreshold, + ) + require.NoError(t, err) + assert.True(t, delivered) + assert.Equal(t, newTip.String(), bareV1(t, bare)) +} + +func TestPrepareOversizedV1ForPush_InitialHistoryBeyondReplayLimit(t *testing.T) { + dir, repo, _, _, tip := bloatedV1Fixture(t) + t.Chdir(dir) + + for range MaxCommitTraversalDepth { + parent := readCommit(t, repo, tip) + tip = makeOrphanCommit(t, repo, parent.TreeHash, []plumbing.Hash{tip}, "Checkpoint: clean suffix") + } + setV1Tip(t, repo, tip) + + bare := t.TempDir() + testutil.RunGit(t, dir, "init", "--bare", bare) + require.NoError(t, prepareOversizedV1ForPush( + context.Background(), repo, "file://"+bare, shrinkTestThreshold, + )) + + newTip := readTip(t, repo) + require.NotEqual(t, tip, newTip) + oversized, err := findOversizedMetadataBlobs(context.Background(), repo, newTip, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized) +} + +func TestPrepareOversizedV1ForPush_BoundsOnlyLocalReplaySuffix(t *testing.T) { + dir, repo, _, _, sharedTip := bloatedV1Fixture(t) + t.Chdir(dir) + + localTip := sharedTip + for i := range MaxCommitTraversalDepth + 1 { + localTip = appendCheckpointFiles(t, repo, localTip, "Checkpoint: local suffix", map[string][]byte{ + "ff/ffffffffff/0/metadata.json": []byte(fmt.Sprintf(`{"checkpoint_id":"%012x"}`, i) + "\n"), + }) + } + setV1Tip(t, repo, localTip) + repairedRemoteTip, err := newMetadataRewriter( + context.Background(), repo, shrinkTestThreshold, + ).rewriteHistory(sharedTip) + require.NoError(t, err) + pushV1ToBare(t, dir, repairedRemoteTip) + setV1Tip(t, repo, localTip) + + err = prepareOversizedV1ForPush(context.Background(), repo, "origin", shrinkTestThreshold) + require.ErrorContains(t, err, fmt.Sprintf("commit chain exceeded %d commits", MaxCommitTraversalDepth)) + assert.Equal(t, localTip, readTip(t, repo), "an over-limit replay must leave the local ref unchanged") +} + +func TestPrepareOversizedV1ForPush_RewritesAfterCleanShallowBoundary(t *testing.T) { + dir, repo, a := setupV1RepoInDir(t) + b := appendCheckpointFiles(t, repo, a, "Checkpoint: clean boundary", map[string][]byte{ + "bb/bbbbbbbbbb/0/metadata.json": []byte(`{"checkpoint_id":"bbbbbbbbbbbb"}` + "\n"), + }) + c := appendCheckpointFiles(t, repo, b, "Checkpoint: oversized", map[string][]byte{ + bloatedMetadataPath: sessionMetadataJSON(t, "shallow-bloat", 200), + }) + d := appendCheckpointFiles(t, repo, c, "Checkpoint: local suffix", map[string][]byte{ + "dd/dddddddddd/0/metadata.json": []byte(`{"checkpoint_id":"dddddddddddd"}` + "\n"), + }) + baredir := pushV1ToBare(t, dir, d) + + cloneParent := t.TempDir() + cloneDir := filepath.Join(cloneParent, "shallow") + testutil.RunGit(t, cloneParent, "clone", "--depth=3", "--branch", paths.MetadataBranchName, "file://"+baredir, cloneDir) + shallowRepo, err := git.PlainOpen(cloneDir) + require.NoError(t, err) + t.Cleanup(func() { _ = shallowRepo.Close() }) + _, err = shallowRepo.CommitObject(a) + require.Error(t, err, "the pre-boundary parent must genuinely be absent") + + emptyRemote := t.TempDir() + testutil.RunGit(t, cloneDir, "init", "--bare", emptyRemote) + t.Chdir(cloneDir) + require.NoError(t, prepareOversizedV1ForPush( + context.Background(), shallowRepo, "file://"+emptyRemote, shrinkTestThreshold, + )) + + newTip := readTip(t, shallowRepo) + require.NotEqual(t, d, newTip) + rewrittenBoundary := commitAt(t, shallowRepo, newTip, 2) + assert.Equal(t, b, rewrittenBoundary.Hash, "clean shallow boundary must remain unchanged") + oversized, err := hasOversizedMetadataIntroducedSince( + context.Background(), shallowRepo, newTip, plumbing.ZeroHash, shrinkTestThreshold, + ) + require.NoError(t, err) + assert.False(t, oversized) +} + +func TestPrepareOversizedV1ForPush_ShallowInitialPushWhenRemoteHasBoundary(t *testing.T) { + dir, repo, a := setupV1RepoInDir(t) + b := appendCheckpointFiles(t, repo, a, "Checkpoint: clean boundary", map[string][]byte{ + "bb/bbbbbbbbbb/0/metadata.json": []byte(`{"checkpoint_id":"bbbbbbbbbbbb"}` + "\n"), + }) + c := appendCheckpointFiles(t, repo, b, "Checkpoint: oversized", map[string][]byte{ + bloatedMetadataPath: sessionMetadataJSON(t, "shallow-bloat", 200), + }) + tip := appendCheckpointFiles(t, repo, c, "Checkpoint: local suffix", map[string][]byte{ + "dd/dddddddddd/0/metadata.json": []byte(`{"checkpoint_id":"dddddddddddd"}` + "\n"), + }) + sourceBare := pushV1ToBare(t, dir, tip) + + cloneDir := filepath.Join(t.TempDir(), "shallow") + testutil.RunGit(t, t.TempDir(), "clone", "--depth=3", "--branch", paths.MetadataBranchName, "file://"+sourceBare, cloneDir) + shallowRepo, err := git.PlainOpen(cloneDir) + require.NoError(t, err) + t.Cleanup(func() { _ = shallowRepo.Close() }) + + targetBare := t.TempDir() + testutil.RunGit(t, cloneDir, "init", "--bare", targetBare) + testutil.RunGit(t, dir, "push", targetBare, b.String()+":refs/heads/checkpoint-boundary") + t.Chdir(cloneDir) + require.NoError(t, prepareOversizedV1ForPush( + context.Background(), shallowRepo, "file://"+targetBare, shrinkTestThreshold, + )) + + delivered, err := pushRefIfNeededWithMetadataThreshold( + context.Background(), "file://"+targetBare, + plumbing.NewBranchReferenceName(paths.MetadataBranchName), shrinkTestThreshold, + ) + require.NoError(t, err) + assert.True(t, delivered) + assert.Equal(t, readTip(t, shallowRepo).String(), bareV1(t, targetBare)) +} + +func TestPrepareOversizedV1ForPush_ShallowCloneConvergesOnRepairedRemote(t *testing.T) { + dir, repo, a := setupV1RepoInDir(t) + b := appendCheckpointFiles(t, repo, a, "Checkpoint: clean boundary", map[string][]byte{ + "bb/bbbbbbbbbb/0/metadata.json": []byte(`{"checkpoint_id":"bbbbbbbbbbbb"}` + "\n"), + }) + c := appendCheckpointFiles(t, repo, b, "Checkpoint: oversized", map[string][]byte{ + bloatedMetadataPath: sessionMetadataJSON(t, "shallow-bloat", 200), + }) + localTip := appendCheckpointFiles(t, repo, c, "Checkpoint: local suffix", map[string][]byte{ + "dd/dddddddddd/0/metadata.json": []byte(`{"checkpoint_id":"dddddddddddd"}` + "\n"), + }) + sourceBare := pushV1ToBare(t, dir, localTip) + + cloneDir := filepath.Join(t.TempDir(), "shallow") + testutil.RunGit(t, t.TempDir(), "clone", "--depth=3", "--branch", paths.MetadataBranchName, "file://"+sourceBare, cloneDir) + shallowRepo, err := git.PlainOpen(cloneDir) + require.NoError(t, err) + t.Cleanup(func() { _ = shallowRepo.Close() }) + + repairedTip, err := newMetadataRewriter(context.Background(), repo, shrinkTestThreshold).rewriteHistory(c) + require.NoError(t, err) + remoteTip := appendCheckpointFiles(t, repo, repairedTip, "Checkpoint: remote suffix", map[string][]byte{ + "ee/eeeeeeeeee/0/metadata.json": []byte(`{"checkpoint_id":"eeeeeeeeeeee"}` + "\n"), + }) + targetBare := t.TempDir() + testutil.RunGit(t, dir, "init", "--bare", targetBare) + testutil.RunGit(t, dir, "push", targetBare, remoteTip.String()+":refs/heads/"+paths.MetadataBranchName) + + t.Chdir(cloneDir) + require.NoError(t, prepareOversizedV1ForPush( + context.Background(), shallowRepo, "file://"+targetBare, shrinkTestThreshold, + )) + got := readTip(t, shallowRepo) + assertCommitHasFile(t, shallowRepo, got, "dd/dddddddddd/0/metadata.json") + assertCommitHasFile(t, shallowRepo, got, "ee/eeeeeeeeee/0/metadata.json") + assert.Equal(t, []string{ + "Checkpoint: local suffix", + "Checkpoint: remote suffix", + "Checkpoint: oversized", + "Checkpoint: clean boundary", + }, []string{ + readCommit(t, shallowRepo, got).Message, + commitAt(t, shallowRepo, got, 1).Message, + commitAt(t, shallowRepo, got, 2).Message, + commitAt(t, shallowRepo, got, 3).Message, + }) +} + +func TestPrepareOversizedV1ForPush_RejectsOversizedShallowBoundary(t *testing.T) { + dir, _, _, _, tip := bloatedV1Fixture(t) + baredir := pushV1ToBare(t, dir, tip) + + cloneParent := t.TempDir() + cloneDir := filepath.Join(cloneParent, "shallow") + testutil.RunGit(t, cloneParent, "clone", "--depth=2", "--branch", paths.MetadataBranchName, "file://"+baredir, cloneDir) + shallowRepo, err := git.PlainOpen(cloneDir) + require.NoError(t, err) + t.Cleanup(func() { _ = shallowRepo.Close() }) + + emptyRemote := t.TempDir() + testutil.RunGit(t, cloneDir, "init", "--bare", emptyRemote) + t.Chdir(cloneDir) + err = prepareOversizedV1ForPush( + context.Background(), shallowRepo, "file://"+emptyRemote, shrinkTestThreshold, + ) + var shallowErr *shallowMetadataRepairError + require.ErrorAs(t, err, &shallowErr) + assert.Equal(t, tip, readTip(t, shallowRepo), "failed shallow repair must not move the local ref") +} + +func TestPrepareOversizedV1ForPush_RepairedRootWithoutHashMergeBase(t *testing.T) { + dir, repo, _ := setupV1RepoInDir(t) + t.Chdir(dir) + ctx := context.Background() + rootFiles := map[string][]byte{ + paths.MetadataFileName: []byte(`{"checkpoints":1}` + "\n"), + bloatedMetadataPath: sessionMetadataJSON(t, "root-bloat", 200), + bloatedTranscript: []byte(`{"role":"user"}` + "\n"), + } + oldRoot := testutil.CommitFiles(t, repo, nil, rootFiles, "Checkpoint: root-bloat") + oldSharedTip := appendCheckpointFiles(t, repo, oldRoot, "Checkpoint: shared", map[string][]byte{ + "20/0000000000/0/metadata.json": []byte(`{"checkpoint_id":"200000000000"}` + "\n"), + }) + localTip := appendCheckpointFiles(t, repo, oldSharedTip, "Checkpoint: local-only", map[string][]byte{ + "30/0000000000/0/metadata.json": []byte(`{"checkpoint_id":"300000000000"}` + "\n"), + }) + + fixedSharedTip, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(oldSharedTip) + require.NoError(t, err) + fixedShared := readCommit(t, repo, fixedSharedTip) + fixedRoot := readCommit(t, repo, fixedShared.ParentHashes[0]) + independentRoot := makeOrphanCommit(t, repo, fixedRoot.TreeHash, nil, fixedRoot.Message) + independentShared := makeOrphanCommit(t, repo, fixedShared.TreeHash, []plumbing.Hash{independentRoot}, fixedShared.Message) + remoteTip := appendCheckpointFiles(t, repo, independentShared, "Checkpoint: remote-only", map[string][]byte{ + "40/0000000000/0/metadata.json": []byte(`{"checkpoint_id":"400000000000"}` + "\n"), + }) + mergeBase, err := computeMergeBase(repo, localTip, remoteTip) + require.NoError(t, err) + require.True(t, mergeBase.IsZero(), "rewriting an affected root leaves no shared commit hash") + + bare := pushV1ToBare(t, dir, remoteTip) + setV1Tip(t, repo, localTip) + require.NoError(t, prepareOversizedV1ForPush(ctx, repo, "origin", shrinkTestThreshold)) + got := readTip(t, repo) + require.Equal(t, []plumbing.Hash{remoteTip}, readCommit(t, repo, got).ParentHashes) + assert.Equal(t, []string{ + "Checkpoint: local-only", + "Checkpoint: remote-only", + "Checkpoint: shared", + "Checkpoint: root-bloat", + }, commitMessages(t, repo, got)) + oversized, err := findOversizedMetadataBlobs(ctx, repo, got, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized) + assertNoFetchTmpRefsWithPurpose(t, repo, metadataCleanupFetchPurpose) + assert.Equal(t, remoteTip.String(), bareV1(t, bare), "preflight must not mutate the remote") +} + +func TestFetchAndRebase_StripsOversizedLocalOnlyHistory(t *testing.T) { + dir, repo, remoteTip, _, localTip := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + + pushV1ToBare(t, dir, remoteTip) + setV1Tip(t, repo, localTip) + require.NoError(t, fetchAndRebaseRefWithMetadataThreshold( + ctx, + "origin", + plumbing.NewBranchReferenceName(paths.MetadataBranchName), + shrinkTestThreshold, + )) + + got := readTip(t, repo) + assert.Equal(t, []string{ + "Checkpoint: eeffffffffff", + "Checkpoint: ccdddddddddd", + "Checkpoint: aabbbbbbbbbb", + }, commitMessages(t, repo, got)) + assert.Equal(t, remoteTip, commitAt(t, repo, got, 2).Hash, + "history before the local-only oversized blob keeps the remote's exact commit") + oversized, err := findOversizedMetadataBlobs(ctx, repo, got, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized) +} + +func TestFetchAndRebase_UnrepairedRemoteUsesExactMergeBase(t *testing.T) { + dir, repo, _, _, sharedTip := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + + localTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: local-only", map[string][]byte{ + "66/7777777777/0/metadata.json": []byte(`{"checkpoint_id":"667777777777"}` + "\n"), + }) + remoteTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: remote-only", map[string][]byte{ + "88/9999999999/0/metadata.json": []byte(`{"checkpoint_id":"889999999999"}` + "\n"), + }) + pushV1ToBare(t, dir, remoteTip) + setV1Tip(t, repo, localTip) + before, handled, err := reconcileOversizedV1ForPush(ctx, repo, localTip, remoteTip, shrinkTestThreshold) + require.NoError(t, err) + assert.False(t, handled) + assert.Equal(t, localTip, before, + "bloat introduced before the exact merge-base is already remote-owned and must not select an older tree boundary") + + require.NoError(t, fetchAndRebaseRefWithMetadataThreshold( + ctx, + "origin", + plumbing.NewBranchReferenceName(paths.MetadataBranchName), + shrinkTestThreshold, + )) + + got := readTip(t, repo) + require.Equal(t, []plumbing.Hash{remoteTip}, readCommit(t, repo, got).ParentHashes) + assert.Equal(t, []string{ + "Checkpoint: local-only", + "Checkpoint: remote-only", + "Checkpoint: eeffffffffff", + "Checkpoint: ccdddddddddd", + "Checkpoint: aabbbbbbbbbb", + }, commitMessages(t, repo, got), + "an older clean tree must not move the replay boundary behind the exact merge-base") + oversized, err := findOversizedMetadataBlobs(ctx, repo, got, shrinkTestThreshold) + require.NoError(t, err) + require.Len(t, oversized, 1, "an unrepaired remote stays on the existing replay path") +} + +func TestPrepareOversizedV1ForPush_DoesNotTrustCleanTipWithHistoricalRemoteBloat(t *testing.T) { + dir, repo, _, _, sharedTip := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + localTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: local-only", map[string][]byte{ + "66/7777777777/0/metadata.json": []byte(`{"checkpoint_id":"667777777777"}` + "\n"), + }) + repairedSharedTip, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(sharedTip) + require.NoError(t, err) + _, repairedMetadata := blobAt(t, repo, repairedSharedTip, bloatedMetadataPath) + remoteTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: remote cleaned tip", map[string][]byte{ + bloatedMetadataPath: repairedMetadata, + }) + require.Equal(t, readCommit(t, repo, repairedSharedTip).TreeHash, readCommit(t, repo, remoteTip).TreeHash) + + got, handled, err := reconcileOversizedV1ForPush(ctx, repo, localTip, remoteTip, shrinkTestThreshold) + require.NoError(t, err) + assert.False(t, handled) + assert.Equal(t, localTip, got, + "a clean cumulative tree must not prove repair while the remote still reaches the old blob") +} + +func TestPrepareOversizedV1ForPush_FindsRepairedBoundaryOnRemoteMergeParent(t *testing.T) { + _, repo, cleanRoot, _, sharedTip := bloatedV1Fixture(t) + ctx := context.Background() + localPath := "66/7777777777/0/metadata.json" + remotePath := "88/9999999999/0/metadata.json" + localTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: local-only", map[string][]byte{ + localPath: []byte(`{"checkpoint_id":"667777777777"}` + "\n"), + }) + repairedSharedTip, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(sharedTip) + require.NoError(t, err) + remoteFirstParent := appendCheckpointFiles(t, repo, cleanRoot, "Checkpoint: remote first parent", map[string][]byte{ + "77/8888888888/0/metadata.json": []byte(`{"checkpoint_id":"778888888888"}` + "\n"), + }) + remoteTreeCommit := appendCheckpointFiles(t, repo, repairedSharedTip, "remote merge tree", map[string][]byte{ + remotePath: []byte(`{"checkpoint_id":"889999999999"}` + "\n"), + }) + remoteTip := makeOrphanCommit( + t, + repo, + readCommit(t, repo, remoteTreeCommit).TreeHash, + []plumbing.Hash{remoteFirstParent, repairedSharedTip}, + "Checkpoint: remote merge", + ) + setV1Tip(t, repo, localTip) + + got, handled, err := reconcileOversizedV1ForPush(ctx, repo, localTip, remoteTip, shrinkTestThreshold) + require.NoError(t, err) + assert.True(t, handled) + assertCommitHasFile(t, repo, got, localPath) + assertCommitHasFile(t, repo, got, remotePath) + require.Equal(t, []plumbing.Hash{remoteTip}, readCommit(t, repo, got).ParentHashes) +} + +func TestRewriteUnpushedV1WithOPF_ReconcilesRepairedRemoteBeforeDivergenceCheck(t *testing.T) { + configureFakeOPF(t, &fakeOPFForRewrite{}) + dir, repo, _, _, sharedTip := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + + localTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: local-after-repair", map[string][]byte{ + "12/3456789abc/0/full.jsonl": []byte(`{"role":"user","content":"PERSONABC"}` + "\n"), + }) + repairedRemoteTip, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(sharedTip) + require.NoError(t, err) + pushV1ToBare(t, dir, repairedRemoteTip) + setV1Tip(t, repo, localTip) + + got, err := rewriteUnpushedV1WithOPF(ctx, repo, "origin", shrinkTestThreshold) + require.NoError(t, err) + gotCommit := readCommit(t, repo, got) + require.Equal(t, []plumbing.Hash{repairedRemoteTip}, gotCommit.ParentHashes) + assert.True(t, trailers.HasOPFApplied(gotCommit.Message)) + assert.Len(t, commitMessages(t, repo, got), 4, "the shared repaired history appears once") + oversized, err := findOversizedMetadataBlobs(ctx, repo, got, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, oversized) +} + +func TestPrepareOversizedV1ForPush_PreservesMergeFinalTree(t *testing.T) { + dir, repo, _, _, sharedTip := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + + localPath := "11/1111111111/0/metadata.json" + sidePath := "22/2222222222/0/metadata.json" + resolutionPath := "33/3333333333/0/metadata.json" + remotePath := "44/4444444444/0/metadata.json" + localTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: local first parent", map[string][]byte{ + localPath: []byte(`{"checkpoint_id":"111111111111"}` + "\n"), + }) + sideTip := appendCheckpointFiles(t, repo, sharedTip, "Checkpoint: local side", map[string][]byte{ + sidePath: []byte(`{"checkpoint_id":"222222222222"}` + "\n"), + }) + sideFile, err := readCommit(t, repo, sideTip).File(sidePath) + require.NoError(t, err) + resolutionBlob, err := checkpoint.CreateBlobFromContent(repo, []byte(`{"checkpoint_id":"333333333333"}`+"\n")) + require.NoError(t, err) + mergeTree, err := checkpoint.ApplyTreeChanges(ctx, repo, readCommit(t, repo, localTip).TreeHash, []checkpoint.TreeChange{ + {Path: sidePath, Entry: &object.TreeEntry{Name: filepath.Base(sidePath), Mode: filemode.Regular, Hash: sideFile.Hash}}, + {Path: resolutionPath, Entry: &object.TreeEntry{Name: filepath.Base(resolutionPath), Mode: filemode.Regular, Hash: resolutionBlob}}, + }) + require.NoError(t, err) + mergeTip := makeOrphanCommit(t, repo, mergeTree, []plumbing.Hash{localTip, sideTip}, "Checkpoint: merge resolution") + + repairedShared, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(sharedTip) + require.NoError(t, err) + remoteTip := appendCheckpointFiles(t, repo, repairedShared, "Checkpoint: remote-only", map[string][]byte{ + remotePath: []byte(`{"checkpoint_id":"444444444444"}` + "\n"), + }) + pushV1ToBare(t, dir, remoteTip) + setV1Tip(t, repo, mergeTip) + + require.NoError(t, prepareOversizedV1ForPush(ctx, repo, "origin", shrinkTestThreshold)) + got := readTip(t, repo) + for _, filePath := range []string{localPath, sidePath, resolutionPath, remotePath} { + assertCommitHasFile(t, repo, got, filePath) + } + assert.Equal(t, []string{ + "Checkpoint: merge resolution", + "Checkpoint: local first parent", + "Checkpoint: remote-only", + "Checkpoint: eeffffffffff", + "Checkpoint: ccdddddddddd", + "Checkpoint: aabbbbbbbbbb", + }, commitMessages(t, repo, got)) +} + +func TestCherryPickOnto_SkipsChangeAlreadyPresentAtTip(t *testing.T) { + t.Parallel() + dir := t.TempDir() + testutil.InitRepo(t, dir) + repo, err := git.PlainOpen(dir) + require.NoError(t, err) + + sourceParent := testutil.CommitFiles(t, repo, nil, map[string][]byte{"metadata.json": []byte("old\n")}, "source parent") + sourceTip := testutil.CommitFiles(t, repo, []plumbing.Hash{sourceParent}, map[string][]byte{"metadata.json": []byte("shared\n")}, "source change") + base := testutil.CommitFiles(t, repo, nil, map[string][]byte{"metadata.json": []byte("shared\n")}, "remote already has change") + sourceCommit := readCommit(t, repo, sourceTip) + + got, err := cherryPickOnto(context.Background(), repo, base, []*object.Commit{sourceCommit}, nil) + require.NoError(t, err) + assert.Equal(t, base, got, "an already-present tree change must not create a duplicate commit") +} + +func appendCheckpointFiles( + t *testing.T, + repo *git.Repository, + parent plumbing.Hash, + message string, + files map[string][]byte, +) plumbing.Hash { + t.Helper() + parentCommit := readCommit(t, repo, parent) + changes := make([]checkpoint.TreeChange, 0, len(files)) + for path, content := range files { + blob, err := checkpoint.CreateBlobFromContent(repo, content) + require.NoError(t, err) + changes = append(changes, checkpoint.TreeChange{ + Path: path, + Entry: &object.TreeEntry{ + Name: filepath.Base(path), + Mode: filemode.Regular, + Hash: blob, + }, + }) + } + tree, err := checkpoint.ApplyTreeChanges(context.Background(), repo, parentCommit.TreeHash, changes) + require.NoError(t, err) + return makeOrphanCommit(t, repo, tree, []plumbing.Hash{parent}, message) +} + +func setV1Tip(t *testing.T, repo *git.Repository, tip plumbing.Hash) { + t.Helper() + require.NoError(t, repo.Storer.SetReference(plumbing.NewHashReference( + plumbing.NewBranchReferenceName(paths.MetadataBranchName), tip, + ))) +} + +func assertCommitHasFile(t *testing.T, repo *git.Repository, commit plumbing.Hash, path string) { + t.Helper() + _, err := readCommit(t, repo, commit).File(path) + require.NoError(t, err) +} + +func assertNoFetchTmpRefsWithPurpose(t *testing.T, repo *git.Repository, purpose string) { + t.Helper() + refs, err := repo.References() + require.NoError(t, err) + defer refs.Close() + err = refs.ForEach(func(ref *plumbing.Reference) error { + if strings.HasPrefix(ref.Name().String(), FetchTmpRefPrefix+purpose+"/") { + t.Errorf("temporary fetch ref was not removed: %s", ref.Name()) + } + return nil + }) + require.NoError(t, err) +} + +func TestFindSharedRewrittenTree_Cancellation(t *testing.T) { + t.Parallel() + _, repo, _, _, tip := bloatedV1Fixture(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + repoPath, pathErr := getRepoPath(repo) + require.NoError(t, pathErr) + _, _, _, err := findSharedRewrittenTree(ctx, repo, repoPath, tip, tip, map[plumbing.Hash]plumbing.Hash{tip: tip}) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestComputeMergeBaseWithGit_RejectsMultipleBases(t *testing.T) { + t.Parallel() + dir, repo, root := setupV1RepoInDir(t) + a := appendCheckpointFiles(t, repo, root, "a", map[string][]byte{ + "a/metadata.json": []byte(`{"checkpoint_id":"aaaaaaaaaaaa"}` + "\n"), + }) + b := appendCheckpointFiles(t, repo, root, "b", map[string][]byte{ + "b/metadata.json": []byte(`{"checkpoint_id":"bbbbbbbbbbbb"}` + "\n"), + }) + union := appendCheckpointFiles(t, repo, a, "union tree", map[string][]byte{ + "b/metadata.json": []byte(`{"checkpoint_id":"bbbbbbbbbbbb"}` + "\n"), + }) + left := makeOrphanCommit(t, repo, readCommit(t, repo, union).TreeHash, []plumbing.Hash{a, b}, "left") + right := makeOrphanCommit(t, repo, readCommit(t, repo, union).TreeHash, []plumbing.Hash{b, a}, "right") + + _, err := computeMergeBaseWithGit(context.Background(), dir, left, right) + require.ErrorContains(t, err, "multiple merge bases") +} + +func TestHasOversizedMetadataIntroducedSince_ScansMergeSideBranches(t *testing.T) { + t.Parallel() + _, repo, cleanRoot, bloatedSide, _ := bloatedV1Fixture(t) + cleanSide := appendCheckpointFiles(t, repo, cleanRoot, "clean side", map[string][]byte{ + "10/0000000000/0/metadata.json": []byte(`{"checkpoint_id":"100000000000"}` + "\n"), + }) + merge := makeOrphanCommit( + t, + repo, + readCommit(t, repo, cleanSide).TreeHash, + []plumbing.Hash{cleanSide, bloatedSide}, + "merge", + ) + + found, err := hasOversizedMetadataIntroducedSince( + context.Background(), repo, merge, cleanRoot, shrinkTestThreshold, + ) + require.NoError(t, err) + assert.True(t, found, "the boundary on the first-parent path must not hide the other parent's bloat") +} + +func TestHasOversizedMetadataIntroducedSince_ScansMergeResolution(t *testing.T) { + t.Parallel() + _, repo, root := setupV1RepoInDir(t) + firstParent := appendCheckpointFiles(t, repo, root, "first parent", map[string][]byte{ + "10/0000000000/0/metadata.json": []byte(`{"checkpoint_id":"100000000000"}` + "\n"), + }) + secondParent := appendCheckpointFiles(t, repo, root, "second parent", map[string][]byte{ + "20/0000000000/0/metadata.json": []byte(`{"checkpoint_id":"200000000000"}` + "\n"), + }) + mergeTree := appendCheckpointFiles(t, repo, firstParent, "merge tree", map[string][]byte{ + bloatedMetadataPath: sessionMetadataJSON(t, "merge-resolution", 200), + }) + merge := makeOrphanCommit( + t, + repo, + readCommit(t, repo, mergeTree).TreeHash, + []plumbing.Hash{firstParent, secondParent}, + "merge resolution", + ) + + found, err := hasOversizedMetadataIntroducedSince( + context.Background(), repo, merge, root, shrinkTestThreshold, + ) + require.NoError(t, err) + assert.True(t, found) +} + +func TestHasOversizedMetadataIntroducedSince_DetectsBlobAliasedByUnrelatedPath(t *testing.T) { + t.Parallel() + dir := t.TempDir() + testutil.InitRepo(t, dir) + repo, err := gitrepo.OpenPath(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = repo.Close() }) + + content := sessionMetadataJSON(t, "aliased", 200) + tip := testutil.CommitFiles(t, repo, nil, map[string][]byte{ + "aaa-unrelated.bin": content, + "metadata.json": content, + }, "aliased metadata") + + found, err := hasOversizedMetadataIntroducedSince( + context.Background(), repo, tip, plumbing.ZeroHash, shrinkTestThreshold, + ) + require.NoError(t, err) + assert.True(t, found) +} + +func BenchmarkHasOversizedMetadataIntroducedSince_CleanHistory(b *testing.B) { + repo, tip := benchmarkCleanCheckpointHistory(b, 500) + b.ResetTimer() + for b.Loop() { + found, err := hasOversizedMetadataIntroducedSince(context.Background(), repo, tip, plumbing.ZeroHash, shrinkTestThreshold) + if err != nil { + b.Fatal(err) + } + if found { + b.Fatal("clean history reported oversized metadata") + } + } +} + +func BenchmarkPerCommitDiffScanner_CleanHistory(b *testing.B) { + repo, tip := benchmarkCleanCheckpointHistory(b, 500) + b.ResetTimer() + for b.Loop() { + history, err := repo.Log(&git.LogOptions{From: tip}) + if err != nil { + b.Fatal(err) + } + err = history.ForEach(func(commit *object.Commit) error { + return forEachMetadataBlobIntroduced(context.Background(), commit, func(_ string, hash plumbing.Hash) error { + blob, err := repo.BlobObject(hash) + if err != nil { + return err + } + if blob.Size > shrinkTestThreshold { + b.Fatal("clean history reported oversized metadata") + } + return nil + }) + }) + history.Close() + if err != nil { + b.Fatal(err) + } + } +} + +func benchmarkCleanCheckpointHistory(b *testing.B, commits int) (*git.Repository, plumbing.Hash) { + b.Helper() + repo, err := git.PlainInit(b.TempDir(), false) + if err != nil { + b.Fatal(err) + } + tree := plumbing.ZeroHash + parent := plumbing.ZeroHash + for i := range commits { + blob, err := checkpoint.CreateBlobFromContent(repo, []byte(fmt.Sprintf(`{"checkpoint_id":"%012d"}`+"\n", i))) + if err != nil { + b.Fatal(err) + } + tree, err = checkpoint.ApplyTreeChanges(context.Background(), repo, tree, []checkpoint.TreeChange{{ + Path: fmt.Sprintf("%02x/%010x/0/metadata.json", i%256, i), + Entry: &object.TreeEntry{ + Name: "metadata.json", + Mode: filemode.Regular, + Hash: blob, + }, + }}) + if err != nil { + b.Fatal(err) + } + commit := &object.Commit{ + Author: object.Signature{Name: "Test", Email: "test@example.com", When: time.Unix(int64(i), 0)}, + Committer: object.Signature{Name: "Test", Email: "test@example.com", When: time.Unix(int64(i), 0)}, + Message: fmt.Sprintf("Checkpoint: %012d", i), + TreeHash: tree, + ParentHashes: nil, + } + if !parent.IsZero() { + commit.ParentHashes = []plumbing.Hash{parent} + } + obj := repo.Storer.NewEncodedObject() + if err := commit.Encode(obj); err != nil { + b.Fatal(err) + } + parent, err = repo.Storer.SetEncodedObject(obj) + if err != nil { + b.Fatal(err) + } + } + return repo, parent +} diff --git a/cmd/entire/cli/strategy/metadata_reconcile.go b/cmd/entire/cli/strategy/metadata_reconcile.go index e274e1dc0b..1334eeb98f 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile.go +++ b/cmd/entire/cli/strategy/metadata_reconcile.go @@ -17,6 +17,7 @@ import ( "github.com/entireio/cli/cmd/entire/cli/gitdir" "github.com/entireio/cli/cmd/entire/cli/logging" "github.com/entireio/cli/cmd/entire/cli/osroot" + "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/plumbing" @@ -196,10 +197,6 @@ func ReconcileDisconnectedMetadataRef( remoteRefName plumbing.ReferenceName, w io.Writer, ) error { - advance := func(hash plumbing.Hash) error { - return setRefHash(repo, localRefName, hash) - } - // Check local ref localRef, err := repo.Reference(localRefName, true) if errors.Is(err, plumbing.ErrReferenceNotFound) { @@ -220,6 +217,12 @@ func ReconcileDisconnectedMetadataRef( localHash := localRef.Hash() remoteHash := remoteRef.Hash() + advance := func(hash plumbing.Hash) error { + if localRefName == plumbing.NewBranchReferenceName(paths.MetadataBranchName) { + return atomicSetV1Ref(ctx, repo, localHash, hash) + } + return setRefHash(repo, localRefName, hash) + } // Same hash — nothing to do if localHash == remoteHash { @@ -426,6 +429,9 @@ func cherryPickOnto(ctx context.Context, repo *git.Repository, base plumbing.Has if err != nil { return plumbing.ZeroHash, fmt.Errorf("failed to apply cherry-pick changes: %w", err) } + if mergedTreeHash.Equal(tipCommit.TreeHash) { + continue + } // Create new commit on top of current tip, preserving original message/author newHash, err := createCherryPickCommit(ctx, repo, mergedTreeHash, currentTip, commit) diff --git a/cmd/entire/cli/strategy/metadata_shrink.go b/cmd/entire/cli/strategy/metadata_shrink.go new file mode 100644 index 0000000000..129e6752b0 --- /dev/null +++ b/cmd/entire/cli/strategy/metadata_shrink.go @@ -0,0 +1,827 @@ +package strategy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "path" + "sort" + "strings" + + git "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" + "github.com/entireio/cli/cmd/entire/cli/jsonutil" + "github.com/entireio/cli/cmd/entire/cli/logging" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/settings" +) + +// OversizedCheckpointMetadataThreshold is the blob size above which a +// per-session metadata.json on entire/checkpoints/v1 is reported by +// `entire doctor`. GitHub warns at 50 MiB and refuses any blob over 100 MiB, and +// a refused blob anywhere in the branch's history makes the whole branch +// unpushable there — including through a forge's push mirror. A healthy +// metadata.json is tens of kilobytes, so anything past this line is the +// pre-v0.10.1 prompt_attributions bloat (see checkpoint.MaxPromptAttributionsBytes), +// not a large session. +const OversizedCheckpointMetadataThreshold int64 = 50 << 20 + +// promptAttributionsField is the metadata.json key the shrink removes. It is +// the only field whose size scales with the working tree rather than the +// session, and nothing reads it back — the attribution summary it fed is a +// separate field that stays. +const promptAttributionsField = "prompt_attributions" + +// OversizedMetadataBlob is one metadata.json blob over the threshold. +type OversizedMetadataBlob struct { + Path string // tree path, e.g. "ab/cdef.../0/metadata.json" + Size int64 // blob size in bytes + Hash plumbing.Hash // blob hash + Commit plumbing.Hash // the commit that introduced this blob version +} + +// MetadataSizeScan is the read-only half of the oversized-metadata repair: what +// is oversized, where, and what the elected sync remote holds, so the report and +// the fix share one fetch. +type MetadataSizeScan struct { + Threshold int64 + LocalTip plumbing.Hash // zero when the branch does not exist locally + RemoteName string // elected checkpoint sync remote; "" when none + // RemoteTip is the remote-tracking tip of the checkpoint branch on + // RemoteName after a refresh. Zero when there is no remote, the remote has + // no such branch, or the refresh failed and no stale tracking ref exists. + RemoteTip plumbing.Hash + // RemoteErr records a failed refresh or an unreadable tracking ref. After a + // failed refresh RemoteTip may still reflect the tracking ref as it was, + // which the force-with-lease push guards against; after an unreadable ref + // the remote fields are cleared and the fix does not push. + RemoteErr error + // RemoteAhead reports that RemoteTip carries commits LocalTip does not. + RemoteAhead bool + PushDisabled bool + // DedicatedCheckpointRemote reports that the branch is pushed to a + // checkpoint_remote URL rather than to RemoteName; the fix is withheld. + DedicatedCheckpointRemote bool + // Local are oversized blobs reachable from LocalTip; Remote those reachable + // from RemoteTip and not already in Local. + Local []OversizedMetadataBlob + Remote []OversizedMetadataBlob +} + +// Empty reports whether nothing is oversized on either side. +func (s *MetadataSizeScan) Empty() bool { return len(s.Local) == 0 && len(s.Remote) == 0 } + +// All returns every oversized blob, largest first. +func (s *MetadataSizeScan) All() []OversizedMetadataBlob { + all := make([]OversizedMetadataBlob, 0, len(s.Local)+len(s.Remote)) + all = append(all, s.Local...) + all = append(all, s.Remote...) + sort.SliceStable(all, func(i, j int) bool { return all[i].Size > all[j].Size }) + return all +} + +// MetadataShrinkResult describes what ShrinkOversizedCheckpointMetadata did. +type MetadataShrinkResult struct { + OldLocalTip plumbing.Hash + NewLocalTip plumbing.Hash + // CommitsRewritten counts commits of the local branch that were rebuilt. + CommitsRewritten int + // RemoteCommitsRewritten counts commits of the remote's history that were + // rebuilt because the remote was ahead of the local branch. + RemoteCommitsRewritten int + BlobsShrunk int + // StillOversized lists blobs that remain over the threshold after the + // prompt_attributions field was removed — bloat of a kind this repair does + // not know about, reported rather than guessed at. + StillOversized []OversizedMetadataBlob + Pushed bool + // PushSkippedReason explains a Pushed == false when no error occurred. + PushSkippedReason string +} + +// ScanOversizedCheckpointMetadata refreshes the elected sync remote's tracking +// ref for the checkpoint branch (best effort) and reports every metadata.json +// blob over threshold reachable from the local branch or from that tracking ref. +// It is read-only: nothing under refs/heads is touched. +// +// The remote side matters because the repair force-pushes: a blob that is only +// on the remote (an earlier local repair whose push failed) still blocks the +// mirror, and an ordinary pre-push would replay the local branch onto that +// remote history and reintroduce it. +// +// The network is touched only when something is oversized. The local branch +// and the remote-tracking ref as last fetched are scanned first; when both are +// clean the answer is "OK" without a fetch, which keeps the routine doctor run +// offline. When either shows a problem the tracking ref is refreshed and the +// remote side re-scanned, so the fix that follows works from the remote's +// current tip. +func ScanOversizedCheckpointMetadata(ctx context.Context, repo *git.Repository, remoteName string, threshold int64) (*MetadataSizeScan, error) { + v1 := plumbing.NewBranchReferenceName(paths.MetadataBranchName) + scan := &MetadataSizeScan{Threshold: threshold, RemoteName: remoteName} + + localTip, err := readV1Tip(repo, v1) + if err != nil { + return nil, fmt.Errorf("read local %s: %w", v1.Short(), err) + } + scan.LocalTip = localTip + scan.Local, err = findOversizedMetadataBlobs(ctx, repo, localTip, threshold) + if err != nil { + return nil, fmt.Errorf("scan local %s: %w", v1.Short(), err) + } + // The tracking ref as last fetched. An unreadable one (dangling, partial + // clone) is reported, not fatal: doctor's other checks must still run, and + // with a clean local branch there is nothing this check would do anyway. + var trackingRef plumbing.ReferenceName + var readErr error + if remoteName != "" { + trackingRef = plumbing.NewRemoteReferenceName(remoteName, paths.MetadataBranchName) + readErr = scan.readRemoteSide(ctx, repo, trackingRef) + } + if len(scan.Local) == 0 && (remoteName == "" || readErr != nil || len(scan.Remote) == 0) { + scan.RemoteErr = readErr + return scan, nil //nolint:nilerr // fail-soft: the unreadable tracking ref is reported on scan.RemoteErr, not fatal to doctor + } + // Something is oversized (or the stale tracking ref could not tell us). + // Where pre-push actually sends the branch decides what happens next: a + // dedicated checkpoint_remote URL has no remote-tracking ref to lease + // against, and pre-push replays local commits onto whatever that URL + // holds, so a local-only rewrite there would be undone by the next push. + // Report it and stop rather than repair half of it — and decide that + // BEFORE the no-remote early return below, so a repository whose only + // checkpoint destination is the dedicated URL is refused the same way. + // The settings are read directly rather than through resolvePushSettings, + // which may fetch and create the local branch as a side effect; this scan + // must stay read-only. They are read for THIS repository's worktree, not + // the process working directory: the scan is handed a repo and must not + // answer for whichever checkout the caller happens to be standing in. + if s, loadErr := loadSettingsForRepo(ctx, repo); loadErr == nil { + scan.PushDisabled = s.IsPushSessionsDisabled() + if s.GetCheckpointRemote() != nil { + scan.DedicatedCheckpointRemote = true + scan.clearRemote() + return scan, nil + } + } else { + // Unknown push policy: do not push. The local rewrite is still useful. + scan.PushDisabled = true + } + if remoteName == "" { + return scan, nil + } + // Refresh from the remote so the fix works from its current tip. + if fetchErr := refreshCheckpointTrackingRef(ctx, remoteName, v1, trackingRef); fetchErr != nil { + scan.RemoteErr = fetchErr + } + if err := scan.readRemoteSide(ctx, repo, trackingRef); err != nil { + scan.RemoteErr = errors.Join(scan.RemoteErr, err) + scan.clearRemote() + } + return scan, nil +} + +// ErrDedicatedCheckpointRemote reports that the repository pushes its +// checkpoint branch to a dedicated checkpoint_remote URL, which the automatic +// repair does not handle (see ScanOversizedCheckpointMetadata). +var ErrDedicatedCheckpointRemote = errors.New("checkpoint branch is pushed to a dedicated checkpoint remote; automatic repair is not available there") + +// loadSettingsForRepo reads Entire settings for the worktree repo was opened on. +func loadSettingsForRepo(ctx context.Context, repo *git.Repository) (*settings.EntireSettings, error) { + wt, err := repo.Worktree() + if err != nil { + return nil, fmt.Errorf("resolve worktree: %w", err) + } + return settings.LoadForWorktreeRoot(ctx, wt.Filesystem().Root()) //nolint:wrapcheck // settings errors carry their own context +} + +// clearRemote leaves the scan with no knowledge of the remote side. +func (s *MetadataSizeScan) clearRemote() { + s.RemoteTip = plumbing.ZeroHash + s.RemoteAhead = false + s.Remote = nil +} + +// readRemoteSide fills RemoteTip, RemoteAhead and Remote from the tracking +// ref as it currently stands. On error the remote fields are cleared. +func (s *MetadataSizeScan) readRemoteSide(ctx context.Context, repo *git.Repository, trackingRef plumbing.ReferenceName) (err error) { + defer func() { + if err != nil { + s.clearRemote() + } + }() + remoteTip, err := readV1Tip(repo, trackingRef) + if err != nil { + return fmt.Errorf("read %s: %w", trackingRef.Short(), err) + } + s.RemoteTip = remoteTip + s.RemoteAhead = false + s.Remote = nil + switch { + case remoteTip.IsZero(): + return nil + case s.LocalTip.IsZero(): + s.RemoteAhead = true + case !remoteTip.Equal(s.LocalTip): + base, mbErr := computeMergeBase(repo, s.LocalTip, remoteTip) + if mbErr != nil { + return fmt.Errorf("compare local and %s: %w", trackingRef.Short(), mbErr) + } + s.RemoteAhead = !base.Equal(remoteTip) + } + if remoteTip.Equal(s.LocalTip) { + return nil + } + remoteBlobs, err := findOversizedMetadataBlobs(ctx, repo, remoteTip, s.Threshold) + if err != nil { + return fmt.Errorf("scan %s: %w", trackingRef.Short(), err) + } + seen := make(map[plumbing.Hash]struct{}, len(s.Local)) + for _, b := range s.Local { + seen[b.Hash] = struct{}{} + } + for _, b := range remoteBlobs { + if _, dup := seen[b.Hash]; !dup { + s.Remote = append(s.Remote, b) + } + } + return nil +} + +// refreshCheckpointTrackingRef fetches the checkpoint branch from remoteName +// into its remote-tracking ref. A remote that has no such branch is not an +// error: the tracking ref is simply left as it was (normally absent). +func refreshCheckpointTrackingRef(ctx context.Context, remoteName string, branch, trackingRef plumbing.ReferenceName) error { + fetchCtx, cancel := context.WithTimeout(ctx, checkpointRemoteForegroundFetchTimeout) + defer cancel() + output, err := remote.Fetch(fetchCtx, remote.FetchOptions{ + Remote: remoteName, + RefSpecs: []string{"+" + branch.String() + ":" + trackingRef.String()}, + NoTags: true, + NoFilter: true, // the scan reads blob sizes, which a blob-filtered fetch would not have + }) + if err == nil { + return nil + } + if strings.Contains(string(output), "couldn't find remote ref") { + return nil + } + if msg := strings.TrimSpace(string(output)); msg != "" { + return fmt.Errorf("fetch %s from %s: %s: %w", branch.Short(), remoteName, msg, err) + } + return fmt.Errorf("fetch %s from %s: %w", branch.Short(), remoteName, err) +} + +// findOversizedMetadataBlobs walks the history behind tip and returns every +// metadata.json blob larger than threshold, largest first. Each commit is +// diffed against its first parent, so the walk costs the size of the changes +// rather than the size of the (cumulative) trees; the root commit's tree is +// listed in full. +func findOversizedMetadataBlobs(ctx context.Context, repo *git.Repository, tip plumbing.Hash, threshold int64) ([]OversizedMetadataBlob, error) { + if tip.IsZero() { + return nil, nil + } + iter, err := repo.Log(&git.LogOptions{From: tip}) + if err != nil { + return nil, fmt.Errorf("log %s: %w", tip, err) + } + defer iter.Close() + + seen := make(map[plumbing.Hash]struct{}) + var found []OversizedMetadataBlob + walkErr := iter.ForEach(func(c *object.Commit) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr //nolint:wrapcheck // context cancellation propagates as-is + } + return forEachMetadataBlobIntroduced(ctx, c, func(blobPath string, hash plumbing.Hash) error { + if _, dup := seen[hash]; dup { + return nil + } + seen[hash] = struct{}{} + blob, blobErr := repo.BlobObject(hash) + if blobErr != nil { + return fmt.Errorf("blob %s at %s: %w", hash, blobPath, blobErr) + } + if blob.Size > threshold { + found = append(found, OversizedMetadataBlob{Path: blobPath, Size: blob.Size, Hash: hash, Commit: c.Hash}) + } + return nil + }) + }) + if walkErr != nil { + return nil, fmt.Errorf("walk history of %s: %w", tip, walkErr) + } + sort.SliceStable(found, func(i, j int) bool { return found[i].Size > found[j].Size }) + return found, nil +} + +// forEachMetadataBlobIntroduced calls fn for every metadata.json blob that +// commit c adds or changes relative to any of its parents. The checkpoint +// branch is linear in practice, but a merge is diffed against every parent so +// a blob that arrived through a side parent is not missed; the caller dedupes +// by hash, so over-reporting costs nothing. +func forEachMetadataBlobIntroduced(ctx context.Context, c *object.Commit, fn func(blobPath string, hash plumbing.Hash) error) error { + tree, err := c.Tree() + if err != nil { + return fmt.Errorf("tree of %s: %w", c.Hash, err) + } + if c.NumParents() == 0 { + files := tree.Files() + defer files.Close() + return files.ForEach(func(f *object.File) error { //nolint:wrapcheck // fn's errors carry their own context + if path.Base(f.Name) != paths.MetadataFileName { + return nil + } + return fn(f.Name, f.Hash) + }) + } + for i := range c.NumParents() { + parent, err := c.Parent(i) + if err != nil { + return fmt.Errorf("parent %d of %s: %w", i, c.Hash, err) + } + parentTree, err := parent.Tree() + if err != nil { + return fmt.Errorf("tree of %s: %w", parent.Hash, err) + } + changes, err := object.DiffTreeContext(ctx, parentTree, tree) + if err != nil { + return fmt.Errorf("diff %s..%s: %w", parent.Hash, c.Hash, err) + } + for _, ch := range changes { + to := ch.To + if to.Name == "" || path.Base(to.Name) != paths.MetadataFileName { + continue + } + if !isFileMode(to.TreeEntry.Mode) { + continue + } + if err := fn(to.Name, to.TreeEntry.Hash); err != nil { + return err + } + } + } + return nil +} + +// isFileMode reports whether a tree entry is a file whose content is a blob. +// filemode.Deprecated (0100664) is a regular file too: go-git documents that it +// "should be treated as Regular", and a metadata.json carrying that mode would +// otherwise be invisible to both the scan and the rewrite. +func isFileMode(mode filemode.FileMode) bool { + return mode == filemode.Regular || mode == filemode.Executable || mode == filemode.Deprecated +} + +// contentReachable reports whether some commit reachable from tip carries +// exactly the tree of commit target. On the cumulative checkpoint branch equal +// trees mean equal checkpoint content, so this answers "does tip's history +// already contain everything target has?" independently of commit hashes — +// which differ across independent rewrites when commit signing is on. +func contentReachable(repo *git.Repository, tip, target plumbing.Hash) (bool, error) { + if tip.IsZero() || target.IsZero() { + return false, nil + } + tc, err := repo.CommitObject(target) + if err != nil { + return false, fmt.Errorf("load commit %s: %w", target, err) + } + iter, err := repo.Log(&git.LogOptions{From: tip}) + if err != nil { + return false, fmt.Errorf("log %s: %w", tip, err) + } + defer iter.Close() + found := false + walkErr := iter.ForEach(func(c *object.Commit) error { + if c.TreeHash.Equal(tc.TreeHash) { + found = true + return errStop + } + return nil + }) + if walkErr != nil && !errors.Is(walkErr, errStop) { + return false, fmt.Errorf("walk history of %s: %w", tip, walkErr) + } + return found, nil +} + +// PushFailedError reports that the local rewrite completed but the push of +// the rewritten branch did not. The local branch is left at the rewritten +// tip; re-running doctor pushes it once the remote is reachable. +type PushFailedError struct { + Remote string + Err error +} + +func (e *PushFailedError) Error() string { + return fmt.Sprintf("force-push %s to %s: %v", paths.MetadataBranchName, e.Remote, e.Err) +} + +func (e *PushFailedError) Unwrap() error { return e.Err } + +// ShrinkOversizedCheckpointMetadata rewrites entire/checkpoints/v1 so that +// every metadata.json blob over scan.Threshold loses its prompt_attributions +// field, then force-pushes the result to the elected sync remote. Commit +// messages, authors, dates and every other blob are preserved; commits whose +// tree and parents are unchanged keep their hash, so history before the first +// oversized blob is untouched. +// +// When the remote carries commits the local branch lacks (RemoteAhead), the +// remote history is rewritten first and the local-only commits are replayed +// onto it through SafelyAdvanceLocalRef — the same reconciliation pre-push +// uses — so nothing either side holds is lost. The push is +// --force-with-lease against the tip the scan observed, so a remote that moved +// in between is refused rather than overwritten; re-running doctor rescans. +// +// On a push failure the returned result still describes the completed local +// rewrite, so the caller can say what state the repository was left in. +func ShrinkOversizedCheckpointMetadata(ctx context.Context, repo *git.Repository, scan *MetadataSizeScan, w io.Writer) (*MetadataShrinkResult, error) { + v1 := plumbing.NewBranchReferenceName(paths.MetadataBranchName) + logCtx := logging.WithComponent(ctx, "checkpoint") + res := &MetadataShrinkResult{OldLocalTip: scan.LocalTip, NewLocalTip: scan.LocalTip} + if scan.DedicatedCheckpointRemote { + return res, ErrDedicatedCheckpointRemote + } + rw := newMetadataRewriter(ctx, repo, scan.Threshold) + + // Local first. When the remote is ahead, its history is rewritten with the + // same memoized rewriter, so every commit the two sides share maps to the + // same rewritten commit and the rewritten local tip is an ancestor of the + // rewritten remote tip (or shares its rewritten merge base). Reconciling + // the ORIGINAL local tip against a rewritten remote would find no common + // history past the first oversized blob and replay every commit since as + // a duplicate. + newTip, err := rw.rewriteHistory(scan.LocalTip) + if err != nil { + return res, fmt.Errorf("rewrite %s: %w", v1.Short(), err) + } + res.CommitsRewritten = rw.commitsRewritten + if !newTip.Equal(scan.LocalTip) { + if err := atomicSetV1Ref(ctx, repo, scan.LocalTip, newTip); err != nil { + return res, err + } + res.NewLocalTip = newTip + } + + if !scan.RemoteTip.IsZero() && scan.RemoteAhead { + before := rw.commitsRewritten + fixedRemote, err := rw.rewriteHistory(scan.RemoteTip) + if err != nil { + return res, fmt.Errorf("rewrite %s/%s: %w", scan.RemoteName, v1.Short(), err) + } + res.RemoteCommitsRewritten = rw.commitsRewritten - before + newTip, err = reconcileRewrittenTips(ctx, repo, w, scan.RemoteName, v1, newTip, fixedRemote) + if err != nil { + return res, err + } + res.NewLocalTip = newTip + } + res.BlobsShrunk = rw.blobsShrunk + res.StillOversized = rw.stillOversized + logging.Info(logCtx, "shrank oversized checkpoint metadata", + slog.String("old_tip", scan.LocalTip.String()), + slog.String("new_tip", newTip.String()), + slog.Int("commits_rewritten", res.CommitsRewritten), + slog.Int("remote_commits_rewritten", res.RemoteCommitsRewritten), + slog.Int("blobs_shrunk", rw.blobsShrunk)) + + switch { + case scan.RemoteName == "": + res.PushSkippedReason = "no checkpoint sync remote is configured" + case scan.RemoteTip.IsZero() && scan.RemoteErr != nil: + res.PushSkippedReason = fmt.Sprintf("the state of %s could not be determined (%v); re-run entire doctor once it is reachable", scan.RemoteName, scan.RemoteErr) + case scan.RemoteTip.IsZero(): + res.PushSkippedReason = fmt.Sprintf("%s has no %s branch yet; the next git push creates it", scan.RemoteName, v1.Short()) + case scan.RemoteTip.Equal(newTip): + res.PushSkippedReason = scan.RemoteName + " already has this history" + case scan.PushDisabled: + res.PushSkippedReason = "checkpoint pushing is disabled in settings; push the branch yourself with --force-with-lease" + default: + if err := forcePushCheckpointBranch(ctx, scan.RemoteName, v1, scan.RemoteTip, newTip); err != nil { + return res, &PushFailedError{Remote: scan.RemoteName, Err: err} + } + res.Pushed = true + } + return res, nil +} + +// reconcileRewrittenTips brings the local branch (at rewritten tip local) and +// the rewritten remote history (fixedRemote) together and returns the new +// local tip. Three cases, decided on content rather than hashes because +// independent rewrites of the same history differ in hash when commit signing +// is on: +// +// - the remote already holds everything local has (local's tree appears in +// the remote's history): adopt the remote tip, no replay; +// - local already holds everything the remote has: keep local, which the +// caller then pushes; +// - genuine divergence: replay the local-only commits onto the remote via +// SafelyAdvanceLocalRef, the same reconciliation pre-push uses. +func reconcileRewrittenTips(ctx context.Context, repo *git.Repository, w io.Writer, remoteName string, v1 plumbing.ReferenceName, local, fixedRemote plumbing.Hash) (plumbing.Hash, error) { + if fixedRemote.Equal(local) { + return local, nil + } + if local.IsZero() { + if err := SafelyAdvanceLocalRef(ctx, repo, v1, fixedRemote); err != nil { + return plumbing.ZeroHash, fmt.Errorf("adopt %s/%s: %w", remoteName, v1.Short(), err) + } + return fixedRemote, nil + } + remoteHasLocal, err := contentReachable(repo, fixedRemote, local) + if err != nil { + return plumbing.ZeroHash, err + } + if remoteHasLocal { + fmt.Fprintf(w, " Remote %s already holds every local checkpoint; adopting its history.\n", remoteName) + if err := atomicSetV1Ref(ctx, repo, local, fixedRemote); err != nil { + return plumbing.ZeroHash, err + } + return fixedRemote, nil + } + localHasRemote, err := contentReachable(repo, local, fixedRemote) + if err != nil { + return plumbing.ZeroHash, err + } + if localHasRemote { + return local, nil + } + fmt.Fprintf(w, " Remote %s has checkpoints not yet local; replaying local checkpoints onto it.\n", remoteName) + if err := SafelyAdvanceLocalRef(ctx, repo, v1, fixedRemote); err != nil { + return plumbing.ZeroHash, fmt.Errorf("reconcile local %s with %s: %w", v1.Short(), remoteName, err) + } + tip, err := readV1Tip(repo, v1) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("re-read local %s: %w", v1.Short(), err) + } + return tip, nil +} + +// forcePushCheckpointBranch replaces the remote's checkpoint branch with newTip, +// refusing (via --force-with-lease) if the remote no longer sits at expected. +func forcePushCheckpointBranch(ctx context.Context, remoteName string, branch plumbing.ReferenceName, expected, newTip plumbing.Hash) error { + pushCtx, cancel := context.WithTimeout(ctx, checkpointPushBudget) + defer cancel() + _, err := remote.PushWithOptions(pushCtx, remote.PushOptions{ + Remote: remoteName, + RefSpecs: []string{newTip.String() + ":" + branch.String()}, + ExtraArgs: []string{"--force-with-lease=" + branch.String() + ":" + expected.String()}, + }) + if err != nil { + return fmt.Errorf("force-push %s to %s: %w", branch.Short(), remoteName, err) + } + return nil +} + +// metadataRewriter rewrites a commit graph replacing oversized metadata.json +// blobs. Every level is memoized: a blob is stripped once however many trees +// reference it, a tree is rebuilt once however many commits share it, and a +// commit is rewritten once however many children it has. +type metadataRewriter struct { + ctx context.Context //nolint:containedctx // scoped to one rewrite; threaded into commit signing + repo *git.Repository + threshold int64 + + blobs map[plumbing.Hash]plumbing.Hash + trees map[plumbing.Hash]plumbing.Hash + commits map[plumbing.Hash]plumbing.Hash + shallow map[plumbing.Hash]bool + + commitsRewritten int + blobsShrunk int + stillOversized []OversizedMetadataBlob +} + +func newMetadataRewriter(ctx context.Context, repo *git.Repository, threshold int64) *metadataRewriter { + return &metadataRewriter{ + ctx: ctx, + repo: repo, + threshold: threshold, + blobs: make(map[plumbing.Hash]plumbing.Hash), + trees: make(map[plumbing.Hash]plumbing.Hash), + commits: make(map[plumbing.Hash]plumbing.Hash), + } +} + +// rewriteHistory returns the rewritten equivalent of tip. Parents are processed +// before children with an explicit stack, so a branch with thousands of +// checkpoints does not recurse thousands deep. +func (r *metadataRewriter) rewriteHistory(tip plumbing.Hash) (plumbing.Hash, error) { + if tip.IsZero() { + return tip, nil + } + type frame struct { + hash plumbing.Hash + expanded bool + } + stack := []frame{{hash: tip}} + for len(stack) > 0 { + if err := r.ctx.Err(); err != nil { + return plumbing.ZeroHash, err //nolint:wrapcheck // context cancellation propagates as-is + } + top := len(stack) - 1 + f := stack[top] + if _, done := r.commits[f.hash]; done { + stack = stack[:top] + continue + } + c, err := r.repo.CommitObject(f.hash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("load commit %s: %w", f.hash, err) + } + if r.shallow[f.hash] { + oversizedPath, err := oversizedMetadataPathInTree(r.repo, c.TreeHash, r.threshold) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("inspect shallow boundary %s: %w", f.hash, err) + } + if oversizedPath != "" { + return plumbing.ZeroHash, &shallowMetadataRepairError{Boundary: f.hash, Path: oversizedPath} + } + r.commits[f.hash] = f.hash + stack = stack[:top] + continue + } + if !f.expanded { + stack[top].expanded = true + for _, p := range c.ParentHashes { + if _, done := r.commits[p]; !done { + stack = append(stack, frame{hash: p}) + } + } + continue + } + newHash, err := r.rewriteCommit(c) + if err != nil { + return plumbing.ZeroHash, err + } + r.commits[f.hash] = newHash + stack = stack[:top] + } + return r.commits[tip], nil +} + +func (r *metadataRewriter) rewriteCommit(c *object.Commit) (plumbing.Hash, error) { + newTree, err := r.rewriteTree(c.TreeHash, "") + if err != nil { + return plumbing.ZeroHash, err + } + parents := make([]plumbing.Hash, 0, len(c.ParentHashes)) + changed := !newTree.Equal(c.TreeHash) + for _, p := range c.ParentHashes { + np, ok := r.commits[p] + if !ok { + return plumbing.ZeroHash, fmt.Errorf("parent %s of %s was not rewritten first", p, c.Hash) + } + if !np.Equal(p) { + changed = true + } + parents = append(parents, np) + } + if !changed { + return c.Hash, nil + } + rewritten := &object.Commit{ + Author: c.Author, + Committer: c.Committer, + Message: c.Message, + TreeHash: newTree, + ParentHashes: parents, + Encoding: c.Encoding, + ExtraHeaders: c.ExtraHeaders, + } + // Re-sign when signing is configured: the original signature covered the + // original tree, so it cannot be carried over. Signing makes the rewritten + // hash non-deterministic across runs, which is why reconciliation below + // compares trees (contentReachable) rather than hashes. + checkpoint.SignCommitBestEffort(r.ctx, rewritten) + obj := r.repo.Storer.NewEncodedObject() + if err := rewritten.Encode(obj); err != nil { + return plumbing.ZeroHash, fmt.Errorf("encode rewritten commit %s: %w", c.Hash, err) + } + hash, err := r.repo.Storer.SetEncodedObject(obj) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("store rewritten commit %s: %w", c.Hash, err) + } + r.commitsRewritten++ + return hash, nil +} + +func (r *metadataRewriter) rewriteTree(hash plumbing.Hash, prefix string) (plumbing.Hash, error) { + if done, ok := r.trees[hash]; ok { + return done, nil + } + tree, err := r.repo.TreeObject(hash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("load tree %s: %w", hash, err) + } + entries := make([]object.TreeEntry, 0, len(tree.Entries)) + changed := false + for _, e := range tree.Entries { + entryPath := e.Name + if prefix != "" { + entryPath = prefix + "/" + e.Name + } + newEntry := e + switch e.Mode { + case filemode.Dir: + sub, subErr := r.rewriteTree(e.Hash, entryPath) + if subErr != nil { + return plumbing.ZeroHash, subErr + } + newEntry.Hash = sub + case filemode.Regular, filemode.Executable, filemode.Deprecated: + if e.Name == paths.MetadataFileName { + blob, blobErr := r.rewriteBlob(e.Hash, entryPath) + if blobErr != nil { + return plumbing.ZeroHash, blobErr + } + newEntry.Hash = blob + } + case filemode.Empty, filemode.Symlink, filemode.Submodule: + // kept verbatim + } + if !newEntry.Hash.Equal(e.Hash) { + changed = true + } + entries = append(entries, newEntry) + } + if !changed { + r.trees[hash] = hash + return hash, nil + } + newTree := &object.Tree{Entries: entries} + obj := r.repo.Storer.NewEncodedObject() + if err := newTree.Encode(obj); err != nil { + return plumbing.ZeroHash, fmt.Errorf("encode tree %s: %w", prefix, err) + } + newHash, err := r.repo.Storer.SetEncodedObject(obj) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("store tree %s: %w", prefix, err) + } + r.trees[hash] = newHash + return newHash, nil +} + +// rewriteBlob returns the replacement for a metadata.json blob: itself when it +// is under the threshold or carries no prompt_attributions, else a copy without +// that field. +func (r *metadataRewriter) rewriteBlob(hash plumbing.Hash, blobPath string) (plumbing.Hash, error) { + if done, ok := r.blobs[hash]; ok { + return done, nil + } + blob, err := r.repo.BlobObject(hash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("load blob %s at %s: %w", hash, blobPath, err) + } + if blob.Size <= r.threshold { + r.blobs[hash] = hash + return hash, nil + } + data, err := readBlob(r.repo, hash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("read blob %s at %s: %w", hash, blobPath, err) + } + stripped, changed, err := stripPromptAttributions(data) + if err != nil || !changed { + // Either not the JSON this repair understands, or large for a reason + // other than the field it knows how to remove: leave it and say so + // rather than guess at what to delete. + r.stillOversized = append(r.stillOversized, OversizedMetadataBlob{Path: blobPath, Size: blob.Size, Hash: hash}) + r.blobs[hash] = hash + return hash, nil //nolint:nilerr // deliberate: an unparseable blob is kept verbatim and reported via stillOversized + } + newHash, err := checkpoint.CreateBlobFromContent(r.repo, stripped) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("write shrunk blob for %s: %w", blobPath, err) + } + r.blobsShrunk++ + if int64(len(stripped)) > r.threshold { + r.stillOversized = append(r.stillOversized, OversizedMetadataBlob{Path: blobPath, Size: int64(len(stripped)), Hash: newHash}) + } + r.blobs[hash] = newHash + return newHash, nil +} + +// stripPromptAttributions removes the prompt_attributions key from a +// metadata.json document, preserving every other field verbatim (as raw JSON), +// and reports whether anything changed. Keys come back sorted, which is the one +// cosmetic difference from the writer's field order. +func stripPromptAttributions(data []byte) ([]byte, bool, error) { + var doc map[string]json.RawMessage + if err := json.Unmarshal(data, &doc); err != nil { + return nil, false, fmt.Errorf("parse metadata.json: %w", err) + } + if _, ok := doc[promptAttributionsField]; !ok { + return data, false, nil + } + delete(doc, promptAttributionsField) + out, err := jsonutil.MarshalIndentWithNewline(doc, "", " ") + if err != nil { + return nil, false, fmt.Errorf("re-encode metadata.json: %w", err) + } + return out, true, nil +} diff --git a/cmd/entire/cli/strategy/metadata_shrink_test.go b/cmd/entire/cli/strategy/metadata_shrink_test.go new file mode 100644 index 0000000000..7446209cf6 --- /dev/null +++ b/cmd/entire/cli/strategy/metadata_shrink_test.go @@ -0,0 +1,429 @@ +package strategy + +import ( + "context" + "encoding/json" + "fmt" + "io" + "maps" + "slices" + "strings" + "testing" + + git "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/testutil" +) + +// shrinkTestThreshold keeps fixtures small: a "bloated" metadata.json in these +// tests is a few kilobytes, not 100 MiB. +const shrinkTestThreshold int64 = 1024 + +const ( + bloatedMetadataPath = "cc/dddddddddd/0/" + paths.MetadataFileName + bloatedTranscript = "cc/dddddddddd/0/" + paths.TranscriptFileName +) + +// sessionMetadataJSON renders a per-session metadata.json. perFile > 0 adds a +// prompt_attributions record whose user_added_per_file map has that many +// entries — the shape the pre-v0.10.1 nested-checkout bug produced. +func sessionMetadataJSON(t *testing.T, sessionID string, perFile int) []byte { + t.Helper() + doc := map[string]any{ + "checkpoint_id": "ccdddddddddd", + "session_id": sessionID, + "agent": "claude-code", + "created_at": "2026-08-01T12:00:00Z", + "attribution": map[string]any{"agent_lines": 10, "human_added": 2}, + } + if perFile > 0 { + added := make(map[string]int, perFile) + for i := range perFile { + added[fmt.Sprintf(".claude/worktrees/agent/src/file%d.go", i)] = 3 + } + doc["prompt_attributions"] = []map[string]any{{ + "checkpoint_number": 1, + "user_lines_added": 3 * perFile, + "user_added_per_file": added, + "user_removed_per_file": map[string]int{}, + }} + } + data, err := json.MarshalIndent(doc, "", " ") + require.NoError(t, err) + return append(data, '\n') +} + +// bloatedV1Fixture builds entire/checkpoints/v1 as three cumulative commits: +// c1 holds one healthy checkpoint, c2 adds a checkpoint whose metadata.json is +// over shrinkTestThreshold, c3 adds another healthy checkpoint (and, being +// cumulative, still references the oversized blob). Returns the three hashes. +func bloatedV1Fixture(t *testing.T) (dir string, repo *git.Repository, c1, c2, c3 plumbing.Hash) { + t.Helper() + dir, r, _ := setupV1RepoInDir(t) + files := map[string][]byte{ + paths.MetadataFileName: []byte(`{"checkpoints":1}` + "\n"), + "aa/bbbbbbbbbb/0/" + paths.MetadataFileName: sessionMetadataJSON(t, "s1", 0), + "aa/bbbbbbbbbb/0/" + paths.TranscriptFileName: []byte(`{"role":"user"}` + "\n"), + } + c1 = testutil.CommitFiles(t, r, nil, files, "Checkpoint: aabbbbbbbbbb") + files[bloatedMetadataPath] = sessionMetadataJSON(t, "s2", 200) + files[bloatedTranscript] = []byte(`{"role":"user","content":"big session"}` + "\n") + c2 = testutil.CommitFiles(t, r, []plumbing.Hash{c1}, files, "Checkpoint: ccdddddddddd") + files["ee/ffffffffff/0/"+paths.MetadataFileName] = sessionMetadataJSON(t, "s3", 0) + files["ee/ffffffffff/0/"+paths.TranscriptFileName] = []byte(`{"role":"user"}` + "\n") + c3 = testutil.CommitFiles(t, r, []plumbing.Hash{c2}, files, "Checkpoint: eeffffffffff") + require.NoError(t, r.Storer.SetReference(plumbing.NewHashReference( + plumbing.NewBranchReferenceName(paths.MetadataBranchName), c3))) + require.Greater(t, int64(len(files[bloatedMetadataPath])), shrinkTestThreshold, "fixture must exceed the test threshold") + return dir, r, c1, c2, c3 +} + +func blobAt(t *testing.T, repo *git.Repository, commit plumbing.Hash, filePath string) (plumbing.Hash, []byte) { + t.Helper() + c, err := repo.CommitObject(commit) + require.NoError(t, err) + f, err := c.File(filePath) + require.NoError(t, err, filePath) + content, err := f.Contents() + require.NoError(t, err) + return f.Hash, []byte(content) +} + +func TestFindOversizedMetadataBlobs_ReportsBlobsOverThreshold(t *testing.T) { + t.Parallel() + _, repo, _, c2, c3 := bloatedV1Fixture(t) + + found, err := findOversizedMetadataBlobs(context.Background(), repo, c3, shrinkTestThreshold) + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, bloatedMetadataPath, found[0].Path) + assert.Equal(t, c2, found[0].Commit, "attributed to the commit that introduced it") + assert.Greater(t, found[0].Size, shrinkTestThreshold) + + none, err := findOversizedMetadataBlobs(context.Background(), repo, plumbing.ZeroHash, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, none, "an absent branch has nothing oversized") +} + +func TestShrinkOversizedCheckpointMetadata_RewritesOnlyAffectedHistory(t *testing.T) { + t.Parallel() + _, repo, c1, _, c3 := bloatedV1Fixture(t) + ctx := context.Background() + v1 := plumbing.NewBranchReferenceName(paths.MetadataBranchName) + transcriptBefore, _ := blobAt(t, repo, c3, bloatedTranscript) + healthyBefore, _ := blobAt(t, repo, c3, "aa/bbbbbbbbbb/0/"+paths.MetadataFileName) + + scan, err := ScanOversizedCheckpointMetadata(ctx, repo, "", shrinkTestThreshold) + require.NoError(t, err) + require.Len(t, scan.Local, 1) + assert.Empty(t, scan.Remote) + + res, err := ShrinkOversizedCheckpointMetadata(ctx, repo, scan, io.Discard) + require.NoError(t, err) + assert.Equal(t, c3, res.OldLocalTip) + assert.NotEqual(t, c3, res.NewLocalTip) + assert.Equal(t, 2, res.CommitsRewritten, "c2 introduced the blob and c3 still referenced it; c1 is untouched") + assert.Equal(t, 1, res.BlobsShrunk) + assert.Empty(t, res.StillOversized) + assert.False(t, res.Pushed) + assert.Contains(t, res.PushSkippedReason, "no checkpoint sync remote") + + ref, err := repo.Reference(v1, true) + require.NoError(t, err) + assert.Equal(t, res.NewLocalTip, ref.Hash(), "local branch moved to the rewritten tip") + + newC3, err := repo.CommitObject(res.NewLocalTip) + require.NoError(t, err) + assert.Equal(t, "Checkpoint: eeffffffffff", newC3.Message, "messages are preserved") + newC2, err := newC3.Parent(0) + require.NoError(t, err) + assert.Equal(t, "Checkpoint: ccdddddddddd", newC2.Message) + assert.Equal(t, []plumbing.Hash{c1}, newC2.ParentHashes, "history before the first oversized blob keeps its hashes") + + _, shrunk := blobAt(t, repo, res.NewLocalTip, bloatedMetadataPath) + var doc map[string]json.RawMessage + require.NoError(t, json.Unmarshal(shrunk, &doc)) + assert.NotContains(t, doc, "prompt_attributions") + assert.Contains(t, doc, "attribution", "the computed summary stays") + assert.Contains(t, doc, "session_id") + assert.LessOrEqual(t, int64(len(shrunk)), shrinkTestThreshold) + + transcriptAfter, _ := blobAt(t, repo, res.NewLocalTip, bloatedTranscript) + assert.Equal(t, transcriptBefore, transcriptAfter, "the transcript blob is byte-identical") + healthyAfter, _ := blobAt(t, repo, res.NewLocalTip, "aa/bbbbbbbbbb/0/"+paths.MetadataFileName) + assert.Equal(t, healthyBefore, healthyAfter, "healthy metadata is untouched") + + // Idempotent: a second scan is clean and a second shrink is a no-op. + again, err := ScanOversizedCheckpointMetadata(ctx, repo, "", shrinkTestThreshold) + require.NoError(t, err) + assert.True(t, again.Empty()) + res2, err := ShrinkOversizedCheckpointMetadata(ctx, repo, again, io.Discard) + require.NoError(t, err) + assert.Equal(t, 0, res2.CommitsRewritten) + assert.Equal(t, res.NewLocalTip, res2.NewLocalTip) +} + +func TestShrinkOversizedCheckpointMetadata_ForcePushesElectedRemoteAndRecoversRemoteOnlyBloat(t *testing.T) { + // Not parallel: remote.Fetch/Push and settings.Load resolve the repository + // from the process working directory. + dir, repo, _, _, c3 := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + v1 := "refs/heads/" + paths.MetadataBranchName + + bare := t.TempDir() + testutil.RunGit(t, dir, "init", "--bare", bare) + testutil.RunGit(t, dir, "remote", "add", "origin", bare) + testutil.RunGit(t, dir, "push", "origin", v1) + require.Equal(t, c3.String(), strings.TrimSpace(testutil.RunGit(t, bare, "rev-parse", v1))) + + scan, err := ScanOversizedCheckpointMetadata(ctx, repo, "origin", shrinkTestThreshold) + require.NoError(t, err) + require.NoError(t, scan.RemoteErr) + assert.Equal(t, c3, scan.RemoteTip) + assert.False(t, scan.RemoteAhead) + require.Len(t, scan.Local, 1) + assert.Empty(t, scan.Remote, "the remote holds the same blob; it is not reported twice") + + res, err := ShrinkOversizedCheckpointMetadata(ctx, repo, scan, io.Discard) + require.NoError(t, err) + assert.True(t, res.Pushed) + assert.Equal(t, res.NewLocalTip.String(), strings.TrimSpace(testutil.RunGit(t, bare, "rev-parse", v1)), + "the remote branch now points at the rewritten history") + fixed := res.NewLocalTip + + // Simulate an earlier repair whose push never landed: the remote and the + // tracking ref still sit on the bloated history while local is clean. + testutil.RunGit(t, bare, "update-ref", v1, c3.String()) + testutil.RunGit(t, dir, "update-ref", "refs/remotes/origin/"+paths.MetadataBranchName, c3.String()) + + scan2, err := ScanOversizedCheckpointMetadata(ctx, repo, "origin", shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, scan2.Local, "local is already clean") + require.Len(t, scan2.Remote, 1, "the bloat that only the remote still carries is reported") + assert.True(t, scan2.RemoteAhead, "the remote history is not an ancestor of the rewritten local one") + + res2, err := ShrinkOversizedCheckpointMetadata(ctx, repo, scan2, io.Discard) + require.NoError(t, err) + assert.Equal(t, fixed, res2.NewLocalTip, "rewriting the remote history reproduces the same fixed commits, so nothing is replayed twice") + assert.True(t, res2.Pushed) + assert.Equal(t, fixed.String(), strings.TrimSpace(testutil.RunGit(t, bare, "rev-parse", v1))) +} + +func TestStripPromptAttributions(t *testing.T) { + t.Parallel() + withField := []byte(`{"session_id":"s","prompt_attributions":[{"x":1}],"attribution":{"agent_lines":1}}`) + out, changed, err := stripPromptAttributions(withField) + require.NoError(t, err) + assert.True(t, changed) + var doc map[string]json.RawMessage + require.NoError(t, json.Unmarshal(out, &doc)) + assert.Equal(t, []string{"attribution", "session_id"}, slices.Sorted(maps.Keys(doc))) + assert.JSONEq(t, `{"agent_lines":1}`, string(doc["attribution"])) + + without := []byte(`{"session_id":"s"}`) + same, changed, err := stripPromptAttributions(without) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, without, same) + + _, _, err = stripPromptAttributions([]byte("not json")) + require.Error(t, err) +} + +// commitMessages returns the messages reachable from tip, newest first. +func commitMessages(t *testing.T, repo *git.Repository, tip plumbing.Hash) []string { + t.Helper() + iter, err := repo.Log(&git.LogOptions{From: tip}) + require.NoError(t, err) + defer iter.Close() + var msgs []string + require.NoError(t, iter.ForEach(func(c *object.Commit) error { + msgs = append(msgs, c.Message) + return nil + })) + return msgs +} + +// pushV1ToBare wires dir to a fresh bare remote named origin holding tip on +// the checkpoint branch, and returns the bare path. +func pushV1ToBare(t *testing.T, dir string, tip plumbing.Hash) string { + t.Helper() + bare := t.TempDir() + testutil.RunGit(t, dir, "init", "--bare", bare) + testutil.RunGit(t, dir, "remote", "add", "origin", bare) + testutil.RunGit(t, dir, "push", "origin", tip.String()+":refs/heads/"+paths.MetadataBranchName) + return bare +} + +func bareV1(t *testing.T, bare string) string { + t.Helper() + return strings.TrimSpace(testutil.RunGit(t, bare, "rev-parse", "refs/heads/"+paths.MetadataBranchName)) +} + +// The remote is ahead by a checkpoint someone else pushed, and both sides +// share the oversized blob. The reconciled history must contain each +// checkpoint exactly once: reconciling the un-rewritten local branch against +// the rewritten remote would replay every commit since the bloat as a +// duplicate. +func TestShrinkOversizedCheckpointMetadata_RemoteAheadByNewCheckpoint_NoDuplicates(t *testing.T) { + // Not parallel: remote.Fetch/Push and settings.Load use the working directory. + dir, repo, c1, _, c3 := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + + // Someone else's checkpoint on top of c3, present only on the remote. + files := map[string][]byte{ + paths.MetadataFileName: []byte(`{"checkpoints":1}` + "\n"), + "aa/bbbbbbbbbb/0/" + paths.MetadataFileName: sessionMetadataJSON(t, "s1", 0), + "aa/bbbbbbbbbb/0/" + paths.TranscriptFileName: []byte(`{"role":"user"}` + "\n"), + bloatedMetadataPath: sessionMetadataJSON(t, "s2", 200), + bloatedTranscript: []byte(`{"role":"user","content":"big session"}` + "\n"), + "ee/ffffffffff/0/" + paths.MetadataFileName: sessionMetadataJSON(t, "s3", 0), + "ee/ffffffffff/0/" + paths.TranscriptFileName: []byte(`{"role":"user"}` + "\n"), + "11/2222222222/0/" + paths.MetadataFileName: sessionMetadataJSON(t, "s4", 0), + "11/2222222222/0/" + paths.TranscriptFileName: []byte(`{"role":"user"}` + "\n"), + } + c4 := testutil.CommitFiles(t, repo, []plumbing.Hash{c3}, files, "Checkpoint: 112222222222") + bare := pushV1ToBare(t, dir, c4) + require.Equal(t, c3, readTip(t, repo), "local stays at c3") + + scan, err := ScanOversizedCheckpointMetadata(ctx, repo, "origin", shrinkTestThreshold) + require.NoError(t, err) + assert.Equal(t, c4, scan.RemoteTip) + assert.True(t, scan.RemoteAhead) + require.Len(t, scan.Local, 1) + assert.Empty(t, scan.Remote, "the shared blob is reported once, on the local side") + + res, err := ShrinkOversizedCheckpointMetadata(ctx, repo, scan, io.Discard) + require.NoError(t, err) + assert.Equal(t, 2, res.CommitsRewritten, "c2 and c3 locally") + assert.Equal(t, 1, res.RemoteCommitsRewritten, "only c4 is new on the remote side; c2 and c3 were already rewritten") + assert.True(t, res.Pushed) + + assert.Equal(t, []string{ + "Checkpoint: 112222222222", + "Checkpoint: eeffffffffff", + "Checkpoint: ccdddddddddd", + "Checkpoint: aabbbbbbbbbb", + }, commitMessages(t, repo, res.NewLocalTip), "every checkpoint exactly once, in order") + assert.Equal(t, c1, commitAt(t, repo, res.NewLocalTip, 3).Hash, "the pre-bloat commit keeps its hash") + assert.Equal(t, res.NewLocalTip.String(), bareV1(t, bare)) + + none, err := findOversizedMetadataBlobs(ctx, repo, res.NewLocalTip, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, none) +} + +// A second clone repaired the same history independently. With commit signing +// on, its rewritten commits carry different hashes but the same trees; the +// repair must recognise the content as already present and adopt it rather +// than replaying no-op duplicates. +func TestShrinkOversizedCheckpointMetadata_IndependentRewriteOfSameContentIsAdopted(t *testing.T) { + // Not parallel: remote.Fetch/Push and settings.Load use the working directory. + dir, repo, c1, _, c3 := bloatedV1Fixture(t) + t.Chdir(dir) + ctx := context.Background() + bare := pushV1ToBare(t, dir, c3) + + // What this run's rewrite of the remote will produce, to borrow its trees. + fixedTip, err := newMetadataRewriter(ctx, repo, shrinkTestThreshold).rewriteHistory(c3) + require.NoError(t, err) + fixedC3 := readCommit(t, repo, fixedTip) + fixedC2 := readCommit(t, repo, fixedC3.ParentHashes[0]) + + // The "other clone's" repair: same trees, different committer identity, so + // different hashes — the shape commit signing produces. + x2 := makeOrphanCommit(t, repo, fixedC2.TreeHash, []plumbing.Hash{c1}, "Checkpoint: ccdddddddddd") + x3 := makeOrphanCommit(t, repo, fixedC3.TreeHash, []plumbing.Hash{x2}, "Checkpoint: eeffffffffff") + require.NotEqual(t, fixedTip, x3) + require.NoError(t, repo.Storer.SetReference(plumbing.NewHashReference( + plumbing.NewBranchReferenceName(paths.MetadataBranchName), x3))) + + scan, err := ScanOversizedCheckpointMetadata(ctx, repo, "origin", shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, scan.Local, "the local rewrite is already clean") + require.Len(t, scan.Remote, 1) + assert.True(t, scan.RemoteAhead, "hashes differ, so by hash the remote looks ahead") + + res, err := ShrinkOversizedCheckpointMetadata(ctx, repo, scan, io.Discard) + require.NoError(t, err) + assert.Equal(t, 0, res.CommitsRewritten) + assert.Equal(t, 2, res.RemoteCommitsRewritten) + assert.Equal(t, fixedTip, res.NewLocalTip, "the remote's rewrite is adopted wholesale") + assert.Len(t, commitMessages(t, repo, res.NewLocalTip), 3, "no duplicates") + assert.True(t, res.Pushed) + assert.Equal(t, fixedTip.String(), bareV1(t, bare)) +} + +// go-git's filemode.Deprecated (0100664) is a regular file; a metadata.json +// carrying it must be found and rewritten like any other. +func TestOversizedMetadata_DeprecatedFileModeIsHandled(t *testing.T) { + t.Parallel() + _, repo, _ := setupV1RepoInDir(t) + ctx := context.Background() + + big, err := checkpoint.CreateBlobFromContent(repo, sessionMetadataJSON(t, "s9", 200)) + require.NoError(t, err) + leaf := encodeTree(t, repo, []object.TreeEntry{{Name: paths.MetadataFileName, Mode: filemode.Deprecated, Hash: big}}) + mid := encodeTree(t, repo, []object.TreeEntry{{Name: "0", Mode: filemode.Dir, Hash: leaf}}) + shard := encodeTree(t, repo, []object.TreeEntry{{Name: "9999999999", Mode: filemode.Dir, Hash: mid}}) + root := encodeTree(t, repo, []object.TreeEntry{{Name: "99", Mode: filemode.Dir, Hash: shard}}) + tip := makeOrphanCommit(t, repo, root, nil, "Checkpoint: 999999999999") + require.NoError(t, repo.Storer.SetReference(plumbing.NewHashReference( + plumbing.NewBranchReferenceName(paths.MetadataBranchName), tip))) + + found, err := findOversizedMetadataBlobs(ctx, repo, tip, shrinkTestThreshold) + require.NoError(t, err) + require.Len(t, found, 1) + + scan, err := ScanOversizedCheckpointMetadata(ctx, repo, "", shrinkTestThreshold) + require.NoError(t, err) + res, err := ShrinkOversizedCheckpointMetadata(ctx, repo, scan, io.Discard) + require.NoError(t, err) + assert.Equal(t, 1, res.BlobsShrunk) + none, err := findOversizedMetadataBlobs(ctx, repo, res.NewLocalTip, shrinkTestThreshold) + require.NoError(t, err) + assert.Empty(t, none) +} + +func encodeTree(t *testing.T, repo *git.Repository, entries []object.TreeEntry) plumbing.Hash { + t.Helper() + obj := repo.Storer.NewEncodedObject() + require.NoError(t, (&object.Tree{Entries: entries}).Encode(obj)) + hash, err := repo.Storer.SetEncodedObject(obj) + require.NoError(t, err) + return hash +} + +func readTip(t *testing.T, repo *git.Repository) plumbing.Hash { + t.Helper() + ref, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true) + require.NoError(t, err) + return ref.Hash() +} + +func readCommit(t *testing.T, repo *git.Repository, h plumbing.Hash) *object.Commit { + t.Helper() + c, err := repo.CommitObject(h) + require.NoError(t, err) + return c +} + +// commitAt walks n first-parents back from tip. +func commitAt(t *testing.T, repo *git.Repository, tip plumbing.Hash, n int) *object.Commit { + t.Helper() + c := readCommit(t, repo, tip) + for range n { + require.NotEmpty(t, c.ParentHashes) + c = readCommit(t, repo, c.ParentHashes[0]) + } + return c +} diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 7d802525de..e1a6587459 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -14,6 +14,7 @@ import ( "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" "github.com/entireio/cli/cmd/entire/cli/logging" + "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/entireio/cli/cmd/entire/cli/settings" "github.com/entireio/cli/perf" @@ -101,6 +102,10 @@ func pushCheckpointRefWithRecovery(ctx context.Context, target string, ref plumb // handle the no-op case. // Does not check any settings — callers are responsible for gating. func pushRefIfNeeded(ctx context.Context, target string, ref plumbing.ReferenceName) (delivered bool, err error) { + return pushRefIfNeededWithMetadataThreshold(ctx, target, ref, OversizedCheckpointMetadataThreshold) +} + +func pushRefIfNeededWithMetadataThreshold(ctx context.Context, target string, ref plumbing.ReferenceName, metadataThreshold int64) (delivered bool, err error) { repo, err := OpenRepository(ctx) if err != nil { logging.Debug(ctx, "push skipped: open repository failed", @@ -124,7 +129,7 @@ func pushRefIfNeeded(ctx context.Context, target string, ref plumbing.ReferenceN return true, nil } - return doPushRef(ctx, target, ref) + return doPushRefWithMetadataThreshold(ctx, target, ref, metadataThreshold) } // hasUnpushedBranchRef checks if the local branch differs from the remote. @@ -166,6 +171,10 @@ var checkpointPushBudget = 2 * time.Minute // delivery, such as latching the captured checkpoint sync remote, must read // delivered and not err. func doPushRef(ctx context.Context, target string, ref plumbing.ReferenceName) (delivered bool, err error) { + return doPushRefWithMetadataThreshold(ctx, target, ref, OversizedCheckpointMetadataThreshold) +} + +func doPushRefWithMetadataThreshold(ctx context.Context, target string, ref plumbing.ReferenceName, metadataThreshold int64) (delivered bool, err error) { ctx, cancel := context.WithTimeout(ctx, checkpointPushBudget) defer cancel() @@ -207,7 +216,7 @@ func doPushRef(ctx context.Context, target string, ref plumbing.ReferenceName) ( stop = startProgressDots(os.Stderr) frCtx, fetchRebaseSpan := perf.Start(ctx, "fetch_and_rebase") - syncErr := fetchAndRebaseRefCommon(frCtx, target, ref) + syncErr := fetchAndRebaseRefWithMetadataThreshold(frCtx, target, ref, metadataThreshold) fetchRebaseSpan.RecordError(syncErr) fetchRebaseSpan.End() if syncErr != nil { @@ -468,6 +477,10 @@ func printProtectedRefBlock(w io.Writer, ref, target string) { // apply cleanly. // The target can be a remote name or a URL. func fetchAndRebaseRefCommon(ctx context.Context, target string, ref plumbing.ReferenceName) error { + return fetchAndRebaseRefWithMetadataThreshold(ctx, target, ref, OversizedCheckpointMetadataThreshold) +} + +func fetchAndRebaseRefWithMetadataThreshold(ctx context.Context, target string, ref plumbing.ReferenceName, metadataThreshold int64) error { // No timeout: runs under doPushRef's shared budget. fetchTarget, err := remote.ResolveFetchTarget(ctx, target) if err != nil { @@ -481,13 +494,23 @@ func fetchAndRebaseRefCommon(ctx context.Context, target string, ref plumbing.Re var refSpec string usedTempRef := remote.IsURL(fetchTarget) || !ref.IsBranch() if usedTempRef { - tmpRef := "refs/entire-fetch-tmp/" + strings.TrimPrefix(ref.String(), "refs/") - refSpec = fmt.Sprintf("+%s:%s", ref.String(), tmpRef) - fetchedRefName = plumbing.ReferenceName(tmpRef) + fetchedRefName, err = newFetchTmpRef("push-recovery") + if err != nil { + return err + } + refSpec = fmt.Sprintf("+%s:%s", ref.String(), fetchedRefName) } else { refSpec = fmt.Sprintf("+%s:refs/remotes/%s/%s", ref.String(), target, ref.Short()) fetchedRefName = plumbing.NewRemoteReferenceName(target, ref.Short()) } + repo, err := OpenRepository(ctx) + if err != nil { + return fmt.Errorf("failed to open git repository: %w", err) + } + defer repo.Close() + if usedTempRef { + defer func() { _ = repo.Storer.RemoveReference(fetchedRefName) }() //nolint:errcheck // cleanup is best-effort + } // Use git CLI for fetch (go-git's fetch can be tricky with auth). // Do NOT --unshallow here: on a shallow repo with deep history (e.g. a @@ -510,23 +533,9 @@ func fetchAndRebaseRefCommon(ctx context.Context, target string, ref plumbing.Re return fmt.Errorf("fetch failed: %s", fetchOutput) } - repo, err := OpenRepository(ctx) - if err != nil { - return fmt.Errorf("failed to open git repository: %w", err) - } - defer repo.Close() - - // Reconcile disconnected metadata branches before rebasing. - // The fetch above updated the remote-tracking ref, so reconciliation - // can compare fresh local vs remote. If disconnected (empty-orphan bug), - // this cherry-picks local commits onto remote tip, updating the local ref. - // If reconciliation fails, abort — proceeding to rebase on disconnected - // refs would silently combine unrelated histories. - if reconcileErr := ReconcileDisconnectedMetadataRef(ctx, repo, ref, fetchedRefName, os.Stderr); reconcileErr != nil { - return fmt.Errorf("metadata reconciliation failed: %w", reconcileErr) - } - - // Get local ref (re-read after potential reconciliation update) + // Read the exact local and fetched tips used by metadata cleanup. Cleanup + // must run before generic disconnected-history recovery because independently + // repaired histories can be hash-disconnected while sharing checkpoint trees. localRef, err := repo.Reference(ref, true) if err != nil { return fmt.Errorf("failed to get local ref: %w", err) @@ -537,15 +546,32 @@ func fetchAndRebaseRefCommon(ctx context.Context, target string, ref plumbing.Re if err != nil { return fmt.Errorf("failed to get remote ref: %w", err) } + if ref == plumbing.NewBranchReferenceName(paths.MetadataBranchName) { + _, handled, reconcileErr := reconcileOversizedV1ForPush( + ctx, repo, localRef.Hash(), remoteRef.Hash(), metadataThreshold, + ) + if reconcileErr != nil { + return fmt.Errorf("reconcile oversized checkpoint metadata: %w", reconcileErr) + } + if handled { + return nil + } + } + // If the content-aware repair did not apply, retain the general recovery for + // legacy disconnected-orphan histories and non-v1 checkpoint refs. + if reconcileErr := ReconcileDisconnectedMetadataRef(ctx, repo, ref, fetchedRefName, os.Stderr); reconcileErr != nil { + return fmt.Errorf("metadata reconciliation failed: %w", reconcileErr) + } + localRef, err = repo.Reference(ref, true) + if err != nil { + return fmt.Errorf("failed to get local ref after metadata reconciliation: %w", err) + } advance := func(hash plumbing.Hash) error { - if err := setRefHash(repo, ref, hash); err != nil { - return err + if ref == plumbing.NewBranchReferenceName(paths.MetadataBranchName) { + return atomicSetV1Ref(ctx, repo, localRef.Hash(), hash) } - if usedTempRef { - _ = repo.Storer.RemoveReference(fetchedRefName) //nolint:errcheck // cleanup is best-effort - } - return nil + return setRefHash(repo, ref, hash) } // If local is already at or behind remote, fast-forward diff --git a/cmd/entire/cli/strategy/push_common_test.go b/cmd/entire/cli/strategy/push_common_test.go index 9b9a8ebb86..6422794af4 100644 --- a/cmd/entire/cli/strategy/push_common_test.go +++ b/cmd/entire/cli/strategy/push_common_test.go @@ -1038,8 +1038,7 @@ func TestFetchAndRebase_URLTarget_ReconcilesFetchedTempRef(t *testing.T) { assert.Contains(t, entries, "aa/aaaaaaaaaa/metadata.json", "remote checkpoint should be preserved") assert.Contains(t, entries, "cc/cccccccccc/metadata.json", "local checkpoint should be preserved") - _, err = repo.Reference(plumbing.ReferenceName("refs/entire-fetch-tmp/"+branchName), true) - assert.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "temporary fetched ref should be cleaned up") + assertNoFetchTmpRefsWithPurpose(t, repo, "push-recovery") } // TestFetchAndRebase_FlaggedOriginTarget_UsesTempRef verifies that enabling @@ -1139,8 +1138,7 @@ func TestFetchAndRebase_FlaggedOriginTarget_UsesTempRef(t *testing.T) { assert.Contains(t, entries, "aa/aaaaaaaaaa/metadata.json", "remote checkpoint should be preserved") assert.Contains(t, entries, "cc/cccccccccc/metadata.json", "local checkpoint should be preserved") - _, err = repo.Reference(plumbing.ReferenceName("refs/entire-fetch-tmp/"+branchName), true) - assert.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "temporary fetched ref should be cleaned up") + assertNoFetchTmpRefsWithPurpose(t, repo, "push-recovery") } // TestIsCheckpointRemoteCommitted verifies that the discoverability check reads diff --git a/cmd/entire/cli/testutil/objects.go b/cmd/entire/cli/testutil/objects.go new file mode 100644 index 0000000000..95e95ac9e1 --- /dev/null +++ b/cmd/entire/cli/testutil/objects.go @@ -0,0 +1,89 @@ +package testutil + +import ( + "sort" + "strings" + "testing" + "time" + + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/stretchr/testify/require" +) + +// CommitFiles writes a commit whose tree is exactly files (slash-separated +// path → content), with the given parents, directly into repo's object store. +// No worktree or index is involved, so it can build history on a ref that is +// never checked out — the checkpoint branch, for instance — without disturbing +// the working tree. Every commit carries its full file set: pass the previous +// commit's files plus the additions to extend a cumulative branch. +func CommitFiles(t *testing.T, repo *git.Repository, parents []plumbing.Hash, files map[string][]byte, message string) plumbing.Hash { + t.Helper() + treeHash := writeTree(t, repo, files) + sig := object.Signature{Name: "test", Email: "test@test.com", When: time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)} + commit := &object.Commit{ + Author: sig, + Committer: sig, + Message: message, + TreeHash: treeHash, + ParentHashes: parents, + } + obj := repo.Storer.NewEncodedObject() + require.NoError(t, commit.Encode(obj)) + hash, err := repo.Storer.SetEncodedObject(obj) + require.NoError(t, err) + return hash +} + +// writeTree encodes files as nested tree objects and returns the root hash. +func writeTree(t *testing.T, repo *git.Repository, files map[string][]byte) plumbing.Hash { + t.Helper() + blobs := make(map[string]plumbing.Hash, len(files)) + subdirs := make(map[string]map[string][]byte) + for p, content := range files { + dir, rest, nested := strings.Cut(p, "/") + if !nested { + obj := repo.Storer.NewEncodedObject() + obj.SetType(plumbing.BlobObject) + w, err := obj.Writer() + require.NoError(t, err) + _, err = w.Write(content) + require.NoError(t, err) + require.NoError(t, w.Close()) + hash, err := repo.Storer.SetEncodedObject(obj) + require.NoError(t, err) + blobs[p] = hash + continue + } + if subdirs[dir] == nil { + subdirs[dir] = make(map[string][]byte) + } + subdirs[dir][rest] = content + } + entries := make([]object.TreeEntry, 0, len(blobs)+len(subdirs)) + for name, hash := range blobs { + entries = append(entries, object.TreeEntry{Name: name, Mode: filemode.Regular, Hash: hash}) + } + for name, sub := range subdirs { + entries = append(entries, object.TreeEntry{Name: name, Mode: filemode.Dir, Hash: writeTree(t, repo, sub)}) + } + // git orders tree entries by name, directories sorting as if suffixed by "/". + sort.Slice(entries, func(i, j int) bool { + return treeSortKey(entries[i]) < treeSortKey(entries[j]) + }) + tree := &object.Tree{Entries: entries} + obj := repo.Storer.NewEncodedObject() + require.NoError(t, tree.Encode(obj)) + hash, err := repo.Storer.SetEncodedObject(obj) + require.NoError(t, err) + return hash +} + +func treeSortKey(e object.TreeEntry) string { + if e.Mode == filemode.Dir { + return e.Name + "/" + } + return e.Name +} diff --git a/cmd/entire/cli/worktree_setup_diagnostic.go b/cmd/entire/cli/worktree_setup_diagnostic.go new file mode 100644 index 0000000000..9c2d5b1510 --- /dev/null +++ b/cmd/entire/cli/worktree_setup_diagnostic.go @@ -0,0 +1,150 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/entireio/cli/cmd/entire/cli/agent/claudecode" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/settings" +) + +const claudeCodeAgentName = "claude-code" + +type worktreeSetupIssue struct { + ConfiguredWorktree string + MissingProjectSettings bool + MissingClaudeProjectHooks bool + CurrentCaptureInactive bool +} + +func inspectWorktreeSetup(ctx context.Context) *worktreeSetupIssue { + currentRoot, err := paths.WorktreeRoot(ctx) + if err != nil { + return nil + } + + project, local, err := settings.FilesPresent(ctx) + if err != nil { + return nil + } + hasAnySettings := project || local + if hasAnySettings { + currentSettings, loadErr := LoadEntireSettings(ctx) + if loadErr != nil || !currentSettings.Enabled { + return nil + } + } + hasPortableProjectSettings := project && settings.ProjectSettingsEnabledForWorktreeRoot(currentRoot) + + hasClaudeProjectHooks, err := claudecode.AreProjectHooksInstalledInWorktree(ctx, currentRoot) + if err != nil { + return nil + } + if hasPortableProjectSettings && hasClaudeProjectHooks { + return nil + } + + for _, sibling := range siblingWorktrees(ctx, currentRoot) { + if !settings.ProjectSettingsEnabledForWorktreeRoot(sibling) || + !settings.IsSetUpAndEnabledForWorktreeRoot(ctx, sibling) { + continue + } + hasHooks, hookErr := claudecode.AreProjectHooksInstalledInWorktree(ctx, sibling) + if hookErr == nil && hasHooks { + return &worktreeSetupIssue{ + ConfiguredWorktree: sibling, + MissingProjectSettings: !hasPortableProjectSettings, + MissingClaudeProjectHooks: !hasClaudeProjectHooks, + CurrentCaptureInactive: !hasAnySettings, + } + } + } + return nil +} + +func siblingWorktrees(ctx context.Context, currentRoot string) []string { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "git", "worktree", "list", "--porcelain", "-z") + cmd.Dir = currentRoot + output, err := cmd.Output() + if err != nil { + return nil + } + return parseSiblingWorktrees(output, normalizeWorktreePath(currentRoot)) +} + +func parseSiblingWorktrees(porcelain []byte, currentRoot string) []string { + var worktrees []string + var path string + prunable := false + bare := false + for _, field := range bytes.Split(porcelain, []byte{0}) { + line := string(field) + switch { + case line == "": + if path != "" && !prunable && !bare && normalizeWorktreePath(path) != currentRoot { + worktrees = append(worktrees, path) + } + path = "" + prunable = false + bare = false + case strings.HasPrefix(line, "worktree "): + path = strings.TrimPrefix(line, "worktree ") + case line == "prunable" || strings.HasPrefix(line, "prunable "): + prunable = true + case line == "bare": + bare = true + } + } + return worktrees +} + +func writeWorktreeSetupIssue(w io.Writer, issue *worktreeSetupIssue) { + if issue == nil { + return + } + fmt.Fprintln(w, "Claude Code worktree portability: INCOMPLETE") + fmt.Fprintf(w, " Entire capture is configured in another worktree: %s\n", issue.ConfiguredWorktree) + fmt.Fprintf(w, " Missing here: %s.\n", strings.Join(issue.missingLabels(), ", ")) + if issue.CurrentCaptureInactive { + fmt.Fprintln(w, " Without Entire settings, every Entire hook in this worktree is inactive.") + fmt.Fprintln(w, " Claude Code sessions started here will not create checkpoints, and commits will not receive Entire trailers.") + fmt.Fprintln(w, " Run `entire enable --agent claude-code` in this worktree.") + } else { + fmt.Fprintln(w, " Fresh worktrees and clones may miss Entire capture until the portable project files are committed.") + fmt.Fprintln(w, " Local Entire settings may still keep Entire active in this worktree.") + fmt.Fprintln(w, " Claude user or local settings may still provide hooks for this worktree.") + fmt.Fprintln(w, " Run `entire enable --agent claude-code --project` to create any missing project setup.") + } + fmt.Fprintln(w, " Commit `.entire/settings.json` and `.claude/settings.json` when the setup should apply across worktrees and clones.") +} + +func (i *worktreeSetupIssue) missingLabels() []string { + missing := make([]string, 0, 2) + if i.MissingProjectSettings { + missing = append(missing, "Entire project settings") + } + if i.MissingClaudeProjectHooks { + missing = append(missing, "shared Claude Code hook config") + } + return missing +} + +func (i *worktreeSetupIssue) missingJSONFields() []string { + missing := make([]string, 0, 2) + if i.MissingProjectSettings { + missing = append(missing, "entire_project_settings") + } + if i.MissingClaudeProjectHooks { + missing = append(missing, "claude_project_hook_config") + } + return missing +} diff --git a/cmd/entire/cli/worktree_setup_diagnostic_test.go b/cmd/entire/cli/worktree_setup_diagnostic_test.go new file mode 100644 index 0000000000..7e4cb8928f --- /dev/null +++ b/cmd/entire/cli/worktree_setup_diagnostic_test.go @@ -0,0 +1,357 @@ +package cli + +import ( + "bytes" + "encoding/json" + "path/filepath" + "slices" + "strings" + "testing" + + agentpkg "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/agent/claudecode" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/strategy" + "github.com/entireio/cli/cmd/entire/cli/testutil" +) + +func TestInspectWorktreeSetup_RequiresConfiguredSibling(t *testing.T) { + t.Run("fresh worktree missing settings and hooks", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + enterWorktree(t, linkedRoot) + + issue := inspectWorktreeSetup(t.Context()) + if issue == nil { + t.Fatal("inspectWorktreeSetup() = nil, want incomplete setup") + } + if !issue.MissingProjectSettings || !issue.MissingClaudeProjectHooks { + t.Errorf("issue = %+v, want both settings and Claude hooks missing", issue) + } + }) + + t.Run("current worktree configured", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + configureClaudeWorktree(t, linkedRoot, testSettingsEnabled, true) + enterWorktree(t, linkedRoot) + + assertNoWorktreeSetupIssue(t, "configured current worktree") + }) + + t.Run("ordinary linked worktrees", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, "", false) + enterWorktree(t, linkedRoot) + + assertNoWorktreeSetupIssue(t, "ordinary linked worktrees") + }) + + t.Run("disabled sibling", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsDisabled, true) + enterWorktree(t, linkedRoot) + + assertNoWorktreeSetupIssue(t, "disabled sibling") + }) + + t.Run("sibling enabled only by local override", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsDisabled, true) + repoRoot := filepath.Join(filepath.Dir(linkedRoot), "repo") + testutil.WriteFile(t, repoRoot, EntireSettingsLocalFile, testSettingsEnabled) + enterWorktree(t, linkedRoot) + + assertNoWorktreeSetupIssue(t, "sibling without portable project settings") + }) + + t.Run("disabled current worktree", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + configureClaudeWorktree(t, linkedRoot, testSettingsDisabled, false) + enterWorktree(t, linkedRoot) + + assertNoWorktreeSetupIssue(t, "disabled current worktree") + }) + + t.Run("current worktree disabled by local override", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + configureClaudeWorktree(t, linkedRoot, testSettingsEnabled, true) + testutil.WriteFile(t, linkedRoot, EntireSettingsLocalFile, testSettingsDisabled) + enterWorktree(t, linkedRoot) + + assertNoWorktreeSetupIssue(t, "current worktree disabled by local override") + }) + + t.Run("current hook config inspection fails", func(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + testutil.WriteFile(t, linkedRoot, ".claude/settings.json", `{`) + enterWorktree(t, linkedRoot) + + assertNoWorktreeSetupIssue(t, "failed current hook config inspection") + }) +} + +func assertNoWorktreeSetupIssue(t *testing.T, scenario string) { + t.Helper() + if issue := inspectWorktreeSetup(t.Context()); issue != nil { + t.Errorf("inspectWorktreeSetup() = %+v, want nil for %s", issue, scenario) + } +} + +func TestInspectWorktreeSetup_ReportsOnlyMissingClaudeProjectHooks(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + configureClaudeWorktree(t, linkedRoot, testSettingsEnabled, false) + enterWorktree(t, linkedRoot) + + issue := inspectWorktreeSetup(t.Context()) + if issue == nil { + t.Fatal("inspectWorktreeSetup() = nil, want incomplete setup") + } + if issue.MissingProjectSettings || !issue.MissingClaudeProjectHooks { + t.Errorf("issue = %+v, want only shared Claude project hooks missing", issue) + } +} + +func TestInspectWorktreeSetup_PreservesCurrentWorktreeVouch(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + testutil.WriteFile(t, linkedRoot, EntireSettingsLocalFile, + `{"enabled":true,"allow_symlinked_agent_dirs":[".claude"]}`) + enterWorktree(t, linkedRoot) + t.Cleanup(func() { agentpkg.SetVouchedSymlinkedDirs("", nil) }) + currentRoot, err := paths.WorktreeRoot(t.Context()) + if err != nil { + t.Fatalf("WorktreeRoot() error = %v", err) + } + + issue := inspectWorktreeSetup(t.Context()) + if issue == nil { + t.Fatal("inspectWorktreeSetup() = nil, want incomplete shared Claude setup") + } + if got := agentpkg.VouchedSymlinkedDirs(currentRoot); !slices.Equal(got, []string{".claude"}) { + t.Errorf("current worktree vouch after sibling inspection = %v, want [.claude]", got) + } +} + +func TestRunStatus_WorktreeSetupWarning(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + enterWorktree(t, linkedRoot) + + var human bytes.Buffer + if err := runStatus(t.Context(), &human, false, false); err != nil { + t.Fatalf("runStatus() error = %v", err) + } + for _, want := range []string{ + "○ not set up", + "Claude Code worktree portability: INCOMPLETE", + "Entire project settings, shared Claude Code hook config", + "Without Entire settings, every Entire hook in this worktree is inactive", + "sessions started here will not create checkpoints", + "entire enable --agent claude-code", + } { + if !strings.Contains(human.String(), want) { + t.Errorf("status output does not contain %q:\n%s", want, human.String()) + } + } + + var detailed bytes.Buffer + if err := runStatus(t.Context(), &detailed, true, false); err != nil { + t.Fatalf("runStatus(--detailed) error = %v", err) + } + if !strings.Contains(detailed.String(), "Claude Code worktree portability: INCOMPLETE") { + t.Errorf("detailed status does not report worktree setup gap:\n%s", detailed.String()) + } + + var machine bytes.Buffer + if err := runStatus(t.Context(), &machine, false, true); err != nil { + t.Fatalf("runStatus(--json) error = %v", err) + } + var result statusJSON + if err := json.Unmarshal(machine.Bytes(), &result); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if result.Error != "not set up" { + t.Errorf("error = %q, want preserved not-set-up contract", result.Error) + } + if result.WorktreeSetup == nil { + t.Fatal("worktree_setup = nil, want incomplete setup details") + } + if result.WorktreeSetup.State != "incomplete" || result.WorktreeSetup.Agent != claudeCodeAgentName { + t.Errorf("worktree_setup = %+v", result.WorktreeSetup) + } + if !slices.Equal(result.WorktreeSetup.Missing, []string{"entire_project_settings", "claude_project_hook_config"}) { + t.Errorf("worktree_setup.missing = %v", result.WorktreeSetup.Missing) + } +} + +func TestRunStatus_SharedClaudeHookWarningAllowsLocalCoverage(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + configureClaudeWorktree(t, linkedRoot, testSettingsEnabled, false) + testutil.WriteFile(t, linkedRoot, ".claude/settings.local.json", + `{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"entire hooks claude-code stop"}]}]}}`) + enterWorktree(t, linkedRoot) + + var stdout bytes.Buffer + if err := runStatus(t.Context(), &stdout, false, false); err != nil { + t.Fatalf("runStatus() error = %v", err) + } + if !strings.Contains(stdout.String(), "Claude user or local settings may still provide hooks") { + t.Errorf("status does not acknowledge local-scope coverage:\n%s", stdout.String()) + } + if strings.Contains(stdout.String(), "sessions started here will not create checkpoints") { + t.Errorf("status makes an absolute capture claim when only shared config is absent:\n%s", stdout.String()) + } + + var machine bytes.Buffer + if err := runStatus(t.Context(), &machine, false, true); err != nil { + t.Fatalf("runStatus(--json) error = %v", err) + } + var result statusJSON + if err := json.Unmarshal(machine.Bytes(), &result); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if result.WorktreeSetup == nil || + !slices.Equal(result.WorktreeSetup.Missing, []string{"claude_project_hook_config"}) { + t.Errorf("worktree_setup = %+v, want only missing shared Claude project config", result.WorktreeSetup) + } +} + +func TestRunStatus_LocalOnlyEntireSettingsWarnsAboutProjectPortability(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + testutil.WriteFile(t, linkedRoot, EntireSettingsLocalFile, testSettingsEnabled) + configureClaudeWorktree(t, linkedRoot, "", true) + enterWorktree(t, linkedRoot) + + var human bytes.Buffer + if err := runStatus(t.Context(), &human, false, false); err != nil { + t.Fatalf("runStatus() error = %v", err) + } + if !strings.Contains(human.String(), "Entire project settings") { + t.Errorf("status does not report missing portable project settings:\n%s", human.String()) + } + if strings.Contains(human.String(), "sessions started here will not create checkpoints") { + t.Errorf("status makes an absolute capture claim for an enabled local setup:\n%s", human.String()) + } + + var machine bytes.Buffer + if err := runStatus(t.Context(), &machine, false, true); err != nil { + t.Fatalf("runStatus(--json) error = %v", err) + } + var result statusJSON + if err := json.Unmarshal(machine.Bytes(), &result); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if result.WorktreeSetup == nil || + !slices.Equal(result.WorktreeSetup.Missing, []string{"entire_project_settings"}) { + t.Errorf("worktree_setup = %+v, want only missing Entire project settings", result.WorktreeSetup) + } +} + +func TestInspectWorktreeSetup_LocalOverrideDoesNotMakeDisabledProjectPortable(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + configureClaudeWorktree(t, linkedRoot, testSettingsDisabled, true) + testutil.WriteFile(t, linkedRoot, EntireSettingsLocalFile, testSettingsEnabled) + enterWorktree(t, linkedRoot) + + issue := inspectWorktreeSetup(t.Context()) + if issue == nil { + t.Fatal("inspectWorktreeSetup() = nil, want missing portable project settings") + } + if !issue.MissingProjectSettings || issue.MissingClaudeProjectHooks || issue.CurrentCaptureInactive { + t.Errorf("issue = %+v, want only inactive project settings reported as non-portable", issue) + } +} + +func TestDoctor_WorktreeWithoutSettingsDoesNotReportHealthyHooks(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + enterWorktree(t, linkedRoot) + issue := inspectWorktreeSetup(t.Context()) + if issue == nil { + t.Fatal("inspectWorktreeSetup() = nil, want incomplete setup") + } + + cmd, stdout := newTestCmd(t) + if err := checkGitHooksWithWorktreeSetup(cmd, false, issue); err != nil { + t.Fatalf("checkGitHooksWithWorktreeSetup() error = %v", err) + } + writeWorktreeSetupIssue(stdout, issue) + if strings.Contains(stdout.String(), "Git hooks: OK") { + t.Errorf("doctor reported inactive hooks as healthy:\n%s", stdout.String()) + } + if !strings.Contains(stdout.String(), "Git hooks: INSTALLED BUT INACTIVE") { + t.Errorf("doctor did not report inactive hooks:\n%s", stdout.String()) + } + if !strings.Contains(stdout.String(), "Claude Code worktree portability: INCOMPLETE") { + t.Errorf("doctor did not report worktree setup gap:\n%s", stdout.String()) + } +} + +func TestDoctor_LocalOnlyEntireSettingsDoesNotReportInactiveHooks(t *testing.T) { + linkedRoot := setupClaudeWorktrees(t, testSettingsEnabled, true) + testutil.WriteFile(t, linkedRoot, EntireSettingsLocalFile, testSettingsEnabled) + configureClaudeWorktree(t, linkedRoot, "", true) + enterWorktree(t, linkedRoot) + issue := inspectWorktreeSetup(t.Context()) + if issue == nil { + t.Fatal("inspectWorktreeSetup() = nil, want project portability warning") + } + + cmd, stdout := newTestCmd(t) + if err := checkGitHooksWithWorktreeSetup(cmd, false, issue); err != nil { + t.Fatalf("checkGitHooksWithWorktreeSetup() error = %v", err) + } + if !strings.Contains(stdout.String(), "Git hooks: OK") { + t.Errorf("doctor did not report active shared Git hooks:\n%s", stdout.String()) + } + if strings.Contains(stdout.String(), "INSTALLED BUT INACTIVE") { + t.Errorf("doctor reported an enabled local-only setup as inactive:\n%s", stdout.String()) + } +} + +func TestParseSiblingWorktrees_SkipsCurrentAndPrunable(t *testing.T) { + t.Parallel() + porcelain := []byte("worktree /repo/current\x00HEAD abc\x00\x00" + + "worktree /repo/ready\x00HEAD def\x00\x00" + + "worktree /repo/common\x00bare\x00\x00" + + "worktree /repo/gone\x00HEAD 123\x00prunable gitdir file points to non-existent location\x00\x00") + + got := parseSiblingWorktrees(porcelain, normalizeWorktreePath("/repo/current")) + if !slices.Equal(got, []string{"/repo/ready"}) { + t.Errorf("parseSiblingWorktrees() = %v, want [/repo/ready]", got) + } +} + +func setupClaudeWorktrees(t *testing.T, sourceSettings string, installClaudeHooks bool) string { + t.Helper() + repoRoot := filepath.Join(t.TempDir(), "repo") + linkedRoot := filepath.Join(filepath.Dir(repoRoot), "linked") + testutil.InitRepo(t, repoRoot) + testutil.WriteFile(t, repoRoot, "README.md", "initial\n") + testutil.GitAdd(t, repoRoot, "README.md") + testutil.GitCommit(t, repoRoot, "initial") + testutil.RunGit(t, repoRoot, "worktree", "add", "-b", "feature", linkedRoot) + if sourceSettings != "" || installClaudeHooks { + configureClaudeWorktree(t, repoRoot, sourceSettings, installClaudeHooks) + } + enterWorktree(t, repoRoot) + if _, err := strategy.InstallGitHook(t.Context(), true, false); err != nil { + t.Fatalf("InstallGitHook() error = %v", err) + } + return linkedRoot +} + +func configureClaudeWorktree(t *testing.T, worktreeRoot, entireSettings string, installClaudeHooks bool) { + t.Helper() + if entireSettings != "" { + testutil.WriteFile(t, worktreeRoot, EntireSettingsFile, entireSettings) + } + if installClaudeHooks { + enterWorktree(t, worktreeRoot) + if _, err := (&claudecode.ClaudeCodeAgent{}).InstallHooks(t.Context(), false); err != nil { + t.Fatalf("InstallHooks() error = %v", err) + } + } +} + +func enterWorktree(t *testing.T, worktreeRoot string) { + t.Helper() + t.Chdir(worktreeRoot) + paths.ClearWorktreeRootCache() + strategy.ClearHooksDirCache() + t.Cleanup(paths.ClearWorktreeRootCache) + t.Cleanup(strategy.ClearHooksDirCache) +} diff --git a/docs/development/checkpoint-implementation.md b/docs/development/checkpoint-implementation.md index 4b06179cd3..9d2f804946 100644 --- a/docs/development/checkpoint-implementation.md +++ b/docs/development/checkpoint-implementation.md @@ -50,6 +50,7 @@ The manual-commit strategy (`manual_commit*.go`) does not modify the active bran - The checkpoint metadata walk reuses the previous checkpoint's redacted blob as a prefix and redacts only appended lines (`checkpoint/redact_cache.go`), turning a per-Stop cost of O(whole transcript) into O(appended). The stored prefix must always end immediately after a `\n`, which is what makes plain byte concatenation reproduce the full result; content with a partial trailing line is therefore never cached. Eligibility is keyed on `paths.TranscriptFileName` (`full.jsonl`), **not** a `.jsonl` suffix — `transcript.jsonl` is regenerated in full each checkpoint and `full.jsonl.001` chunks are not appended, so neither should qualify. Reuse requires the prefix bytes to still hash the same and `redactionFingerprint()` (CLI version + commit + `redact.ConfigFingerprint()`) to match, so a rewritten transcript, changed custom rules, or a CLI upgrade all fall back to a full redaction. Bump `configFingerprintVersion` in `redact/fingerprint.go` whenever the regex layers change behaviour, or stale output can be reused. The cache lives in the git common dir, resolved without caching from the explicit worktree root through `gitrepo.ResolveWorktreeMetadata`, never under `.entire/`, because anything in the metadata directory would be walked into the checkpoint tree and committed. All three whole-transcript paths are covered: the shadow write walks files through `createRedactedBlobFromFile`, while condensation and the Stop finalize rewrite hold the transcript in memory and go through `checkpoint.RedactTranscriptCached`. Those paths do **not** redact the same bytes — the shadow write stores a sanitized transcript, condensation and finalize a sanitized *and* image-externalized one — and they stay separate simply because their keys are different strings: the walk uses its real tree path, the in-memory callers a synthetic key carrying the session ID (so concurrent sessions never share an entry). There is deliberately no scope enum; sharing a key would be safe (the prefix hash rejects a mismatch) but would miss on every checkpoint. The in-memory prefix is stored as a **file** in the cache dir, not a git blob: go-git deflates the whole payload before discovering the object exists (dotgit dedups the rename, not the compression), and above `agent.MaxChunkSize` the whole-transcript blob matches no chunk the store writes, so it would linger unreachable until `git gc` pruned it and silently reverted the cache to full redaction. `redactIncrementally` owns the whole-content fallback and takes the redactor as a parameter, so prefix and suffix cannot come from different pipelines; condensation and finalize share that pipeline by both routing through `redactSessionTranscript`. A per-subagent task transcript opts out with a nil repo: it is written once per task rather than appended across checkpoints. - Each committed session stores the (sanitized, redacted) transcript (`full.jsonl`, read by CLI resume/explain) plus a best-effort compact transcript (`transcript.jsonl`, generated via `transcript/compact`). Like `full.jsonl`, `transcript.jsonl` stores the **full compacted session** on every checkpoint (via `compact.FullWithBoundary`), so each checkpoint is self-contained and the session survives a mid-history checkpoint being lost/reverted/rebased. This checkpoint's slice begins at the session metadata's `compact_transcript_start` (a line offset in compact-output coordinates, distinct from `checkpoint_transcript_start` which indexes raw `full.jsonl` lines); a nil/absent marker means a legacy delta-only `transcript.jsonl` (read from line 0). The marker rounds toward inclusion when a streaming message straddles the boundary, so the slice never drops this checkpoint's content but may repeat ≤1 merged line at its head. Compact generation is best-effort and is skipped when the compacted output exceeds the 50MB blob cap (unlike `full.jsonl`, `transcript.jsonl` is not chunked — `full.jsonl` stays authoritative and the compact is regenerable); in the OPF finalize rewrite a failed/skipped regeneration drops the prior `transcript.jsonl` and clears the marker rather than shipping a stale, less-redacted compact. Both files are pushed with the v1 branch. The root `metadata.json` `sessions[].transcript` pointer keeps targeting `full.jsonl`; when the compact transcript was generated the session entry also carries a `compact_transcript` path pointing at `transcript.jsonl` (omitted otherwise) so external readers can locate it next to `full.jsonl`. - **Subagent task records** - subagent work (Claude Code's Task tool) is captured as durable `session.TaskRecord` entries on session state, a pointer mid-turn (declared transcript path + labels; background launches record at launch, completions attach files/tokens/path exactly-once via `strategy.CompleteTaskRecord`, Factory Droid Workers upsert). Condensation materializes each record — declared path first, agent-layout fallback, same sanitize → externalize → redact pipeline — into `tasks//{agent-.jsonl, task.json}` inside the parent session's checkpoint (unavailable transcript → `task.json` with a stable path-free reason; the writer redacts `task.json`'s free-text `task_description` itself, since the record carries it verbatim); live records store transcript-so-far each condensation, completed records are removed after a successful write. `State.HasTaskContent()` is the trigger currency: records-only sessions condense, records never live on the shadow branch, and shadow-branch existence does not imply task content (shadow pinning keys on `StepCount` only). `SaveTaskStep` is incremental-only (post-todo). +- **Per-session `metadata.json` is capped, and doctor repairs the history that predates the cap.** `prompt_attributions` is the one metadata field whose size scales with the working tree rather than the session (CLI versions before v0.10.1 walked nested git checkouts — agent worktrees — into it on every prompt and produced 70MB and 106MB blobs that made `entire/checkpoints/v1` unpushable to GitHub, whose blob limit is 100 MiB). Nothing reads the field back — the CLI never does, and entire-api's ingest explicitly salvages a metadata.json clipped at that field — so `checkpoint.CapPromptAttributions` drops it above `MaxPromptAttributionsBytes` (4 MiB) at the write boundary — on the fresh session write **and** in `updateSessionMetadata`, the finalize/backfill re-marshal, which would otherwise copy a pre-cap field from an older CLI verbatim into every later rewrite — and `entire doctor` (`checkOversizedCheckpointMetadata`, backed by `strategy.ScanOversizedCheckpointMetadata` / `ShrinkOversizedCheckpointMetadata`) reports the v1 branch wherever a metadata.json exceeds `OversizedCheckpointMetadataThreshold` (50 MiB) and offers to rewrite it without the field, then force-push with `--force-with-lease` against the tip it observed. **`doctor --force` does not apply this fix**: it rewrites history and writes to a remote, and `--force` is run routinely by scripts and the e2e harness; an interactive yes or `entire doctor shrink-checkpoint-metadata [--yes]` is the opt-in. The rewrite is memoized at blob, tree and commit level and keeps every hash before the first oversized blob. **Local is rewritten before the remote is reconciled**: when the remote is ahead, its history is rewritten with the same memoized rewriter so shared commits map to the same rewritten commits, and the two rewritten tips are compared by *tree* (`contentReachable`), not hash — commit signing makes independent rewrites differ in hash, and reconciling the original local tip against a rewritten remote would replay every commit since the first oversized blob as a duplicate (the reviewer's critical finding on the first draft). Only genuine divergence falls through to `SafelyAdvanceLocalRef`. The repair is per clone: another clone still holding the bloated local branch replays it onto the repaired remote on its next push (`fetchAndRebaseRefCommon`), so the fix tells the user to run it in every clone. The scan touches the network only when something is already oversized locally or in the last-fetched tracking ref, so the routine doctor run stays offline. Two deliberate limits: branch backend only (with git-refs primary the v1 branch is no longer what gets pushed), and report-only when a dedicated `checkpoint_remote` URL is configured — there is no remote-tracking ref to lease against there, and pre-push's replay onto that URL would undo a local-only rewrite, so half a repair is worse than naming the problem. - Uses the `post-rewrite` Git hook to keep local session linkage aligned after amend/rebase rewrites - Builds git trees in-memory using go-git plumbing APIs - **Location-independent transcript resolution** - transcript paths are always computed dynamically from the current repo location (via `agent.GetSessionDir` + `agent.ResolveSessionFile`), never stored in checkpoint metadata. This ensures log restore (`RestoreLogsOnly`) works after repo relocation or across machines.