diff --git a/scripts/test-github-adapter.ts b/scripts/test-github-adapter.ts new file mode 100644 index 0000000000..9b96005c94 --- /dev/null +++ b/scripts/test-github-adapter.ts @@ -0,0 +1,114 @@ +#!/usr/bin/env -S deno run --allow-net --allow-env + +/** + * Manual test script for GitHubFSAdapter with a real repository + * + * Usage: + * GITHUB_TOKEN=$(gh auth token) deno run --allow-net --allow-env scripts/test-github-adapter.ts + */ + +import { GitHubFSAdapter } from "../src/platform/adapters/github-fs-adapter/index.ts"; + +const token = Deno.env.get("GITHUB_TOKEN"); +if (!token) { + console.error("ERROR: GITHUB_TOKEN environment variable required"); + console.error("Run: GITHUB_TOKEN=$(gh auth token) deno run --allow-net --allow-env scripts/test-github-adapter.ts"); + Deno.exit(1); +} + +const owner = "veryfront"; +const repo = "codersociety"; + +console.log(`\n๐Ÿงช Testing GitHubFSAdapter with ${owner}/${repo}\n`); + +const adapter = new GitHubFSAdapter({ + type: "github", + github: { + token, + owner, + repo, + ref: "main", + }, +}); + +try { + // Test 1: Initialize + console.log("1๏ธโƒฃ Initializing adapter..."); + const startInit = Date.now(); + await adapter.initialize(); + console.log(` โœ… Initialized in ${Date.now() - startInit}ms`); + + // Test 2: List root directory + console.log("\n2๏ธโƒฃ Listing root directory..."); + const rootEntries = await adapter.readdir(""); + console.log(` โœ… Found ${rootEntries.length} entries:`); + for (const entry of rootEntries.slice(0, 10)) { + const icon = entry.isDirectory ? "๐Ÿ“" : "๐Ÿ“„"; + console.log(` ${icon} ${entry.name}`); + } + if (rootEntries.length > 10) { + console.log(` ... and ${rootEntries.length - 10} more`); + } + + // Test 3: Check if common files exist + console.log("\n3๏ธโƒฃ Checking common files..."); + const filesToCheck = ["package.json", "README.md", "veryfront.config.ts", "tsconfig.json"]; + for (const file of filesToCheck) { + const exists = await adapter.exists(file); + console.log(` ${exists ? "โœ…" : "โŒ"} ${file}`); + } + + // Test 4: Read a file + console.log("\n4๏ธโƒฃ Reading package.json..."); + if (await adapter.exists("package.json")) { + const content = await adapter.readTextFile("package.json"); + const pkg = JSON.parse(content); + console.log(` โœ… Project: ${pkg.name || "unnamed"}`); + console.log(` โœ… Content length: ${content.length} bytes`); + } else { + console.log(" โญ๏ธ Skipped (file doesn't exist)"); + } + + // Test 5: File resolution + console.log("\n5๏ธโƒฃ Testing file resolution..."); + const pathsToResolve = ["pages/index", "app/page", "src/index"]; + for (const path of pathsToResolve) { + const resolved = await adapter.resolveFile(path); + if (resolved) { + console.log(` โœ… ${path} โ†’ ${resolved}`); + } else { + console.log(` โž– ${path} โ†’ not found`); + } + } + + // Test 6: Stat a file + console.log("\n6๏ธโƒฃ Getting file stats..."); + const firstFile = rootEntries.find(e => e.isFile); + if (firstFile) { + const stat = await adapter.stat(firstFile.path); + console.log(` โœ… ${firstFile.name}: ${stat.size} bytes, isFile=${stat.isFile}`); + } + + // Test 7: Cache stats + console.log("\n7๏ธโƒฃ Cache statistics:"); + const cacheStats = adapter.getCacheStats(); + console.log(` ๐Ÿ“Š Entries: ${cacheStats.cache.size}`); + console.log(` ๐Ÿ“Š Memory: ${(cacheStats.cache.memoryUsed / 1024).toFixed(1)} KB`); + console.log(` ๐Ÿ“Š Hit rate: ${(cacheStats.cache.hitRate * 100).toFixed(1)}%`); + + // Test 8: Rate limit info + console.log("\n8๏ธโƒฃ Rate limit status:"); + const rateLimit = adapter.getRateLimitInfo(); + if (rateLimit) { + console.log(` ๐Ÿ“Š Remaining: ${rateLimit.remaining}/${rateLimit.limit}`); + console.log(` ๐Ÿ“Š Resets: ${rateLimit.reset.toLocaleTimeString()}`); + } else { + console.log(" โž– No rate limit info yet"); + } + + console.log("\nโœ… All tests passed!\n"); + +} catch (error) { + console.error("\nโŒ Test failed:", error); + Deno.exit(1); +} diff --git a/src/core/config/loader.ts b/src/core/config/loader.ts index 202db7e5cd..933882c4f0 100644 --- a/src/core/config/loader.ts +++ b/src/core/config/loader.ts @@ -159,13 +159,12 @@ class ConfigValidationError extends Error { } /** - * Check if the adapter is using a virtual filesystem (e.g., Veryfront API) - * Supports both single-project (VeryfrontFSAdapter) and multi-project (MultiProjectFSAdapter) modes + * Check if the adapter is using a virtual filesystem (e.g., Veryfront API, GitHub) */ function isVirtualFilesystem(adapter: RuntimeAdapter): boolean { const wrappedAdapter = (adapter?.fs as { fsAdapter?: unknown })?.fsAdapter; const adapterName = (wrappedAdapter as { constructor?: { name?: string } })?.constructor?.name; - return adapterName === "VeryfrontFSAdapter" || adapterName === "MultiProjectFSAdapter"; + return adapterName === "VeryfrontFSAdapter" || adapterName === "MultiProjectFSAdapter" || adapterName === "GitHubFSAdapter"; } /** diff --git a/src/core/config/schema.ts b/src/core/config/schema.ts index fd2921d925..5bc83621aa 100644 --- a/src/core/config/schema.ts +++ b/src/core/config/schema.ts @@ -156,7 +156,7 @@ export const veryfrontConfigSchema = z .optional(), fs: z .object({ - type: z.enum(["local", "veryfront-api", "memory"]).optional(), + type: z.enum(["local", "veryfront-api", "memory", "github"]).optional(), local: z .object({ baseDir: z.string().optional(), @@ -195,6 +195,32 @@ export const veryfrontConfigSchema = z }) .partial() .optional(), + github: z + .object({ + token: z.string(), + owner: z.string(), + repo: z.string(), + ref: z.string().optional(), + cache: z + .object({ + enabled: z.boolean().optional(), + ttl: z.number().int().positive().optional(), + maxSize: z.number().int().positive().optional(), + maxMemory: z.number().int().positive().optional(), + }) + .partial() + .optional(), + retry: z + .object({ + maxRetries: z.number().int().min(0).optional(), + initialDelay: z.number().int().positive().optional(), + maxDelay: z.number().int().positive().optional(), + }) + .partial() + .optional(), + }) + .partial() + .optional(), }) .partial() .optional(), diff --git a/src/core/config/types.ts b/src/core/config/types.ts index 0419186c1e..b81fed0467 100644 --- a/src/core/config/types.ts +++ b/src/core/config/types.ts @@ -124,7 +124,7 @@ export interface VeryfrontConfig { }; }; fs?: { - type?: "local" | "veryfront-api" | "memory"; + type?: "local" | "veryfront-api" | "memory" | "github"; local?: { baseDir?: string; }; @@ -152,6 +152,27 @@ export interface VeryfrontConfig { memory?: { files?: Record; }; + github?: { + /** GitHub Personal Access Token */ + token: string; + /** Repository owner (user or organization) */ + owner: string; + /** Repository name */ + repo: string; + /** Branch, tag, or commit SHA (default: "main") */ + ref?: string; + cache?: { + enabled?: boolean; + ttl?: number; + maxSize?: number; + maxMemory?: number; + }; + retry?: { + maxRetries?: number; + initialDelay?: number; + maxDelay?: number; + }; + }; }; ai?: { enabled?: boolean; diff --git a/src/platform/adapters/fs-adapter-factory.ts b/src/platform/adapters/fs-adapter-factory.ts index 5837c8a2cd..6439656eda 100644 --- a/src/platform/adapters/fs-adapter-factory.ts +++ b/src/platform/adapters/fs-adapter-factory.ts @@ -34,11 +34,28 @@ export async function createFSAdapter(config: FSAdapterConfig): Promise1MB) +4. Results are cached with configurable TTL + +## Rate Limits + +GitHub API allows 5,000 requests/hour for authenticated requests. The adapter: + +- Caches aggressively to minimize API calls +- Warns when approaching rate limit +- Includes rate limit info in errors + +## Limitations + +- **Read-only**: No write operations supported +- **Single repo**: One repository per adapter instance +- **No webhooks**: Cache invalidation is TTL-based only diff --git a/src/platform/adapters/github-fs-adapter/adapter.test.ts b/src/platform/adapters/github-fs-adapter/adapter.test.ts new file mode 100644 index 0000000000..24cd9ef3a6 --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/adapter.test.ts @@ -0,0 +1,275 @@ +import { assertEquals, assertRejects } from "std/assert/mod.ts"; +import { afterEach, beforeEach, describe, it } from "std/testing/bdd.ts"; +import { GitHubFSAdapter } from "./adapter.ts"; +import { createGitHubConfig } from "./types.ts"; + +// Mock tree response +const mockTreeResponse = { + sha: "abc123", + tree: [ + { path: "README.md", type: "blob", sha: "sha1", size: 100 }, + { path: "src/index.ts", type: "blob", sha: "sha2", size: 200 }, + { path: "src/utils/helper.ts", type: "blob", sha: "sha3", size: 150 }, + { path: "src", type: "tree", sha: "sha4" }, + { path: "src/utils", type: "tree", sha: "sha5" }, + ], + truncated: false, +}; + +// Mock file content (base64 encoded "hello world") +const mockFileContent = { + type: "file", + name: "README.md", + path: "README.md", + sha: "sha1", + size: 11, + content: btoa("hello world"), + encoding: "base64", +}; + +describe("GitHubFSAdapter", () => { + let originalFetch: typeof fetch; + + beforeEach(() => { + originalFetch = globalThis.fetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + describe("createGitHubConfig", () => { + it("should throw if token is missing", () => { + try { + createGitHubConfig({ token: "", owner: "test", repo: "test" }); + throw new Error("Should have thrown"); + } catch (error) { + assertEquals((error as Error).message.includes("token"), true); + } + }); + + it("should throw if owner/repo is missing", () => { + try { + createGitHubConfig({ token: "token", owner: "", repo: "test" }); + throw new Error("Should have thrown"); + } catch (error) { + assertEquals((error as Error).message.includes("owner"), true); + } + }); + + it("should apply defaults", () => { + const config = createGitHubConfig({ + token: "token", + owner: "owner", + repo: "repo", + }); + assertEquals(config.ref, "main"); + assertEquals(config.cache.enabled, true); + assertEquals(config.cache.ttl, 60_000); + assertEquals(config.retry.maxRetries, 3); + }); + }); + + describe("initialization", () => { + it("should fetch tree on initialize", async () => { + let treeRequested = false; + + globalThis.fetch = (url) => { + if (String(url).includes("/git/trees/")) { + treeRequested = true; + return Promise.resolve( + new Response(JSON.stringify(mockTreeResponse), { status: 200 }), + ); + } + return Promise.resolve(new Response("Not found", { status: 404 })); + }; + + const adapter = new GitHubFSAdapter({ + type: "github", + github: { token: "test", owner: "owner", repo: "repo" }, + }); + + await adapter.initialize(); + assertEquals(treeRequested, true); + }); + }); + + describe("file operations", () => { + let adapter: GitHubFSAdapter; + + beforeEach(async () => { + globalThis.fetch = (url) => { + const urlStr = String(url); + if (urlStr.includes("/git/trees/")) { + return Promise.resolve( + new Response(JSON.stringify(mockTreeResponse), { status: 200 }), + ); + } + if (urlStr.includes("/contents/README.md")) { + return Promise.resolve( + new Response(JSON.stringify(mockFileContent), { status: 200 }), + ); + } + return Promise.resolve(new Response("Not found", { status: 404 })); + }; + + adapter = new GitHubFSAdapter({ + type: "github", + github: { token: "test", owner: "owner", repo: "repo" }, + }); + await adapter.initialize(); + }); + + it("should check file exists from index", async () => { + assertEquals(await adapter.exists("README.md"), true); + assertEquals(await adapter.exists("src/index.ts"), true); + assertEquals(await adapter.exists("nonexistent.ts"), false); + }); + + it("should check directory exists from index", async () => { + assertEquals(await adapter.exists("src"), true); + assertEquals(await adapter.exists("src/utils"), true); + assertEquals(await adapter.exists("nonexistent"), false); + }); + + it("should stat file", async () => { + const stat = await adapter.stat("README.md"); + assertEquals(stat.isFile, true); + assertEquals(stat.isDirectory, false); + assertEquals(stat.size, 100); + }); + + it("should stat directory", async () => { + const stat = await adapter.stat("src"); + assertEquals(stat.isFile, false); + assertEquals(stat.isDirectory, true); + }); + + it("should read file content", async () => { + const content = await adapter.readTextFile("README.md"); + assertEquals(content, "hello world"); + }); + + it("should throw on nonexistent file", async () => { + await assertRejects( + () => adapter.stat("nonexistent.ts"), + Error, + "not found", + ); + }); + }); + + describe("directory operations", () => { + let adapter: GitHubFSAdapter; + + beforeEach(async () => { + globalThis.fetch = (url) => { + if (String(url).includes("/git/trees/")) { + return Promise.resolve( + new Response(JSON.stringify(mockTreeResponse), { status: 200 }), + ); + } + return Promise.resolve(new Response("Not found", { status: 404 })); + }; + + adapter = new GitHubFSAdapter({ + type: "github", + github: { token: "test", owner: "owner", repo: "repo" }, + }); + await adapter.initialize(); + }); + + it("should list root directory", async () => { + const entries = await adapter.readdir(""); + const names = entries.map((e) => e.name); + assertEquals(names.includes("README.md"), true); + assertEquals(names.includes("src"), true); + }); + + it("should list subdirectory", async () => { + const entries = await adapter.readdir("src"); + const names = entries.map((e) => e.name); + assertEquals(names.includes("index.ts"), true); + assertEquals(names.includes("utils"), true); + }); + }); + + describe("file resolution", () => { + let adapter: GitHubFSAdapter; + + beforeEach(async () => { + const treeWithExtensions = { + ...mockTreeResponse, + tree: [ + { path: "pages/index.tsx", type: "blob", sha: "s1", size: 100 }, + { path: "pages/about.mdx", type: "blob", sha: "s2", size: 100 }, + { path: "lib/utils.ts", type: "blob", sha: "s3", size: 100 }, + { path: "pages", type: "tree", sha: "s4" }, + { path: "lib", type: "tree", sha: "s5" }, + ], + }; + + globalThis.fetch = (url) => { + if (String(url).includes("/git/trees/")) { + return Promise.resolve( + new Response(JSON.stringify(treeWithExtensions), { status: 200 }), + ); + } + return Promise.resolve(new Response("Not found", { status: 404 })); + }; + + adapter = new GitHubFSAdapter({ + type: "github", + github: { token: "test", owner: "owner", repo: "repo" }, + }); + await adapter.initialize(); + }); + + it("should resolve file with extension", async () => { + const resolved = await adapter.resolveFile("lib/utils"); + assertEquals(resolved, "lib/utils.ts"); + }); + + it("should resolve index file", async () => { + const resolved = await adapter.resolveFile("pages"); + assertEquals(resolved, "pages/index.tsx"); + }); + + it("should return null for unresolvable path", async () => { + const resolved = await adapter.resolveFile("nonexistent"); + assertEquals(resolved, null); + }); + }); + + describe("error handling", () => { + it("should handle 401 authentication error", async () => { + globalThis.fetch = () => { + return Promise.resolve(new Response("Unauthorized", { status: 401 })); + }; + + const adapter = new GitHubFSAdapter({ + type: "github", + github: { token: "bad-token", owner: "owner", repo: "repo" }, + }); + + await assertRejects( + () => adapter.initialize(), + Error, + "authentication", + ); + }); + + it("should handle 404 repo not found", async () => { + globalThis.fetch = () => { + return Promise.resolve(new Response("Not found", { status: 404 })); + }; + + const adapter = new GitHubFSAdapter({ + type: "github", + github: { token: "token", owner: "owner", repo: "nonexistent" }, + }); + + await assertRejects(() => adapter.initialize(), Error, "Not found"); + }); + }); +}); diff --git a/src/platform/adapters/github-fs-adapter/adapter.ts b/src/platform/adapters/github-fs-adapter/adapter.ts new file mode 100644 index 0000000000..457274ce12 --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/adapter.ts @@ -0,0 +1,212 @@ +import { logger } from "@veryfront/utils"; +import { FileCache } from "../file-cache/file-cache.ts"; +import type { FSAdapter, FSAdapterConfig } from "../veryfront-fs-adapter/types.ts"; +import { GitHubAPIClient } from "./github-api-client.ts"; +import { GitHubDirectoryOperations } from "./directory-operations.ts"; +import { GitHubReadOperations } from "./read-operations.ts"; +import { GitHubStatOperations } from "./stat-operations.ts"; +import { + createGitHubConfig, + type DirectoryEntry, + type FileInfo, + type GitHubConfig, + type ResolvedGitHubConfig, +} from "./types.ts"; + +const LOG_PREFIX = "[GitHubFSAdapter]"; + +/** + * GitHub filesystem adapter for veryfront-renderer + * + * Provides read-only access to files in a GitHub repository via the GitHub API. + * Uses tree-based indexing for efficient file resolution and caching. + */ +export class GitHubFSAdapter implements FSAdapter { + private readonly config: ResolvedGitHubConfig; + private readonly client: GitHubAPIClient; + private readonly cache: FileCache; + private readonly statOps: GitHubStatOperations; + private readonly readOps: GitHubReadOperations; + private readonly dirOps: GitHubDirectoryOperations; + private readonly projectDir: string; + + private initialized = false; + + constructor(adapterConfig: FSAdapterConfig) { + if (!adapterConfig.github) { + throw new Error("GitHub adapter requires github configuration"); + } + + // Store projectDir to strip from absolute paths + this.projectDir = adapterConfig.projectDir || ""; + + // Resolve config from raw config + environment + const rawConfig: GitHubConfig = { + token: adapterConfig.github.token || Deno.env.get("GITHUB_TOKEN") || "", + owner: adapterConfig.github.owner || Deno.env.get("GITHUB_OWNER") || "", + repo: adapterConfig.github.repo || Deno.env.get("GITHUB_REPO") || "", + ref: adapterConfig.github.ref || Deno.env.get("GITHUB_REF") || "main", + cache: adapterConfig.github.cache, + retry: adapterConfig.github.retry, + }; + + this.config = createGitHubConfig(rawConfig); + + // Initialize components + this.client = new GitHubAPIClient(this.config); + + this.cache = new FileCache({ + enabled: this.config.cache.enabled, + ttl: this.config.cache.ttl, + maxSize: this.config.cache.maxSize, + maxMemory: this.config.cache.maxMemory, + }); + + this.statOps = new GitHubStatOperations(this.config, this.client, this.cache, this.projectDir); + this.readOps = new GitHubReadOperations( + this.config, + this.client, + this.cache, + this.statOps, + this.projectDir, + ); + this.dirOps = new GitHubDirectoryOperations(this.config, this.cache, this.statOps, this.projectDir); + + logger.info(`${LOG_PREFIX} Created adapter`, { + repo: this.client.repoId, + ref: this.config.ref, + }); + } + + /** + * Initialize the adapter by fetching the repository tree + */ + async initialize(): Promise { + if (this.initialized) { + return; + } + + logger.info(`${LOG_PREFIX} Initializing`, { + repo: this.client.repoId, + ref: this.config.ref, + }); + + // Build the file index from repository tree + await this.statOps.buildIndex(); + + this.initialized = true; + + logger.info(`${LOG_PREFIX} Initialized successfully`); + } + + /** + * Read file content + */ + async readFile(path: string): Promise { + await this.ensureInitialized(); + return this.readOps.readFile(path); + } + + /** + * Read file content as text + */ + async readTextFile(path: string): Promise { + await this.ensureInitialized(); + return this.readOps.readTextFile(path); + } + + /** + * Check if file or directory exists + */ + async exists(path: string): Promise { + await this.ensureInitialized(); + return this.statOps.exists(path); + } + + /** + * Get file or directory stat information + */ + async stat(path: string): Promise { + await this.ensureInitialized(); + return this.statOps.stat(path); + } + + /** + * Read directory contents (async iterable) + */ + async *readDir(path: string): AsyncIterable { + await this.ensureInitialized(); + yield* this.dirOps.readDir(path); + } + + /** + * Read directory contents (array) + */ + async readdir(path: string): Promise { + await this.ensureInitialized(); + return this.dirOps.readdir(path); + } + + /** + * Resolve a file path, trying various extensions + */ + async resolveFile(basePath: string): Promise { + await this.ensureInitialized(); + return this.statOps.resolveFile(basePath); + } + + /** + * Get cache statistics + */ + getCacheStats(): { + cache: { + size: number; + memoryUsed: number; + hits: number; + misses: number; + hitRate: number; + }; + } { + const stats = this.cache.stats(); + return { + cache: { + size: stats.size, + memoryUsed: stats.memoryUsed, + hits: stats.hits, + misses: stats.misses, + hitRate: stats.hitRate, + }, + }; + } + + /** + * Get rate limit information from GitHub API + */ + getRateLimitInfo(): { + limit: number; + remaining: number; + reset: Date; + } | null { + return this.client.getRateLimitInfo(); + } + + /** + * Clear all caches and reset state + */ + dispose(): void { + this.cache.clear(); + this.statOps.clearIndex(); + this.initialized = false; + + logger.info(`${LOG_PREFIX} Disposed`); + } + + /** + * Ensure adapter is initialized before operations + */ + private async ensureInitialized(): Promise { + if (!this.initialized) { + await this.initialize(); + } + } +} diff --git a/src/platform/adapters/github-fs-adapter/directory-operations.ts b/src/platform/adapters/github-fs-adapter/directory-operations.ts new file mode 100644 index 0000000000..41fe15b8ac --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/directory-operations.ts @@ -0,0 +1,119 @@ +import { logger } from "@veryfront/utils"; +import type { FileCache } from "../file-cache/file-cache.ts"; +import type { GitHubStatOperations } from "./stat-operations.ts"; +import type { DirectoryEntry, ResolvedGitHubConfig } from "./types.ts"; + +const LOG_PREFIX = "[GitHubDirectoryOperations]"; + +/** + * Handles directory listing operations for GitHub adapter + */ +export class GitHubDirectoryOperations { + private readonly config: ResolvedGitHubConfig; + private readonly cache: FileCache; + private readonly statOps: GitHubStatOperations; + private readonly projectDir: string; + + constructor( + config: ResolvedGitHubConfig, + cache: FileCache, + statOps: GitHubStatOperations, + projectDir: string = "", + ) { + this.config = config; + this.cache = cache; + this.statOps = statOps; + this.projectDir = projectDir; + } + + /** + * Read directory contents + */ + readdir(path: string): DirectoryEntry[] { + const normalizedPath = this.normalizePath(path); + + // Check cache + const cacheKey = `github:dir:${this.config.ref}:${normalizedPath}`; + const cached = this.cache.get(cacheKey); + if (cached) { + return cached; + } + + logger.debug(`${LOG_PREFIX} Reading directory`, { path: normalizedPath }); + + // Check if directory exists + if (normalizedPath && !this.statOps.isDirectory(normalizedPath)) { + logger.debug(`${LOG_PREFIX} Directory not found`, { + path: normalizedPath, + }); + return []; + } + + const entries: DirectoryEntry[] = []; + + // Get files in directory + const files = this.statOps.getFilesInDirectory(normalizedPath); + for (const file of files) { + const name = file.path.split("/").pop() || file.path; + entries.push({ + name, + path: file.path, + isFile: true, + isDirectory: false, + isSymlink: false, + }); + } + + // Get subdirectories + const subdirs = this.statOps.getSubdirectories(normalizedPath); + for (const subdir of subdirs) { + const fullPath = normalizedPath ? `${normalizedPath}/${subdir}` : subdir; + entries.push({ + name: subdir, + path: fullPath, + isFile: false, + isDirectory: true, + isSymlink: false, + }); + } + + // Sort entries: directories first, then files, alphabetically + entries.sort((a, b) => { + if (a.isDirectory && !b.isDirectory) return -1; + if (!a.isDirectory && b.isDirectory) return 1; + return a.name.localeCompare(b.name); + }); + + // Cache the result + this.cache.set(cacheKey, entries); + + return entries; + } + + /** + * Async generator for readDir compatibility + */ + async *readDir(path: string): AsyncIterable { + const entries = this.readdir(path); + for (const entry of entries) { + yield entry; + } + } + + /** + * Normalize a file path, stripping projectDir prefix if present + */ + private normalizePath(path: string): string { + let normalized = path; + + // Strip projectDir prefix if present (handles absolute paths from renderer) + if (this.projectDir && normalized.startsWith(this.projectDir)) { + normalized = normalized.slice(this.projectDir.length); + } + + return normalized + .replace(/^\/+/, "") // Remove leading slashes + .replace(/\/+$/, "") // Remove trailing slashes + .replace(/\/+/g, "/"); // Collapse multiple slashes + } +} diff --git a/src/platform/adapters/github-fs-adapter/github-api-client.ts b/src/platform/adapters/github-fs-adapter/github-api-client.ts new file mode 100644 index 0000000000..4406b6f465 --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/github-api-client.ts @@ -0,0 +1,280 @@ +import { createError, toError } from "../../../core/errors/veryfront-error.ts"; +import { logger } from "@veryfront/utils"; +import type { + GitHubBlobResponse, + GitHubContentItem, + GitHubTreeResponse, + ResolvedGitHubConfig, +} from "./types.ts"; + +const LOG_PREFIX = "[GitHubAPIClient]"; + +/** + * Rate limit info from GitHub API response headers + */ +interface RateLimitInfo { + limit: number; + remaining: number; + reset: Date; + used: number; +} + +/** + * GitHub API client for repository operations + */ +export class GitHubAPIClient { + private readonly baseUrl = "https://api.github.com"; + private readonly config: ResolvedGitHubConfig; + private rateLimitInfo: RateLimitInfo | null = null; + + constructor(config: ResolvedGitHubConfig) { + this.config = config; + } + + /** + * Get the repository identifier string + */ + get repoId(): string { + return `${this.config.owner}/${this.config.repo}`; + } + + /** + * Fetch the full repository tree recursively + */ + async getTree(ref?: string): Promise { + const treeRef = ref || this.config.ref; + const endpoint = + `/repos/${this.config.owner}/${this.config.repo}/git/trees/${treeRef}?recursive=1`; + + logger.debug(`${LOG_PREFIX} Fetching tree`, { ref: treeRef }); + + const response = await this.request(endpoint); + + if (response.truncated) { + logger.warn( + `${LOG_PREFIX} Repository tree is truncated. Large repos may have incomplete file listing.`, + ); + } + + return response; + } + + /** + * Get file or directory contents + */ + getContents( + path: string, + ref?: string, + ): Promise { + const contentRef = ref || this.config.ref; + const normalizedPath = path.replace(/^\/+/, ""); + const endpoint = + `/repos/${this.config.owner}/${this.config.repo}/contents/${normalizedPath}?ref=${contentRef}`; + + logger.debug(`${LOG_PREFIX} Fetching contents`, { path: normalizedPath }); + + return this.request(endpoint); + } + + /** + * Get blob content by SHA (for files >1MB) + */ + getBlob(sha: string): Promise { + const endpoint = `/repos/${this.config.owner}/${this.config.repo}/git/blobs/${sha}`; + + logger.debug(`${LOG_PREFIX} Fetching blob`, { sha }); + + return this.request(endpoint); + } + + /** + * Get current rate limit status + */ + getRateLimitInfo(): RateLimitInfo | null { + return this.rateLimitInfo; + } + + /** + * Make an authenticated request to the GitHub API + */ + private async request(endpoint: string): Promise { + const url = `${this.baseUrl}${endpoint}`; + + let lastError: Error | null = null; + let attempt = 0; + + while (attempt < this.config.retry.maxRetries) { + attempt++; + + try { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${this.config.token}`, + Accept: "application/vnd.github.v3+json", + "User-Agent": "veryfront-renderer", + "X-GitHub-Api-Version": "2022-11-28", + }, + }); + + // Update rate limit info from headers + this.updateRateLimitInfo(response); + + if (!response.ok) { + const errorBody = await response.text(); + throw this.createAPIError(response.status, errorBody, endpoint); + } + + return (await response.json()) as T; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + // Don't retry client errors (4xx) except rate limits + if (this.isClientError(lastError) && !this.isRateLimitError(lastError)) { + throw lastError; + } + + // Check if we should retry + if (attempt < this.config.retry.maxRetries) { + const delay = this.calculateRetryDelay(attempt, lastError); + logger.warn(`${LOG_PREFIX} Request failed, retrying`, { + attempt, + delay, + error: lastError.message, + }); + await this.sleep(delay); + } + } + } + + throw lastError || new Error("Request failed after retries"); + } + + /** + * Update rate limit info from response headers + */ + private updateRateLimitInfo(response: Response): void { + const limit = response.headers.get("X-RateLimit-Limit"); + const remaining = response.headers.get("X-RateLimit-Remaining"); + const reset = response.headers.get("X-RateLimit-Reset"); + const used = response.headers.get("X-RateLimit-Used"); + + if (limit && remaining && reset) { + this.rateLimitInfo = { + limit: parseInt(limit, 10), + remaining: parseInt(remaining, 10), + reset: new Date(parseInt(reset, 10) * 1000), + used: used ? parseInt(used, 10) : 0, + }; + + // Warn when approaching rate limit + if (this.rateLimitInfo.remaining < 100) { + logger.warn(`${LOG_PREFIX} Approaching rate limit`, { + remaining: this.rateLimitInfo.remaining, + reset: this.rateLimitInfo.reset.toISOString(), + }); + } + } + } + + /** + * Create an appropriate error for API responses + */ + private createAPIError( + status: number, + body: string, + endpoint: string, + ): Error { + let message: string; + let errorType: "config" | "file" | "network" = "network"; + + switch (status) { + case 401: + errorType = "config"; + message = "GitHub API authentication failed. Check your GITHUB_TOKEN is valid."; + break; + case 403: + if (this.rateLimitInfo && this.rateLimitInfo.remaining === 0) { + message = + `GitHub API rate limit exceeded. Resets at ${this.rateLimitInfo.reset.toISOString()}`; + } else { + errorType = "config"; + message = "GitHub API access forbidden. Check token permissions for this repository."; + } + break; + case 404: + errorType = "file"; + message = `Not found: ${endpoint}`; + break; + case 422: + errorType = "config"; + message = `Invalid request to GitHub API: ${body}`; + break; + default: + message = `GitHub API error (${status}): ${body}`; + } + + const error = toError( + createError({ + type: errorType, + message, + }), + ); + + // Add status code and context for error handling + (error as Error & { statusCode?: number; endpoint?: string; repo?: string }).statusCode = + status; + (error as Error & { endpoint?: string }).endpoint = endpoint; + (error as Error & { repo?: string }).repo = this.repoId; + + return error; + } + + /** + * Check if error is a client error (4xx) + */ + private isClientError(error: Error): boolean { + return ( + (error as Error & { statusCode?: number }).statusCode !== undefined && + (error as Error & { statusCode?: number }).statusCode! >= 400 && + (error as Error & { statusCode?: number }).statusCode! < 500 + ); + } + + /** + * Check if error is a rate limit error + */ + private isRateLimitError(error: Error): boolean { + return ( + (error as Error & { statusCode?: number }).statusCode === 403 && + this.rateLimitInfo !== null && + this.rateLimitInfo.remaining === 0 + ); + } + + /** + * Calculate retry delay with exponential backoff + */ + private calculateRetryDelay(attempt: number, error: Error): number { + // If rate limited, wait until reset + if (this.isRateLimitError(error) && this.rateLimitInfo) { + const waitMs = this.rateLimitInfo.reset.getTime() - Date.now(); + return Math.max(waitMs, this.config.retry.initialDelay); + } + + // Exponential backoff + const delay = Math.min( + this.config.retry.initialDelay * Math.pow(2, attempt - 1), + this.config.retry.maxDelay, + ); + + // Add jitter + return delay + Math.random() * 1000; + } + + /** + * Sleep for the specified milliseconds + */ + private sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/src/platform/adapters/github-fs-adapter/index.ts b/src/platform/adapters/github-fs-adapter/index.ts new file mode 100644 index 0000000000..c7cf70cac2 --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/index.ts @@ -0,0 +1,19 @@ +export { GitHubFSAdapter } from "./adapter.ts"; +export { GitHubAPIClient } from "./github-api-client.ts"; +export { GitHubStatOperations } from "./stat-operations.ts"; +export { GitHubReadOperations } from "./read-operations.ts"; +export { GitHubDirectoryOperations } from "./directory-operations.ts"; + +export type { + DirectoryEntry, + FileIndexEntry, + FileInfo, + GitHubBlobResponse, + GitHubConfig, + GitHubContentItem, + GitHubTreeEntry, + GitHubTreeResponse, + ResolvedGitHubConfig, +} from "./types.ts"; + +export { createGitHubConfig } from "./types.ts"; diff --git a/src/platform/adapters/github-fs-adapter/read-operations.ts b/src/platform/adapters/github-fs-adapter/read-operations.ts new file mode 100644 index 0000000000..2456195697 --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/read-operations.ts @@ -0,0 +1,284 @@ +import { createError, toError } from "../../../core/errors/veryfront-error.ts"; +import { logger } from "@veryfront/utils"; +import type { FileCache } from "../file-cache/file-cache.ts"; +import type { GitHubAPIClient } from "./github-api-client.ts"; +import type { GitHubStatOperations } from "./stat-operations.ts"; +import type { GitHubContentItem, ResolvedGitHubConfig } from "./types.ts"; + +const LOG_PREFIX = "[GitHubReadOperations]"; + +/** Max file size for Contents API (1MB) */ +const MAX_CONTENTS_SIZE = 1024 * 1024; + +/** + * Handles file read operations for GitHub adapter + */ +export class GitHubReadOperations { + private readonly config: ResolvedGitHubConfig; + private readonly client: GitHubAPIClient; + private readonly cache: FileCache; + private readonly statOps: GitHubStatOperations; + private readonly projectDir: string; + + constructor( + config: ResolvedGitHubConfig, + client: GitHubAPIClient, + cache: FileCache, + statOps: GitHubStatOperations, + projectDir: string = "", + ) { + this.config = config; + this.client = client; + this.cache = cache; + this.statOps = statOps; + this.projectDir = projectDir; + } + + /** + * Read file content as text + */ + async readTextFile(path: string): Promise { + const normalizedPath = this.normalizePath(path); + + // Check cache + const cacheKey = `github:content:${this.config.ref}:${normalizedPath}`; + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + + logger.debug(`${LOG_PREFIX} Reading file`, { path: normalizedPath }); + + // Get file entry from index for size check + const fileEntry = this.statOps.getFileEntry(normalizedPath); + + let content: string; + + if (fileEntry && fileEntry.size > MAX_CONTENTS_SIZE) { + // Large file: use Blob API + content = await this.readLargeFile(fileEntry.sha); + } else { + // Normal file: use Contents API + content = await this.readContentsFile(normalizedPath); + } + + // Cache the content + this.cache.set(cacheKey, content); + + return content; + } + + /** + * Read file content as bytes + */ + async readFile(path: string): Promise { + const normalizedPath = this.normalizePath(path); + + // Check cache for bytes + const cacheKey = `github:bytes:${this.config.ref}:${normalizedPath}`; + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + + logger.debug(`${LOG_PREFIX} Reading file as bytes`, { path: normalizedPath }); + + // Get file entry from index for size check + const fileEntry = this.statOps.getFileEntry(normalizedPath); + + let bytes: Uint8Array; + + if (fileEntry && fileEntry.size > MAX_CONTENTS_SIZE) { + // Large file: use Blob API + bytes = await this.readLargeFileBytes(fileEntry.sha); + } else { + // Normal file: use Contents API + bytes = await this.readContentsFileBytes(normalizedPath); + } + + // Cache the bytes + this.cache.set(cacheKey, bytes); + + return bytes; + } + + /** + * Read file using Contents API + */ + private async readContentsFile(path: string): Promise { + try { + const response = await this.client.getContents(path); + + // Handle array response (directory) + if (Array.isArray(response)) { + throw toError( + createError({ + type: "file", + message: `Path is a directory: ${path}`, + }), + ); + } + + const item = response as GitHubContentItem; + + // Check type + if (item.type !== "file") { + throw toError( + createError({ + type: "file", + message: `Not a file: ${path} (type: ${item.type})`, + }), + ); + } + + // Decode content + if (!item.content) { + throw toError( + createError({ + type: "file", + message: `File has no content: ${path}`, + }), + ); + } + + return this.decodeBase64(item.content); + } catch (error) { + // Re-throw with more context if needed + if ( + error instanceof Error && + (error as Error & { statusCode?: number }).statusCode === 404 + ) { + throw toError( + createError({ + type: "file", + message: `File not found: ${path}`, + context: { + path, + operation: "read", + }, + }), + ); + } + throw error; + } + } + + /** + * Read large file using Blob API + */ + private async readLargeFile(sha: string): Promise { + // Check blob cache (blobs are immutable, cache forever) + const blobCacheKey = `github:blob:${sha}`; + const cachedBlob = this.cache.get(blobCacheKey); + if (cachedBlob !== undefined) { + return cachedBlob; + } + + logger.debug(`${LOG_PREFIX} Reading large file via Blob API`, { sha }); + + const blob = await this.client.getBlob(sha); + + const content = blob.encoding === "base64" ? this.decodeBase64(blob.content) : blob.content; + + // Cache blob (immutable content, uses default TTL) + this.cache.set(blobCacheKey, content); + + return content; + } + + /** + * Read file as bytes using Contents API + */ + private async readContentsFileBytes(path: string): Promise { + try { + const response = await this.client.getContents(path); + + if (Array.isArray(response)) { + throw toError( + createError({ + type: "file", + message: `Path is a directory: ${path}`, + }), + ); + } + + const item = response as GitHubContentItem; + + if (item.type !== "file") { + throw toError( + createError({ + type: "file", + message: `Not a file: ${path} (type: ${item.type})`, + }), + ); + } + + if (!item.content) { + throw toError( + createError({ + type: "file", + message: `File has no content: ${path}`, + }), + ); + } + + return this.decodeBase64ToBytes(item.content); + } catch (error) { + if ( + error instanceof Error && + (error as Error & { statusCode?: number }).statusCode === 404 + ) { + throw toError( + createError({ + type: "file", + message: `File not found: ${path}`, + context: { path, operation: "read" }, + }), + ); + } + throw error; + } + } + + /** + * Read large file as bytes using Blob API + */ + private async readLargeFileBytes(sha: string): Promise { + const blobCacheKey = `github:blob:bytes:${sha}`; + const cachedBlob = this.cache.get(blobCacheKey); + if (cachedBlob !== undefined) { + return cachedBlob; + } + + logger.debug(`${LOG_PREFIX} Reading large file via Blob API`, { sha }); + + const blob = await this.client.getBlob(sha); + const bytes = blob.encoding === "base64" + ? this.decodeBase64ToBytes(blob.content) + : new TextEncoder().encode(blob.content); + + this.cache.set(blobCacheKey, bytes); + return bytes; + } + + private decodeBase64ToBytes(content: string): Uint8Array { + const binaryString = atob(content.replace(/\n/g, "")); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; + } + + private decodeBase64(content: string): string { + return new TextDecoder().decode(this.decodeBase64ToBytes(content)); + } + + private normalizePath(path: string): string { + let normalized = path; + if (this.projectDir && normalized.startsWith(this.projectDir)) { + normalized = normalized.slice(this.projectDir.length); + } + return normalized.replace(/^\/+/, "").replace(/\/+$/, "").replace(/\/+/g, "/"); + } +} diff --git a/src/platform/adapters/github-fs-adapter/stat-operations.ts b/src/platform/adapters/github-fs-adapter/stat-operations.ts new file mode 100644 index 0000000000..8fca6bea2b --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/stat-operations.ts @@ -0,0 +1,393 @@ +import { createError, toError } from "../../../core/errors/veryfront-error.ts"; +import { logger } from "@veryfront/utils"; +import type { FileCache } from "../file-cache/file-cache.ts"; +import type { GitHubAPIClient } from "./github-api-client.ts"; +import type { FileIndexEntry, FileInfo, GitHubTreeEntry, ResolvedGitHubConfig } from "./types.ts"; + +const LOG_PREFIX = "[GitHubStatOperations]"; + +/** Extensions to try when resolving files */ +const RESOLVE_EXTENSIONS = [".tsx", ".ts", ".jsx", ".js", ".mdx", ".md"]; + +/** + * Handles file stat operations and index building for GitHub adapter + */ +export class GitHubStatOperations { + private readonly config: ResolvedGitHubConfig; + private readonly client: GitHubAPIClient; + private readonly cache: FileCache; + private readonly projectDir: string; + + /** File index built from tree */ + private fileIndex: Map = new Map(); + + /** Directory index (set of directory paths) */ + private directoryIndex: Set = new Set(); + + /** Promise guard for concurrent index building */ + private buildingIndex: Promise | null = null; + + /** Whether index has been built */ + private indexBuilt = false; + + constructor( + config: ResolvedGitHubConfig, + client: GitHubAPIClient, + cache: FileCache, + projectDir: string = "", + ) { + this.config = config; + this.client = client; + this.cache = cache; + this.projectDir = projectDir; + } + + /** + * Build the file index from GitHub tree + */ + async buildIndex(): Promise { + // Return existing promise if already building + if (this.buildingIndex) { + return this.buildingIndex; + } + + // Skip if already built + if (this.indexBuilt) { + return; + } + + this.buildingIndex = this.doBuildIndex(); + + try { + await this.buildingIndex; + } finally { + this.buildingIndex = null; + } + } + + /** + * Internal index building logic + */ + private async doBuildIndex(): Promise { + const cacheKey = `github:tree:${this.client.repoId}:${this.config.ref}`; + + // Try cache first + const cached = this.cache.get(cacheKey); + if (cached) { + logger.debug(`${LOG_PREFIX} Using cached tree`); + this.buildIndexFromEntries(cached); + this.indexBuilt = true; + return; + } + + // Fetch from API + logger.info(`${LOG_PREFIX} Fetching repository tree`, { + repo: this.client.repoId, + ref: this.config.ref, + }); + + const tree = await this.client.getTree(); + + // Cache tree entries + this.cache.set(cacheKey, tree.tree); + + this.buildIndexFromEntries(tree.tree); + this.indexBuilt = true; + + logger.info(`${LOG_PREFIX} Index built`, { + files: this.fileIndex.size, + directories: this.directoryIndex.size, + }); + } + + /** + * Build indexes from tree entries + */ + private buildIndexFromEntries(entries: GitHubTreeEntry[]): void { + this.fileIndex.clear(); + this.directoryIndex.clear(); + + // Root is always a directory + this.directoryIndex.add(""); + + for (const entry of entries) { + if (entry.type === "blob") { + // Add file to index + this.fileIndex.set(entry.path, { + path: entry.path, + sha: entry.sha, + size: entry.size ?? 0, + type: "blob", + }); + + // Build directory hierarchy + this.addDirectoryHierarchy(entry.path); + } else if (entry.type === "tree") { + // Add directory + this.directoryIndex.add(entry.path); + } + } + } + + /** + * Add all parent directories for a file path + */ + private addDirectoryHierarchy(filePath: string): void { + const parts = filePath.split("/"); + let current = ""; + + for (let i = 0; i < parts.length - 1; i++) { + const part = parts[i]; + if (part) { + current = current ? `${current}/${part}` : part; + this.directoryIndex.add(current); + } + } + } + + /** + * Get file stat information + */ + async stat(path: string): Promise { + await this.ensureIndex(); + + const normalizedPath = this.normalizePath(path); + + logger.debug(`${LOG_PREFIX} stat called`, { + inputPath: path, + normalizedPath, + projectDir: this.projectDir, + indexSize: this.fileIndex.size, + }); + + // Check cache + const cacheKey = `github:stat:${this.config.ref}:${normalizedPath}`; + const cached = this.cache.get(cacheKey); + if (cached) { + return cached; + } + + // Check file index + const fileEntry = this.fileIndex.get(normalizedPath); + if (fileEntry) { + const info: FileInfo = { + isFile: true, + isDirectory: false, + isSymlink: false, + size: fileEntry.size, + mtime: null, // GitHub doesn't provide mtime in tree/contents API + }; + this.cache.set(cacheKey, info); + return info; + } + + // Check directory index + if (this.directoryIndex.has(normalizedPath)) { + const info: FileInfo = { + isFile: false, + isDirectory: true, + isSymlink: false, + size: 0, + mtime: null, + }; + this.cache.set(cacheKey, info); + return info; + } + + // Not found + logger.debug(`${LOG_PREFIX} File not found`, { + path: normalizedPath, + indexSize: this.fileIndex.size, + }); + + throw toError( + createError({ + type: "file", + message: `File not found: ${normalizedPath}`, + context: { + path: normalizedPath, + operation: "read", + }, + }), + ); + } + + /** + * Check if a file or directory exists + */ + async exists(path: string): Promise { + try { + await this.stat(path); + return true; + } catch { + return false; + } + } + + /** + * Resolve a file path, trying various extensions + */ + async resolveFile(basePath: string): Promise { + await this.ensureIndex(); + + const normalizedPath = this.normalizePath(basePath); + + // Check cache + const cacheKey = `github:resolve:${this.config.ref}:${normalizedPath}`; + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + + // Try exact match + if (this.fileIndex.has(normalizedPath)) { + this.cache.set(cacheKey, normalizedPath); + return normalizedPath; + } + + // Try with extensions + for (const ext of RESOLVE_EXTENSIONS) { + const pathWithExt = normalizedPath + ext; + if (this.fileIndex.has(pathWithExt)) { + this.cache.set(cacheKey, pathWithExt); + return pathWithExt; + } + } + + // Try index files + const indexPaths = RESOLVE_EXTENSIONS.map( + (ext) => `${normalizedPath}/index${ext}`, + ); + for (const indexPath of indexPaths) { + if (this.fileIndex.has(indexPath)) { + this.cache.set(cacheKey, indexPath); + return indexPath; + } + } + + // Try with pages/ prefix + if (!normalizedPath.startsWith("pages/")) { + const withPages = `pages/${normalizedPath}`; + + // Try exact + if (this.fileIndex.has(withPages)) { + this.cache.set(cacheKey, withPages); + return withPages; + } + + // Try with extensions + for (const ext of RESOLVE_EXTENSIONS) { + const pathWithExt = withPages + ext; + if (this.fileIndex.has(pathWithExt)) { + this.cache.set(cacheKey, pathWithExt); + return pathWithExt; + } + } + + // Try index files + for (const ext of RESOLVE_EXTENSIONS) { + const indexPath = `${withPages}/index${ext}`; + if (this.fileIndex.has(indexPath)) { + this.cache.set(cacheKey, indexPath); + return indexPath; + } + } + } + + // Not found + this.cache.set(cacheKey, null); + return null; + } + + /** + * Get file entry from index + */ + getFileEntry(path: string): FileIndexEntry | undefined { + return this.fileIndex.get(this.normalizePath(path)); + } + + /** + * Get all files in a directory + */ + getFilesInDirectory(dirPath: string): FileIndexEntry[] { + const normalizedDir = this.normalizePath(dirPath); + const prefix = normalizedDir ? `${normalizedDir}/` : ""; + const files: FileIndexEntry[] = []; + + for (const [path, entry] of this.fileIndex) { + if (path.startsWith(prefix)) { + // Only include direct children + const relativePath = path.slice(prefix.length); + if (!relativePath.includes("/")) { + files.push(entry); + } + } + } + + return files; + } + + /** + * Get subdirectories of a directory + */ + getSubdirectories(dirPath: string): string[] { + const normalizedDir = this.normalizePath(dirPath); + const prefix = normalizedDir ? `${normalizedDir}/` : ""; + const subdirs: Set = new Set(); + + for (const dir of this.directoryIndex) { + if (dir.startsWith(prefix) && dir !== normalizedDir) { + const relativePath = dir.slice(prefix.length); + const firstPart = relativePath.split("/")[0]; + if (firstPart) { + subdirs.add(firstPart); + } + } + } + + return Array.from(subdirs); + } + + /** + * Check if directory exists + */ + isDirectory(path: string): boolean { + return this.directoryIndex.has(this.normalizePath(path)); + } + + /** + * Clear the index + */ + clearIndex(): void { + this.fileIndex.clear(); + this.directoryIndex.clear(); + this.indexBuilt = false; + this.buildingIndex = null; + } + + /** + * Ensure index is built + */ + private async ensureIndex(): Promise { + if (!this.indexBuilt) { + await this.buildIndex(); + } + } + + /** + * Normalize a file path, stripping projectDir prefix if present + */ + private normalizePath(path: string): string { + let normalized = path; + + // Strip projectDir prefix if present (handles absolute paths from renderer) + if (this.projectDir && normalized.startsWith(this.projectDir)) { + normalized = normalized.slice(this.projectDir.length); + } + + return normalized + .replace(/^\/+/, "") // Remove leading slashes + .replace(/\/+$/, "") // Remove trailing slashes + .replace(/\/+/g, "/"); // Collapse multiple slashes + } +} diff --git a/src/platform/adapters/github-fs-adapter/types.ts b/src/platform/adapters/github-fs-adapter/types.ts new file mode 100644 index 0000000000..41e3cff0b0 --- /dev/null +++ b/src/platform/adapters/github-fs-adapter/types.ts @@ -0,0 +1,165 @@ +import { createError, toError } from "../../../core/errors/veryfront-error.ts"; +import type { DirectoryEntry } from "../veryfront-fs-adapter/types.ts"; + +// Re-export DirectoryEntry for convenience +export type { DirectoryEntry }; + +/** + * GitHub repository configuration + */ +export interface GitHubConfig { + /** Personal Access Token for GitHub API authentication */ + token: string; + /** Repository owner (user or organization) */ + owner: string; + /** Repository name */ + repo: string; + /** Branch, tag, or commit SHA (default: "main") */ + ref?: string; + /** Cache configuration */ + cache?: { + enabled?: boolean; + /** TTL in milliseconds (default: 60000) */ + ttl?: number; + /** Max entries (default: 1000) */ + maxSize?: number; + /** Max memory in bytes (default: 100MB) */ + maxMemory?: number; + }; + /** Retry configuration */ + retry?: { + maxRetries?: number; + initialDelay?: number; + maxDelay?: number; + }; +} + +/** + * Resolved GitHub configuration with defaults applied + */ +export interface ResolvedGitHubConfig { + token: string; + owner: string; + repo: string; + ref: string; + cache: { + enabled: boolean; + ttl: number; + maxSize: number; + maxMemory: number; + }; + retry: { + maxRetries: number; + initialDelay: number; + maxDelay: number; + }; +} + +/** + * GitHub tree entry from Git Trees API + */ +export interface GitHubTreeEntry { + path: string; + mode: string; + type: "blob" | "tree" | "commit"; + sha: string; + size?: number; +} + +/** + * GitHub tree response + */ +export interface GitHubTreeResponse { + sha: string; + url: string; + tree: GitHubTreeEntry[]; + truncated: boolean; +} + +/** + * GitHub content item from Contents API + */ +export interface GitHubContentItem { + type: "file" | "dir" | "symlink" | "submodule"; + name: string; + path: string; + sha: string; + size: number; + content?: string; + encoding?: "base64"; + download_url?: string | null; +} + +/** + * GitHub blob response + */ +export interface GitHubBlobResponse { + sha: string; + size: number; + content: string; + encoding: "base64" | "utf-8"; +} + +/** + * File info structure matching FSAdapter.stat return type + */ +export interface FileInfo { + isFile: boolean; + isDirectory: boolean; + isSymlink: boolean; + size: number; + mtime: Date | null; +} + +/** + * Internal file index entry + */ +export interface FileIndexEntry { + path: string; + sha: string; + size: number; + type: "blob" | "tree"; +} + +/** + * Create resolved config with defaults from raw GitHubConfig + */ +export function createGitHubConfig(config: GitHubConfig): ResolvedGitHubConfig { + if (!config.token) { + throw toError( + createError({ + type: "config", + message: + "GitHub adapter requires a token. Set GITHUB_TOKEN environment variable or provide token in config.", + }), + ); + } + + if (!config.owner || !config.repo) { + throw toError( + createError({ + type: "config", + message: + "GitHub adapter requires owner and repo. Provide them in config or via GITHUB_OWNER and GITHUB_REPO environment variables.", + }), + ); + } + + return { + token: config.token, + owner: config.owner, + repo: config.repo, + ref: config.ref || "main", + cache: { + enabled: config.cache?.enabled ?? true, + ttl: config.cache?.ttl ?? 60_000, + maxSize: config.cache?.maxSize ?? 1000, + maxMemory: config.cache?.maxMemory ?? 100 * 1024 * 1024, + }, + retry: { + maxRetries: config.retry?.maxRetries ?? 3, + initialDelay: config.retry?.initialDelay ?? 1000, + maxDelay: config.retry?.maxDelay ?? 10000, + }, + }; +} diff --git a/src/platform/adapters/veryfront-fs-adapter/types.ts b/src/platform/adapters/veryfront-fs-adapter/types.ts index 0f58f413c6..5bf20ed5df 100644 --- a/src/platform/adapters/veryfront-fs-adapter/types.ts +++ b/src/platform/adapters/veryfront-fs-adapter/types.ts @@ -1,5 +1,6 @@ import type { Project } from "../veryfront-api-client.ts"; import { createError, toError } from "../../../core/errors/veryfront-error.ts"; +import type { GitHubConfig } from "../github-fs-adapter/types.ts"; export interface DirectoryEntry { name: string; @@ -30,7 +31,7 @@ export interface FSAdapter { } export interface FSAdapterConfig { - type?: "local" | "veryfront-api" | "memory"; + type?: "local" | "veryfront-api" | "memory" | "github"; projectDir?: string; veryfront?: { apiKey?: string; @@ -50,6 +51,7 @@ export interface FSAdapterConfig { retryDelay?: number; }; }; + github?: GitHubConfig; } export interface VeryfrontConfig { diff --git a/src/server/dev-server/route-discovery.ts b/src/server/dev-server/route-discovery.ts index 98554ef6c3..e7f36d004e 100644 --- a/src/server/dev-server/route-discovery.ts +++ b/src/server/dev-server/route-discovery.ts @@ -8,18 +8,30 @@ import { withFallback } from "@veryfront/platform/adapters/index.ts"; import { createFileSystem } from "../../platform/compat/fs.ts"; export class RouteDiscovery { + private useRelativePaths: boolean; + constructor( private projectDir: string, private adapter: RuntimeAdapter, private router: DynamicRouter, private config?: VeryfrontConfig, - ) {} + ) { + // For remote FS adapters (github, veryfront-api), use relative paths + const fsType = config?.fs?.type; + this.useRelativePaths = fsType === "github" || fsType === "veryfront-api"; + } async discoverRoutes(): Promise { this.router.clear(); this.router.clearCache(); + logger.debug("[SERVER] Starting route discovery", { + useRelativePaths: this.useRelativePaths, + fsType: this.config?.fs?.type, + }); + const routeDirs = await this.resolveRouteDirectories(); + logger.debug("[SERVER] Route directories resolved", { count: routeDirs.length, dirs: routeDirs }); if (routeDirs.length === 0) { logger.warn("[SERVER] No route directories found; skipping discovery"); return; @@ -54,14 +66,15 @@ export class RouteDiscovery { ]; for (const candidate of candidates) { - const absolute = join(this.projectDir, candidate.dir); - if (await this.directoryExists(absolute)) { - results.push({ type: candidate.type, path: absolute }); + // For remote FS adapters, use relative paths; for local, use absolute + const pathToCheck = this.useRelativePaths ? candidate.dir : join(this.projectDir, candidate.dir); + if (await this.directoryExists(pathToCheck)) { + results.push({ type: candidate.type, path: pathToCheck }); } } if (results.length === 0 && preferredRouter === "app") { - const pagesFallback = join(this.projectDir, "pages"); + const pagesFallback = this.useRelativePaths ? "pages" : join(this.projectDir, "pages"); if (await this.directoryExists(pagesFallback)) { logger.warn('[SERVER] router="app" but app/ directory missing; falling back to pages/'); results.push({ type: "pages", path: pagesFallback }); @@ -69,7 +82,7 @@ export class RouteDiscovery { } if (results.length === 0 && preferredRouter === "pages") { - const appFallback = join(this.projectDir, "app"); + const appFallback = this.useRelativePaths ? "app" : join(this.projectDir, "app"); if (await this.directoryExists(appFallback)) { logger.warn('[SERVER] router="pages" but pages/ directory missing; using app/'); results.push({ type: "app", path: appFallback }); @@ -78,8 +91,8 @@ export class RouteDiscovery { if (results.length === 0 && preferredRouter === undefined) { const fallbackDirs = [ - { type: "app" as const, path: join(this.projectDir, "app") }, - { type: "pages" as const, path: join(this.projectDir, "pages") }, + { type: "app" as const, path: this.useRelativePaths ? "app" : join(this.projectDir, "app") }, + { type: "pages" as const, path: this.useRelativePaths ? "pages" : join(this.projectDir, "pages") }, ]; for (const fallback of fallbackDirs) { if (await this.directoryExists(fallback.path)) { @@ -93,13 +106,22 @@ export class RouteDiscovery { private async directoryExists(path: string): Promise { try { + logger.debug("[SERVER] Checking directory exists", { path, useRelativePaths: this.useRelativePaths }); + // For remote FS adapters, don't fall back to local filesystem + if (this.useRelativePaths) { + const stat = await this.adapter.fs.stat(path); + logger.debug("[SERVER] Directory stat result", { path, isDirectory: stat.isDirectory }); + return stat.isDirectory; + } + // For local filesystem, use fallback const stat = await withFallback( () => this.adapter.fs.stat(path), () => createFileSystem().stat(path), { operationName: "stat:routeDiscovery:directoryExists", logError: false }, ); return stat.isDirectory; - } catch { + } catch (error) { + logger.debug("[SERVER] Directory check failed", { path, error: String(error) }); return false; } } @@ -185,6 +207,10 @@ export class RouteDiscovery { } private toProjectRelativePath(fullPath: string): string { + // For remote FS adapters, paths are already relative + if (this.useRelativePaths) { + return fullPath; + } return fullPath.startsWith(this.projectDir) ? fullPath.slice(this.projectDir.length + 1) : fullPath; diff --git a/src/server/handlers/request/static.ts b/src/server/handlers/request/static.ts index 782497d1d7..8eca18d81e 100644 --- a/src/server/handlers/request/static.ts +++ b/src/server/handlers/request/static.ts @@ -168,9 +168,16 @@ export class StaticHandler extends BaseHandler { pushCandidate(abs, dir); } + this.logDebug(`Trying static file candidates`, { + reqPath, + candidateCount: candidates.length, + candidates: candidates.map(c => ({ source: c.source, path: c.abs })), + }, ctx); + for (const candidate of candidates) { try { // Use secure filesystem wrapper (automatic path validation) + this.logDebug(`Checking candidate`, { path: candidate.abs, source: candidate.source }, ctx); const info = await secureFs.stat(candidate.abs); if (!info.isFile) continue; diff --git a/veryfront.config.ts b/veryfront.config.ts index 1fccbdb945..6700529683 100644 --- a/veryfront.config.ts +++ b/veryfront.config.ts @@ -66,7 +66,22 @@ if (proxyMode) { console.log(`[Config] API Base URL: ${apiBaseUrl}`); } -export default { +// GitHub test mode: Use GITHUB_TOKEN env var to enable +const useGitHub = !!Deno.env.get("GITHUB_TOKEN"); + +export default useGitHub ? { + fs: { + type: "github" as const, + github: { + token: Deno.env.get("GITHUB_TOKEN"), + owner: "veryfront", + repo: "codersociety", + ref: "main", + cache: { enabled: true, ttl: 60000 }, + }, + }, + dev: { port: 3001, host: "localhost", hmr: false }, +} : { // Use veryfront-api filesystem adapter fs: { type: "veryfront-api" as const,