From e029e3ac04ee1f66be4da3d9144db01d01d32049 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Sat, 12 Sep 2026 23:39:07 +0400 Subject: [PATCH 01/17] feat: add a version registry that keeps release lists between runs fix: request every releases page once with `per_page=100` fix: stop `destroy()` from loading the remaining release pages Every run asked GitHub or GitLab for the release list and spent the API rate limit on it. Releases are now kept in a local version registry: a provider-neutral database of the releases and assets of every known repository, one JSON file per repository, on by default in the per-user cache directory. Versions never expire; only the last check of a repository has a TTL (`cache-ttl`, default 600 s), within which `dload get` costs no API request. A stale check fetches only the newest pages until a stored release is reached. Pages stay lazy: the first run loads what the requested version needs, older releases are fetched on demand and appended. A failed check falls back to the stored releases. `dload get --refresh` ignores the TTL once, `dload cache:clear [software...]` drops records. The page loader used to build a paginator per page and probe the next one, so every page but the first was requested twice; `destroy()` iterated the whole lazy collection and loaded every remaining page after each download. Co-Authored-By: Dmitriy Derepko Assisted-By: Claude Fable 5.1 --- README-es.md | 65 ++++++ README-ru.md | 66 ++++++ README-zh.md | 59 +++++ README.md | 68 ++++++ bin/dload | 1 + dload.xsd | 10 + src/Bootstrap.php | 34 +++ src/Command/CacheClear.php | 92 ++++++++ src/Command/Get.php | 9 + src/Module/Config/Schema/Cache.php | 43 ++++ src/Module/Downloader/Downloader.php | 6 + .../Registry/Internal/CacheDirectory.php | 57 +++++ .../Registry/Internal/FileRegistryStorage.php | 148 ++++++++++++ .../Registry/Internal/PassThroughRegistry.php | 32 +++ .../Internal/StoredVersionRegistry.php | 178 ++++++++++++++ src/Module/Registry/Record/AssetRecord.php | 69 ++++++ src/Module/Registry/Record/ReleasePage.php | 23 ++ src/Module/Registry/Record/ReleaseRecord.php | 77 ++++++ .../Registry/Record/RepositoryRecord.php | 213 +++++++++++++++++ src/Module/Registry/RegistryStorage.php | 46 ++++ src/Module/Registry/ReleaseSource.php | 29 +++ src/Module/Registry/RepositoryId.php | 48 ++++ src/Module/Registry/VersionRegistry.php | 44 ++++ .../Repository/Internal/CachedGenerator.php | 10 + src/Module/Repository/Internal/Collection.php | 12 + .../Internal/GitHub/Api/RepositoryApi.php | 138 +++++------ .../GitHub/Api/Response/AssetInfo.php | 15 ++ .../GitHub/Api/Response/ReleaseInfo.php | 17 ++ .../Repository/Internal/GitHub/Factory.php | 4 +- .../Internal/GitHub/GitHubAsset.php | 8 +- .../Internal/GitHub/GitHubRelease.php | 19 +- .../Internal/GitHub/GitHubReleaseSource.php | 34 +++ .../Internal/GitHub/GitHubRepository.php | 52 +++-- .../Internal/GitLab/Api/RepositoryApi.php | 142 +++++------ .../GitLab/Api/Response/AssetInfo.php | 10 + .../GitLab/Api/Response/ReleaseInfo.php | 17 ++ .../Repository/Internal/GitLab/Factory.php | 4 +- .../Internal/GitLab/GitLabAsset.php | 8 +- .../Internal/GitLab/GitLabRelease.php | 19 +- .../Internal/GitLab/GitLabReleaseSource.php | 34 +++ .../Internal/GitLab/GitLabRepository.php | 54 +++-- tests/Acceptance/DLoadTest.php | 6 +- .../Registry/VersionRegistryBindingTest.php | 92 ++++++++ .../Module/Registry/CacheDirectoryTest.php | 36 +++ .../Registry/FileRegistryStorageTest.php | 116 +++++++++ .../Module/Registry/RepositoryRecordTest.php | 140 +++++++++++ .../Registry/StoredVersionRegistryTest.php | 220 ++++++++++++++++++ .../Registry/Stub/ArrayReleaseSource.php | 81 +++++++ .../Registry/Stub/InMemoryRegistryStorage.php | 52 +++++ .../Internal/GitHub/GitHubRepositoryTest.php | 152 ++++++++++++ .../Internal/GitHub/Stub/PagedClientStub.php | 105 +++++++++ .../Internal/GitLab/FactoryTest.php | 8 +- .../Internal/GitLab/GitLabRepositoryTest.php | 114 +++++++++ .../Internal/GitLab/Stub/PagedClientStub.php | 106 +++++++++ 54 files changed, 3043 insertions(+), 199 deletions(-) create mode 100644 src/Command/CacheClear.php create mode 100644 src/Module/Config/Schema/Cache.php create mode 100644 src/Module/Registry/Internal/CacheDirectory.php create mode 100644 src/Module/Registry/Internal/FileRegistryStorage.php create mode 100644 src/Module/Registry/Internal/PassThroughRegistry.php create mode 100644 src/Module/Registry/Internal/StoredVersionRegistry.php create mode 100644 src/Module/Registry/Record/AssetRecord.php create mode 100644 src/Module/Registry/Record/ReleasePage.php create mode 100644 src/Module/Registry/Record/ReleaseRecord.php create mode 100644 src/Module/Registry/Record/RepositoryRecord.php create mode 100644 src/Module/Registry/RegistryStorage.php create mode 100644 src/Module/Registry/ReleaseSource.php create mode 100644 src/Module/Registry/RepositoryId.php create mode 100644 src/Module/Registry/VersionRegistry.php create mode 100644 src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php create mode 100644 src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php create mode 100644 tests/Integration/Module/Registry/VersionRegistryBindingTest.php create mode 100644 tests/Unit/Module/Registry/CacheDirectoryTest.php create mode 100644 tests/Unit/Module/Registry/FileRegistryStorageTest.php create mode 100644 tests/Unit/Module/Registry/RepositoryRecordTest.php create mode 100644 tests/Unit/Module/Registry/StoredVersionRegistryTest.php create mode 100644 tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php create mode 100644 tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php create mode 100644 tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php create mode 100644 tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php create mode 100644 tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php create mode 100644 tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php diff --git a/README-es.md b/README-es.md index 47426d0..9c9fc35 100644 --- a/README-es.md +++ b/README-es.md @@ -48,6 +48,7 @@ Con DLoad puedes: - [Tipos de Descarga](#tipos-de-descarga) - [Restricciones de Versión](#restricciones-de-versión) - [Opciones de Configuración Avanzadas](#opciones-de-configuración-avanzadas) + - [Registro de Versiones](#registro-de-versiones) - [Construir RoadRunner Personalizado](#construir-roadrunner-personalizado) - [Configuración de Acción de Construcción](#configuración-de-acción-de-construcción) - [Atributos de Acción Velox](#atributos-de-acción-velox) @@ -348,6 +349,70 @@ Usa restricciones de versión estilo Composer: ``` +### Registro de Versiones + +Resolver una versión significa pedir a GitHub o GitLab la lista de releases del repositorio. DLoad +guarda lo que aprende en un **registro de versiones** local: una pequeña base de datos con los releases +y assets de cada repositorio conocido, un archivo JSON por repositorio. Las versiones nunca expiran. +Lo que expira es la *última comprobación* del repositorio: mientras sea más reciente que `cache-ttl`, +`dload get` se responde desde el registro sin una sola petición a la API. Cuando es más antigua, DLoad +pide a la API solo los releases publicados desde entonces, normalmente una única petición. + +Las páginas de releases se siguen cargando de forma perezosa. La primera ejecución obtiene solo las +páginas necesarias para encontrar un release que cumpla la versión pedida; los releases más antiguos se +cargan después, bajo demanda. + +El registro está activado por defecto y vive en el directorio de caché del usuario +(`$XDG_CACHE_HOME/dload`, `%LOCALAPPDATA%\dload\cache` en Windows, `~/.cache/dload` en otros casos): + +```xml + + + + + +``` + +| Atributo | Variable de entorno | Por defecto | Significado | +|-------------|---------------------|--------------------------------|--------------------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | directorio de caché del usuario | Directorio del registro de versiones. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Segundos que sigue siendo válida la última comprobación. `0` desactiva el registro. | + +```bash +# Comprobar si hay nuevos releases aunque la última comprobación siga vigente +./vendor/bin/dload get rr --refresh + +# Olvidar los repositorios de un software, o todo el registro +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> El registro solo contiene metadatos de releases: tags, nombres y enlaces de descarga. Las descargas +> no pasan por él y nunca guarda credenciales, así que el directorio puede compartirse o guardarse en +> la caché de CI sin problemas. Si una comprobación falla por un error de red o un límite de la API, se +> usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. + +En GitHub Actions el directorio puede conservarse entre ejecuciones del workflow, de modo que cada +ejecución gasta el límite de la API solo en los releases publicados desde la anterior: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +El `github.run_id` en la clave hace que cada ejecución guarde su registro, y `restore-keys` permite +que la siguiente parta del más reciente. Los jobs paralelos de un mismo workflow no ven la caché de los +demás, ya que `actions/cache` la guarda al terminar cada job. + ## Construir RoadRunner Personalizado DLoad soporta la construcción de binarios personalizados de RoadRunner usando la herramienta Velox. Esto es útil cuando necesitas RoadRunner con combinaciones específicas de plugins que no están disponibles en las versiones pre-construidas. diff --git a/README-ru.md b/README-ru.md index eea812f..b9a21fe 100644 --- a/README-ru.md +++ b/README-ru.md @@ -49,6 +49,7 @@ DLoad решает распространённую проблему в PHP-пр - [Типы загрузки](#типы-загрузки) - [Ограничения версий](#ограничения-версий) - [Расширенные настройки](#расширенные-настройки) + - [Реестр версий](#реестр-версий) - [Сборка кастомного RoadRunner](#сборка-кастомного-roadrunner) - [Настройка действия сборки](#настройка-действия-сборки) - [Атрибуты Velox-действия](#атрибуты-velox-действия) @@ -349,6 +350,71 @@ DLoad поддерживает три типа загрузки, которые ``` +### Реестр версий + +Чтобы определить версию, DLoad запрашивает у GitHub или GitLab список релизов репозитория. Всё, +что он узнаёт, сохраняется в локальном **реестре версий**: небольшой базе релизов и ассетов каждого +известного репозитория, по одному JSON-файлу на репозиторий. Версии из реестра не устаревают. +Устаревает только *последняя проверка* репозитория: пока она моложе `cache-ttl`, `dload get` +отвечает из реестра без единого запроса к API. Когда проверка устарела, DLoad запрашивает у API +только релизы, вышедшие после неё, и обычно это один запрос. + +Страницы релизов по-прежнему загружаются лениво. Первый запуск получает столько страниц, сколько +нужно, чтобы найти релиз под запрошенную версию, а более старые релизы догружаются позже, по мере +надобности. + +Реестр включён по умолчанию и живёт в пользовательском каталоге кэша (`$XDG_CACHE_HOME/dload`, +`%LOCALAPPDATA%\dload\cache` в Windows, иначе `~/.cache/dload`): + +```xml + + + + + +``` + +| Атрибут | Переменная окружения | По умолчанию | Значение | +|-------------|----------------------|---------------------------|---------------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | каталог кэша пользователя | Каталог реестра версий. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Сколько секунд действует последняя проверка репозитория. `0` отключает реестр. | + +```bash +# Проверить репозитории на новые релизы, даже если последняя проверка ещё свежая +./vendor/bin/dload get rr --refresh + +# Забыть репозитории, из которых берётся программа, или весь реестр целиком +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> В реестре хранятся только метаданные релизов: теги, имена и ссылки на ассеты. Загрузки через него +> не проходят, учётные данные в нём не сохраняются, поэтому каталог можно свободно передавать между +> машинами и складывать в кэш CI. Если проверка не удалась из-за сетевой ошибки или лимита API, +> используются сохранённые релизы, а репозиторий, который раньше не встречался, по-прежнему +> завершится ошибкой. + +В GitHub Actions каталог можно переносить между запусками workflow, тогда запуск тратит лимит API +только на релизы, вышедшие после предыдущего: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +`github.run_id` в ключе заставляет каждый запуск сохранять свой реестр, а `restore-keys` позволяет +следующему запуску начать с самого свежего. Параллельные джобы одного workflow кэш друг друга не +видят: `actions/cache` сохраняет его по завершении джобы. + ## Сборка кастомного RoadRunner DLoad поддерживает сборку кастомных бинарников RoadRunner с помощью инструмента сборки Velox. Это полезно когда нужен RoadRunner с определёнными комбинациями плагинов, которые недоступны в готовых релизах. diff --git a/README-zh.md b/README-zh.md index cd65fff..2b63606 100644 --- a/README-zh.md +++ b/README-zh.md @@ -48,6 +48,7 @@ DLoad 解决了 PHP 项目中的一个实际问题：如何在分发 PHP 代码 - [下载类型](#下载类型) - [版本约束](#版本约束) - [高级配置选项](#高级配置选项) + - [版本注册表](#版本注册表) - [构建自定义 RoadRunner](#构建自定义-roadrunner) - [构建动作配置](#构建动作配置) - [Velox 动作属性](#velox-动作属性) @@ -348,6 +349,64 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： ``` +### 版本注册表 + +解析版本意味着向 GitHub 或 GitLab 请求仓库的发布列表。DLoad 会把获取到的信息保存在本地的 +**版本注册表**中：这是一个小型数据库，记录每个已知仓库的发布版本和资产，每个仓库一个 JSON 文件。 +其中的版本永不过期，过期的只是仓库的*最近一次检查*：只要检查时间比 `cache-ttl` 更新，`dload get` +就直接从注册表返回结果，不会发出任何 API 请求。检查过期后，DLoad 只向 API 请求此后发布的版本， +通常只需一次请求。 + +发布页面仍然按需加载。首次运行只获取找到满足所需版本的发布所需的页面，更早的发布会在之后真正需要时再加载。 + +注册表默认启用，位于用户缓存目录（`$XDG_CACHE_HOME/dload`，Windows 下为 `%LOCALAPPDATA%\dload\cache`， +其他情况为 `~/.cache/dload`）： + +```xml + + + + + +``` + +| 属性 | 环境变量 | 默认值 | 含义 | +|-------------|--------------------|--------------|----------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | 用户缓存目录 | 版本注册表所在目录。 | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | 最近一次检查保持有效的秒数。`0` 表示禁用注册表。 | + +```bash +# 即使最近一次检查仍然有效，也强制检查仓库是否有新发布 +./vendor/bin/dload get rr --refresh + +# 忘记某个软件所使用的仓库，或清空整个注册表 +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> 注册表只保存发布的元数据：标签、名称和资产下载链接。下载不会经过注册表，也不会保存任何凭据， +> 因此该目录可以自由共享或放入 CI 缓存。若因网络错误或 API 速率限制导致检查失败，会使用已保存的发布； +> 从未见过的仓库仍会明确报错。 + +在 GitHub Actions 中可以在多次工作流运行之间保留该目录，这样每次运行只为上次运行之后发布的版本消耗速率限制： + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +键中的 `github.run_id` 使每次运行都保存自己的注册表，而 `restore-keys` 让下一次运行从最新的注册表开始。 +同一工作流中并行运行的作业彼此看不到缓存，因为 `actions/cache` 在作业结束时才保存缓存。 + ## 构建自定义 RoadRunner DLoad 支持使用 Velox 构建工具来构建自定义 RoadRunner 二进制文件。当你需要包含特定插件组合的 RoadRunner，而这些组合在预构建版本中不可用时，这功能就很有用了。 diff --git a/README.md b/README.md index 628c879..92f62e3 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ With DLoad, you can: - [Download Types](#download-types) - [Version Constraints](#version-constraints) - [Advanced Configuration Options](#advanced-configuration-options) + - [Version Registry](#version-registry) - [Building Custom RoadRunner](#building-custom-roadrunner) - [Build Action Configuration](#build-action-configuration) - [Velox Action Attributes](#velox-action-attributes) @@ -350,6 +351,70 @@ Use Composer-style version constraints: ``` +### Version Registry + +Resolving a version means asking GitHub or GitLab for the repository's release list. DLoad keeps +what it learns in a local **version registry**: a small database of the releases and assets every +known repository offers, one JSON file per repository. Versions never expire from it. What expires +is the *last check* of a repository: while the check is younger than `cache-ttl`, `dload get` is +answered from the registry without a single API request. When it is older, DLoad asks the API only +for the releases published since the last check, which is usually one request. + +Release pages are still loaded lazily. The first run fetches only as many pages as it takes to find +a release that satisfies the requested version, and older releases are fetched later, on demand, +when a run actually needs one of them. + +The registry is on by default and lives in the per-user cache directory (`$XDG_CACHE_HOME/dload`, +`%LOCALAPPDATA%\dload\cache` on Windows, `~/.cache/dload` otherwise): + +```xml + + + + + +``` + +| Attribute | Environment variable | Default | Meaning | +|-------------|----------------------|----------------------|------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | user cache directory | Directory of the version registry. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Seconds the last check of a repository stays valid. `0` disables the registry. | + +```bash +# Check the repositories for new releases even if the last check is still fresh +./vendor/bin/dload get rr --refresh + +# Forget the repositories a software package is served from, or the whole registry +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> The registry holds release metadata only: tags, names and asset download links. Downloads never +> go through it and credentials are never stored in it, so the directory can be shared or committed +> to a CI cache freely. When a check fails because of a network error or a rate limit, the stored +> releases are used instead, and a repository that was never seen before still fails loudly. + +In GitHub Actions the directory can be carried between workflow runs, so a run spends the rate limit +only on releases published since the previous one: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +The `github.run_id` in the key makes every workflow run save its registry, while `restore-keys` +lets the next run start from the most recent one. Jobs that run in parallel within one workflow do not +see each other's cache, since `actions/cache` saves it when a job ends. + ## Building Custom RoadRunner DLoad supports building custom RoadRunner binaries using the Velox build tool. This is useful when you need RoadRunner with custom plugin combinations that aren't available in pre-built releases. @@ -600,6 +665,9 @@ Add to CI/CD environment variables for automated downloads. > 1,000 requests per hour across all jobs of the repository. With a large job matrix the limit may run out, > and downloads from other repositories may be rejected. Use a personal access token if that happens. +Release lists are also kept in a local version registry, so repeated runs and runs that carry the +registry between them spend the rate limit only on new releases: see [Version Registry](#version-registry). + ## Failure Reporting `dload get` exits with a non-zero code when at least one requested package was not installed, and prints diff --git a/bin/dload b/bin/dload index 053b8e8..612ec56 100755 --- a/bin/dload +++ b/bin/dload @@ -51,6 +51,7 @@ use Symfony\Component\Console\CommandLoader\FactoryCommandLoader; Command\Show::getCommandName() => static fn() => new Command\Show(), Command\Init::getCommandName() => static fn() => new Command\Init(), Command\Build::getCommandName() => static fn() => new Command\Build(), + Command\CacheClear::getCommandName() => static fn() => new Command\CacheClear(), ]), ); $application->setDefaultCommand(Command\Get::getCommandName(), false); diff --git a/dload.xsd b/dload.xsd index 793255e..3246f4f 100644 --- a/dload.xsd +++ b/dload.xsd @@ -249,6 +249,16 @@ Temporary directory for downloads + + + Directory of the version registry (release lists database); the per-user cache directory when not set + + + + + Number of seconds the last check of a repository for new releases stays valid; 0 disables the version registry + + diff --git a/src/Bootstrap.php b/src/Bootstrap.php index 887153a..bdbc579 100644 --- a/src/Bootstrap.php +++ b/src/Bootstrap.php @@ -12,8 +12,15 @@ use Internal\DLoad\Module\Common\Internal\Injection\ConfigInflector; use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\Common\Stability; +use Internal\DLoad\Module\Config\Schema\Cache as CacheConfig; use Internal\DLoad\Module\HttpClient\Factory; use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; +use Internal\DLoad\Module\Registry\Internal\CacheDirectory; +use Internal\DLoad\Module\Registry\Internal\FileRegistryStorage; +use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; +use Internal\DLoad\Module\Registry\Internal\StoredVersionRegistry; +use Internal\DLoad\Module\Registry\RegistryStorage; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitHub\Factory as GithubRepositoryFactory; use Internal\DLoad\Module\Repository\Internal\GitLab\Factory as GitLabRepositoryFactory; use Internal\DLoad\Module\Repository\RepositoryProvider; @@ -21,6 +28,7 @@ use Internal\DLoad\Module\Velox\Builder; use Internal\DLoad\Module\Velox\Internal\Client\BuildRoadRunner; use Internal\DLoad\Module\Velox\Internal\VeloxBuilder; +use Internal\DLoad\Service\Logger; /** * Bootstraps the application by configuring the dependency container. @@ -113,6 +121,32 @@ public function withConfig( ->addRepositoryFactory($container->get(GithubRepositoryFactory::class)) ->addRepositoryFactory($container->get(GitLabRepositoryFactory::class)), ); + $this->container->bind( + RegistryStorage::class, + static function (Container $container) use ($environment): RegistryStorage { + $config = $container->get(CacheConfig::class); + + return new FileRegistryStorage( + $config->dir ?? CacheDirectory::resolve($environment), + $container->get(Logger::class), + ); + }, + ); + $this->container->bind( + VersionRegistry::class, + static function (Container $container): VersionRegistry { + $config = $container->get(CacheConfig::class); + + return $config->ttl <= 0 + ? new PassThroughRegistry() + : new StoredVersionRegistry( + $container->get(RegistryStorage::class), + $config->ttl, + $container->get(Logger::class), + $config->refresh, + ); + }, + ); $this->container->bind(BinaryProvider::class, BinaryProviderImpl::class); $this->container->bind(Factory::class, NyholmFactoryImpl::class); $this->container->bind(Builder::class, VeloxBuilder::class); diff --git a/src/Command/CacheClear.php b/src/Command/CacheClear.php new file mode 100644 index 0000000..c94d3b7 --- /dev/null +++ b/src/Command/CacheClear.php @@ -0,0 +1,92 @@ +addArgument( + self::ARG_SOFTWARE, + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'Software whose repositories must be forgotten, e.g. "rr", "dolt". Everything when omitted.', + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + parent::execute($input, $output); + + $storage = $this->container->get(RegistryStorage::class); + + /** @var list $software */ + $software = \array_values(\array_filter( + (array) $input->getArgument(self::ARG_SOFTWARE), + static fn(mixed $name): bool => \is_string($name) && $name !== '', + )); + + if ($software === []) { + $storage->clear(); + $output->writeln('The version registry has been cleared.'); + + return Command::SUCCESS; + } + + $removed = 0; + foreach ($this->recordsOf($storage, $software) as $record) { + $storage->remove($record->id); + $output->writeln(\sprintf('Forgot %s', OutputFormatter::escape((string) $record->id))); + ++$removed; + } + + $output->writeln(\sprintf('%d repository listing(s) removed.', $removed)); + + return Command::SUCCESS; + } + + /** + * @param list $software + * @return \Generator + */ + private function recordsOf(RegistryStorage $storage, array $software): \Generator + { + foreach ($storage->all() as $record) { + \array_intersect($record->software, $software) === [] or yield $record; + } + } +} diff --git a/src/Command/Get.php b/src/Command/Get.php index ac23e49..3beaecf 100644 --- a/src/Command/Get.php +++ b/src/Command/Get.php @@ -43,6 +43,9 @@ * * # Force download even if binary exists * ./vendor/bin/dload get rr --force + * + * # Check for new releases even if the version registry is still fresh + * ./vendor/bin/dload get rr --refresh * ``` * * @internal @@ -72,6 +75,12 @@ public function configure(): void $this->addOption('os', null, InputOption::VALUE_OPTIONAL, 'Operating system, e.g. "linux", "darwin" etc.'); $this->addOption('stability', null, InputOption::VALUE_OPTIONAL, 'Minimum stability, e.g. "rc", "beta" etc.'); $this->addOption('force', 'f', InputOption::VALUE_NONE, 'Force download even if binary exists'); + $this->addOption( + 'refresh', + null, + InputOption::VALUE_NONE, + 'Check repositories for new releases even if the version registry is still fresh', + ); } /** diff --git a/src/Module/Config/Schema/Cache.php b/src/Module/Config/Schema/Cache.php new file mode 100644 index 0000000..2fdcb22 --- /dev/null +++ b/src/Module/Config/Schema/Cache.php @@ -0,0 +1,43 @@ +repoConfig = \array_shift($repositories); $repository = $this->repositoryProvider->getByConfig($context->repoConfig); + + // The registry keeps track of which software is served from which repository + $this->registry->attach($context->software->getId(), RepositoryId::fromConfig($context->repoConfig)); $context->repositoryAttempt = $context->diagnostics->addRepository( type: $context->repoConfig->type, name: $repository->getName(), diff --git a/src/Module/Registry/Internal/CacheDirectory.php b/src/Module/Registry/Internal/CacheDirectory.php new file mode 100644 index 0000000..1b87644 --- /dev/null +++ b/src/Module/Registry/Internal/CacheDirectory.php @@ -0,0 +1,57 @@ + $env Environment variables. + * @return non-empty-string + */ + public static function resolve(array $env): string + { + $xdg = self::variable($env, 'XDG_CACHE_HOME'); + if ($xdg !== null) { + return $xdg . \DIRECTORY_SEPARATOR . 'dload'; + } + + $localAppData = self::variable($env, 'LOCALAPPDATA'); + if ($localAppData !== null && \DIRECTORY_SEPARATOR === '\\') { + return $localAppData . \DIRECTORY_SEPARATOR . 'dload' . \DIRECTORY_SEPARATOR . 'cache'; + } + + $home = self::variable($env, 'HOME') ?? self::variable($env, 'USERPROFILE'); + if ($home !== null) { + return $home . \DIRECTORY_SEPARATOR . '.cache' . \DIRECTORY_SEPARATOR . 'dload'; + } + + return \sys_get_temp_dir() . \DIRECTORY_SEPARATOR . 'dload-cache'; + } + + /** + * @param array $env + * @return non-empty-string|null + */ + private static function variable(array $env, string $name): ?string + { + $value = $env[$name] ?? null; + + return \is_string($value) && \trim($value) !== '' ? \rtrim($value, '/\\') : null; + } +} diff --git a/src/Module/Registry/Internal/FileRegistryStorage.php b/src/Module/Registry/Internal/FileRegistryStorage.php new file mode 100644 index 0000000..f9b2a1a --- /dev/null +++ b/src/Module/Registry/Internal/FileRegistryStorage.php @@ -0,0 +1,148 @@ +/repositories/github/roadrunner-server/roadrunner.json + * /repositories/gitlab/group/project.json + * ``` + * + * Files are written aside and renamed into place, so an interrupted or parallel run cannot + * leave a half-written record for anyone to read. + * + * @internal + * @psalm-internal Internal\DLoad + */ +final class FileRegistryStorage implements RegistryStorage +{ + private const REPOSITORIES_DIR = 'repositories'; + private const EXTENSION = '.json'; + + private readonly Path $root; + + public function __construct( + Path|string $directory, + private readonly Logger $logger, + ) { + $this->root = Path::create($directory)->join(self::REPOSITORIES_DIR); + } + + public function load(RepositoryId $id): ?RepositoryRecord + { + return $this->read($this->fileOf($id)); + } + + public function save(RepositoryRecord $record): void + { + $file = $this->fileOf($record->id); + $directory = $file->parent(); + + $directory->isDir() or FS::mkdir($directory); + + $payload = \json_encode($record->toArray(), \JSON_THROW_ON_ERROR | \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES); + + $temp = Path::create((string) $file . '.' . \getmypid() . '.tmp'); + @\file_put_contents((string) $temp, $payload) === false and throw new \RuntimeException( + \sprintf('Failed to write registry record `%s`.', $temp), + ); + + if (!FS::moveFile($temp, $file)) { + FS::removeFile($temp); + throw new \RuntimeException(\sprintf('Failed to store registry record `%s`.', $file)); + } + } + + public function all(): iterable + { + if (!$this->root->isDir()) { + return; + } + + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator((string) $this->root, \FilesystemIterator::SKIP_DOTS), + ); + + /** @var \SplFileInfo $file */ + foreach ($iterator as $file) { + if (!$file->isFile() || !\str_ends_with($file->getFilename(), self::EXTENSION)) { + continue; + } + + $record = $this->read(Path::create($file->getPathname())); + $record === null or yield $record; + } + } + + public function remove(RepositoryId $id): void + { + $file = $this->fileOf($id); + $file->isFile() and FS::removeFile($file); + } + + public function clear(): void + { + $this->root->isDir() and FS::removeDir($this->root); + } + + /** + * Keeps a path segment safe for every file system: anything but plain ASCII is replaced, + * and a segment that would otherwise be empty or a directory reference gets a placeholder. + * + * @return non-empty-string + */ + private static function sanitize(string $segment): string + { + $safe = (string) \preg_replace('/[^A-Za-z0-9._-]+/', '_', $segment); + + return $safe === '' || \trim($safe, '.') === '' ? '_' : $safe; + } + + /** + * Reads a record, or returns `null` when there is none or it cannot be used. + */ + private function read(Path $file): ?RepositoryRecord + { + if (!$file->isFile()) { + return null; + } + + try { + $content = @\file_get_contents((string) $file); + $content === false and throw new \RuntimeException(\sprintf('Failed to read registry record `%s`.', $file)); + + /** @var mixed $payload */ + $payload = \json_decode($content, true, 512, \JSON_THROW_ON_ERROR); + \is_array($payload) or throw new \UnexpectedValueException('Registry record must be a JSON object.'); + + return RepositoryRecord::fromArray($payload); + } catch (\Throwable $e) { + // A half-written, hand-edited or outdated record is not worth a failed download: + // report it and let the registry fetch the releases again. + $this->logger->exception($e, important: false); + + return null; + } + } + + private function fileOf(RepositoryId $id): Path + { + $segments = \array_map(self::sanitize(...), [$id->type, ...\explode('/', $id->uri)]); + $segments[\array_key_last($segments)] .= self::EXTENSION; + + return $this->root->join(...$segments); + } +} diff --git a/src/Module/Registry/Internal/PassThroughRegistry.php b/src/Module/Registry/Internal/PassThroughRegistry.php new file mode 100644 index 0000000..d6df289 --- /dev/null +++ b/src/Module/Registry/Internal/PassThroughRegistry.php @@ -0,0 +1,32 @@ +pages() as $page) { + yield $page->releases; + } + } + + public function attach(string $software, RepositoryId $id): void + { + // Nothing to record + } +} diff --git a/src/Module/Registry/Internal/StoredVersionRegistry.php b/src/Module/Registry/Internal/StoredVersionRegistry.php new file mode 100644 index 0000000..868a9c4 --- /dev/null +++ b/src/Module/Registry/Internal/StoredVersionRegistry.php @@ -0,0 +1,178 @@ + $ttl Seconds the last check stays valid. + * @param bool $refresh Ignore the TTL and check the source for every repository once. + * @param null|\Closure(): int $clock Current unix time; defaults to `time()`. + */ + public function __construct( + private readonly RegistryStorage $storage, + private readonly int $ttl, + private readonly Logger $logger, + private readonly bool $refresh = false, + ?\Closure $clock = null, + ) { + $this->clock = $clock ?? static fn(): int => \time(); + } + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); + + if ($this->refresh || $record->isStale(($this->clock)(), $this->ttl)) { + $record = $this->check($record, $source); + } else { + $this->logger->debug('Releases of `%s` are served from the version registry.', (string) $id); + } + + $stored = $record->releases(); + $stored === [] or yield $stored; + + if ($record->complete) { + return; + } + + // Older releases are loaded only when the consumer actually needs them + yield from $this->extend($record, $source); + } + + public function attach(string $software, RepositoryId $id): void + { + $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); + $updated = $record->withSoftware($software); + + $updated === $record or $this->persist($updated); + } + + /** + * @param list $page + */ + private static function hasKnown(RepositoryRecord $record, array $page): bool + { + foreach ($page as $release) { + if ($record->has($release->tag)) { + return true; + } + } + + return false; + } + + /** + * Fetches the releases published since the last check and stores the result. + * + * @throws RepositoryException When the source fails and nothing is stored to fall back on. + */ + private function check(RepositoryRecord $record, ReleaseSource $source): RepositoryRecord + { + try { + $fetched = []; + $complete = $record->complete; + + foreach ($source->pages() as $page) { + $fetched = [...$fetched, ...$page->releases]; + + // The listing ended during the check: everything is known now + $page->last and $complete = true; + + // Reaching a known release means everything newer has been fetched. A record + // without releases cannot hit one, so its check is the first page only. + if ($page->last || $record->count() === 0 || self::hasKnown($record, $page->releases)) { + break; + } + } + + $updated = $record + ->withHead($fetched) + ->withComplete($complete) + ->withCheckedAt(($this->clock)()); + + $this->persist($updated); + + return $updated; + } catch (RepositoryException $e) { + $record->count() > 0 or throw $e; + + $this->logger->exception($e, important: false); + $this->logger->info( + 'Failed to check `%s` for new releases, %d stored release(s) are used instead.', + (string) $record->id, + $record->count(), + ); + + return $record; + } + } + + /** + * Loads the releases older than the stored ones page by page, persisting every page. + * + * @return \Generator, mixed, void> + * @throws RepositoryException + */ + private function extend(RepositoryRecord $record, ReleaseSource $source): \Generator + { + foreach ($source->pages($record->count()) as $page) { + $new = \array_values(\array_filter( + $page->releases, + static fn(ReleaseRecord $release): bool => !$record->has($release->tag), + )); + + $record = $record->withTail($new)->withComplete($page->last); + $this->persist($record); + + $new === [] or yield $new; + } + + $record->complete or $this->persist($record->withComplete(true)); + } + + /** + * Stores the record; a storage failure is reported and swallowed. + */ + private function persist(RepositoryRecord $record): void + { + try { + $this->storage->save($record); + } catch (\Throwable $e) { + $this->logger->exception($e, important: false); + } + } +} diff --git a/src/Module/Registry/Record/AssetRecord.php b/src/Module/Registry/Record/AssetRecord.php new file mode 100644 index 0000000..0643f33 --- /dev/null +++ b/src/Module/Registry/Record/AssetRecord.php @@ -0,0 +1,69 @@ +|null $size Size in bytes when the source reports it. + * @param non-empty-string|null $contentType MIME type when the source reports it. + */ + public function __construct( + public readonly string $name, + public readonly string $uri, + public readonly ?int $size = null, + public readonly ?string $contentType = null, + ) {} + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe an asset. + */ + public static function fromArray(array $data): self + { + $name = $data['name'] ?? null; + $uri = $data['uri'] ?? null; + \is_string($name) && $name !== '' && \is_string($uri) && $uri !== '' or throw new \InvalidArgumentException( + 'Asset record requires non-empty `name` and `uri`.', + ); + + $size = $data['size'] ?? null; + $contentType = $data['content_type'] ?? null; + + return new self( + name: $name, + uri: $uri, + size: \is_int($size) && $size >= 0 ? $size : null, + contentType: \is_string($contentType) && $contentType !== '' ? $contentType : null, + ); + } + + /** + * @return AssetArray + */ + public function toArray(): array + { + $result = ['name' => $this->name, 'uri' => $this->uri]; + $this->size === null or $result['size'] = $this->size; + $this->contentType === null or $result['content_type'] = $this->contentType; + + return $result; + } +} diff --git a/src/Module/Registry/Record/ReleasePage.php b/src/Module/Registry/Record/ReleasePage.php new file mode 100644 index 0000000..64aa0db --- /dev/null +++ b/src/Module/Registry/Record/ReleasePage.php @@ -0,0 +1,23 @@ + $releases Releases of the page, newest first. + * @param bool $last Whether the listing has no page after this one. + */ + public function __construct( + public readonly array $releases, + public readonly bool $last, + ) {} +} diff --git a/src/Module/Registry/Record/ReleaseRecord.php b/src/Module/Registry/Record/ReleaseRecord.php new file mode 100644 index 0000000..4a01b28 --- /dev/null +++ b/src/Module/Registry/Record/ReleaseRecord.php @@ -0,0 +1,77 @@ +, + * } + */ +final class ReleaseRecord +{ + /** + * @param non-empty-string $tag Tag the release was made from; identifies the release within a repository. + * @param non-empty-string $name Human-readable release name. + * @param list $assets + */ + public function __construct( + public readonly string $tag, + public readonly string $name, + public readonly ?\DateTimeImmutable $publishedAt = null, + public readonly bool $prerelease = false, + public readonly array $assets = [], + ) {} + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe a release. + */ + public static function fromArray(array $data): self + { + $tag = $data['tag'] ?? null; + \is_string($tag) && $tag !== '' or throw new \InvalidArgumentException('Release record requires a non-empty `tag`.'); + + $name = $data['name'] ?? null; + \is_string($name) && $name !== '' or $name = $tag; + + $publishedAt = $data['published_at'] ?? null; + $assets = []; + foreach (\is_array($data['assets'] ?? null) ? $data['assets'] : [] as $asset) { + \is_array($asset) and $assets[] = AssetRecord::fromArray($asset); + } + + return new self( + tag: $tag, + name: $name, + publishedAt: \is_string($publishedAt) && $publishedAt !== '' ? new \DateTimeImmutable($publishedAt) : null, + prerelease: (bool) ($data['prerelease'] ?? false), + assets: $assets, + ); + } + + /** + * @return ReleaseArray + */ + public function toArray(): array + { + return [ + 'tag' => $this->tag, + 'name' => $this->name, + 'published_at' => $this->publishedAt?->format(\DateTimeInterface::ATOM), + 'prerelease' => $this->prerelease, + 'assets' => \array_map(static fn(AssetRecord $asset): array => $asset->toArray(), $this->assets), + ]; + } +} diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php new file mode 100644 index 0000000..2ecc5a1 --- /dev/null +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -0,0 +1,213 @@ +, + * releases: list, + * } + */ +final class RepositoryRecord +{ + /** Format version of the stored payload; bump when the structure changes incompatibly. */ + public const FORMAT_VERSION = 1; + + /** @var array Releases keyed by tag, newest first. */ + private readonly array $releases; + + /** + * @param int|null $checkedAt Unix timestamp of the last successful check against the source. + * @param bool $complete Whether the stored releases reach the end of the source listing. + * @param list $software Identifiers of the software packages served from this repository. + * @param list $releases Releases newest first. + */ + public function __construct( + public readonly RepositoryId $id, + public readonly ?int $checkedAt = null, + public readonly bool $complete = false, + public readonly array $software = [], + array $releases = [], + ) { + $indexed = []; + foreach ($releases as $release) { + $indexed[$release->tag] ??= $release; + } + + $this->releases = $indexed; + } + + public static function empty(RepositoryId $id): self + { + return new self($id); + } + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe a repository record. + */ + public static function fromArray(array $data): self + { + ($data['version'] ?? null) === self::FORMAT_VERSION or throw new \InvalidArgumentException( + 'Unsupported repository record format.', + ); + + $repository = $data['repository'] ?? null; + $type = \is_array($repository) ? ($repository['type'] ?? null) : null; + $uri = \is_array($repository) ? ($repository['uri'] ?? null) : null; + \is_string($type) && $type !== '' && \is_string($uri) && $uri !== '' or throw new \InvalidArgumentException( + 'Repository record requires a repository type and URI.', + ); + + $releases = []; + foreach (\is_array($data['releases'] ?? null) ? $data['releases'] : [] as $release) { + \is_array($release) and $releases[] = ReleaseRecord::fromArray($release); + } + + /** @var list $software */ + $software = \array_values(\array_filter( + \is_array($data['software'] ?? null) ? $data['software'] : [], + static fn(mixed $name): bool => \is_string($name) && $name !== '', + )); + + $checkedAt = $data['checked_at'] ?? null; + + return new self( + id: new RepositoryId($type, $uri), + checkedAt: \is_int($checkedAt) ? $checkedAt : null, + complete: (bool) ($data['complete'] ?? false), + software: $software, + releases: $releases, + ); + } + + /** + * @return list Releases newest first. + */ + public function releases(): array + { + return \array_values($this->releases); + } + + /** + * @return int<0, max> + */ + public function count(): int + { + return \count($this->releases); + } + + /** + * @param non-empty-string $tag + */ + public function has(string $tag): bool + { + return isset($this->releases[$tag]); + } + + /** + * Whether the last check is older than the given number of seconds, or never happened. + * + * @param int<0, max> $ttl + */ + public function isStale(int $now, int $ttl): bool + { + return $this->checkedAt === null || $now - $this->checkedAt > $ttl; + } + + /** + * Replaces the head of the list with freshly fetched releases. + * + * The fetched releases are the newest ones; they overwrite the stored entries with the same + * tags (assets may have been attached after the release was created) and the remaining stored + * releases follow them, so the list stays newest first. + * + * @param list $fetched Newest first. + */ + public function withHead(array $fetched): self + { + return $this->with(releases: [...$fetched, ...$this->releases()]); + } + + /** + * Appends older releases loaded on demand; already known tags are ignored. + * + * @param list $fetched + */ + public function withTail(array $fetched): self + { + return $this->with(releases: [...$this->releases(), ...$fetched]); + } + + public function withCheckedAt(int $checkedAt): self + { + return $this->with(checkedAt: $checkedAt); + } + + public function withComplete(bool $complete): self + { + return $this->with(complete: $complete); + } + + /** + * @param non-empty-string $software + */ + public function withSoftware(string $software): self + { + return \in_array($software, $this->software, true) + ? $this + : $this->with(software: [...$this->software, $software]); + } + + /** + * @return RepositoryArray + */ + public function toArray(): array + { + return [ + 'version' => self::FORMAT_VERSION, + 'repository' => ['type' => $this->id->type, 'uri' => $this->id->uri], + 'checked_at' => $this->checkedAt, + 'complete' => $this->complete, + 'software' => $this->software, + 'releases' => \array_map(static fn(ReleaseRecord $release): array => $release->toArray(), $this->releases()), + ]; + } + + /** + * @param list|null $software + * @param list|null $releases + */ + private function with( + ?int $checkedAt = null, + ?bool $complete = null, + ?array $software = null, + ?array $releases = null, + ): self { + return new self( + id: $this->id, + checkedAt: $checkedAt ?? $this->checkedAt, + complete: $complete ?? $this->complete, + software: $software ?? $this->software, + releases: $releases ?? $this->releases(), + ); + } +} diff --git a/src/Module/Registry/RegistryStorage.php b/src/Module/Registry/RegistryStorage.php new file mode 100644 index 0000000..f4d6077 --- /dev/null +++ b/src/Module/Registry/RegistryStorage.php @@ -0,0 +1,46 @@ + + */ + public function all(): iterable; + + /** + * Removes the record of a repository; a missing record is not an error. + */ + public function remove(RepositoryId $id): void; + + /** + * Removes every record. + */ + public function clear(): void; +} diff --git a/src/Module/Registry/ReleaseSource.php b/src/Module/Registry/ReleaseSource.php new file mode 100644 index 0000000..23954e3 --- /dev/null +++ b/src/Module/Registry/ReleaseSource.php @@ -0,0 +1,29 @@ + $offset Number of newest releases to skip. + * @return \Generator Pages of releases. + * @throws RepositoryException When a page cannot be loaded. + */ + public function pages(int $offset = 0): \Generator; +} diff --git a/src/Module/Registry/RepositoryId.php b/src/Module/Registry/RepositoryId.php new file mode 100644 index 0000000..8177676 --- /dev/null +++ b/src/Module/Registry/RepositoryId.php @@ -0,0 +1,48 @@ +type, $config->uri); + } + + public function equals(self $other): bool + { + return $this->type === $other->type && $this->uri === $other->uri; + } + + /** + * @return non-empty-string + */ + public function __toString(): string + { + return $this->type . ':' . $this->uri; + } +} diff --git a/src/Module/Registry/VersionRegistry.php b/src/Module/Registry/VersionRegistry.php new file mode 100644 index 0000000..986419a --- /dev/null +++ b/src/Module/Registry/VersionRegistry.php @@ -0,0 +1,44 @@ +releases($id, $source) as $page) { + * foreach ($page as $record) { + * // ... + * } + * } + * ``` + */ +interface VersionRegistry +{ + /** + * Lists the releases of a repository newest first, page by page. + * + * Older releases that are not in the database yet are loaded from the source only when the + * iteration reaches them, so a consumer that stops early costs no extra request. + * + * @return \Generator, mixed, void> + * @throws RepositoryException When releases cannot be obtained from either the database or the source. + */ + public function releases(RepositoryId $id, ReleaseSource $source): \Generator; + + /** + * Records that a software package is served from the repository. + * + * @param non-empty-string $software Software identifier. + */ + public function attach(string $software, RepositoryId $id): void; +} diff --git a/src/Module/Repository/Internal/CachedGenerator.php b/src/Module/Repository/Internal/CachedGenerator.php index b304174..2c2ca51 100644 --- a/src/Module/Repository/Internal/CachedGenerator.php +++ b/src/Module/Repository/Internal/CachedGenerator.php @@ -63,6 +63,16 @@ public function getIterator(): \Traversable goto start; } + /** + * Returns the items produced so far without pulling anything more from the generator. + * + * @return list + */ + public function loaded(): array + { + return \array_values($this->cache); + } + /** * Returns the first item in the cache or from the generator. * diff --git a/src/Module/Repository/Internal/Collection.php b/src/Module/Repository/Internal/Collection.php index b8033b0..32dbda0 100644 --- a/src/Module/Repository/Internal/Collection.php +++ b/src/Module/Repository/Internal/Collection.php @@ -94,6 +94,18 @@ public function filter(callable $filter): static return $clone; } + /** + * Returns the items loaded so far, ignoring filters and without loading anything more. + * + * Iterating a lazy collection may cost requests; releasing what has been loaded must not. + * + * @return list + */ + public function loaded(): array + { + return \is_array($this->items) ? \array_values($this->items) : $this->items->loaded(); + } + /** * Maps each item in the collection using the provided callback. * diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index c3ce8fb..10f87ea 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -6,11 +6,11 @@ use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; use Internal\DLoad\Module\HttpClient\Method; +use Internal\DLoad\Module\Registry\Record\ReleasePage; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\RepositoryInfo; -use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Service\Logger; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; @@ -28,6 +28,12 @@ final class RepositoryApi private const URL_REPOSITORY = 'https://api.github.com/repos/%s'; private const URL_RELEASES = 'https://api.github.com/repos/%s/releases'; + /** + * Number of releases to ask for in a single page. GitHub serves 30 by default and allows up to + * 100, so the maximum keeps the release list within as few requests as the API permits. + */ + public const RELEASES_PER_PAGE = 100; + /** * @var non-empty-string */ @@ -79,74 +85,75 @@ public function getRepository(): RepositoryInfo } /** + * Lists releases newest first, page by page, starting from the given page. + * + * A page is requested only when the generator advances to it, so a consumer that stops early + * costs no extra request. + * * @param int<1, max> $page - * @return Paginator + * @return \Generator * @throws RepositoryException */ - public function getReleases(int $page = 1): Paginator + public function releasePages(int $page = 1): \Generator { - $pageLoader = function () use ($page): \Generator { - $currentPage = $page; - - do { - $response = $this->releasesRequest($currentPage); - - /** @var list, - * prerelease: bool, - * draft: bool - * }> $data */ - $data = $this->decodeReleasesResponse($response); - - // If empty response, no more pages - if ($data === []) { - return; + $currentPage = $page; + + do { + $response = $this->releasesRequest($currentPage); + + /** @var list, + * prerelease: bool, + * draft: bool + * }> $data */ + $data = $this->decodeReleasesResponse($response); + + // If empty response, no more pages + if ($data === []) { + return; + } + + $releases = []; + $failure = null; + foreach ($data as $releaseData) { + try { + $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); + } catch (\Throwable $e) { + $failure ??= $e; + $this->logger->exception($e, important: false); + // Skip invalid releases + continue; } - - $releases = []; - $failure = null; - foreach ($data as $releaseData) { - try { - $releases[] = ReleaseInfo::fromApiResponse($releaseData); - } catch (\Throwable $e) { - $failure ??= $e; - $this->logger->exception($e, important: false); - // Skip invalid releases - continue; - } - } - - // The whole page is unreadable: the response structure is not what we expect - if ($releases === [] && $failure !== null) { - throw new ApiException( - \sprintf( - 'GitHub API returned %d release(s) for repository `%s`, but none of them could be read: %s', - \count($data), - $this->repositoryPath, - $failure->getMessage(), - ), + } + + // The whole page is unreadable: the response structure is not what we expect + if ($releases === [] && $failure !== null) { + throw new ApiException( + \sprintf( + 'GitHub API returned %d release(s) for repository `%s`, but none of them could be read: %s', + \count($data), $this->repositoryPath, - $failure, - ); - } + $failure->getMessage(), + ), + $this->repositoryPath, + $failure, + ); + } - yield $releases; + $hasMorePages = $this->hasNextPage($response); - // Check if there are more pages - $hasMorePages = $this->hasNextPage($response); - $currentPage++; - } while ($hasMorePages); - }; + yield new ReleasePage($releases, !$hasMorePages); - return Paginator::createFromGenerator($pageLoader(), null); + $currentPage++; + } while ($hasMorePages); } /** @@ -196,13 +203,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, $this->repositoryPath), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, $this->repositoryPath), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->request(Method::Get, $uri); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php index 43225da..af168f6 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php @@ -4,6 +4,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; + /** * GitHub Asset Data Transfer Object. * @@ -42,4 +44,17 @@ public static function fromApiResponse(array $data): self contentType: $data['content_type'], ); } + + /** + * Maps the asset into the provider-neutral registry record. + */ + public function toRecord(): AssetRecord + { + return new AssetRecord( + name: $this->name, + uri: $this->downloadUrl, + size: $this->size, + contentType: $this->contentType, + ); + } } diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php index 6478ea4..e145b7d 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php @@ -4,6 +4,9 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; + /** * GitHub Release Data Transfer Object. * @@ -57,4 +60,18 @@ public static function fromApiResponse(array $data): self draft: $data['draft'], ); } + + /** + * Maps the release into the provider-neutral registry record. + */ + public function toRecord(): ReleaseRecord + { + return new ReleaseRecord( + tag: $this->tagName, + name: $this->name, + publishedAt: $this->publishedAt, + prerelease: $this->prerelease, + assets: \array_map(static fn(AssetInfo $asset): AssetRecord => $asset->toRecord(), $this->assets), + ); + } } diff --git a/src/Module/Repository/Internal/GitHub/Factory.php b/src/Module/Repository/Internal/GitHub/Factory.php index 77fd1c4..6773153 100644 --- a/src/Module/Repository/Internal/GitHub/Factory.php +++ b/src/Module/Repository/Internal/GitHub/Factory.php @@ -7,6 +7,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitHub; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\RepositoryFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitHub $gitHubConfig, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->gitHubClient = new Client( $httpFactory, @@ -50,7 +52,7 @@ public function create(RepositoryConfig $config): GitHubRepository $api = $this->createRepositoryApi($org, $repo); - return new GitHubRepository($api, $org, $repo, $this->logger); + return new GitHubRepository($api, $org, $repo, $this->logger, $this->registry); } /** diff --git a/src/Module/Repository/Internal/GitHub/GitHubAsset.php b/src/Module/Repository/Internal/GitHub/GitHubAsset.php index 020916c..41548f4 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubAsset.php +++ b/src/Module/Repository/Internal/GitHub/GitHubAsset.php @@ -9,8 +9,8 @@ use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\HttpClient\Method; use Internal\DLoad\Module\HttpClient\StreamReader; +use Internal\DLoad\Module\Registry\Record\AssetRecord; use Internal\DLoad\Module\Repository\Internal\Asset; -use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\AssetInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Exception\RepositoryException; @@ -41,12 +41,12 @@ private function __construct( ); } - public static function fromDTO( + public static function fromRecord( RepositoryApi $api, GitHubRelease $release, - AssetInfo $dto, + AssetRecord $record, ): self { - return new self($api, $release, $dto->name, $dto->downloadUrl); + return new self($api, $release, $record->name, $record->uri); } /** diff --git a/src/Module/Repository/Internal/GitHub/GitHubRelease.php b/src/Module/Repository/Internal/GitHub/GitHubRelease.php index 8f0aef9..adffc49 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRelease.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRelease.php @@ -5,8 +5,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; -use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\Release; use Internal\DLoad\Module\Version\Version; @@ -30,17 +30,20 @@ private function __construct( parent::__construct($repository, $name, $version); } - public static function fromDTO( + /** + * @throws \InvalidArgumentException When the release tag is not a version. + */ + public static function fromRecord( RepositoryApi $api, GitHubRepository $repository, - ReleaseInfo $dto, + ReleaseRecord $record, ): self { - $version = Version::fromVersionString($dto->tagName); - $result = new self($repository, $dto->name, $version); + $version = Version::fromVersionString($record->tag); + $result = new self($repository, $record->name, $version); - $result->assets = AssetsCollection::create(static function () use ($api, $result, $dto): \Generator { - foreach ($dto->assets as $assetDTO) { - yield GitHubAsset::fromDTO($api, $result, $assetDTO); + $result->assets = AssetsCollection::create(static function () use ($api, $result, $record): \Generator { + foreach ($record->assets as $asset) { + yield GitHubAsset::fromRecord($api, $result, $asset); } }); diff --git a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php new file mode 100644 index 0000000..572073d --- /dev/null +++ b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php @@ -0,0 +1,34 @@ +api->releasePages(\intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1) as $page) { + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); + $skip = 0; + } + } +} diff --git a/src/Module/Repository/Internal/GitHub/GitHubRepository.php b/src/Module/Repository/Internal/GitHub/GitHubRepository.php index 2c4d9be..12a7fdf 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRepository.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRepository.php @@ -5,9 +5,12 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\RepositoryId; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -19,6 +22,9 @@ */ final class GitHubRepository implements Repository, Destroyable { + /** Repository type identifier in the version registry. */ + public const TYPE = 'github'; + private ?ReleasesCollection $releases = null; /** @@ -37,13 +43,17 @@ public function __construct( string $org, string $repo, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->name = $org . '/' . $repo; } /** * Returns a lazily loaded collection of repository releases. - * Pages are loaded only when needed during iteration or filtering. + * + * Releases come from the version registry, which serves stored ones without a request and + * asks the API only for what it does not know yet. Pages are loaded only when needed during + * iteration or filtering. */ public function getReleases(): ReleasesCollection { @@ -53,32 +63,37 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + // to avoid first eager loading because of generator + yield []; + + $pages = $this->registry->releases( + new RepositoryId(self::TYPE, $this->name), + new GitHubReleaseSource($this->api), + ); $anyPageLoaded = false; - do { + while (true) { try { - // to avoid first eager loading because of generator - yield []; + // Advancing the generator is what requests the next page + $anyPageLoaded ? $pages->next() : $pages->rewind(); - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + if (!$pages->valid()) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($pages->current() as $record) { try { - $toYield[] = GitHubRelease::fromDTO($this->api, $this, $releaseDTO); + $toYield[] = GitHubRelease::fromRecord($this->api, $this, $record); } catch (\Throwable $e) { $this->logger->exception($e, important: false); // Skip invalid releases continue; } } - yield $toYield; - $anyPageLoaded = true; - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; + $anyPageLoaded = true; + yield $toYield; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing repository, etc.) must reach the user. @@ -93,11 +108,11 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } }; // Create paginator - $paginator = \Internal\DLoad\Module\Repository\Internal\Paginator::createFromGenerator($pageLoader(), null); + $paginator = Paginator::createFromGenerator($pageLoader(), null); // Create a collection with the paginator $this->releases = ReleasesCollection::create($paginator); @@ -112,9 +127,10 @@ public function getName(): string public function destroy(): void { - $this->releases === null or $this->releases->map( - static fn(object $release) => $release instanceof Destroyable and $release->destroy(), - ); + // Only what was loaded is released: iterating the collection would request the remaining pages + foreach ($this->releases?->loaded() ?? [] as $release) { + $release instanceof Destroyable and $release->destroy(); + } unset($this->releases); } diff --git a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php index 6af6c71..bdd6c9d 100644 --- a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php @@ -6,11 +6,11 @@ use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; use Internal\DLoad\Module\HttpClient\Method; +use Internal\DLoad\Module\Registry\Record\ReleasePage; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\RepositoryInfo; -use Internal\DLoad\Module\Repository\Internal\Paginator; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; @@ -28,6 +28,12 @@ final class RepositoryApi private const URL_RELEASES = 'https://gitlab.com/api/v4/projects/%s/releases'; private const URL_RELEASE_ASSET = 'https://gitlab.com/api/v4/projects/%s/releases/%s/downloads/%s'; + /** + * Number of releases to ask for in a single page. GitLab serves 20 by default and allows up to + * 100, so the maximum keeps the release list within as few requests as the API permits. + */ + public const RELEASES_PER_PAGE = 100; + /** * @var non-empty-string */ @@ -88,76 +94,77 @@ public function getRepository(): RepositoryInfo } /** + * Lists releases newest first, page by page, starting from the given page. + * + * A page is requested only when the generator advances to it, so a consumer that stops early + * costs no extra request. + * * @param int<1, max> $page - * @return Paginator + * @return \Generator * @throws RepositoryException */ - public function getReleases(int $page = 1): Paginator + public function releasePages(int $page = 1): \Generator { - $pageLoader = function () use ($page): \Generator { - $currentPage = $page; - - do { - $response = $this->releasesRequest($currentPage); - - /** @var list - * }, - * upcoming_release: bool - * }> $data */ - $data = $this->decodeReleasesResponse($response); - - // If empty response, no more pages - if ($data === []) { - return; + $currentPage = $page; + + do { + $response = $this->releasesRequest($currentPage); + + /** @var list + * }, + * upcoming_release: bool + * }> $data */ + $data = $this->decodeReleasesResponse($response); + + // If empty response, no more pages + if ($data === []) { + return; + } + + $releases = []; + $failure = null; + foreach ($data as $releaseData) { + try { + $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); + } catch (\Throwable $e) { + $failure ??= $e; + // Skip invalid releases + continue; } - - $releases = []; - $failure = null; - foreach ($data as $releaseData) { - try { - $releases[] = ReleaseInfo::fromApiResponse($releaseData); - } catch (\Throwable $e) { - $failure ??= $e; - // Skip invalid releases - continue; - } - } - - // The whole page is unreadable: the response structure is not what we expect - if ($releases === [] && $failure !== null) { - throw new ApiException( - \sprintf( - 'GitLab API returned %d release(s) for project `%s`, but none of them could be read: %s', - \count($data), - $this->repositoryPath, - $failure->getMessage(), - ), + } + + // The whole page is unreadable: the response structure is not what we expect + if ($releases === [] && $failure !== null) { + throw new ApiException( + \sprintf( + 'GitLab API returned %d release(s) for project `%s`, but none of them could be read: %s', + \count($data), $this->repositoryPath, - $failure, - ); - } + $failure->getMessage(), + ), + $this->repositoryPath, + $failure, + ); + } - yield $releases; + $hasMorePages = $this->hasNextPage($response); - // Check if there are more pages - $hasMorePages = $this->hasNextPage($response); - $currentPage++; - } while ($hasMorePages); - }; + yield new ReleasePage($releases, !$hasMorePages); - return Paginator::createFromGenerator($pageLoader(), null); + $currentPage++; + } while ($hasMorePages); } /** @@ -207,13 +214,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->request(Method::Get, $uri); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php b/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php index 46512ea..0c1a025 100644 --- a/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php +++ b/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php @@ -4,6 +4,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; + /** * GitLab Asset Data Transfer Object. * @@ -39,4 +41,12 @@ public static function fromApiResponse(array $data): self linkType: $data['link_type'] ?? null, ); } + + /** + * Maps the asset into the provider-neutral registry record. + */ + public function toRecord(): AssetRecord + { + return new AssetRecord(name: $this->name, uri: $this->downloadUrl); + } } diff --git a/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php index 07d6ee2..236a7da 100644 --- a/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php @@ -4,6 +4,9 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; + /** * GitLab Release Data Transfer Object. * @@ -58,4 +61,18 @@ public static function fromApiResponse(array $data): self prerelease: $data['upcoming_release'], ); } + + /** + * Maps the release into the provider-neutral registry record. + */ + public function toRecord(): ReleaseRecord + { + return new ReleaseRecord( + tag: $this->tagName, + name: $this->name, + publishedAt: $this->publishedAt, + prerelease: $this->prerelease, + assets: \array_map(static fn(AssetInfo $asset): AssetRecord => $asset->toRecord(), $this->assets), + ); + } } diff --git a/src/Module/Repository/Internal/GitLab/Factory.php b/src/Module/Repository/Internal/GitLab/Factory.php index 68d601d..2ccf644 100644 --- a/src/Module/Repository/Internal/GitLab/Factory.php +++ b/src/Module/Repository/Internal/GitLab/Factory.php @@ -7,6 +7,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\RepositoryFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitLab $gitLabConfig, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->gitLabClient = new Client( $httpFactory, @@ -49,7 +51,7 @@ public function create(RepositoryConfig $config): GitLabRepository $uri = \is_string($path) && $path !== '' ? $path : $config->uri; $api = $this->createRepositoryApi($uri); - return new GitLabRepository($api, $uri, $this->logger); + return new GitLabRepository($api, $uri, $this->logger, $this->registry); } /** diff --git a/src/Module/Repository/Internal/GitLab/GitLabAsset.php b/src/Module/Repository/Internal/GitLab/GitLabAsset.php index 5b0f7a7..927c1b6 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabAsset.php +++ b/src/Module/Repository/Internal/GitLab/GitLabAsset.php @@ -8,8 +8,8 @@ use Internal\DLoad\Module\Common\Architecture; use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\HttpClient\StreamReader; +use Internal\DLoad\Module\Registry\Record\AssetRecord; use Internal\DLoad\Module\Repository\Internal\Asset; -use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\AssetInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Exception\RepositoryException; @@ -40,12 +40,12 @@ private function __construct( ); } - public static function fromDTO( + public static function fromRecord( RepositoryApi $api, GitLabRelease $release, - AssetInfo $dto, + AssetRecord $record, ): self { - return new self($api, $release, $dto->name, $dto->downloadUrl); + return new self($api, $release, $record->name, $record->uri); } /** diff --git a/src/Module/Repository/Internal/GitLab/GitLabRelease.php b/src/Module/Repository/Internal/GitLab/GitLabRelease.php index f4bc854..8760d2c 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRelease.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRelease.php @@ -5,8 +5,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; -use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\Release; use Internal\DLoad\Module\Version\Version; @@ -30,17 +30,20 @@ private function __construct( parent::__construct($repository, $name, $version); } - public static function fromDTO( + /** + * @throws \InvalidArgumentException When the release tag is not a version. + */ + public static function fromRecord( RepositoryApi $api, GitLabRepository $repository, - ReleaseInfo $dto, + ReleaseRecord $record, ): self { - $version = Version::fromVersionString($dto->tagName); - $result = new self($repository, $dto->name, $version); + $version = Version::fromVersionString($record->tag); + $result = new self($repository, $record->name, $version); - $result->assets = AssetsCollection::create(static function () use ($api, $result, $dto): \Generator { - foreach ($dto->assets as $assetDTO) { - yield GitLabAsset::fromDTO($api, $result, $assetDTO); + $result->assets = AssetsCollection::create(static function () use ($api, $result, $record): \Generator { + foreach ($record->assets as $asset) { + yield GitLabAsset::fromRecord($api, $result, $asset); } }); diff --git a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php new file mode 100644 index 0000000..d07d9b6 --- /dev/null +++ b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php @@ -0,0 +1,34 @@ +api->releasePages(\intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1) as $page) { + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); + $skip = 0; + } + } +} diff --git a/src/Module/Repository/Internal/GitLab/GitLabRepository.php b/src/Module/Repository/Internal/GitLab/GitLabRepository.php index 94f3dff..5fad466 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRepository.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRepository.php @@ -5,9 +5,12 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\RepositoryId; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -19,6 +22,9 @@ */ final class GitLabRepository implements Repository, Destroyable { + /** Repository type identifier in the version registry. */ + public const TYPE = 'gitlab'; + private ?ReleasesCollection $releases = null; /** @@ -35,13 +41,17 @@ public function __construct( private readonly RepositoryApi $api, string $projectPath, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->name = $projectPath; } /** * Returns a lazily loaded collection of repository releases. - * Pages are loaded only when needed during iteration or filtering. + * + * Releases come from the version registry, which serves stored ones without a request and + * asks the API only for what it does not know yet. Pages are loaded only when needed during + * iteration or filtering. */ public function getReleases(): ReleasesCollection { @@ -51,38 +61,43 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + // to avoid first eager loading because of generator + yield []; + + $pages = $this->registry->releases( + new RepositoryId(self::TYPE, $this->name), + new GitLabReleaseSource($this->api), + ); $anyPageLoaded = false; - do { + while (true) { try { - // to avoid first eager loading because of generator - yield []; + // Advancing the generator is what requests the next page + $anyPageLoaded ? $pages->next() : $pages->rewind(); - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + if (!$pages->valid()) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($pages->current() as $record) { try { - $toYield[] = GitLabRelease::fromDTO($this->api, $this, $releaseDTO); + $toYield[] = GitLabRelease::fromRecord($this->api, $this, $record); } catch (\Throwable) { // Skip invalid releases continue; } } - yield $toYield; - $anyPageLoaded = true; - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; + $anyPageLoaded = true; + yield $toYield; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing project, etc.) must reach the user. $anyPageLoaded or throw $e; # A rate limit leaves the release list incomplete: hiding it would produce a report - # that claims the project has nothing more, so it must reach the user as well. + # that claims the repository has nothing more, so it must reach the user as well. $e instanceof RateLimitException and throw $e; # Already loaded releases are enough to continue, so a failure of a subsequent page @@ -90,11 +105,11 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } }; // Create paginator - $paginator = \Internal\DLoad\Module\Repository\Internal\Paginator::createFromGenerator($pageLoader(), null); + $paginator = Paginator::createFromGenerator($pageLoader(), null); // Create a collection with the paginator $this->releases = ReleasesCollection::create($paginator); @@ -109,9 +124,10 @@ public function getName(): string public function destroy(): void { - $this->releases === null or $this->releases->map( - static fn(object $release) => $release instanceof Destroyable and $release->destroy(), - ); + // Only what was loaded is released: iterating the collection would request the remaining pages + foreach ($this->releases?->loaded() ?? [] as $release) { + $release instanceof Destroyable and $release->destroy(); + } unset($this->releases); } diff --git a/tests/Acceptance/DLoadTest.php b/tests/Acceptance/DLoadTest.php index 4c7c19e..b22078f 100644 --- a/tests/Acceptance/DLoadTest.php +++ b/tests/Acceptance/DLoadTest.php @@ -231,8 +231,12 @@ protected function cleanup(): void */ private function buildDLoad(string $xmlConfig): DLoad { + // The version registry must not leak into the user's cache directory from a test run + $environment = \getenv() + ['DLOAD_CACHE_DIR' => (string) $this->testRuntimeDir->join('registry')]; + $environment['DLOAD_CACHE_DIR'] = (string) $this->testRuntimeDir->join('registry'); + $container = Bootstrap::init() - ->withConfig($xmlConfig, [], [], \getenv()) + ->withConfig($xmlConfig, [], [], $environment) ->finish(); $container->set($input = new ArgvInput(), InputInterface::class); $container->set($output = new BufferedOutput(), OutputInterface::class); diff --git a/tests/Integration/Module/Registry/VersionRegistryBindingTest.php b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php new file mode 100644 index 0000000..0d84010 --- /dev/null +++ b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php @@ -0,0 +1,92 @@ + $this->directory]); + + Assert::instanceOf($container->get(VersionRegistry::class), StoredVersionRegistry::class); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/dload/repositories/github/a/b.json')); + } + + #[Test] + public function environmentVariableSetsTheDirectory(): void + { + $container = self::bootstrap(environment: ['DLOAD_CACHE_DIR' => $this->directory]); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b.json')); + } + + #[Test] + public function xmlAttributeSetsTheDirectory(): void + { + $container = self::bootstrap(xml: \sprintf('', $this->directory)); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b.json')); + } + + #[Test] + public function zeroTtlDisablesTheRegistry(): void + { + $container = self::bootstrap(environment: ['DLOAD_CACHE_DIR' => $this->directory, 'DLOAD_CACHE_TTL' => '0']); + + Assert::instanceOf($container->get(VersionRegistry::class), PassThroughRegistry::class); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-registry-binding-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @param array $environment + */ + private static function bootstrap(?string $xml = null, array $environment = []): Container + { + return Bootstrap::init() + ->withConfig(xml: $xml, environment: $environment) + ->finish(); + } +} diff --git a/tests/Unit/Module/Registry/CacheDirectoryTest.php b/tests/Unit/Module/Registry/CacheDirectoryTest.php new file mode 100644 index 0000000..3cfab9a --- /dev/null +++ b/tests/Unit/Module/Registry/CacheDirectoryTest.php @@ -0,0 +1,36 @@ + '/var/cache/', 'HOME' => '/home/u', 'LOCALAPPDATA' => 'C:\\x']); + + Assert::same($dir, '/var/cache' . \DIRECTORY_SEPARATOR . 'dload'); + } + + #[Test] + public function homeIsUsedWhenNothingElseIsSet(): void + { + $dir = CacheDirectory::resolve(['HOME' => '/home/u', 'XDG_CACHE_HOME' => ' ']); + + Assert::same($dir, '/home/u' . \DIRECTORY_SEPARATOR . '.cache' . \DIRECTORY_SEPARATOR . 'dload'); + } + + #[Test] + public function fallsBackToTheTemporaryDirectory(): void + { + Assert::same(CacheDirectory::resolve([]), \sys_get_temp_dir() . \DIRECTORY_SEPARATOR . 'dload-cache'); + } +} diff --git a/tests/Unit/Module/Registry/FileRegistryStorageTest.php b/tests/Unit/Module/Registry/FileRegistryStorageTest.php new file mode 100644 index 0000000..6832128 --- /dev/null +++ b/tests/Unit/Module/Registry/FileRegistryStorageTest.php @@ -0,0 +1,116 @@ +storage(); + + $storage->save(self::record('github', 'roadrunner-server/roadrunner', ['v1'])); + $storage->save(self::record('gitlab', 'group/sub/project', ['v2'])); + + Assert::true(\is_file($this->directory . '/repositories/github/roadrunner-server/roadrunner.json')); + Assert::true(\is_file($this->directory . '/repositories/gitlab/group/sub/project.json')); + + $loaded = $storage->load(new RepositoryId('github', 'roadrunner-server/roadrunner')); + Assert::same($loaded->releases()[0]->tag, 'v1'); + Assert::same($loaded->software, ['rr']); + } + + #[Test] + public function missingAndCorruptedRecordsReadAsNull(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + + Assert::null($storage->load($id)); + + $storage->save(self::record('github', 'owner/repo', ['v1'])); + \file_put_contents($this->directory . '/repositories/github/owner/repo.json', '{not json'); + + Assert::null($storage->load($id)); + } + + #[Test] + public function listsRemovesAndClears(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'a/b', ['v1'])); + $storage->save(self::record('github', 'c/d', ['v1'])); + + Assert::count(\iterator_to_array($storage->all(), false), 2); + + $storage->remove(new RepositoryId('github', 'a/b')); + Assert::count(\iterator_to_array($storage->all(), false), 1); + + $storage->clear(); + Assert::count(\iterator_to_array($storage->all(), false), 0); + Assert::null($storage->load(new RepositoryId('github', 'c/d'))); + } + + #[Test] + public function unsafePathSegmentsAreSanitized(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', '../owner/re po:x'); + + $storage->save(new RepositoryRecord($id)); + + Assert::false(\is_dir(\dirname($this->directory) . '/owner')); + Assert::true(\is_file($this->directory . '/repositories/github/_/owner/re_po_x.json')); + Assert::true($storage->load($id)?->id->equals($id) ?? false); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-registry-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @param non-empty-string $type + * @param non-empty-string $uri + * @param list $tags + */ + private static function record(string $type, string $uri, array $tags): RepositoryRecord + { + return new RepositoryRecord( + id: new RepositoryId($type, $uri), + checkedAt: 1_000, + software: ['rr'], + releases: \array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), + ); + } + + private function storage(): FileRegistryStorage + { + return new FileRegistryStorage($this->directory, new Logger()); + } +} diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php new file mode 100644 index 0000000..9f1ab33 --- /dev/null +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -0,0 +1,140 @@ +withHead([new ReleaseRecord('v3', 'v3'), new ReleaseRecord('v2', 'new v2')]); + + Assert::same(self::tags($updated), ['v3', 'v2', 'v1']); + Assert::same($updated->releases()[1]->name, 'new v2'); + } + + #[Test] + public function tailIgnoresKnownReleases(): void + { + $record = new RepositoryRecord(self::id(), releases: [new ReleaseRecord('v2', 'v2')]); + + $updated = $record->withTail([new ReleaseRecord('v2', 'dup'), new ReleaseRecord('v1', 'v1')]); + + Assert::same(self::tags($updated), ['v2', 'v1']); + Assert::same($updated->releases()[0]->name, 'v2'); + } + + #[Test] + public function stalenessDependsOnTheLastCheck(): void + { + $never = RepositoryRecord::empty(self::id()); + $checked = $never->withCheckedAt(1_000); + + Assert::true($never->isStale(1_000, 600)); + Assert::false($checked->isStale(1_600, 600)); + Assert::true($checked->isStale(1_601, 600)); + } + + #[Test] + public function softwareIsAttachedOnce(): void + { + $record = RepositoryRecord::empty(self::id())->withSoftware('rr'); + + Assert::same($record->withSoftware('rr'), $record); + Assert::same($record->withSoftware('temporal')->software, ['rr', 'temporal']); + } + + #[Test] + public function survivesTheArrayRoundTrip(): void + { + $record = new RepositoryRecord( + id: self::id(), + checkedAt: 1_000, + complete: true, + software: ['rr'], + releases: [ + new ReleaseRecord( + tag: 'v2.0.0', + name: 'Release 2', + publishedAt: new \DateTimeImmutable('2024-01-02T03:04:05+00:00'), + prerelease: true, + assets: [new AssetRecord('rr-linux-amd64.tar.gz', 'https://x/rr.tar.gz', 42, 'application/gzip')], + ), + new ReleaseRecord('v1.0.0', 'v1.0.0'), + ], + ); + + $restored = RepositoryRecord::fromArray(\json_decode(\json_encode($record->toArray()), true)); + + Assert::same($restored->toArray(), $record->toArray()); + Assert::true($restored->id->equals(self::id())); + Assert::same($restored->releases()[0]->assets[0]->size, 42); + Assert::same($restored->releases()[0]->publishedAt?->format(\DATE_ATOM), '2024-01-02T03:04:05+00:00'); + Assert::null($restored->releases()[1]->publishedAt); + } + + #[Test] + public function rejectsUnknownFormatVersion(): void + { + try { + RepositoryRecord::fromArray(['version' => 99, 'repository' => ['type' => 'github', 'uri' => 'a/b']]); + Assert::fail('An unknown format version must be rejected.'); + } catch (\InvalidArgumentException $e) { + Assert::same($e->getMessage(), 'Unsupported repository record format.'); + } + } + + #[Test] + public function skipsBrokenReleasesButRequiresATag(): void + { + try { + ReleaseRecord::fromArray(['name' => 'no tag']); + Assert::fail('A release without a tag must be rejected.'); + } catch (\InvalidArgumentException $e) { + Assert::same($e->getMessage(), 'Release record requires a non-empty `tag`.'); + } + + // A broken asset makes the whole release unusable rather than silently dropping the asset + try { + ReleaseRecord::fromArray(['tag' => 'v1', 'assets' => [['name' => 'x']]]); + Assert::fail('An asset without a URI must be rejected.'); + } catch (\InvalidArgumentException) { + } + + $release = ReleaseRecord::fromArray(['tag' => 'v1', 'assets' => [['name' => 'ok', 'uri' => 'https://x']]]); + Assert::same($release->name, 'v1'); + Assert::count($release->assets, 1); + } + + private static function id(): RepositoryId + { + return new RepositoryId('github', 'owner/repo'); + } + + /** + * @return list + */ + private static function tags(RepositoryRecord $record): array + { + return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $record->releases()); + } +} diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php new file mode 100644 index 0000000..834460f --- /dev/null +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -0,0 +1,220 @@ +registry(); + + $pages = $registry->releases($this->id, $source); + $first = self::tagsOf($pages->current()); + + Assert::same($first, ['v6', 'v5']); + Assert::same($source->served, [0]); + + // The record already holds what was fetched, marked as incomplete + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v6', 'v5']); + Assert::false($record->complete); + Assert::same($record->checkedAt, $this->now); + } + + #[Test] + public function olderReleasesAreLoadedOnDemandAndPersisted(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $registry = $this->registry(); + + $all = self::flatten($registry->releases($this->id, $source)); + + Assert::same($all, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2, 4]); + + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::true($record->complete); + } + + #[Test] + public function freshRecordIsServedWithoutAnyRequest(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $this->now += 100; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + Assert::same($source->served, []); + } + + #[Test] + public function staleRecordIsCheckedWithASinglePageWhenNothingIsNew(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + Assert::same($source->served, [0]); + Assert::same($this->storage->load($this->id)->checkedAt, $this->now); + } + + #[Test] + public function newReleasesAreFetchedUntilAKnownOneIsReached(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + // Three releases were published: they span two pages, the second one reaches `v3` + $source->publish('v6', 'v5', 'v4'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2]); + } + + #[Test] + public function firstPageOverwritesStoredReleasesOnCheck(): void + { + $source = new ArrayReleaseSource([new ReleaseRecord('v1', 'v1', assets: [])]); + self::flatten($this->registry()->releases($this->id, $source)); + + // Assets were attached after the release had been stored + $updated = new ArrayReleaseSource([ + new ReleaseRecord('v1', 'v1', assets: [new \Internal\DLoad\Module\Registry\Record\AssetRecord('rr.zip', 'https://x/rr.zip')]), + ]); + $this->now += 601; + self::flatten($this->registry()->releases($this->id, $updated)); + + Assert::count($this->storage->load($this->id)->releases()[0]->assets, 1); + } + + #[Test] + public function refreshFlagIgnoresTheTtl(): void + { + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $source->publish('v3'); + self::flatten($this->registry(refresh: true)->releases($this->id, $source)); + + Assert::same($source->served, [0]); + Assert::same(self::tagsOf($this->storage->load($this->id)->releases()), ['v3', 'v2', 'v1']); + } + + #[Test] + public function failedCheckFallsBackToStoredReleases(): void + { + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + + $source->fail(); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v2', 'v1']); + } + + #[Test] + public function failedCheckWithoutStoredReleasesIsReported(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + $source->fail(); + + try { + self::flatten($this->registry()->releases($this->id, $source)); + Assert::fail('The failure of the source must reach the caller when nothing is stored.'); + } catch (ApiException) { + Assert::same($this->storage->records, []); + } + } + + #[Test] + public function storageFailureDoesNotBreakTheListing(): void + { + $this->storage->failOnSave = true; + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); + } + + #[Test] + public function attachRecordsTheSoftwareOnce(): void + { + $registry = $this->registry(); + + $registry->attach('rr', $this->id); + $registry->attach('rr', $this->id); + $registry->attach('roadrunner', $this->id); + + Assert::same($this->storage->load($this->id)->software, ['rr', 'roadrunner']); + Assert::same($this->storage->saves, 2); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->storage = new InMemoryRegistryStorage(); + $this->id = new RepositoryId('github', 'owner/repo'); + $this->now = 1_000_000; + } + + /** + * @param iterable> $pages + * @return list + */ + private static function flatten(iterable $pages): array + { + $tags = []; + foreach ($pages as $page) { + $tags = [...$tags, ...self::tagsOf($page)]; + } + + return $tags; + } + + /** + * @param list $releases + * @return list + */ + private static function tagsOf(array $releases): array + { + return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $releases); + } + + private function registry(bool $refresh = false): StoredVersionRegistry + { + return new StoredVersionRegistry($this->storage, 600, new Logger(), $refresh, fn(): int => $this->now); + } +} diff --git a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php new file mode 100644 index 0000000..93dda14 --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php @@ -0,0 +1,81 @@ + + */ + public array $served = []; + + /** When set, every page request fails with this exception. */ + public ?\Throwable $failure = null; + + /** + * @param list $releases Newest first. + * @param int<1, max> $perPage + */ + public function __construct( + private array $releases, + private readonly int $perPage = 2, + ) {} + + /** + * @param list $tags Newest first. + */ + public static function ofTags(array $tags, int $perPage = 2): self + { + return new self(\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), $perPage); + } + + /** + * Publishes releases on top of the list, as a repository would between two runs. + * + * @param non-empty-string ...$tags Newest first. + */ + public function publish(string ...$tags): void + { + $this->releases = [...\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), ...$this->releases]; + } + + public function fail(?\Throwable $failure = null): void + { + $this->failure = $failure ?? new ApiException('API is unavailable.', 'stub/stub'); + } + + public function pages(int $offset = 0): \Generator + { + // Align with a page boundary and skip within the page, like a real paged API + $page = \intdiv($offset, $this->perPage); + $skip = $offset % $this->perPage; + + do { + $this->failure === null or throw $this->failure; + + $this->served[] = $page * $this->perPage; + $items = \array_slice($this->releases, $page * $this->perPage, $this->perPage); + $last = ($page + 1) * $this->perPage >= \count($this->releases); + + yield new ReleasePage(\array_slice($items, $skip), $last); + + $skip = 0; + ++$page; + } while (!$last); + } +} diff --git a/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php new file mode 100644 index 0000000..6a4a76b --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php @@ -0,0 +1,52 @@ + */ + public array $records = []; + + /** @var int<0, max> */ + public int $saves = 0; + + public bool $failOnSave = false; + + public function load(RepositoryId $id): ?RepositoryRecord + { + return $this->records[(string) $id] ?? null; + } + + public function save(RepositoryRecord $record): void + { + $this->failOnSave and throw new \RuntimeException('Storage is read-only.'); + + ++$this->saves; + $this->records[(string) $record->id] = $record; + } + + public function all(): iterable + { + yield from \array_values($this->records); + } + + public function remove(RepositoryId $id): void + { + unset($this->records[(string) $id]); + } + + public function clear(): void + { + $this->records = []; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php new file mode 100644 index 0000000..13a50b7 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -0,0 +1,152 @@ +getReleases(), false); + + Assert::same(\count($releases), 300); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function pagesAreLoadedOnlyWhenNeeded(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + + // Consume the whole first page, but nothing beyond it + $seen = 0; + foreach ($repository->getReleases() as $release) { + unset($release); + if (++$seen === 100) { + break; + } + } + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function destroyDoesNotLoadTheRemainingPages(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + $repository->getReleases()->first(); + + $repository->destroy(); + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + #[Test] + public function secondRunIsServedFromTheRegistryWithoutRequests(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 2); + $firstRun = self::names(self::createRepository($firstClient, self::registry($storage))); + + // A second run in a fresh process with the registry carried over + $secondClient = new PagedClientStub(pages: 2); + $secondRun = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1, 2]); + Assert::same($secondClient->requestedPages(), []); + Assert::same($secondRun, $firstRun); + Assert::same(\count($secondRun), 200); + } + + #[Test] + public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void + { + $storage = new InMemoryRegistryStorage(); + + // The first run needs the first page only + $firstClient = new PagedClientStub(pages: 3); + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + break; + } + + // The second run needs everything: the stored page costs nothing, the rest is fetched + $secondClient = new PagedClientStub(pages: 3); + $all = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1]); + Assert::same($secondClient->requestedPages(), [2, 3]); + Assert::same(\count($all), 300); + } + + private static function createRepository( + PagedClientStub $client, + VersionRegistry $registry = new PassThroughRegistry(), + ): GitHubRepository { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + $api = new RepositoryApi( + new Client($httpFactory, $client, new GitHubConfig()), + $httpFactory, + 'owner', + 'repo', + $logger, + ); + + return new GitHubRepository($api, 'owner', 'repo', $logger, $registry); + } + + private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry + { + return new StoredVersionRegistry($storage, 600, new Logger()); + } + + /** + * @return list + */ + private static function names(GitHubRepository $repository): array + { + return \array_map( + static fn(ReleaseInterface $release): string => $release->getName(), + \iterator_to_array($repository->getReleases(), false), + ); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php new file mode 100644 index 0000000..b6a04da --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -0,0 +1,105 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<1, max> $releasesPerPage Number of releases on every such page. + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 100, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + $perPage = self::perPageOf($query); + $all = $this->allReleases(); + + // Serve the slice the real API would serve for the requested page size + $releases = \array_slice($all, ($page - 1) * $perPage, $perPage); + if ($releases === []) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page * $perPage < \count($all) + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($releases)); + } + + /** + * Page number of every received request, in order. + * + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + private static function perPageOf(string $query): int + { + \parse_str($query, $params); + + return \max(1, (int) ($params['per_page'] ?? 30)); + } + + /** + * @return list> + */ + private function allReleases(): array + { + $releases = []; + $total = $this->pages * $this->releasesPerPage; + + for ($i = 1; $i <= $total; $i++) { + $tag = \sprintf('v1.0.%d', $i); + $releases[] = [ + 'name' => $tag, + 'tag_name' => $tag, + 'published_at' => '2024-01-01T00:00:00Z', + 'assets' => [], + 'prerelease' => false, + 'draft' => false, + ]; + } + + return $releases; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php index 96b9527..8a25c34 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab as GitLabConfig; +use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; use Internal\DLoad\Module\Repository\Internal\GitLab\Factory; use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab\Stub\HttpFactoryStub; @@ -69,6 +70,11 @@ public function createDerivesTheProjectPathFromTheUri(string $uri, string $expec #[BeforeTest] protected function prepare(): void { - $this->factory = new Factory(new HttpFactoryStub(), new GitLabConfig(), new Logger()); + $this->factory = new Factory( + new HttpFactoryStub(), + new GitLabConfig(), + new Logger(), + new PassThroughRegistry(), + ); } } diff --git a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php new file mode 100644 index 0000000..c94c5b0 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php @@ -0,0 +1,114 @@ +getReleases(), false); + + Assert::same(\count($releases), 300); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function pagesAreLoadedOnlyWhenNeeded(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + + $seen = 0; + foreach ($repository->getReleases() as $release) { + unset($release); + if (++$seen === 100) { + break; + } + } + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function destroyDoesNotLoadTheRemainingPages(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + $repository->getReleases()->first(); + + $repository->destroy(); + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + #[Test] + public function secondRunIsServedFromTheRegistryWithoutRequests(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 2); + \iterator_to_array(self::createRepository($firstClient, self::registry($storage))->getReleases(), false); + + $secondClient = new PagedClientStub(pages: 2); + $secondRun = \iterator_to_array(self::createRepository($secondClient, self::registry($storage))->getReleases(), false); + + Assert::same($firstClient->requestedPages(), [1, 2]); + Assert::same($secondClient->requestedPages(), []); + Assert::same(\count($secondRun), 200); + } + + private static function createRepository( + PagedClientStub $client, + VersionRegistry $registry = new PassThroughRegistry(), + ): GitLabRepository { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + $api = new RepositoryApi( + new Client($httpFactory, $client, new GitLabConfig()), + $httpFactory, + 'group/project', + ); + + return new GitLabRepository($api, 'group/project', $logger, $registry); + } + + private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry + { + return new StoredVersionRegistry($storage, 600, new Logger()); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php new file mode 100644 index 0000000..eca2b51 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php @@ -0,0 +1,106 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<1, max> $releasesPerPage Number of releases on every such page. + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 100, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + $perPage = self::perPageOf($query); + $all = $this->allReleases(); + + // Serve the slice the real API would serve for the requested page size + $releases = \array_slice($all, ($page - 1) * $perPage, $perPage); + if ($releases === []) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page * $perPage < \count($all) + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($releases)); + } + + /** + * Page number of every received request, in order. + * + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + private static function perPageOf(string $query): int + { + \parse_str($query, $params); + + return \max(1, (int) ($params['per_page'] ?? 20)); + } + + /** + * @return list> + */ + private function allReleases(): array + { + $releases = []; + $total = $this->pages * $this->releasesPerPage; + + for ($i = 1; $i <= $total; $i++) { + $tag = \sprintf('v1.0.%d', $i); + $releases[] = [ + 'name' => $tag, + 'tag_name' => $tag, + 'description' => 'Release ' . $tag, + 'created_at' => '2024-01-01T00:00:00Z', + 'released_at' => '2024-01-01T00:00:00Z', + 'assets' => ['links' => []], + 'upcoming_release' => false, + ]; + } + + return $releases; + } +} From ea36d7e47902ccc80eb3ad6c368abb87ad7438b9 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Sat, 12 Sep 2026 23:58:13 +0400 Subject: [PATCH 02/17] fix: recover from releases deleted after the version registry stored them A 404 for a release asset was reported as a missing repository, with advice about tokens and addresses, and the deleted release stayed in the registry until a later check happened to overwrite it. Asset URLs now raise `AssetNotFoundException`; when every matching asset of a release is gone, the downloader drops the release from the registry, marks the repository for a check, and fetches the list once more before giving up. Assisted-By: Claude Fable 5.1 --- README-es.md | 4 +- README-ru.md | 4 +- README-zh.md | 3 +- README.md | 2 + src/Module/Downloader/Downloader.php | 58 +++++- .../Downloader/Exception/ReleaseGone.php | 13 ++ .../Registry/Internal/PassThroughRegistry.php | 5 + .../Internal/StoredVersionRegistry.php | 13 ++ .../Registry/Record/RepositoryRecord.php | 31 +++ src/Module/Registry/VersionRegistry.php | 10 + .../Exception/AssetNotFoundException.php | 13 ++ .../Internal/GitHub/Api/ResponseValidator.php | 6 + .../Internal/GitLab/Api/ResponseValidator.php | 6 + .../Repository/Internal/ResponseValidator.php | 18 ++ .../Unit/Module/Downloader/DownloaderTest.php | 181 ++++++++++++++++++ .../Module/Downloader/Stub/GoneAssetStub.php | 58 ++++++ .../Stub/SequenceRepositoryFactoryStub.php | 39 ++++ .../Module/Registry/RepositoryRecordTest.php | 20 ++ .../Registry/StoredVersionRegistryTest.php | 35 ++++ .../Registry/Stub/RecordingRegistry.php | 39 ++++ .../GitHub/Api/ResponseValidatorTest.php | 24 ++- .../GitLab/Api/ResponseValidatorTest.php | 21 ++ 22 files changed, 592 insertions(+), 11 deletions(-) create mode 100644 src/Module/Downloader/Exception/ReleaseGone.php create mode 100644 src/Module/Repository/Exception/AssetNotFoundException.php create mode 100644 tests/Unit/Module/Downloader/DownloaderTest.php create mode 100644 tests/Unit/Module/Downloader/Stub/GoneAssetStub.php create mode 100644 tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php create mode 100644 tests/Unit/Module/Registry/Stub/RecordingRegistry.php diff --git a/README-es.md b/README-es.md index 9c9fc35..a4e61a5 100644 --- a/README-es.md +++ b/README-es.md @@ -391,7 +391,9 @@ El registro está activado por defecto y vive en el directorio de caché del usu > El registro solo contiene metadatos de releases: tags, nombres y enlaces de descarga. Las descargas > no pasan por él y nunca guarda credenciales, así que el directorio puede compartirse o guardarse en > la caché de CI sin problemas. Si una comprobación falla por un error de red o un límite de la API, se -> usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. +> usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. Un +> release almacenado cuyos assets desaparecieron del origen se elimina del registro en cuanto falla +> su descarga, y la lista de releases se vuelve a obtener antes de que la ejecución se dé por vencida. En GitHub Actions el directorio puede conservarse entre ejecuciones del workflow, de modo que cada ejecución gasta el límite de la API solo en los releases publicados desde la anterior: diff --git a/README-ru.md b/README-ru.md index b9a21fe..1b3cd39 100644 --- a/README-ru.md +++ b/README-ru.md @@ -393,7 +393,9 @@ DLoad поддерживает три типа загрузки, которые > не проходят, учётные данные в нём не сохраняются, поэтому каталог можно свободно передавать между > машинами и складывать в кэш CI. Если проверка не удалась из-за сетевой ошибки или лимита API, > используются сохранённые релизы, а репозиторий, который раньше не встречался, по-прежнему -> завершится ошибкой. +> завершится ошибкой. Сохранённый релиз, ассеты которого исчезли из источника, удаляется из +> реестра сразу после неудачной загрузки, а список релизов запрашивается заново, прежде чем +> запуск завершится ошибкой. В GitHub Actions каталог можно переносить между запусками workflow, тогда запуск тратит лимит API только на релизы, вышедшие после предыдущего: diff --git a/README-zh.md b/README-zh.md index 2b63606..67c810e 100644 --- a/README-zh.md +++ b/README-zh.md @@ -387,7 +387,8 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： > [!NOTE] > 注册表只保存发布的元数据：标签、名称和资产下载链接。下载不会经过注册表，也不会保存任何凭据， > 因此该目录可以自由共享或放入 CI 缓存。若因网络错误或 API 速率限制导致检查失败，会使用已保存的发布； -> 从未见过的仓库仍会明确报错。 +> 从未见过的仓库仍会明确报错。若某个已保存发布的资产在上游已被删除，下载失败后它会立即从注册表中移除， +> 并在本次运行放弃之前重新获取发布列表。 在 GitHub Actions 中可以在多次工作流运行之间保留该目录，这样每次运行只为上次运行之后发布的版本消耗速率限制： diff --git a/README.md b/README.md index 92f62e3..220c218 100644 --- a/README.md +++ b/README.md @@ -394,6 +394,8 @@ The registry is on by default and lives in the per-user cache directory (`$XDG_C > go through it and credentials are never stored in it, so the directory can be shared or committed > to a CI cache freely. When a check fails because of a network error or a rate limit, the stored > releases are used instead, and a repository that was never seen before still fails loudly. +> A stored release whose assets have disappeared upstream is dropped from the registry as soon as +> its download fails, and the release list is fetched again before the run gives up. In GitHub Actions the directory can be carried between workflow runs, so a run spends the rate limit only on releases published since the previous one: diff --git a/src/Module/Downloader/Downloader.php b/src/Module/Downloader/Downloader.php index 4e95530..c317a95 100644 --- a/src/Module/Downloader/Downloader.php +++ b/src/Module/Downloader/Downloader.php @@ -16,6 +16,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Software; use Internal\DLoad\Module\Downloader\Exception\DownloadFailed; use Internal\DLoad\Module\Downloader\Exception\NotFound; +use Internal\DLoad\Module\Downloader\Exception\ReleaseGone; use Internal\DLoad\Module\Downloader\Internal\Diagnostics\DownloadDiagnostics; use Internal\DLoad\Module\Downloader\Internal\DownloadContext; use Internal\DLoad\Module\Downloader\Task\DownloadResult; @@ -25,6 +26,7 @@ use Internal\DLoad\Module\Repository\AssetInterface; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\ReleaseInterface; @@ -167,9 +169,12 @@ public function download( * @param DownloadContext $context Download context information * @return \Closure(): ReleaseInterface Closure that returns the selected release */ - private function processRepository(Repository $repository, DownloadContext $context): \Closure + private function processRepository(Repository $repository, DownloadContext $context, bool $mayRetry = true): \Closure { - return function () use ($repository, $context): ReleaseInterface { + return function () use ($repository, $context, $mayRetry): ReleaseInterface { + // Set when a release turned out to be deleted: the release list is outdated then + $forgotten = false; + $this->logger->info( 'Loading releases from `%s` repository %s', $context->repoConfig->type, @@ -206,7 +211,15 @@ private function processRepository(Repository $repository, DownloadContext $cont } process_release: - $releases === [] and throw new NotFound('No relevant release found.'); + if ($releases === []) { + // The list was outdated: ask the repository again once, with the deleted releases forgotten + if ($forgotten && $mayRetry) { + return $this->retryRepository($context); + } + + throw new NotFound('No relevant release found.'); + } + $context->release = \array_shift($releases); $context->releaseAttempt = $context->repositoryAttempt->addRelease($context->release->getName()); @@ -215,6 +228,17 @@ private function processRepository(Repository $repository, DownloadContext $cont try { await(coroutine($this->processRelease($context))); return $context->release; + } catch (ReleaseGone $e) { + // The registry must not offer this release again, and the list needs a fresh check + $this->registry->forget( + RepositoryId::fromConfig($context->repoConfig), + $context->release->getVersion()->string, + ); + $forgotten = true; + + $context->releaseAttempt->reason ??= $e->getMessage(); + $this->logger->debug($e->getMessage()); + goto process_release; } catch (NotFound $e) { $context->releaseAttempt->reason ??= $e->getMessage(); $this->logger->debug($e->getMessage()); @@ -224,6 +248,23 @@ private function processRepository(Repository $repository, DownloadContext $cont }; } + /** + * Fetches the release list anew after deleted releases were forgotten and tries once more. + * + * @throws NotFound When the fresh list has nothing suitable either. + */ + private function retryRepository(DownloadContext $context): ReleaseInterface + { + $this->logger->info('Release list of `%s` is outdated, fetching it again.', $context->repoConfig->uri); + $repository = $this->repositoryProvider->getByConfig($context->repoConfig); + + try { + return await(coroutine($this->processRepository($repository, $context, mayRetry: false))); + } finally { + $repository instanceof Destroyable and $repository->destroy(); + } + } + /** * Processes a release to find suitable assets. * @@ -397,8 +438,16 @@ private function findAssetWithGradualFiltering(DownloadContext $context): AssetI */ private function tryProcessAssets(array $assets, DownloadContext $context): AssetInterface { + // Stays true while every failed asset answered "not found": then the release itself is gone + $gone = $assets !== []; + process_asset: - $assets === [] and throw new NotFound('none of the matching assets could be downloaded'); + if ($assets === []) { + $gone and throw new ReleaseGone('every matching asset of the release is no longer available'); + + throw new NotFound('none of the matching assets could be downloaded'); + } + $context->asset = \array_shift($assets); $this->logger->debug('Trying to load asset `%s`', $context->asset->getName()); try { @@ -408,6 +457,7 @@ private function tryProcessAssets(array $assets, DownloadContext $context): Asse // Retrying other assets makes the situation worse: report the limit immediately throw $e; } catch (\Throwable $e) { + $gone = $gone && $e instanceof AssetNotFoundException; $context->releaseAttempt->addFailure($context->asset->getName(), $e); $this->logger->exception($e, important: false); goto process_asset; diff --git a/src/Module/Downloader/Exception/ReleaseGone.php b/src/Module/Downloader/Exception/ReleaseGone.php new file mode 100644 index 0000000..30b1c3f --- /dev/null +++ b/src/Module/Downloader/Exception/ReleaseGone.php @@ -0,0 +1,13 @@ +persist($updated); } + public function forget(RepositoryId $id, string $tag): void + { + $record = $this->storage->load($id); + if ($record === null || !$record->has($tag)) { + return; + } + + $this->logger->debug('Release `%s` of `%s` is gone: dropped from the version registry.', $tag, (string) $id); + + // Without the last check the next listing asks the source again + $this->persist($record->withoutRelease($tag)->withoutCheck()); + } + /** * @param list $page */ diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php index 2ecc5a1..91c6ce6 100644 --- a/src/Module/Registry/Record/RepositoryRecord.php +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -162,6 +162,37 @@ public function withCheckedAt(int $checkedAt): self return $this->with(checkedAt: $checkedAt); } + /** + * Forgets the last check, so the record counts as stale until the source is asked again. + */ + public function withoutCheck(): self + { + return new self( + id: $this->id, + checkedAt: null, + complete: $this->complete, + software: $this->software, + releases: $this->releases(), + ); + } + + /** + * Drops a release; an unknown tag leaves the record as it is. + * + * @param non-empty-string $tag + */ + public function withoutRelease(string $tag): self + { + if (!$this->has($tag)) { + return $this; + } + + return $this->with(releases: \array_values(\array_filter( + $this->releases(), + static fn(ReleaseRecord $release): bool => $release->tag !== $tag, + ))); + } + public function withComplete(bool $complete): self { return $this->with(complete: $complete); diff --git a/src/Module/Registry/VersionRegistry.php b/src/Module/Registry/VersionRegistry.php index 986419a..a0929be 100644 --- a/src/Module/Registry/VersionRegistry.php +++ b/src/Module/Registry/VersionRegistry.php @@ -41,4 +41,14 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator; * @param non-empty-string $software Software identifier. */ public function attach(string $software, RepositoryId $id): void; + + /** + * Drops a release that turned out to be gone and marks the repository for a check. + * + * Called when the assets of a stored release cannot be downloaded any more: the next listing + * asks the source again instead of trusting the stored record. + * + * @param non-empty-string $tag Tag of the release as stored in the registry. + */ + public function forget(RepositoryId $id, string $tag): void; } diff --git a/src/Module/Repository/Exception/AssetNotFoundException.php b/src/Module/Repository/Exception/AssetNotFoundException.php new file mode 100644 index 0000000..df3a1a9 --- /dev/null +++ b/src/Module/Repository/Exception/AssetNotFoundException.php @@ -0,0 +1,13 @@ +accessDeniedMessage($apiMessage, $repository), $repository, ), + // A missing asset is not a missing repository: the listing was fine, the file is gone + $status === 404 && $this->isAssetUri((string) $request->getUri()) => new AssetNotFoundException( + \sprintf( + '%s asset is no longer available: HTTP 404 for %s. ' + . 'The release may have been deleted or its assets replaced since the release list was fetched.', + $this->providerName(), + (string) $request->getUri(), + ), + $repository, + ), $status === 404 => new RepositoryNotFoundException( $this->notFoundMessage($apiMessage, $repository, $endpoint), $repository, @@ -135,6 +146,13 @@ protected function repositoryTerm(): string */ abstract protected function repositoryFromUri(string $uri): ?string; + /** + * Whether the URI points to a release asset rather than to the API. + * + * A 404 for an asset means the release is gone, not that the repository does not exist. + */ + abstract protected function isAssetUri(string $uri): bool; + /** * @return positive-int|null Requests per hour allowed without a token. */ diff --git a/tests/Unit/Module/Downloader/DownloaderTest.php b/tests/Unit/Module/Downloader/DownloaderTest.php new file mode 100644 index 0000000..f0b459a --- /dev/null +++ b/tests/Unit/Module/Downloader/DownloaderTest.php @@ -0,0 +1,181 @@ +download([$repository]); + + Assert::same($result->version->string, 'v1.9.0'); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + Assert::same($this->registry->attached, [['rr', 'github:owner/repo']]); + } + + #[Test] + public function outdatedListIsFetchedAgainWhenNothingIsLeft(): void + { + // The stored list knows only the deleted release; a fresh list has its replacement + $stale = new RepositoryStub('owner/repo'); + $stale = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($stale, 'v2.0.0', assets: false), + ])); + $fresh = new RepositoryStub('owner/repo'); + $fresh = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($fresh, 'v2.0.1', assets: true), + ])); + $factory = new SequenceRepositoryFactoryStub([$stale, $fresh]); + + $result = $this->download($factory); + + Assert::same($result->version->string, 'v2.0.1'); + Assert::same($factory->created, 2); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + } + + #[Test] + public function outdatedListIsFetchedAgainOnlyOnce(): void + { + $stale = new RepositoryStub('owner/repo'); + $stale = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($stale, 'v2.0.0', assets: false), + ])); + $factory = new SequenceRepositoryFactoryStub([$stale]); + + try { + $this->download($factory); + Assert::fail('DownloadFailed is expected when the fresh list has nothing suitable either.'); + } catch (DownloadFailed $e) { + Assert::same($factory->created, 2); + Assert::string($e->report)->contains('no longer available'); + } + } + + #[Test] + public function releaseWithOtherFailuresIsNotForgotten(): void + { + $repository = new RepositoryStub('owner/repo'); + $broken = new ReleaseStub($repository, 'v2.0.0', Version::fromVersionString('v2.0.0')); + $broken->setAssets([ + new GoneAssetStub($broken, 'rr-linux-amd64.tar.gz'), + // A working asset: the release is not gone, the first asset just was + new AssetStub($broken, 'rr-linux-amd64.zip', 'https://x/rr.zip'), + ]); + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create([$broken])); + + $result = $this->download([$repository]); + + Assert::same($result->version->string, 'v2.0.0'); + Assert::same($this->registry->forgotten, []); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->tempDir = \sys_get_temp_dir() . '/dload-downloader-' . \bin2hex(\random_bytes(6)); + $this->registry = new RecordingRegistry(); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->tempDir) and FS::removeDir(Path::create($this->tempDir)); + } + + /** + * @param non-empty-string $tag + * @param bool $assets `true` for a downloadable asset, `false` for one that is gone. + */ + private static function release(RepositoryStub $repository, string $tag, bool $assets): ReleaseStub + { + $release = new ReleaseStub($repository, $tag, Version::fromVersionString($tag)); + $release->setAssets([ + $assets + ? new AssetStub($release, 'rr-linux-amd64.tar.gz', 'https://x/' . $tag . '/rr.tar.gz') + : new GoneAssetStub($release, 'rr-linux-amd64.tar.gz'), + ]); + + return $release; + } + + /** + * @param list|SequenceRepositoryFactoryStub $repositories + */ + private function download(array|SequenceRepositoryFactoryStub $repositories): DownloadResult + { + $factory = $repositories instanceof SequenceRepositoryFactoryStub + ? $repositories + : new SequenceRepositoryFactoryStub($repositories); + + $config = new DownloaderConfig(); + $config->tmpDir = $this->tempDir; + + $downloader = new Downloader( + config: $config, + logger: new Logger(), + repositoryProvider: (new RepositoryProvider())->addRepositoryFactory($factory), + architecture: Architecture::tryFromString('amd64') ?? throw new \LogicException(), + operatingSystem: OperatingSystem::tryFromString('linux') ?? throw new \LogicException(), + stability: Stability::Stable, + archiveService: new ArchiveFactory(), + registry: $this->registry, + ); + + $software = Software::fromArray([ + 'name' => 'rr', + 'repositories' => [['type' => 'github', 'uri' => 'owner/repo']], + ]); + $task = $downloader->download($software, DownloadConfig::fromSoftwareId('rr'), static fn(): null => null); + + /** @var DownloadResult */ + return await(($task->handler)()); + } +} diff --git a/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php new file mode 100644 index 0000000..b33d2d9 --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php @@ -0,0 +1,58 @@ +release; + } + + public function getName(): string + { + return $this->name; + } + + public function getUri(): string + { + return 'https://github.com/owner/repo/releases/download/' . $this->release->getName() . '/' . $this->name; + } + + public function getOperatingSystem(): ?OperatingSystem + { + return null; + } + + public function getArchitecture(): ?Architecture + { + return null; + } + + public function download(): \Traversable + { + throw new AssetNotFoundException('GitHub asset is no longer available: HTTP 404 for ' . $this->getUri(), 'owner/repo'); + + /** @psalm-suppress UnevaluatedCode */ + yield ''; + } +} diff --git a/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php b/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php new file mode 100644 index 0000000..a89bb6c --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php @@ -0,0 +1,39 @@ + Number of repositories created so far. */ + public int $created = 0; + + /** + * @param list $repositories Repositories to return, in order; the last one repeats. + */ + public function __construct( + private readonly array $repositories, + ) {} + + public function supports(RepositoryConfig $config): bool + { + return true; + } + + public function create(RepositoryConfig $config): Repository + { + $repository = $this->repositories[\min($this->created, \count($this->repositories) - 1)]; + ++$this->created; + + return $repository; + } +} diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php index 9f1ab33..ae6570c 100644 --- a/tests/Unit/Module/Registry/RepositoryRecordTest.php +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -43,6 +43,26 @@ public function tailIgnoresKnownReleases(): void Assert::same($updated->releases()[0]->name, 'v2'); } + #[Test] + public function releaseCanBeDroppedAndTheCheckForgotten(): void + { + $record = new RepositoryRecord(self::id(), checkedAt: 1_000, releases: [ + new ReleaseRecord('v2', 'v2'), + new ReleaseRecord('v1', 'v1'), + ]); + + $dropped = $record->withoutRelease('v2'); + + Assert::same(self::tags($dropped), ['v1']); + Assert::same($dropped->checkedAt, 1_000); + Assert::same($record->withoutRelease('v9'), $record); + + $stale = $dropped->withoutCheck(); + Assert::null($stale->checkedAt); + Assert::same(self::tags($stale), ['v1']); + Assert::true($stale->isStale(1_000, 600)); + } + #[Test] public function stalenessDependsOnTheLastCheck(): void { diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php index 834460f..5a32f23 100644 --- a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -169,6 +169,41 @@ public function storageFailureDoesNotBreakTheListing(): void Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); } + #[Test] + public function forgetDropsTheReleaseAndForcesTheNextCheck(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + $registry = $this->registry(); + self::flatten($registry->releases($this->id, $source)); + $source->served = []; + + // `v3` was deleted upstream and its download failed + $registry->forget($this->id, 'v3'); + + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v2', 'v1']); + Assert::null($record->checkedAt); + + // The record is fresh by time, yet the next listing asks the source again + $again = self::flatten($this->registry()->releases($this->id, $source)); + Assert::same($source->served, [0]); + Assert::same($again, ['v3', 'v2', 'v1']); + } + + #[Test] + public function forgetOfAnUnknownReleaseChangesNothing(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $saves = $this->storage->saves; + + $this->registry()->forget($this->id, 'v9'); + $this->registry()->forget(new RepositoryId('github', 'other/repo'), 'v1'); + + Assert::same($this->storage->saves, $saves); + Assert::same($this->storage->load($this->id)->checkedAt, $this->now); + } + #[Test] public function attachRecordsTheSoftwareOnce(): void { diff --git a/tests/Unit/Module/Registry/Stub/RecordingRegistry.php b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php new file mode 100644 index 0000000..95a7603 --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php @@ -0,0 +1,39 @@ + Software attached, as `[software, repository id]`. */ + public array $attached = []; + + /** @var list Releases forgotten, as `[repository id, tag]`. */ + public array $forgotten = []; + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + foreach ($source->pages() as $page) { + yield $page->releases; + } + } + + public function attach(string $software, RepositoryId $id): void + { + $this->attached[] = [$software, (string) $id]; + } + + public function forget(RepositoryId $id, string $tag): void + { + $this->forgotten[] = [(string) $id, $tag]; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php b/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php index 6d13d98..a5f51e7 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Repository\Exception\AccessDeniedException; use Internal\DLoad\Module\Repository\Exception\ApiException; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\ResponseValidator; @@ -126,20 +127,35 @@ public function serverErrorIsReportedAsTemporaryFailure(): void } #[Test] - public function repositoryIsResolvedFromAssetDownloadUrl(): void + public function missingAssetIsNotReportedAsMissingRepository(): void { $validator = new ResponseValidator(authenticated: false); $request = new Request('GET', 'https://github.com/owner/repo/releases/download/v1.0.0/asset.zip'); - $response = new ResponseStub(404, [], \json_encode(['message' => 'Not Found'])); + $response = new ResponseStub(404, [], 'Not Found'); try { $validator->validate($request, $response); - Assert::fail('RepositoryNotFoundException is expected.'); - } catch (RepositoryNotFoundException $e) { + Assert::fail('AssetNotFoundException is expected.'); + } catch (AssetNotFoundException $e) { + // The repository is still known, but the advice about tokens and addresses would mislead Assert::same($e->repository, 'owner/repo'); + Assert::string($e->getMessage())->contains('asset is no longer available'); + Assert::string($e->getMessage())->contains('release may have been deleted'); + Assert::string($e->getMessage())->notContains('GITHUB_TOKEN'); } } + #[Test] + public function forbiddenAssetIsStillAnAccessProblem(): void + { + $validator = new ResponseValidator(authenticated: false); + $request = new Request('GET', 'https://github.com/owner/repo/releases/download/v1.0.0/asset.zip'); + + Expect::exception(AccessDeniedException::class); + + $validator->validate($request, new ResponseStub(403, [], 'Forbidden')); + } + #[Test] public function transportFailureKeepsTheOriginalError(): void { diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php b/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php index fd8e2ea..ad45478 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab\Api; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\ResponseValidator; @@ -16,6 +17,26 @@ #[Covers(ResponseValidator::class)] final class ResponseValidatorTest { + #[Test] + public function missingAssetIsNotReportedAsMissingProject(): void + { + $validator = new ResponseValidator(authenticated: false); + $request = new Request( + 'GET', + 'https://gitlab.com/api/v4/projects/group%2Fproject/releases/v1.0.0/downloads/asset.zip', + ); + $response = new ResponseStub(404, [], \json_encode(['message' => '404 Not Found'])); + + try { + $validator->validate($request, $response); + Assert::fail('AssetNotFoundException is expected.'); + } catch (AssetNotFoundException $e) { + Assert::same($e->repository, 'group/project'); + Assert::string($e->getMessage())->contains('asset is no longer available'); + Assert::string($e->getMessage())->notContains('GITLAB_TOKEN'); + } + } + #[Test] public function projectPathIsDecodedFromApiUrl(): void { From afeaa29b97e454cd4bfb6b10fbe9f37619edb1b3 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Mon, 14 Sep 2026 19:34:02 +0400 Subject: [PATCH 03/17] fix: drop releases deleted upstream when the version registry checks the head A release deleted from the middle of the stored list survived the check as a phantom at the end of the list. It inflated the release count the registry uses as the offset for loading older pages, so one real release was skipped and never loaded until the cache was cleared. The repository identity is normalized to lower case without surrounding slashes, as GitHub and GitLab resolve paths case-insensitively, and the file storage rejects a record whose identity differs from the requested one: sanitized names and case-insensitive file systems can map two identities onto one file. Windows device names such as `nul` or `com1` get a prefix, because Windows opens the device whatever the extension and the write silently failed. Assisted-By: Claude Fable 5.1 --- .../Registry/Internal/FileRegistryStorage.php | 17 ++++-- .../Registry/Record/RepositoryRecord.php | 59 ++++++++++++++++--- src/Module/Registry/RepositoryId.php | 25 ++++++-- .../Registry/FileRegistryStorageTest.php | 23 ++++++++ .../Module/Registry/RepositoryRecordTest.php | 53 +++++++++++++++++ .../Registry/StoredVersionRegistryTest.php | 51 ++++++++++++++++ .../Registry/Stub/ArrayReleaseSource.php | 13 ++++ 7 files changed, 222 insertions(+), 19 deletions(-) diff --git a/src/Module/Registry/Internal/FileRegistryStorage.php b/src/Module/Registry/Internal/FileRegistryStorage.php index f9b2a1a..e51741f 100644 --- a/src/Module/Registry/Internal/FileRegistryStorage.php +++ b/src/Module/Registry/Internal/FileRegistryStorage.php @@ -43,7 +43,10 @@ public function __construct( public function load(RepositoryId $id): ?RepositoryRecord { - return $this->read($this->fileOf($id)); + $record = $this->read($this->fileOf($id)); + + // Sanitizing and case-insensitive file systems may map two identities onto one file + return $record?->id->equals($id) === true ? $record : null; } public function save(RepositoryRecord $record): void @@ -55,7 +58,7 @@ public function save(RepositoryRecord $record): void $payload = \json_encode($record->toArray(), \JSON_THROW_ON_ERROR | \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES); - $temp = Path::create((string) $file . '.' . \getmypid() . '.tmp'); + $temp = Path::create((string) $file . '.' . (int) \getmypid() . '.tmp'); @\file_put_contents((string) $temp, $payload) === false and throw new \RuntimeException( \sprintf('Failed to write registry record `%s`.', $temp), ); @@ -100,7 +103,9 @@ public function clear(): void /** * Keeps a path segment safe for every file system: anything but plain ASCII is replaced, - * and a segment that would otherwise be empty or a directory reference gets a placeholder. + * a segment that would otherwise be empty or a directory reference gets a placeholder, and + * a Windows device name (`nul`, `con`, `com1`...) gets a prefix, as Windows opens the device + * whatever the extension. * * @return non-empty-string */ @@ -108,7 +113,11 @@ private static function sanitize(string $segment): string { $safe = (string) \preg_replace('/[^A-Za-z0-9._-]+/', '_', $segment); - return $safe === '' || \trim($safe, '.') === '' ? '_' : $safe; + if ($safe === '' || \trim($safe, '.') === '') { + return '_'; + } + + return \preg_match('/^(?:con|prn|aux|nul|com[0-9]|lpt[0-9])(?:\.|$)/i', $safe) === 1 ? '_' . $safe : $safe; } /** diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php index 91c6ce6..84d11f7 100644 --- a/src/Module/Registry/Record/RepositoryRecord.php +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -70,24 +70,29 @@ public static function fromArray(array $data): self 'Unsupported repository record format.', ); + /** @var mixed $repository */ $repository = $data['repository'] ?? null; + /** @var mixed $type */ $type = \is_array($repository) ? ($repository['type'] ?? null) : null; + /** @var mixed $uri */ $uri = \is_array($repository) ? ($repository['uri'] ?? null) : null; \is_string($type) && $type !== '' && \is_string($uri) && $uri !== '' or throw new \InvalidArgumentException( 'Repository record requires a repository type and URI.', ); $releases = []; + /** @var mixed $release */ foreach (\is_array($data['releases'] ?? null) ? $data['releases'] : [] as $release) { \is_array($release) and $releases[] = ReleaseRecord::fromArray($release); } - /** @var list $software */ - $software = \array_values(\array_filter( - \is_array($data['software'] ?? null) ? $data['software'] : [], - static fn(mixed $name): bool => \is_string($name) && $name !== '', - )); + $software = []; + /** @var mixed $name */ + foreach (\is_array($data['software'] ?? null) ? $data['software'] : [] as $name) { + \is_string($name) && $name !== '' and $software[] = $name; + } + /** @var mixed $checkedAt */ $checkedAt = $data['checked_at'] ?? null; return new self( @@ -136,15 +141,51 @@ public function isStale(int $now, int $ttl): bool /** * Replaces the head of the list with freshly fetched releases. * - * The fetched releases are the newest ones; they overwrite the stored entries with the same - * tags (assets may have been attached after the release was created) and the remaining stored - * releases follow them, so the list stays newest first. + * The fetched releases are the newest ones, and sources only ever add releases at the top, + * so from the first stored release they contain onwards both lists walk the same listing + * positions. A stored release missing from the fetched span at its position was deleted + * upstream and is dropped: keeping it would inflate `count()`, which the registry uses as + * the offset for loading the tail. Stored releases beyond the span are kept, and when the + * fetched releases contain no stored one they are the whole listing. * * @param list $fetched Newest first. */ public function withHead(array $fetched): self { - return $this->with(releases: [...$fetched, ...$this->releases()]); + $fetchedTags = \array_fill_keys(\array_map(static fn(ReleaseRecord $release): string => $release->tag, $fetched), true); + + $stored = $this->releases(); + $overlap = null; + foreach ($stored as $position => $release) { + if (isset($fetchedTags[$release->tag])) { + $overlap = $position; + break; + } + } + + if ($overlap === null) { + return $this->with(releases: $fetched); + } + + // Listing positions the fetched releases still cover, counting from the overlap + $spanned = 0; + foreach ($fetched as $position => $release) { + if ($release->tag === $stored[$overlap]->tag) { + $spanned = \count($fetched) - $position; + break; + } + } + + $kept = []; + foreach (\array_slice($stored, $overlap) as $release) { + if ($spanned <= 0) { + $kept[] = $release; + } elseif (isset($fetchedTags[$release->tag])) { + --$spanned; + } + } + + return $this->with(releases: [...$fetched, ...$kept]); } /** diff --git a/src/Module/Registry/RepositoryId.php b/src/Module/Registry/RepositoryId.php index 8177676..1972be0 100644 --- a/src/Module/Registry/RepositoryId.php +++ b/src/Module/Registry/RepositoryId.php @@ -11,6 +11,8 @@ * * The same repository may be referenced by several software packages and by several configs, * so the registry keys its records by the repository type and URI rather than by software name. + * GitHub and GitLab resolve paths case-insensitively, so the identity is normalized: lower case, + * no surrounding slashes. * * ```php * $id = RepositoryId::fromConfig($repositoryConfig); @@ -19,14 +21,25 @@ */ final class RepositoryId implements \Stringable { + /** @var non-empty-string Repository type, e.g. `github` or `gitlab`. */ + public readonly string $type; + + /** @var non-empty-string Repository path within the type, e.g. `owner/repo`. */ + public readonly string $uri; + /** - * @param non-empty-string $type Repository type, e.g. `github` or `gitlab`. - * @param non-empty-string $uri Repository URI within the type, e.g. `owner/repo`. + * @param non-empty-string $type + * @param non-empty-string $uri + * @throws \InvalidArgumentException When the URI has nothing but slashes and spaces. */ - public function __construct( - public readonly string $type, - public readonly string $uri, - ) {} + public function __construct(string $type, string $uri) + { + $normalized = \strtolower(\trim($uri, " \t\n\r/")); + $normalized === '' and throw new \InvalidArgumentException(\sprintf('Repository URI `%s` is empty.', $uri)); + + $this->type = \strtolower($type); + $this->uri = $normalized; + } public static function fromConfig(RepositoryConfig $config): self { diff --git a/tests/Unit/Module/Registry/FileRegistryStorageTest.php b/tests/Unit/Module/Registry/FileRegistryStorageTest.php index 6832128..651cc54 100644 --- a/tests/Unit/Module/Registry/FileRegistryStorageTest.php +++ b/tests/Unit/Module/Registry/FileRegistryStorageTest.php @@ -82,6 +82,29 @@ public function unsafePathSegmentsAreSanitized(): void Assert::true($storage->load($id)?->id->equals($id) ?? false); } + #[Test] + public function recordOfAnotherRepositoryInTheSameFileIsIgnored(): void + { + // Two identities sanitize to one file name + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/re?po', ['v1'])); + + Assert::null($storage->load(new RepositoryId('github', 'owner/re*po'))); + Assert::notNull($storage->load(new RepositoryId('github', 'owner/re?po'))); + } + + #[Test] + public function windowsDeviceNamesAreEscaped(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'nul/com1'); + + $storage->save(self::record('github', 'nul/com1', ['v1'])); + + Assert::true(\is_file($this->directory . '/repositories/github/_nul/_com1.json')); + Assert::same($storage->load($id)?->releases()[0]->tag, 'v1'); + } + #[BeforeTest] protected function prepare(): void { diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php index ae6570c..b037775 100644 --- a/tests/Unit/Module/Registry/RepositoryRecordTest.php +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -32,6 +32,59 @@ public function headReplacesKnownReleasesAndKeepsNewestFirst(): void Assert::same($updated->releases()[1]->name, 'new v2'); } + #[Test] + public function headDropsStoredReleasesMissingFromTheFetchedSpan(): void + { + // `v3` was deleted upstream: the fresh head reaches `v2`, and `v3` is not in it + $record = new RepositoryRecord(self::id(), releases: [ + new ReleaseRecord('v4', 'v4'), + new ReleaseRecord('v3', 'v3'), + new ReleaseRecord('v2', 'v2'), + new ReleaseRecord('v1', 'v1'), + ]); + + $updated = $record->withHead([new ReleaseRecord('v5', 'v5'), new ReleaseRecord('v4', 'v4'), new ReleaseRecord('v2', 'v2')]); + + Assert::same(self::tags($updated), ['v5', 'v4', 'v2', 'v1']); + Assert::same($updated->count(), 4); + } + + #[Test] + public function headDropsTheStoredReleaseRightAfterTheOnlyOneItReaches(): void + { + // `v5` was deleted: the fresh page reads `v6, v4`, so `v5` no longer follows `v6` + $record = new RepositoryRecord(self::id(), releases: [new ReleaseRecord('v6', 'v6'), new ReleaseRecord('v5', 'v5')]); + + $updated = $record->withHead([new ReleaseRecord('v6', 'v6'), new ReleaseRecord('v4', 'v4')]); + + Assert::same(self::tags($updated), ['v6', 'v4']); + } + + #[Test] + public function headReachingNoStoredReleaseIsTheWholeListing(): void + { + $record = new RepositoryRecord(self::id(), releases: [new ReleaseRecord('v1', 'v1')]); + + $updated = $record->withHead([new ReleaseRecord('v2', 'v2')]); + + Assert::same(self::tags($updated), ['v2']); + } + + #[Test] + public function repositoryIdIsNormalized(): void + { + $id = new RepositoryId('GitHub', '/Owner/Repo/'); + + Assert::same((string) $id, 'github:owner/repo'); + Assert::true($id->equals(new RepositoryId('github', 'owner/repo'))); + + try { + new RepositoryId('github', '/'); + Assert::fail('A URI without a path must be rejected.'); + } catch (\InvalidArgumentException) { + } + } + #[Test] public function tailIgnoresKnownReleases(): void { diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php index 5a32f23..b92967c 100644 --- a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -8,6 +8,7 @@ use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Registry\RepositoryId; use Internal\DLoad\Module\Repository\Exception\ApiException; +use Internal\DLoad\Module\Repository\Internal\GitHub\Exception\GitHubRateLimitException; use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Registry\Stub\ArrayReleaseSource; use Internal\DLoad\Tests\Unit\Module\Registry\Stub\InMemoryRegistryStorage; @@ -103,6 +104,56 @@ public function newReleasesAreFetchedUntilAKnownOneIsReached(): void Assert::same($source->served, [0, 2]); } + #[Test] + public function releaseDeletedUpstreamDoesNotShiftTheTail(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + + // The first run stores the first page only + $this->registry()->releases($this->id, $source)->current(); + $source->served = []; + + // `v5` is deleted upstream, so every older release moves up one position + $source->delete('v5'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + // The check drops `v5`; the tail continues from the true offset and `v3` is not skipped + Assert::same($again, ['v6', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2, 4]); + } + + #[Test] + public function failedTailLoadingStillServesTheStoredReleases(): void + { + $source = ArrayReleaseSource::ofTags(['v4', 'v3', 'v2', 'v1']); + $this->registry()->releases($this->id, $source)->current(); + + $source->fail(); + $pages = $this->registry()->releases($this->id, $source); + + // The stored page comes first; only the request for the tail fails + Assert::same(self::tagsOf($pages->current()), ['v4', 'v3']); + try { + $pages->next(); + Assert::fail('The failure of the tail request must reach the caller.'); + } catch (ApiException) { + Assert::false($this->storage->load($this->id)->complete); + } + } + + #[Test] + public function rateLimitOnCheckFallsBackToStoredReleases(): void + { + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + + $source->fail(new GitHubRateLimitException('Rate limit exceeded.', 'owner/repo', null)); + $this->now += 601; + + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); + } + #[Test] public function firstPageOverwritesStoredReleasesOnCheck(): void { diff --git a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php index 93dda14..4332ee8 100644 --- a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php +++ b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php @@ -54,6 +54,19 @@ public function publish(string ...$tags): void $this->releases = [...\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), ...$this->releases]; } + /** + * Deletes a release from the list, as a repository owner would between two runs. + * + * @param non-empty-string $tag + */ + public function delete(string $tag): void + { + $this->releases = \array_values(\array_filter( + $this->releases, + static fn(ReleaseRecord $release): bool => $release->tag !== $tag, + )); + } + public function fail(?\Throwable $failure = null): void { $this->failure = $failure ?? new ApiException('API is unavailable.', 'stub/stub'); From d1b559ea14fd14ab28ebc9eb930a023970a33b3d Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Mon, 14 Sep 2026 19:34:34 +0400 Subject: [PATCH 04/17] feat(cache:clear): ask before dropping the whole registry docs: state that XML cache attributes take precedence over the environment Without arguments the command wiped the registry immediately, unlike `init`, which confirms destructive actions in a terminal. A non-interactive run and `--force` still go ahead, so CI scripts need no change. Assisted-By: Claude Fable 5.1 --- README-es.md | 4 +- README-ru.md | 4 +- README-zh.md | 4 +- README.md | 4 +- dload.xsd | 4 +- src/Command/CacheClear.php | 35 +++++ tests/Integration/Command/CacheClearTest.php | 151 +++++++++++++++++++ 7 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 tests/Integration/Command/CacheClearTest.php diff --git a/README-es.md b/README-es.md index a4e61a5..31ed7e8 100644 --- a/README-es.md +++ b/README-es.md @@ -378,13 +378,15 @@ El registro está activado por defecto y vive en el directorio de caché del usu | `cache-dir` | `DLOAD_CACHE_DIR` | directorio de caché del usuario | Directorio del registro de versiones. | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Segundos que sigue siendo válida la última comprobación. `0` desactiva el registro. | +Un atributo definido en `dload.xml` tiene prioridad sobre la variable de entorno. + ```bash # Comprobar si hay nuevos releases aunque la última comprobación siga vigente ./vendor/bin/dload get rr --refresh # Olvidar los repositorios de un software, o todo el registro ./vendor/bin/dload cache:clear rr -./vendor/bin/dload cache:clear +./vendor/bin/dload cache:clear --force ``` > [!NOTE] diff --git a/README-ru.md b/README-ru.md index 1b3cd39..e86a4f2 100644 --- a/README-ru.md +++ b/README-ru.md @@ -379,13 +379,15 @@ DLoad поддерживает три типа загрузки, которые | `cache-dir` | `DLOAD_CACHE_DIR` | каталог кэша пользователя | Каталог реестра версий. | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Сколько секунд действует последняя проверка репозитория. `0` отключает реестр. | +Атрибут в `dload.xml` имеет приоритет над переменной окружения. + ```bash # Проверить репозитории на новые релизы, даже если последняя проверка ещё свежая ./vendor/bin/dload get rr --refresh # Забыть репозитории, из которых берётся программа, или весь реестр целиком ./vendor/bin/dload cache:clear rr -./vendor/bin/dload cache:clear +./vendor/bin/dload cache:clear --force ``` > [!NOTE] diff --git a/README-zh.md b/README-zh.md index 67c810e..d33d90a 100644 --- a/README-zh.md +++ b/README-zh.md @@ -375,13 +375,15 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： | `cache-dir` | `DLOAD_CACHE_DIR` | 用户缓存目录 | 版本注册表所在目录。 | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | 最近一次检查保持有效的秒数。`0` 表示禁用注册表。 | +`dload.xml` 中的属性优先于环境变量。 + ```bash # 即使最近一次检查仍然有效，也强制检查仓库是否有新发布 ./vendor/bin/dload get rr --refresh # 忘记某个软件所使用的仓库，或清空整个注册表 ./vendor/bin/dload cache:clear rr -./vendor/bin/dload cache:clear +./vendor/bin/dload cache:clear --force ``` > [!NOTE] diff --git a/README.md b/README.md index 220c218..504e621 100644 --- a/README.md +++ b/README.md @@ -380,13 +380,15 @@ The registry is on by default and lives in the per-user cache directory (`$XDG_C | `cache-dir` | `DLOAD_CACHE_DIR` | user cache directory | Directory of the version registry. | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Seconds the last check of a repository stays valid. `0` disables the registry. | +An attribute set in `dload.xml` takes precedence over the environment variable. + ```bash # Check the repositories for new releases even if the last check is still fresh ./vendor/bin/dload get rr --refresh # Forget the repositories a software package is served from, or the whole registry ./vendor/bin/dload cache:clear rr -./vendor/bin/dload cache:clear +./vendor/bin/dload cache:clear --force ``` > [!NOTE] diff --git a/dload.xsd b/dload.xsd index 3246f4f..a6e146a 100644 --- a/dload.xsd +++ b/dload.xsd @@ -251,12 +251,12 @@ - Directory of the version registry (release lists database); the per-user cache directory when not set + Directory of the version registry (release lists database); the per-user cache directory when not set. Takes precedence over DLOAD_CACHE_DIR - Number of seconds the last check of a repository for new releases stays valid; 0 disables the version registry + Number of seconds the last check of a repository for new releases stays valid; 0 disables the version registry. Takes precedence over DLOAD_CACHE_TTL diff --git a/src/Command/CacheClear.php b/src/Command/CacheClear.php index c94d3b7..67d62af 100644 --- a/src/Command/CacheClear.php +++ b/src/Command/CacheClear.php @@ -9,9 +9,12 @@ use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Formatter\OutputFormatter; +use Symfony\Component\Console\Helper\QuestionHelper; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Question\ConfirmationQuestion; /** * Removes stored release listings from the version registry. @@ -37,6 +40,7 @@ final class CacheClear extends Base { private const ARG_SOFTWARE = 'software'; + private const OPTION_FORCE = 'force'; public function configure(): void { @@ -46,6 +50,12 @@ public function configure(): void InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'Software whose repositories must be forgotten, e.g. "rr", "dolt". Everything when omitted.', ); + $this->addOption( + self::OPTION_FORCE, + 'f', + InputOption::VALUE_NONE, + 'Clear the whole registry without asking for confirmation', + ); } protected function execute(InputInterface $input, OutputInterface $output): int @@ -61,6 +71,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int )); if ($software === []) { + if (!$this->confirmed($input, $output)) { + $output->writeln('The version registry is left as it is.'); + + return Command::SUCCESS; + } + $storage->clear(); $output->writeln('The version registry has been cleared.'); @@ -79,6 +95,25 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::SUCCESS; } + /** + * Whether the whole registry may be dropped: a non-interactive run goes ahead, a person is asked. + */ + private function confirmed(InputInterface $input, OutputInterface $output): bool + { + if ((bool) $input->getOption(self::OPTION_FORCE) || !$input->isInteractive()) { + return true; + } + + /** @var QuestionHelper $helper */ + $helper = $this->getHelper('question'); + + return (bool) $helper->ask( + $input, + $output, + new ConfirmationQuestion('Forget every stored release listing? [y/N] ', false), + ); + } + /** * @param list $software * @return \Generator diff --git a/tests/Integration/Command/CacheClearTest.php b/tests/Integration/Command/CacheClearTest.php new file mode 100644 index 0000000..2dd8e32 --- /dev/null +++ b/tests/Integration/Command/CacheClearTest.php @@ -0,0 +1,151 @@ +seed('a/b', ['rr']); + $this->seed('c/d', ['temporal', 'tctl']); + $this->seed('e/f', ['dolt']); + + $tester = $this->run(['software' => ['rr', 'tctl']]); + + Assert::string($tester->getDisplay())->contains('2 repository listing(s) removed.'); + Assert::same(self::uris($this->storage), ['e/f']); + } + + #[Test] + public function unknownSoftwareRemovesNothing(): void + { + $this->seed('a/b', ['rr']); + + $tester = $this->run(['software' => ['unknown']]); + + Assert::string($tester->getDisplay())->contains('0 repository listing(s) removed.'); + Assert::same(self::uris($this->storage), ['a/b']); + } + + #[Test] + public function clearsEverythingWithoutAskingWhenNotInteractive(): void + { + $this->seed('a/b', ['rr']); + + $tester = $this->run([], interactive: false); + + Assert::string($tester->getDisplay())->contains('has been cleared'); + Assert::same(self::uris($this->storage), []); + } + + #[Test] + public function clearingEverythingInteractivelyRequiresConfirmation(): void + { + $this->seed('a/b', ['rr']); + + $declined = $this->run([], interactive: true, answers: ['n']); + Assert::string($declined->getDisplay())->contains('left as it is'); + Assert::same(self::uris($this->storage), ['a/b']); + + $confirmed = $this->run([], interactive: true, answers: ['y']); + Assert::string($confirmed->getDisplay())->contains('has been cleared'); + Assert::same(self::uris($this->storage), []); + } + + #[Test] + public function forceSkipsTheConfirmation(): void + { + $this->seed('a/b', ['rr']); + + $tester = $this->run(['--force' => true], interactive: true); + + Assert::string($tester->getDisplay())->contains('has been cleared'); + Assert::same(self::uris($this->storage), []); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-cache-clear-' . \bin2hex(\random_bytes(6)); + $this->storage = new FileRegistryStorage($this->directory, new Logger()); + \mkdir($this->directory, recursive: true); + \file_put_contents($this->directory . '/dload.xml', ''); + \putenv('DLOAD_CACHE_DIR=' . $this->directory); + } + + #[AfterTest] + protected function cleanup(): void + { + \putenv('DLOAD_CACHE_DIR'); + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @return list + */ + private static function uris(FileRegistryStorage $storage): array + { + $uris = \array_map( + static fn(RepositoryRecord $record): string => $record->id->uri, + \iterator_to_array($storage->all(), false), + ); + \sort($uris); + + return $uris; + } + + /** + * @param non-empty-string $uri + * @param list $software + */ + private function seed(string $uri, array $software): void + { + $this->storage->save(new RepositoryRecord(new RepositoryId('github', $uri), software: $software)); + } + + /** + * @param array $input + * @param list $answers + */ + private function run(array $input, bool $interactive = false, array $answers = []): CommandTester + { + $application = new Application(); + // Symfony Console 8 renamed `add()` to `addCommand()` + \method_exists($application, 'addCommand') + ? $application->addCommand(new CacheClear()) + : $application->add(new CacheClear()); + + $tester = new CommandTester($application->find('cache:clear')); + $answers === [] or $tester->setInputs($answers); + $tester->execute( + $input + ['--config' => $this->directory . '/dload.xml'], + ['interactive' => $interactive], + ); + + return $tester; + } +} From 3c49450cd35fb978779e7b12f93e926907948d7f Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Mon, 14 Sep 2026 19:34:35 +0400 Subject: [PATCH 05/17] chore(psalm): make the static analysis pass on the version registry code The psalm job of the pull request failed on level 1 findings: mixed assignments while decoding stored records, flow analysis across `goto` in the downloader, and generator return types. Assisted-By: Claude Fable 5.1 --- src/Module/Downloader/Downloader.php | 8 ++++---- src/Module/Registry/Internal/CacheDirectory.php | 4 +++- src/Module/Registry/Record/AssetRecord.php | 2 ++ src/Module/Registry/Record/ReleaseRecord.php | 4 ++++ .../Repository/Internal/GitHub/Api/RepositoryApi.php | 2 +- .../Repository/Internal/GitHub/GitHubReleaseSource.php | 4 +++- .../Repository/Internal/GitHub/GitHubRepository.php | 2 +- .../Repository/Internal/GitLab/Api/RepositoryApi.php | 2 +- .../Repository/Internal/GitLab/GitLabReleaseSource.php | 4 +++- .../Repository/Internal/GitLab/GitLabRepository.php | 2 +- 10 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/Module/Downloader/Downloader.php b/src/Module/Downloader/Downloader.php index c317a95..60c80e2 100644 --- a/src/Module/Downloader/Downloader.php +++ b/src/Module/Downloader/Downloader.php @@ -213,6 +213,7 @@ private function processRepository(Repository $repository, DownloadContext $cont process_release: if ($releases === []) { // The list was outdated: ask the repository again once, with the deleted releases forgotten + /** @var bool $forgotten */ if ($forgotten && $mayRetry) { return $this->retryRepository($context); } @@ -230,10 +231,8 @@ private function processRepository(Repository $repository, DownloadContext $cont return $context->release; } catch (ReleaseGone $e) { // The registry must not offer this release again, and the list needs a fresh check - $this->registry->forget( - RepositoryId::fromConfig($context->repoConfig), - $context->release->getVersion()->string, - ); + $tag = $context->release->getVersion()->string; + $tag === '' or $this->registry->forget(RepositoryId::fromConfig($context->repoConfig), $tag); $forgotten = true; $context->releaseAttempt->reason ??= $e->getMessage(); @@ -443,6 +442,7 @@ private function tryProcessAssets(array $assets, DownloadContext $context): Asse process_asset: if ($assets === []) { + /** @var bool $gone */ $gone and throw new ReleaseGone('every matching asset of the release is no longer available'); throw new NotFound('none of the matching assets could be downloaded'); diff --git a/src/Module/Registry/Internal/CacheDirectory.php b/src/Module/Registry/Internal/CacheDirectory.php index 1b87644..49b4ca1 100644 --- a/src/Module/Registry/Internal/CacheDirectory.php +++ b/src/Module/Registry/Internal/CacheDirectory.php @@ -50,8 +50,10 @@ public static function resolve(array $env): string */ private static function variable(array $env, string $name): ?string { + /** @var mixed $value */ $value = $env[$name] ?? null; + $path = \is_string($value) ? \rtrim(\trim($value), '/\\') : ''; - return \is_string($value) && \trim($value) !== '' ? \rtrim($value, '/\\') : null; + return $path === '' ? null : $path; } } diff --git a/src/Module/Registry/Record/AssetRecord.php b/src/Module/Registry/Record/AssetRecord.php index 0643f33..9f50988 100644 --- a/src/Module/Registry/Record/AssetRecord.php +++ b/src/Module/Registry/Record/AssetRecord.php @@ -44,7 +44,9 @@ public static function fromArray(array $data): self 'Asset record requires non-empty `name` and `uri`.', ); + /** @var mixed $size */ $size = $data['size'] ?? null; + /** @var mixed $contentType */ $contentType = $data['content_type'] ?? null; return new self( diff --git a/src/Module/Registry/Record/ReleaseRecord.php b/src/Module/Registry/Record/ReleaseRecord.php index 4a01b28..48f6021 100644 --- a/src/Module/Registry/Record/ReleaseRecord.php +++ b/src/Module/Registry/Record/ReleaseRecord.php @@ -43,11 +43,15 @@ public static function fromArray(array $data): self $tag = $data['tag'] ?? null; \is_string($tag) && $tag !== '' or throw new \InvalidArgumentException('Release record requires a non-empty `tag`.'); + /** @var mixed $name */ $name = $data['name'] ?? null; \is_string($name) && $name !== '' or $name = $tag; + /** @var mixed $publishedAt */ $publishedAt = $data['published_at'] ?? null; + $assets = []; + /** @var mixed $asset */ foreach (\is_array($data['assets'] ?? null) ? $data['assets'] : [] as $asset) { \is_array($asset) and $assets[] = AssetRecord::fromArray($asset); } diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index 10f87ea..c32502f 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -91,7 +91,7 @@ public function getRepository(): RepositoryInfo * costs no extra request. * * @param int<1, max> $page - * @return \Generator + * @return \Generator * @throws RepositoryException */ public function releasePages(int $page = 1): \Generator diff --git a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php index 572073d..b6ab6c5 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php +++ b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php @@ -25,8 +25,10 @@ public function __construct( public function pages(int $offset = 0): \Generator { $skip = $offset % RepositoryApi::RELEASES_PER_PAGE; + /** @var int<1, max> $first */ + $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; - foreach ($this->api->releasePages(\intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1) as $page) { + foreach ($this->api->releasePages($first) as $page) { yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); $skip = 0; } diff --git a/src/Module/Repository/Internal/GitHub/GitHubRepository.php b/src/Module/Repository/Internal/GitHub/GitHubRepository.php index 12a7fdf..a0c80e0 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRepository.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRepository.php @@ -82,7 +82,7 @@ public function getReleases(): ReleasesCollection } $toYield = []; - foreach ($pages->current() as $record) { + foreach ($pages->current() ?? [] as $record) { try { $toYield[] = GitHubRelease::fromRecord($this->api, $this, $record); } catch (\Throwable $e) { diff --git a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php index bdd6c9d..8f28209 100644 --- a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php @@ -100,7 +100,7 @@ public function getRepository(): RepositoryInfo * costs no extra request. * * @param int<1, max> $page - * @return \Generator + * @return \Generator * @throws RepositoryException */ public function releasePages(int $page = 1): \Generator diff --git a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php index d07d9b6..da7ea74 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php +++ b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php @@ -25,8 +25,10 @@ public function __construct( public function pages(int $offset = 0): \Generator { $skip = $offset % RepositoryApi::RELEASES_PER_PAGE; + /** @var int<1, max> $first */ + $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; - foreach ($this->api->releasePages(\intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1) as $page) { + foreach ($this->api->releasePages($first) as $page) { yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); $skip = 0; } diff --git a/src/Module/Repository/Internal/GitLab/GitLabRepository.php b/src/Module/Repository/Internal/GitLab/GitLabRepository.php index 5fad466..6410e4a 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRepository.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRepository.php @@ -80,7 +80,7 @@ public function getReleases(): ReleasesCollection } $toYield = []; - foreach ($pages->current() as $record) { + foreach ($pages->current() ?? [] as $record) { try { $toYield[] = GitLabRelease::fromRecord($this->api, $this, $record); } catch (\Throwable) { From 86d6b00f55a70b914f8c55c31684dda28a9bc976 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Mon, 14 Sep 2026 19:34:36 +0400 Subject: [PATCH 06/17] test: cover the registry paths the first tests passed trivially The stale-on-error test used a complete record, so the tail loading path was never entered; the downloader test with a working asset never reached the branch where every asset fails for a reason other than a missing file. Tail loading from inside a page and the GitLab tail path had no test at all. Assisted-By: Claude Fable 5.1 --- tests/Acceptance/DLoadTest.php | 2 +- .../Unit/Module/Downloader/DownloaderTest.php | 23 ++++++++ .../Module/Downloader/Stub/GoneAssetStub.php | 4 +- .../Internal/GitHub/GitHubRepositoryTest.php | 26 +++++++++ .../Internal/GitLab/GitLabRepositoryTest.php | 56 +++++++++++++++++++ 5 files changed, 109 insertions(+), 2 deletions(-) diff --git a/tests/Acceptance/DLoadTest.php b/tests/Acceptance/DLoadTest.php index b22078f..5b21c00 100644 --- a/tests/Acceptance/DLoadTest.php +++ b/tests/Acceptance/DLoadTest.php @@ -232,7 +232,7 @@ protected function cleanup(): void private function buildDLoad(string $xmlConfig): DLoad { // The version registry must not leak into the user's cache directory from a test run - $environment = \getenv() + ['DLOAD_CACHE_DIR' => (string) $this->testRuntimeDir->join('registry')]; + $environment = \getenv(); $environment['DLOAD_CACHE_DIR'] = (string) $this->testRuntimeDir->join('registry'); $container = Bootstrap::init() diff --git a/tests/Unit/Module/Downloader/DownloaderTest.php b/tests/Unit/Module/Downloader/DownloaderTest.php index f0b459a..17e7d93 100644 --- a/tests/Unit/Module/Downloader/DownloaderTest.php +++ b/tests/Unit/Module/Downloader/DownloaderTest.php @@ -117,6 +117,29 @@ public function releaseWithOtherFailuresIsNotForgotten(): void Assert::same($this->registry->forgotten, []); } + #[Test] + public function releaseFailingForOtherReasonsIsNotForgotten(): void + { + $repository = new RepositoryStub('owner/repo'); + $broken = new ReleaseStub($repository, 'v2.0.0', Version::fromVersionString('v2.0.0')); + $broken->setAssets([ + new GoneAssetStub($broken, 'rr-linux-amd64.tar.gz'), + // Every asset fails, but a network error says nothing about the release being gone + new GoneAssetStub($broken, 'rr-linux-amd64.zip', new \RuntimeException('Connection reset by peer')), + ]); + $factory = new SequenceRepositoryFactoryStub([ + new RepositoryStub('owner/repo', ReleasesCollection::create([$broken])), + ]); + + try { + $this->download($factory); + Assert::fail('DownloadFailed is expected when no asset could be downloaded.'); + } catch (DownloadFailed) { + Assert::same($this->registry->forgotten, []); + Assert::same($factory->created, 1); + } + } + #[BeforeTest] protected function prepare(): void { diff --git a/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php index b33d2d9..aca547d 100644 --- a/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php +++ b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php @@ -17,10 +17,12 @@ final class GoneAssetStub implements AssetInterface { /** * @param non-empty-string $name + * @param \Throwable|null $failure What the download throws; "not found" by default. */ public function __construct( private readonly ReleaseInterface $release, private readonly string $name, + private readonly ?\Throwable $failure = null, ) {} public function getRelease(): ReleaseInterface @@ -50,7 +52,7 @@ public function getArchitecture(): ?Architecture public function download(): \Traversable { - throw new AssetNotFoundException('GitHub asset is no longer available: HTTP 404 for ' . $this->getUri(), 'owner/repo'); + throw $this->failure ?? new AssetNotFoundException('GitHub asset is no longer available: HTTP 404 for ' . $this->getUri(), 'owner/repo'); /** @psalm-suppress UnevaluatedCode */ yield ''; diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php index 13a50b7..06e8f96 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -8,6 +8,9 @@ use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; use Internal\DLoad\Module\Registry\Internal\StoredVersionRegistry; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; +use Internal\DLoad\Module\Registry\Record\RepositoryRecord; +use Internal\DLoad\Module\Registry\RepositoryId; use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; @@ -117,6 +120,29 @@ public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void Assert::same(\count($all), 300); } + #[Test] + public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): void + { + // A fresh record holds the first 50 releases: the tail starts in the middle of API page 1 + $storage = new InMemoryRegistryStorage(); + $storage->save(new RepositoryRecord( + id: new RepositoryId(GitHubRepository::TYPE, 'owner/repo'), + checkedAt: \time(), + releases: \array_map( + static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), + \range(1, 50), + ), + )); + + $client = new PagedClientStub(pages: 2); + $all = self::names(self::createRepository($client, self::registry($storage))); + + Assert::same($client->requestedPages(), [1, 2]); + Assert::same(\count($all), 200); + Assert::same(\count(\array_unique($all)), 200); + Assert::same($all[50], 'v1.0.51'); + } + private static function createRepository( PagedClientStub $client, VersionRegistry $registry = new PassThroughRegistry(), diff --git a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php index c94c5b0..9f036fe 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php @@ -8,11 +8,15 @@ use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; use Internal\DLoad\Module\Registry\Internal\StoredVersionRegistry; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; +use Internal\DLoad\Module\Registry\Record\RepositoryRecord; +use Internal\DLoad\Module\Registry\RepositoryId; use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\GitLab\GitLabReleaseSource; use Internal\DLoad\Module\Repository\Internal\GitLab\GitLabRepository; +use Internal\DLoad\Module\Repository\ReleaseInterface; use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Registry\Stub\InMemoryRegistryStorage; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab\Stub\PagedClientStub; @@ -92,6 +96,47 @@ public function secondRunIsServedFromTheRegistryWithoutRequests(): void Assert::same(\count($secondRun), 200); } + #[Test] + public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 3); + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + break; + } + + $secondClient = new PagedClientStub(pages: 3); + $all = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1]); + Assert::same($secondClient->requestedPages(), [2, 3]); + Assert::same(\count($all), 300); + } + + #[Test] + public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): void + { + $storage = new InMemoryRegistryStorage(); + $storage->save(new RepositoryRecord( + id: new RepositoryId(GitLabRepository::TYPE, 'group/project'), + checkedAt: \time(), + releases: \array_map( + static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), + \range(1, 50), + ), + )); + + $client = new PagedClientStub(pages: 2); + $all = self::names(self::createRepository($client, self::registry($storage))); + + Assert::same($client->requestedPages(), [1, 2]); + Assert::same(\count($all), 200); + Assert::same(\count(\array_unique($all)), 200); + Assert::same($all[50], 'v1.0.51'); + } + private static function createRepository( PagedClientStub $client, VersionRegistry $registry = new PassThroughRegistry(), @@ -111,4 +156,15 @@ private static function registry(InMemoryRegistryStorage $storage): StoredVersio { return new StoredVersionRegistry($storage, 600, new Logger()); } + + /** + * @return list + */ + private static function names(GitLabRepository $repository): array + { + return \array_map( + static fn(ReleaseInterface $release): string => $release->getName(), + \iterator_to_array($repository->getReleases(), false), + ); + } } From 244e38fea0c7be56b21051f19c625dc98b258827 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Mon, 14 Sep 2026 19:43:27 +0400 Subject: [PATCH 07/17] feat(registry): store the asset digest reported by GitHub GitHub returns a `sha256:` digest for release assets. Nothing reads it yet: the field is stored now so that a future binary cache can verify a downloaded file without re-fetching the release lists. The field is optional, so existing registry files stay readable. Assisted-By: Claude Fable 5.1 --- src/Module/Registry/Record/AssetRecord.php | 7 +++++++ .../Repository/Internal/GitHub/Api/RepositoryApi.php | 3 ++- .../Internal/GitHub/Api/Response/AssetInfo.php | 9 ++++++++- .../Internal/GitHub/Api/Response/ReleaseInfo.php | 3 ++- tests/Unit/Module/Registry/RepositoryRecordTest.php | 4 +++- .../Internal/GitHub/GitHubRepositoryTest.php | 11 +++++++++++ .../Internal/GitHub/Stub/PagedClientStub.php | 8 +++++++- 7 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/Module/Registry/Record/AssetRecord.php b/src/Module/Registry/Record/AssetRecord.php index 9f50988..f76311e 100644 --- a/src/Module/Registry/Record/AssetRecord.php +++ b/src/Module/Registry/Record/AssetRecord.php @@ -15,6 +15,7 @@ * uri: non-empty-string, * size?: int|null, * content_type?: string|null, + * digest?: string|null, * } */ final class AssetRecord @@ -24,12 +25,14 @@ final class AssetRecord * @param non-empty-string $uri Download URI. * @param int<0, max>|null $size Size in bytes when the source reports it. * @param non-empty-string|null $contentType MIME type when the source reports it. + * @param non-empty-string|null $digest Content checksum as `:`, e.g. `sha256:9f86d0…`, when the source reports it. */ public function __construct( public readonly string $name, public readonly string $uri, public readonly ?int $size = null, public readonly ?string $contentType = null, + public readonly ?string $digest = null, ) {} /** @@ -48,12 +51,15 @@ public static function fromArray(array $data): self $size = $data['size'] ?? null; /** @var mixed $contentType */ $contentType = $data['content_type'] ?? null; + /** @var mixed $digest */ + $digest = $data['digest'] ?? null; return new self( name: $name, uri: $uri, size: \is_int($size) && $size >= 0 ? $size : null, contentType: \is_string($contentType) && $contentType !== '' ? $contentType : null, + digest: \is_string($digest) && $digest !== '' ? $digest : null, ); } @@ -65,6 +71,7 @@ public function toArray(): array $result = ['name' => $this->name, 'uri' => $this->uri]; $this->size === null or $result['size'] = $this->size; $this->contentType === null or $result['content_type'] = $this->contentType; + $this->digest === null or $result['digest'] = $this->digest; return $result; } diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index c32502f..373e595 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -109,7 +109,8 @@ public function releasePages(int $page = 1): \Generator * name: string, * browser_download_url: string, * size: int, - * content_type: string + * content_type: string, + * digest?: string|null * }>, * prerelease: bool, * draft: bool diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php index af168f6..adba8ae 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php @@ -19,12 +19,14 @@ final class AssetInfo * @param non-empty-string $downloadUrl * @param int<0, max> $size * @param non-empty-string $contentType + * @param non-empty-string|null $digest Checksum as `sha256:`; GitHub reports it for assets uploaded since 2025. */ public function __construct( public readonly string $name, public readonly string $downloadUrl, public readonly int $size, public readonly string $contentType, + public readonly ?string $digest = null, ) {} /** @@ -32,16 +34,20 @@ public function __construct( * name: string, * browser_download_url: string, * size: int, - * content_type: string + * content_type: string, + * digest?: string|null * } $data */ public static function fromApiResponse(array $data): self { + $digest = $data['digest'] ?? null; + return new self( name: $data['name'], downloadUrl: $data['browser_download_url'], size: $data['size'], contentType: $data['content_type'], + digest: $digest === '' ? null : $digest, ); } @@ -55,6 +61,7 @@ public function toRecord(): AssetRecord uri: $this->downloadUrl, size: $this->size, contentType: $this->contentType, + digest: $this->digest, ); } } diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php index e145b7d..e06e58d 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php @@ -38,7 +38,8 @@ public function __construct( * name: string, * browser_download_url: string, * size: int, - * content_type: string + * content_type: string, + * digest?: string|null * }>, * prerelease: bool, * draft: bool diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php index b037775..54d8f1c 100644 --- a/tests/Unit/Module/Registry/RepositoryRecordTest.php +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -150,7 +150,7 @@ public function survivesTheArrayRoundTrip(): void name: 'Release 2', publishedAt: new \DateTimeImmutable('2024-01-02T03:04:05+00:00'), prerelease: true, - assets: [new AssetRecord('rr-linux-amd64.tar.gz', 'https://x/rr.tar.gz', 42, 'application/gzip')], + assets: [new AssetRecord('rr-linux-amd64.tar.gz', 'https://x/rr.tar.gz', 42, 'application/gzip', 'sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08')], ), new ReleaseRecord('v1.0.0', 'v1.0.0'), ], @@ -161,6 +161,8 @@ public function survivesTheArrayRoundTrip(): void Assert::same($restored->toArray(), $record->toArray()); Assert::true($restored->id->equals(self::id())); Assert::same($restored->releases()[0]->assets[0]->size, 42); + Assert::same($restored->releases()[0]->assets[0]->digest, 'sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'); + Assert::null(AssetRecord::fromArray(['name' => 'x', 'uri' => 'https://x', 'digest' => ''])->digest); Assert::same($restored->releases()[0]->publishedAt?->format(\DATE_ATOM), '2024-01-02T03:04:05+00:00'); Assert::null($restored->releases()[1]->publishedAt); } diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php index 06e8f96..0e41a81 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -120,6 +120,17 @@ public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void Assert::same(\count($all), 300); } + #[Test] + public function assetDigestReportedByTheApiIsStored(): void + { + $storage = new InMemoryRegistryStorage(); + + self::createRepository(new PagedClientStub(pages: 1), self::registry($storage))->getReleases()->first(); + + $asset = $storage->load(new RepositoryId(GitHubRepository::TYPE, 'owner/repo'))?->releases()[0]->assets[0]; + Assert::same($asset?->digest, 'sha256:' . \hash('sha256', 'v1.0.1')); + } + #[Test] public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): void { diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php index b6a04da..55982e6 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -94,7 +94,13 @@ private function allReleases(): array 'name' => $tag, 'tag_name' => $tag, 'published_at' => '2024-01-01T00:00:00Z', - 'assets' => [], + 'assets' => [[ + 'name' => 'rr-linux-amd64.tar.gz', + 'browser_download_url' => 'https://github.com/owner/repo/releases/download/' . $tag . '/rr-linux-amd64.tar.gz', + 'size' => 1024, + 'content_type' => 'application/gzip', + 'digest' => 'sha256:' . \hash('sha256', $tag), + ]], 'prerelease' => false, 'draft' => false, ]; From 73efb9198d7755a1d09d4a8115605b2558c3c2fd Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Mon, 14 Sep 2026 20:25:19 +0400 Subject: [PATCH 08/17] feat(registry): split release lists into segments read on demand fix(registry): keep GitHub draft releases out of the shared registry feat(registry): report an API rate limit once per run in plain sight refactor(registry): pass Path between services and keep the temp fallback per user docs: note draft releases and mid-list insertions as registry limits One JSON file per repository was read and rewritten whole on every run, which grows to megabytes for repositories with thousands of releases and assets. The record now holds an index with the tags of every segment, so counting and membership never touch the releases, and each segment of at most 100 releases is a file of its own that is read when the iteration reaches it and written only when it changed. The partial segment sits at the head, where every check adds its releases, so the full segments behind it are never rewritten. Drafts are visible to the token holder only, yet the registry is meant to be shared and cached in CI, so they are not stored. The `attach()` argument order now matches `forget()`, and the registry types are `@internal` like the rest of the tool. Assisted-By: Claude Fable 5.1 --- README-es.md | 1 + README-ru.md | 1 + README-zh.md | 1 + README.md | 1 + src/Bootstrap.php | 3 +- src/Module/Downloader/Downloader.php | 2 +- .../Registry/Internal/CacheDirectory.php | 35 +- .../Registry/Internal/FileRegistryStorage.php | 192 ++++++++--- .../Registry/Internal/PassThroughRegistry.php | 2 +- .../Internal/StoredVersionRegistry.php | 71 +++-- src/Module/Registry/Record/AssetRecord.php | 2 + src/Module/Registry/Record/ReleasePage.php | 2 + src/Module/Registry/Record/ReleaseRecord.php | 2 + src/Module/Registry/Record/ReleaseSegment.php | 111 +++++++ .../Registry/Record/RepositoryRecord.php | 298 +++++++++++++++--- src/Module/Registry/RegistryStorage.php | 2 + src/Module/Registry/ReleaseSource.php | 2 + src/Module/Registry/RepositoryId.php | 2 + src/Module/Registry/VersionRegistry.php | 4 +- .../Internal/GitHub/Api/RepositoryApi.php | 4 +- tests/Integration/Command/CacheClearTest.php | 2 +- .../Registry/VersionRegistryBindingTest.php | 6 +- .../Module/Registry/CacheDirectoryTest.php | 44 ++- .../Registry/FileRegistryStorageTest.php | 150 +++++++-- .../Registry/PassThroughRegistryTest.php | 37 +++ .../Module/Registry/RepositoryRecordTest.php | 267 +++++++++++++--- .../Registry/StoredVersionRegistryTest.php | 37 ++- .../Registry/Stub/InMemoryRegistryStorage.php | 2 +- .../Registry/Stub/RecordingRegistry.php | 2 +- .../Internal/GitHub/GitHubRepositoryTest.php | 24 +- .../Internal/GitHub/Stub/PagedClientStub.php | 8 +- .../Internal/GitLab/GitLabRepositoryTest.php | 12 +- 32 files changed, 1117 insertions(+), 212 deletions(-) create mode 100644 src/Module/Registry/Record/ReleaseSegment.php create mode 100644 tests/Unit/Module/Registry/PassThroughRegistryTest.php diff --git a/README-es.md b/README-es.md index 31ed7e8..46f2fb0 100644 --- a/README-es.md +++ b/README-es.md @@ -396,6 +396,7 @@ Un atributo definido en `dload.xml` tiene prioridad sobre la variable de entorno > usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. Un > release almacenado cuyos assets desaparecieron del origen se elimina del registro en cuanto falla > su descarga, y la lista de releases se vuelve a obtener antes de que la ejecución se dé por vencida. +> Los borradores de releases de GitHub nunca se almacenan. Un release que el proveedor inserta por debajo del inicio de su lista, como un release de GitLab con un `released_at` retroactivo, no lo detecta la comprobación; ejecute `cache:clear` para ese software para incorporarlo. En GitHub Actions el directorio puede conservarse entre ejecuciones del workflow, de modo que cada ejecución gasta el límite de la API solo en los releases publicados desde la anterior: diff --git a/README-ru.md b/README-ru.md index e86a4f2..3ce56f0 100644 --- a/README-ru.md +++ b/README-ru.md @@ -398,6 +398,7 @@ DLoad поддерживает три типа загрузки, которые > завершится ошибкой. Сохранённый релиз, ассеты которого исчезли из источника, удаляется из > реестра сразу после неудачной загрузки, а список релизов запрашивается заново, прежде чем > запуск завершится ошибкой. +> Черновики релизов GitHub в реестр не попадают. Релиз, который провайдер вставляет не в начало списка, например релиз GitLab с задним числом в `released_at`, проверка не замечает; чтобы его подхватить, выполните `cache:clear` для этой программы. В GitHub Actions каталог можно переносить между запусками workflow, тогда запуск тратит лимит API только на релизы, вышедшие после предыдущего: diff --git a/README-zh.md b/README-zh.md index d33d90a..5beae95 100644 --- a/README-zh.md +++ b/README-zh.md @@ -391,6 +391,7 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： > 因此该目录可以自由共享或放入 CI 缓存。若因网络错误或 API 速率限制导致检查失败，会使用已保存的发布； > 从未见过的仓库仍会明确报错。若某个已保存发布的资产在上游已被删除，下载失败后它会立即从注册表中移除， > 并在本次运行放弃之前重新获取发布列表。 +> GitHub 的草稿发布不会被保存。若提供方将某个发布插入到列表开头以下的位置，例如 GitLab 中 `released_at` 被回填的发布，检查不会发现它；请对该软件运行 `cache:clear` 以获取它。 在 GitHub Actions 中可以在多次工作流运行之间保留该目录，这样每次运行只为上次运行之后发布的版本消耗速率限制： diff --git a/README.md b/README.md index 504e621..8f35d97 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,7 @@ An attribute set in `dload.xml` takes precedence over the environment variable. > releases are used instead, and a repository that was never seen before still fails loudly. > A stored release whose assets have disappeared upstream is dropped from the registry as soon as > its download fails, and the release list is fetched again before the run gives up. +> GitHub draft releases are never stored. A release that a provider inserts below the top of its listing, such as a GitLab release with a backdated `released_at`, is not noticed by a check; run `cache:clear` for that software to pick it up. In GitHub Actions the directory can be carried between workflow runs, so a run spends the rate limit only on releases published since the previous one: diff --git a/src/Bootstrap.php b/src/Bootstrap.php index bdbc579..65ab415 100644 --- a/src/Bootstrap.php +++ b/src/Bootstrap.php @@ -29,6 +29,7 @@ use Internal\DLoad\Module\Velox\Internal\Client\BuildRoadRunner; use Internal\DLoad\Module\Velox\Internal\VeloxBuilder; use Internal\DLoad\Service\Logger; +use Internal\Path; /** * Bootstraps the application by configuring the dependency container. @@ -127,7 +128,7 @@ static function (Container $container) use ($environment): RegistryStorage { $config = $container->get(CacheConfig::class); return new FileRegistryStorage( - $config->dir ?? CacheDirectory::resolve($environment), + $config->dir === null ? CacheDirectory::resolve($environment) : Path::create($config->dir), $container->get(Logger::class), ); }, diff --git a/src/Module/Downloader/Downloader.php b/src/Module/Downloader/Downloader.php index 60c80e2..7d84e80 100644 --- a/src/Module/Downloader/Downloader.php +++ b/src/Module/Downloader/Downloader.php @@ -117,7 +117,7 @@ public function download( $repository = $this->repositoryProvider->getByConfig($context->repoConfig); // The registry keeps track of which software is served from which repository - $this->registry->attach($context->software->getId(), RepositoryId::fromConfig($context->repoConfig)); + $this->registry->attach(RepositoryId::fromConfig($context->repoConfig), $context->software->getId()); $context->repositoryAttempt = $context->diagnostics->addRepository( type: $context->repoConfig->type, name: $repository->getName(), diff --git a/src/Module/Registry/Internal/CacheDirectory.php b/src/Module/Registry/Internal/CacheDirectory.php index 49b4ca1..010281e 100644 --- a/src/Module/Registry/Internal/CacheDirectory.php +++ b/src/Module/Registry/Internal/CacheDirectory.php @@ -4,6 +4,8 @@ namespace Internal\DLoad\Module\Registry\Internal; +use Internal\Path; + /** * Resolves the default directory of the version registry. * @@ -13,7 +15,7 @@ * - `$XDG_CACHE_HOME/dload` when the variable is set; * - `%LOCALAPPDATA%\dload\cache` on Windows; * - `$HOME/.cache/dload` otherwise; - * - the system temporary directory as the last resort. + * - a per-user directory under the system temporary directory as the last resort. * * @internal * @psalm-internal Internal\DLoad @@ -22,26 +24,43 @@ final class CacheDirectory { /** * @param array $env Environment variables. - * @return non-empty-string + * @param bool $windows Whether the platform conventions of Windows apply. */ - public static function resolve(array $env): string + public static function resolve(array $env, bool $windows = \DIRECTORY_SEPARATOR === '\\'): Path { $xdg = self::variable($env, 'XDG_CACHE_HOME'); if ($xdg !== null) { - return $xdg . \DIRECTORY_SEPARATOR . 'dload'; + return Path::create($xdg)->join('dload'); } $localAppData = self::variable($env, 'LOCALAPPDATA'); - if ($localAppData !== null && \DIRECTORY_SEPARATOR === '\\') { - return $localAppData . \DIRECTORY_SEPARATOR . 'dload' . \DIRECTORY_SEPARATOR . 'cache'; + if ($localAppData !== null && $windows) { + return Path::create($localAppData)->join('dload', 'cache'); } $home = self::variable($env, 'HOME') ?? self::variable($env, 'USERPROFILE'); if ($home !== null) { - return $home . \DIRECTORY_SEPARATOR . '.cache' . \DIRECTORY_SEPARATOR . 'dload'; + return Path::create($home)->join('.cache', 'dload'); + } + + // The temporary directory is shared by every user of the host: keep the registries apart + $user = self::variable($env, 'USER') ?? self::variable($env, 'USERNAME') ?? self::processOwner(); + + return Path::create(\sys_get_temp_dir())->join('dload-cache-' . (string) \preg_replace('/[^A-Za-z0-9._-]+/', '_', $user)); + } + + /** + * @return non-empty-string + */ + private static function processOwner(): string + { + if (\function_exists('posix_geteuid')) { + return (string) \posix_geteuid(); } - return \sys_get_temp_dir() . \DIRECTORY_SEPARATOR . 'dload-cache'; + $owner = \get_current_user(); + + return $owner === '' ? 'default' : $owner; } /** diff --git a/src/Module/Registry/Internal/FileRegistryStorage.php b/src/Module/Registry/Internal/FileRegistryStorage.php index e51741f..7dc73d0 100644 --- a/src/Module/Registry/Internal/FileRegistryStorage.php +++ b/src/Module/Registry/Internal/FileRegistryStorage.php @@ -5,6 +5,7 @@ namespace Internal\DLoad\Module\Registry\Internal; use Internal\DLoad\Module\Common\FileSystem\FS; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Registry\Record\RepositoryRecord; use Internal\DLoad\Module\Registry\RegistryStorage; use Internal\DLoad\Module\Registry\RepositoryId; @@ -12,17 +13,21 @@ use Internal\Path; /** - * Storage that keeps one JSON file per repository. + * Storage that keeps one directory per repository. * - * The layout is meant to be readable and portable between machines: + * The layout is meant to be readable and portable between machines: the index holds the + * metadata and the tags of every segment, and each segment of releases is a file of its own, + * so a run reads only the segments it iterates. * * ``` - * /repositories/github/roadrunner-server/roadrunner.json - * /repositories/gitlab/group/project.json + * /repositories/github/roadrunner-server/roadrunner/index.json + * /repositories/github/roadrunner-server/roadrunner/releases-0001.json + * /repositories/gitlab/group/project/index.json * ``` * * Files are written aside and renamed into place, so an interrupted or parallel run cannot - * leave a half-written record for anyone to read. + * leave a half-written file for anyone to read. Segments are written before the index, so the + * index never points at a file that is not there yet. * * @internal * @psalm-internal Internal\DLoad @@ -30,43 +35,47 @@ final class FileRegistryStorage implements RegistryStorage { private const REPOSITORIES_DIR = 'repositories'; + private const INDEX_FILE = 'index.json'; + private const SEGMENT_PREFIX = 'releases-'; private const EXTENSION = '.json'; + private const TEMP_EXTENSION = '.tmp'; + + /** Age after which a leftover temporary file of a crashed run is removed, in seconds. */ + private const STALE_TEMP_AGE = 3600; private readonly Path $root; public function __construct( - Path|string $directory, + Path $directory, private readonly Logger $logger, ) { - $this->root = Path::create($directory)->join(self::REPOSITORIES_DIR); + $this->root = $directory->join(self::REPOSITORIES_DIR); } public function load(RepositoryId $id): ?RepositoryRecord { - $record = $this->read($this->fileOf($id)); + $record = $this->readIndex($this->directoryOf($id)); - // Sanitizing and case-insensitive file systems may map two identities onto one file + // Sanitizing and case-insensitive file systems may map two identities onto one directory return $record?->id->equals($id) === true ? $record : null; } public function save(RepositoryRecord $record): void { - $file = $this->fileOf($record->id); - $directory = $file->parent(); - + $directory = $this->directoryOf($record->id); $directory->isDir() or FS::mkdir($directory); + $directory->isDir() or throw new \RuntimeException(\sprintf('Failed to create registry directory `%s`.', $directory)); - $payload = \json_encode($record->toArray(), \JSON_THROW_ON_ERROR | \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES); + foreach ($record->segments as $segment) { + $segment->dirty and $this->write( + $this->segmentFile($directory, $segment->key), + \array_map(static fn(ReleaseRecord $release): array => $release->toArray(), $segment->releases()), + ); + } - $temp = Path::create((string) $file . '.' . (int) \getmypid() . '.tmp'); - @\file_put_contents((string) $temp, $payload) === false and throw new \RuntimeException( - \sprintf('Failed to write registry record `%s`.', $temp), - ); + $this->write($directory->join(self::INDEX_FILE), $record->toArray()); - if (!FS::moveFile($temp, $file)) { - FS::removeFile($temp); - throw new \RuntimeException(\sprintf('Failed to store registry record `%s`.', $file)); - } + $this->removeOrphans($directory, $record); } public function all(): iterable @@ -81,19 +90,28 @@ public function all(): iterable /** @var \SplFileInfo $file */ foreach ($iterator as $file) { - if (!$file->isFile() || !\str_ends_with($file->getFilename(), self::EXTENSION)) { + if (!$file->isFile() || $file->getFilename() !== self::INDEX_FILE) { continue; } - $record = $this->read(Path::create($file->getPathname())); + $record = $this->readIndex(Path::create($file->getPath())); $record === null or yield $record; } } public function remove(RepositoryId $id): void { - $file = $this->fileOf($id); - $file->isFile() and FS::removeFile($file); + $directory = $this->directoryOf($id); + if (!$directory->isDir()) { + return; + } + + FS::removeDir($directory); + + // Owner and type directories are worth nothing once empty + for ($parent = $directory->parent(); (string) $parent !== (string) $this->root && $this->isEmptyDir($parent); $parent = $parent->parent()) { + FS::removeDir($parent); + } } public function clear(): void @@ -121,23 +139,29 @@ private static function sanitize(string $segment): string } /** - * Reads a record, or returns `null` when there is none or it cannot be used. + * Reads the index of a record, or returns `null` when there is none or it cannot be used. */ - private function read(Path $file): ?RepositoryRecord + private function readIndex(Path $directory): ?RepositoryRecord { + $file = $directory->join(self::INDEX_FILE); if (!$file->isFile()) { return null; } try { - $content = @\file_get_contents((string) $file); - $content === false and throw new \RuntimeException(\sprintf('Failed to read registry record `%s`.', $file)); + $record = RepositoryRecord::fromArray( + $this->decode($file), + fn(string $key): array => $this->readSegment($directory, $key), + ); - /** @var mixed $payload */ - $payload = \json_decode($content, true, 512, \JSON_THROW_ON_ERROR); - \is_array($payload) or throw new \UnexpectedValueException('Registry record must be a JSON object.'); + // A segment file missing now would fail the run when its turn comes + foreach ($record->segments as $segment) { + $this->segmentFile($directory, $segment->key)->isFile() or throw new \RuntimeException( + \sprintf('Registry record `%s` refers to a missing segment `%s`.', $directory, $segment->key), + ); + } - return RepositoryRecord::fromArray($payload); + return $record; } catch (\Throwable $e) { // A half-written, hand-edited or outdated record is not worth a failed download: // report it and let the registry fetch the releases again. @@ -147,11 +171,105 @@ private function read(Path $file): ?RepositoryRecord } } - private function fileOf(RepositoryId $id): Path + /** + * @param non-empty-string $key + * @return list + * @throws \RuntimeException When the segment cannot be read; the record is dropped, so the next run starts afresh. + */ + private function readSegment(Path $directory, string $key): array + { + try { + $releases = []; + /** @var mixed $item */ + foreach ($this->decode($this->segmentFile($directory, $key)) as $item) { + \is_array($item) or throw new \UnexpectedValueException('Registry segment must hold release objects.'); + $releases[] = ReleaseRecord::fromArray($item); + } + + return $releases; + } catch (\Throwable $e) { + $directory->isDir() and FS::removeDir($directory); + + throw new \RuntimeException( + \sprintf('Registry segment `%s` of `%s` is unreadable; the record was dropped.', $key, $directory), + previous: $e, + ); + } + } + + /** + * @return array + * @throws \RuntimeException + * @throws \JsonException + */ + private function decode(Path $file): array { - $segments = \array_map(self::sanitize(...), [$id->type, ...\explode('/', $id->uri)]); - $segments[\array_key_last($segments)] .= self::EXTENSION; + $content = @\file_get_contents((string) $file); + $content === false and throw new \RuntimeException(\sprintf('Failed to read registry file `%s`.', $file)); + + /** @var mixed $payload */ + $payload = \json_decode($content, true, 512, \JSON_THROW_ON_ERROR); + \is_array($payload) or throw new \UnexpectedValueException(\sprintf('Registry file `%s` must hold a JSON structure.', $file)); - return $this->root->join(...$segments); + return $payload; + } + + /** + * Writes the payload aside and renames it into place. + * + * @throws \RuntimeException When the file cannot be written. + */ + private function write(Path $file, array $payload): void + { + $json = \json_encode($payload, \JSON_THROW_ON_ERROR | \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES); + + $temp = Path::create((string) $file . '.' . (int) \getmypid() . self::TEMP_EXTENSION); + @\file_put_contents((string) $temp, $json) === false and throw new \RuntimeException( + \sprintf('Failed to write registry file `%s`.', $temp), + ); + + if (!FS::moveFile($temp, $file)) { + FS::removeFile($temp); + throw new \RuntimeException(\sprintf('Failed to store registry file `%s`.', $file)); + } + } + + /** + * Removes segment files the index no longer refers to and temporary files a crashed run left behind. + */ + private function removeOrphans(Path $directory, RepositoryRecord $record): void + { + $known = [self::INDEX_FILE]; + foreach ($record->segments as $segment) { + $known[] = self::SEGMENT_PREFIX . self::sanitize($segment->key) . self::EXTENSION; + } + + foreach (new \FilesystemIterator((string) $directory, \FilesystemIterator::SKIP_DOTS) as $file) { + /** @var \SplFileInfo $file */ + $name = $file->getFilename(); + $stale = \str_ends_with($name, self::TEMP_EXTENSION) + ? \time() - $file->getMTime() > self::STALE_TEMP_AGE + : !\in_array($name, $known, true); + + $stale && $file->isFile() and FS::removeFile(Path::create($file->getPathname())); + } + } + + private function isEmptyDir(Path $directory): bool + { + return $directory->isDir() && !(new \FilesystemIterator((string) $directory, \FilesystemIterator::SKIP_DOTS))->valid(); + } + + private function directoryOf(RepositoryId $id): Path + { + return $this->root->join(...\array_map(self::sanitize(...), [$id->type, ...\explode('/', $id->uri)])); + } + + /** + * @param non-empty-string $key + */ + private function segmentFile(Path $directory, string $key): Path + { + return $directory->join(self::SEGMENT_PREFIX . self::sanitize($key) . self::EXTENSION); } } diff --git a/src/Module/Registry/Internal/PassThroughRegistry.php b/src/Module/Registry/Internal/PassThroughRegistry.php index 210e71a..a1326da 100644 --- a/src/Module/Registry/Internal/PassThroughRegistry.php +++ b/src/Module/Registry/Internal/PassThroughRegistry.php @@ -25,7 +25,7 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator } } - public function attach(string $software, RepositoryId $id): void + public function attach(RepositoryId $id, string $software): void { // Nothing to record } diff --git a/src/Module/Registry/Internal/StoredVersionRegistry.php b/src/Module/Registry/Internal/StoredVersionRegistry.php index 85d8bf3..356689d 100644 --- a/src/Module/Registry/Internal/StoredVersionRegistry.php +++ b/src/Module/Registry/Internal/StoredVersionRegistry.php @@ -10,6 +10,7 @@ use Internal\DLoad\Module\Registry\ReleaseSource; use Internal\DLoad\Module\Registry\RepositoryId; use Internal\DLoad\Module\Registry\VersionRegistry; +use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Service\Logger; @@ -22,12 +23,13 @@ * pages are fetched until a page contains a release that is already stored. Usually that is * one request. The first page is always taken from the source, so releases whose assets were * attached after the check are updated too. - * 2. **Serve.** The stored releases are yielded without any request. + * 2. **Serve.** The stored releases are yielded segment by segment without any request. * 3. **Extend.** When the consumer runs past the stored releases and the listing is not known * to be complete, older pages are fetched one by one and appended to the record. * * A check that fails while releases are stored falls back to the stored ones: an outage or a rate - * limit should not break what worked a minute ago. + * limit should not break what worked a minute ago. A rate limit is reported once per run in plain + * sight, as the stored list may be missing newer releases until the limit resets. * * @internal * @psalm-internal Internal\DLoad @@ -37,6 +39,8 @@ final class StoredVersionRegistry implements VersionRegistry /** @var \Closure(): int */ private readonly \Closure $clock; + private bool $rateLimitReported = false; + /** * @param int<0, max> $ttl Seconds the last check stays valid. * @param bool $refresh Ignore the TTL and check the source for every repository once. @@ -62,8 +66,7 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator $this->logger->debug('Releases of `%s` are served from the version registry.', (string) $id); } - $stored = $record->releases(); - $stored === [] or yield $stored; + yield from $record->pages(); if ($record->complete) { return; @@ -73,7 +76,7 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator yield from $this->extend($record, $source); } - public function attach(string $software, RepositoryId $id): void + public function attach(RepositoryId $id, string $software): void { $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); $updated = $record->withSoftware($software); @@ -132,23 +135,16 @@ private function check(RepositoryRecord $record, ReleaseSource $source): Reposit } } - $updated = $record - ->withHead($fetched) - ->withComplete($complete) - ->withCheckedAt(($this->clock)()); - - $this->persist($updated); - - return $updated; + return $this->persist( + $record + ->withHead($fetched) + ->withComplete($complete) + ->withCheckedAt(($this->clock)()), + ); } catch (RepositoryException $e) { $record->count() > 0 or throw $e; - $this->logger->exception($e, important: false); - $this->logger->info( - 'Failed to check `%s` for new releases, %d stored release(s) are used instead.', - (string) $record->id, - $record->count(), - ); + $this->report($e, $record); return $record; } @@ -168,8 +164,7 @@ private function extend(RepositoryRecord $record, ReleaseSource $source): \Gener static fn(ReleaseRecord $release): bool => !$record->has($release->tag), )); - $record = $record->withTail($new)->withComplete($page->last); - $this->persist($record); + $record = $this->persist($record->withTail($new)->withComplete($page->last)); $new === [] or yield $new; } @@ -177,15 +172,47 @@ private function extend(RepositoryRecord $record, ReleaseSource $source): \Gener $record->complete or $this->persist($record->withComplete(true)); } + /** + * Tells why the check was skipped. A rate limit is shown to the user once per run: the stored + * list still works, but it may lack newer releases until the limit resets. Anything else is + * an ordinary transient failure and stays in the debug output. + */ + private function report(RepositoryException $e, RepositoryRecord $record): void + { + $this->logger->exception($e, important: false); + + if ($e instanceof RateLimitException && !$this->rateLimitReported) { + $this->rateLimitReported = true; + $this->logger->error( + 'The API rate limit prevents checking `%s` for new releases; %d stored release(s) are used, newer ones may be missing. %s', + (string) $record->id, + $record->count(), + $e->getMessage(), + ); + + return; + } + + $this->logger->debug( + 'Failed to check `%s` for new releases, %d stored release(s) are used instead.', + (string) $record->id, + $record->count(), + ); + } + /** * Stores the record; a storage failure is reported and swallowed. + * + * Returns the record as stored, so later writes do not repeat the segments already written. */ - private function persist(RepositoryRecord $record): void + private function persist(RepositoryRecord $record): RepositoryRecord { try { $this->storage->save($record); } catch (\Throwable $e) { $this->logger->exception($e, important: false); } + + return $record->persisted(); } } diff --git a/src/Module/Registry/Record/AssetRecord.php b/src/Module/Registry/Record/AssetRecord.php index f76311e..3c9e601 100644 --- a/src/Module/Registry/Record/AssetRecord.php +++ b/src/Module/Registry/Record/AssetRecord.php @@ -17,6 +17,8 @@ * content_type?: string|null, * digest?: string|null, * } + * + * @internal */ final class AssetRecord { diff --git a/src/Module/Registry/Record/ReleasePage.php b/src/Module/Registry/Record/ReleasePage.php index 64aa0db..93f25b1 100644 --- a/src/Module/Registry/Record/ReleasePage.php +++ b/src/Module/Registry/Record/ReleasePage.php @@ -9,6 +9,8 @@ * * Besides the releases, the page tells whether the listing ends with it: the registry needs to * know that at the moment the page arrives, without requesting the next one to find out. + * + * @internal */ final class ReleasePage { diff --git a/src/Module/Registry/Record/ReleaseRecord.php b/src/Module/Registry/Record/ReleaseRecord.php index 48f6021..ab82a37 100644 --- a/src/Module/Registry/Record/ReleaseRecord.php +++ b/src/Module/Registry/Record/ReleaseRecord.php @@ -18,6 +18,8 @@ * prerelease?: bool, * assets?: list, * } + * + * @internal */ final class ReleaseRecord { diff --git a/src/Module/Registry/Record/ReleaseSegment.php b/src/Module/Registry/Record/ReleaseSegment.php new file mode 100644 index 0000000..660b7fc --- /dev/null +++ b/src/Module/Registry/Record/ReleaseSegment.php @@ -0,0 +1,111 @@ +|null */ + private ?array $releases; + + /** @var (\Closure(): list)|null */ + private ?\Closure $loader; + + /** + * @param non-empty-string $key Identifier of the segment, unique within the repository record. + * @param list $tags Tags of the releases, newest first. + * @param list|null $releases Releases, newest first; `null` when a loader provides them. + * @param (\Closure(): list)|null $loader Reads the releases from storage. + * @param bool $dirty Whether the storage does not hold this content yet. + */ + private function __construct( + public readonly string $key, + public readonly array $tags, + ?array $releases, + ?\Closure $loader, + public readonly bool $dirty, + ) { + $this->releases = $releases; + $this->loader = $loader; + } + + /** + * Segment assembled from releases that are not stored yet. + * + * @param non-empty-string $key + * @param non-empty-list $releases Newest first. + */ + public static function fresh(string $key, array $releases): self + { + return new self( + key: $key, + tags: \array_map(static fn(ReleaseRecord $release): string => $release->tag, $releases), + releases: $releases, + loader: null, + dirty: true, + ); + } + + /** + * Segment whose releases are read from storage when first needed. + * + * @param non-empty-string $key + * @param non-empty-list $tags Newest first. + * @param \Closure(): list $loader + */ + public static function stored(string $key, array $tags, \Closure $loader): self + { + return new self(key: $key, tags: $tags, releases: null, loader: $loader, dirty: false); + } + + /** + * @return list Newest first. + * @throws \RuntimeException When the releases cannot be read from storage. + */ + public function releases(): array + { + if ($this->releases === null) { + $this->releases = ($this->loader ?? static fn(): array => [])(); + $this->loader = null; + } + + return $this->releases; + } + + /** + * @return int<0, max> + */ + public function count(): int + { + return \count($this->tags); + } + + /** + * @param non-empty-string $tag + */ + public function has(string $tag): bool + { + return \in_array($tag, $this->tags, true); + } + + /** + * The same segment as the storage holds it now. + */ + public function persisted(): self + { + return $this->dirty + ? new self(key: $this->key, tags: $this->tags, releases: $this->releases, loader: null, dirty: false) + : $this; + } +} diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php index 84d11f7..949b85e 100644 --- a/src/Module/Registry/Record/RepositoryRecord.php +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -14,45 +14,61 @@ * releases found on a check) or at the tail (older releases loaded on demand). `complete` tells * whether the tail has reached the end of the listing. * + * The list is split into segments of at most `SEGMENT_SIZE` releases, so a repository with + * thousands of releases costs one small index read plus the segments the run actually iterates. + * New releases are packed into the neighbouring segment while it has room and start a new one + * when it is full. The tags of every segment live in the index, so counting and membership never + * load a segment. + * * The record is immutable; every change produces a new instance. * - * @psalm-import-type ReleaseArray from ReleaseRecord * @psalm-type RepositoryArray = array{ * version: int, * repository: array{type: non-empty-string, uri: non-empty-string}, * checked_at: int|null, * complete: bool, * software: list, - * releases: list, + * segments: list}>, * } + * + * @internal */ final class RepositoryRecord { /** Format version of the stored payload; bump when the structure changes incompatibly. */ - public const FORMAT_VERSION = 1; + public const FORMAT_VERSION = 2; + + /** Greatest number of releases a segment holds. */ + public const SEGMENT_SIZE = 100; - /** @var array Releases keyed by tag, newest first. */ - private readonly array $releases; + /** @var list Newest first. */ + public readonly array $segments; + + /** @var array Segment position of every stored tag. */ + private readonly array $index; /** * @param int|null $checkedAt Unix timestamp of the last successful check against the source. * @param bool $complete Whether the stored releases reach the end of the source listing. * @param list $software Identifiers of the software packages served from this repository. - * @param list $releases Releases newest first. + * @param list $segments Newest first. */ public function __construct( public readonly RepositoryId $id, public readonly ?int $checkedAt = null, public readonly bool $complete = false, public readonly array $software = [], - array $releases = [], + array $segments = [], ) { - $indexed = []; - foreach ($releases as $release) { - $indexed[$release->tag] ??= $release; + $index = []; + foreach ($segments as $position => $segment) { + foreach ($segment->tags as $tag) { + $index[$tag] ??= $position; + } } - $this->releases = $indexed; + $this->segments = $segments; + $this->index = $index; } public static function empty(RepositoryId $id): self @@ -61,10 +77,13 @@ public static function empty(RepositoryId $id): self } /** + * Restores the index of a record; the releases of every segment come through the loader. + * * @param array $data + * @param \Closure(non-empty-string): list $loader Reads the releases of a segment by its key. * @throws \InvalidArgumentException When the array does not describe a repository record. */ - public static function fromArray(array $data): self + public static function fromArray(array $data, \Closure $loader): self { ($data['version'] ?? null) === self::FORMAT_VERSION or throw new \InvalidArgumentException( 'Unsupported repository record format.', @@ -80,10 +99,23 @@ public static function fromArray(array $data): self 'Repository record requires a repository type and URI.', ); - $releases = []; - /** @var mixed $release */ - foreach (\is_array($data['releases'] ?? null) ? $data['releases'] : [] as $release) { - \is_array($release) and $releases[] = ReleaseRecord::fromArray($release); + $segments = []; + /** @var mixed $segment */ + foreach (\is_array($data['segments'] ?? null) ? $data['segments'] : [] as $segment) { + \is_array($segment) or throw new \InvalidArgumentException('Repository record segment must be an object.'); + + /** @var mixed $key */ + $key = $segment['key'] ?? null; + \is_string($key) && $key !== '' or throw new \InvalidArgumentException('Repository record segment requires a `key`.'); + + $tags = []; + /** @var mixed $tag */ + foreach (\is_array($segment['tags'] ?? null) ? $segment['tags'] : [] as $tag) { + \is_string($tag) && $tag !== '' or throw new \InvalidArgumentException('Repository record segment tags must be non-empty strings.'); + $tags[] = $tag; + } + + $tags === [] or $segments[] = ReleaseSegment::stored($key, $tags, static fn(): array => $loader($key)); } $software = []; @@ -100,16 +132,37 @@ public static function fromArray(array $data): self checkedAt: \is_int($checkedAt) ? $checkedAt : null, complete: (bool) ($data['complete'] ?? false), software: $software, - releases: $releases, + segments: $segments, ); } /** - * @return list Releases newest first. + * Releases segment by segment, newest first; a segment is read from storage when reached. + * + * @return \Generator, mixed, void> + * @throws \RuntimeException When a segment cannot be read from storage. + */ + public function pages(): \Generator + { + foreach ($this->segments as $segment) { + yield $segment->releases(); + } + } + + /** + * Every release, newest first; reads every segment. + * + * @return list + * @throws \RuntimeException When a segment cannot be read from storage. */ public function releases(): array { - return \array_values($this->releases); + $releases = []; + foreach ($this->pages() as $page) { + $releases = [...$releases, ...$page]; + } + + return $releases; } /** @@ -117,7 +170,7 @@ public function releases(): array */ public function count(): int { - return \count($this->releases); + return \count($this->index); } /** @@ -125,7 +178,7 @@ public function count(): int */ public function has(string $tag): bool { - return isset($this->releases[$tag]); + return isset($this->index[$tag]); } /** @@ -152,50 +205,92 @@ public function isStale(int $now, int $ttl): bool */ public function withHead(array $fetched): self { + $fetched = self::unique($fetched); $fetchedTags = \array_fill_keys(\array_map(static fn(ReleaseRecord $release): string => $release->tag, $fetched), true); - $stored = $this->releases(); + // Flat view of the stored tags with the segment each one belongs to + $stored = []; + foreach ($this->segments as $position => $segment) { + foreach ($segment->tags as $tag) { + $stored[] = [$tag, $position]; + } + } + $overlap = null; - foreach ($stored as $position => $release) { - if (isset($fetchedTags[$release->tag])) { - $overlap = $position; + foreach ($stored as $offset => [$tag]) { + if (isset($fetchedTags[$tag])) { + $overlap = $offset; break; } } if ($overlap === null) { - return $this->with(releases: $fetched); + return $this->with(segments: self::pack($fetched, [], $this->nextKey())); } // Listing positions the fetched releases still cover, counting from the overlap $spanned = 0; foreach ($fetched as $position => $release) { - if ($release->tag === $stored[$overlap]->tag) { + if ($release->tag === $stored[$overlap][0]) { $spanned = \count($fetched) - $position; break; } } - $kept = []; - foreach (\array_slice($stored, $overlap) as $release) { + $keepFrom = \count($stored); + foreach (\array_slice($stored, $overlap, preserve_keys: true) as $offset => [$tag]) { if ($spanned <= 0) { - $kept[] = $release; - } elseif (isset($fetchedTags[$release->tag])) { - --$spanned; + $keepFrom = $offset; + break; } + + isset($fetchedTags[$tag]) and --$spanned; } - return $this->with(releases: [...$fetched, ...$kept]); + // The segment holding the first kept release is split unless the release opens it; + // the segments after it stay as they are + $loose = $fetched; + $following = []; + if ($keepFrom < \count($stored)) { + [$tag, $position] = $stored[$keepFrom]; + $segment = $this->segments[$position]; + $start = (int) \array_search($tag, $segment->tags, true); + $start === 0 or $loose = [...$loose, ...\array_slice($segment->releases(), $start)]; + $following = \array_slice($this->segments, $start === 0 ? $position : $position + 1); + } + + return $this->with(segments: self::pack($loose, $following, $this->nextKey())); } /** * Appends older releases loaded on demand; already known tags are ignored. * - * @param list $fetched + * The last segment is filled up before a new one starts. + * + * @param list $fetched Newest first. */ public function withTail(array $fetched): self { - return $this->with(releases: [...$this->releases(), ...$fetched]); + $new = \array_values(\array_filter( + self::unique($fetched), + fn(ReleaseRecord $release): bool => !$this->has($release->tag), + )); + if ($new === []) { + return $this; + } + + $segments = $this->segments; + $last = \array_pop($segments); + if ($last === null || $last->count() >= self::SEGMENT_SIZE) { + $last === null or $segments[] = $last; + + return $this->with(segments: [...$segments, ...self::chunk($new, $this->nextKey())]); + } + + return $this->with(segments: [ + ...$segments, + ...self::chunk([...$last->releases(), ...$new], $this->nextKey(), $last->key), + ]); } public function withCheckedAt(int $checkedAt): self @@ -213,7 +308,7 @@ public function withoutCheck(): self checkedAt: null, complete: $this->complete, software: $this->software, - releases: $this->releases(), + segments: $this->segments, ); } @@ -228,10 +323,18 @@ public function withoutRelease(string $tag): self return $this; } - return $this->with(releases: \array_values(\array_filter( - $this->releases(), + $position = $this->index[$tag]; + $segments = $this->segments; + $remaining = \array_values(\array_filter( + $segments[$position]->releases(), static fn(ReleaseRecord $release): bool => $release->tag !== $tag, - ))); + )); + + $remaining === [] + ? \array_splice($segments, $position, 1) + : $segments[$position] = ReleaseSegment::fresh($segments[$position]->key, $remaining); + + return $this->with(segments: $segments); } public function withComplete(bool $complete): self @@ -250,6 +353,16 @@ public function withSoftware(string $software): self } /** + * The record as the storage holds it now: no segment is dirty any more. + */ + public function persisted(): self + { + return $this->with(segments: \array_map(static fn(ReleaseSegment $segment): ReleaseSegment => $segment->persisted(), $this->segments)); + } + + /** + * The index of the record; the releases of the segments are stored separately. + * * @return RepositoryArray */ public function toArray(): array @@ -260,26 +373,125 @@ public function toArray(): array 'checked_at' => $this->checkedAt, 'complete' => $this->complete, 'software' => $this->software, - 'releases' => \array_map(static fn(ReleaseRecord $release): array => $release->toArray(), $this->releases()), + 'segments' => \array_map( + static fn(ReleaseSegment $segment): array => ['key' => $segment->key, 'tags' => $segment->tags], + $this->segments, + ), ]; } + /** + * Packs loose releases into segments in front of the given ones. + * + * The short remainder goes first, where the next check adds its releases, so the head segment + * fills up over several checks and the full segments behind it are never rewritten. When the + * remainder and the first following segment fit into one, they are joined. + * + * @param list $loose Newest first. + * @param list $following Segments that stay as they are. + * @param int<1, max> $nextKey + * @return list + */ + private static function pack(array $loose, array $following, int $nextKey): array + { + if ($loose === []) { + return $following; + } + + $remainder = \count($loose) % self::SEGMENT_SIZE; + $first = $following[0] ?? null; + if ($first !== null && $remainder > 0 && $remainder + $first->count() <= self::SEGMENT_SIZE) { + $loose = [...$loose, ...$first->releases()]; + \array_shift($following); + $remainder = \count($loose) % self::SEGMENT_SIZE; + } + + $head = []; + if ($remainder > 0) { + /** @var non-empty-list $partial */ + $partial = \array_slice($loose, 0, $remainder); + $head[] = ReleaseSegment::fresh(self::key($nextKey++), $partial); + } + + return [...$head, ...self::chunk(\array_slice($loose, $remainder), $nextKey), ...$following]; + } + + /** + * Splits releases into segments of at most `SEGMENT_SIZE`, keying them from `$nextKey` on. + * + * @param list $releases + * @param int<1, max> $nextKey + * @param non-empty-string|null $reuseKey Key for the first segment, when it replaces an existing one. + * @return list + */ + private static function chunk(array $releases, int $nextKey, ?string $reuseKey = null): array + { + $segments = []; + foreach (\array_chunk($releases, self::SEGMENT_SIZE) as $chunk) { + $segments[] = ReleaseSegment::fresh($reuseKey ?? self::key($nextKey++), $chunk); + $reuseKey = null; + } + + return $segments; + } + + /** + * @param int<1, max> $number + * @return non-empty-string + */ + private static function key(int $number): string + { + /** @var non-empty-string */ + return \sprintf('%04d', $number); + } + + /** + * Drops the releases repeating a tag seen before, so no tag lands in two segments. + * + * @param list $releases + * @return list + */ + private static function unique(array $releases): array + { + $seen = []; + $unique = []; + foreach ($releases as $release) { + isset($seen[$release->tag]) or $unique[] = $release; + $seen[$release->tag] = true; + } + + return $unique; + } + + /** + * @return int<1, max> + */ + private function nextKey(): int + { + $max = 0; + foreach ($this->segments as $segment) { + $max = \max($max, (int) $segment->key); + } + + return $max + 1; + } + /** * @param list|null $software - * @param list|null $releases + * @param list|null $segments */ private function with( ?int $checkedAt = null, ?bool $complete = null, ?array $software = null, - ?array $releases = null, + ?array $segments = null, ): self { return new self( id: $this->id, checkedAt: $checkedAt ?? $this->checkedAt, complete: $complete ?? $this->complete, software: $software ?? $this->software, - releases: $releases ?? $this->releases(), + segments: $segments ?? $this->segments, ); } } diff --git a/src/Module/Registry/RegistryStorage.php b/src/Module/Registry/RegistryStorage.php index f4d6077..4219ad2 100644 --- a/src/Module/Registry/RegistryStorage.php +++ b/src/Module/Registry/RegistryStorage.php @@ -12,6 +12,8 @@ * Implementations must never let a storage failure escape as an exception from `load()`: * a broken or unreadable record is reported as missing, because the registry is an optimisation * and must not turn a working download into a failed one. + * + * @internal */ interface RegistryStorage { diff --git a/src/Module/Registry/ReleaseSource.php b/src/Module/Registry/ReleaseSource.php index 23954e3..1ae4e4c 100644 --- a/src/Module/Registry/ReleaseSource.php +++ b/src/Module/Registry/ReleaseSource.php @@ -12,6 +12,8 @@ * * The source lists releases newest first, page by page, and must not perform a request before * the corresponding page is actually iterated, so the registry can stop as early as it likes. + * + * @internal */ interface ReleaseSource { diff --git a/src/Module/Registry/RepositoryId.php b/src/Module/Registry/RepositoryId.php index 1972be0..1edb936 100644 --- a/src/Module/Registry/RepositoryId.php +++ b/src/Module/Registry/RepositoryId.php @@ -18,6 +18,8 @@ * $id = RepositoryId::fromConfig($repositoryConfig); * echo $id; // github:roadrunner-server/roadrunner * ``` + * + * @internal */ final class RepositoryId implements \Stringable { diff --git a/src/Module/Registry/VersionRegistry.php b/src/Module/Registry/VersionRegistry.php index a0929be..e123b12 100644 --- a/src/Module/Registry/VersionRegistry.php +++ b/src/Module/Registry/VersionRegistry.php @@ -21,6 +21,8 @@ * } * } * ``` + * + * @internal */ interface VersionRegistry { @@ -40,7 +42,7 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator; * * @param non-empty-string $software Software identifier. */ - public function attach(string $software, RepositoryId $id): void; + public function attach(RepositoryId $id, string $software): void; /** * Drops a release that turned out to be gone and marks the repository for a check. diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index 373e595..baf4b89 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -126,7 +126,9 @@ public function releasePages(int $page = 1): \Generator $failure = null; foreach ($data as $releaseData) { try { - $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); + $release = ReleaseInfo::fromApiResponse($releaseData); + // A draft is visible to the token holder only; the registry may be shared + $release->draft or $releases[] = $release->toRecord(); } catch (\Throwable $e) { $failure ??= $e; $this->logger->exception($e, important: false); diff --git a/tests/Integration/Command/CacheClearTest.php b/tests/Integration/Command/CacheClearTest.php index 2dd8e32..592bbef 100644 --- a/tests/Integration/Command/CacheClearTest.php +++ b/tests/Integration/Command/CacheClearTest.php @@ -91,7 +91,7 @@ public function forceSkipsTheConfirmation(): void protected function prepare(): void { $this->directory = \sys_get_temp_dir() . '/dload-cache-clear-' . \bin2hex(\random_bytes(6)); - $this->storage = new FileRegistryStorage($this->directory, new Logger()); + $this->storage = new FileRegistryStorage(Path::create($this->directory), new Logger()); \mkdir($this->directory, recursive: true); \file_put_contents($this->directory . '/dload.xml', ''); \putenv('DLOAD_CACHE_DIR=' . $this->directory); diff --git a/tests/Integration/Module/Registry/VersionRegistryBindingTest.php b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php index 0d84010..1b8a4ba 100644 --- a/tests/Integration/Module/Registry/VersionRegistryBindingTest.php +++ b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php @@ -39,7 +39,7 @@ public function registryIsEnabledByDefaultInTheUserCacheDirectory(): void Assert::instanceOf($container->get(VersionRegistry::class), StoredVersionRegistry::class); $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); - Assert::true(\is_file($this->directory . '/dload/repositories/github/a/b.json')); + Assert::true(\is_file($this->directory . '/dload/repositories/github/a/b/index.json')); } #[Test] @@ -48,7 +48,7 @@ public function environmentVariableSetsTheDirectory(): void $container = self::bootstrap(environment: ['DLOAD_CACHE_DIR' => $this->directory]); $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); - Assert::true(\is_file($this->directory . '/repositories/github/a/b.json')); + Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); } #[Test] @@ -57,7 +57,7 @@ public function xmlAttributeSetsTheDirectory(): void $container = self::bootstrap(xml: \sprintf('', $this->directory)); $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); - Assert::true(\is_file($this->directory . '/repositories/github/a/b.json')); + Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); } #[Test] diff --git a/tests/Unit/Module/Registry/CacheDirectoryTest.php b/tests/Unit/Module/Registry/CacheDirectoryTest.php index 3cfab9a..746aec7 100644 --- a/tests/Unit/Module/Registry/CacheDirectoryTest.php +++ b/tests/Unit/Module/Registry/CacheDirectoryTest.php @@ -5,6 +5,7 @@ namespace Internal\DLoad\Tests\Unit\Module\Registry; use Internal\DLoad\Module\Registry\Internal\CacheDirectory; +use Internal\Path; use Testo\Assert; use Testo\Codecov\Covers; use Testo\Test; @@ -15,22 +16,53 @@ final class CacheDirectoryTest #[Test] public function xdgCacheHomeWins(): void { - $dir = CacheDirectory::resolve(['XDG_CACHE_HOME' => '/var/cache/', 'HOME' => '/home/u', 'LOCALAPPDATA' => 'C:\\x']); + $dir = CacheDirectory::resolve(['XDG_CACHE_HOME' => '/var/cache/', 'HOME' => '/home/u', 'LOCALAPPDATA' => 'C:\\x'], windows: true); - Assert::same($dir, '/var/cache' . \DIRECTORY_SEPARATOR . 'dload'); + Assert::same((string) $dir, (string) Path::create('/var/cache')->join('dload')); + } + + #[Test] + public function localAppDataIsUsedOnWindowsOnly(): void + { + $env = ['LOCALAPPDATA' => 'C:\\Users\\u\\AppData\\Local', 'HOME' => '/home/u']; + + Assert::same( + (string) CacheDirectory::resolve($env, windows: true), + (string) Path::create('C:\\Users\\u\\AppData\\Local')->join('dload', 'cache'), + ); + Assert::same( + (string) CacheDirectory::resolve($env, windows: false), + (string) Path::create('/home/u')->join('.cache', 'dload'), + ); } #[Test] public function homeIsUsedWhenNothingElseIsSet(): void { - $dir = CacheDirectory::resolve(['HOME' => '/home/u', 'XDG_CACHE_HOME' => ' ']); + $dir = CacheDirectory::resolve(['HOME' => '/home/u', 'XDG_CACHE_HOME' => ' '], windows: false); + + Assert::same((string) $dir, (string) Path::create('/home/u')->join('.cache', 'dload')); + } + + #[Test] + public function userProfileStandsInForHome(): void + { + $dir = CacheDirectory::resolve(['USERPROFILE' => 'C:\\Users\\u'], windows: true); - Assert::same($dir, '/home/u' . \DIRECTORY_SEPARATOR . '.cache' . \DIRECTORY_SEPARATOR . 'dload'); + Assert::same((string) $dir, (string) Path::create('C:\\Users\\u')->join('.cache', 'dload')); } #[Test] - public function fallsBackToTheTemporaryDirectory(): void + public function fallsBackToAPerUserTemporaryDirectory(): void { - Assert::same(CacheDirectory::resolve([]), \sys_get_temp_dir() . \DIRECTORY_SEPARATOR . 'dload-cache'); + $dir = CacheDirectory::resolve(['USER' => 'j doe'], windows: false); + + Assert::same((string) $dir, (string) Path::create(\sys_get_temp_dir())->join('dload-cache-j_doe')); + + // Without a user name in the environment the process owner keeps the directories apart + $anonymous = (string) CacheDirectory::resolve([], windows: false); + $prefix = (string) Path::create(\sys_get_temp_dir())->join('dload-cache-'); + Assert::true(\str_starts_with($anonymous, $prefix)); + Assert::true(\strlen($anonymous) > \strlen($prefix)); } } diff --git a/tests/Unit/Module/Registry/FileRegistryStorageTest.php b/tests/Unit/Module/Registry/FileRegistryStorageTest.php index 651cc54..91a5556 100644 --- a/tests/Unit/Module/Registry/FileRegistryStorageTest.php +++ b/tests/Unit/Module/Registry/FileRegistryStorageTest.php @@ -23,21 +23,64 @@ final class FileRegistryStorageTest private string $directory; #[Test] - public function storesOneReadableFilePerRepository(): void + public function storesAnIndexAndOneFilePerSegment(): void { $storage = $this->storage(); - $storage->save(self::record('github', 'roadrunner-server/roadrunner', ['v1'])); + $storage->save(self::record('github', 'roadrunner-server/roadrunner', self::range(150, 1))); $storage->save(self::record('gitlab', 'group/sub/project', ['v2'])); - Assert::true(\is_file($this->directory . '/repositories/github/roadrunner-server/roadrunner.json')); - Assert::true(\is_file($this->directory . '/repositories/gitlab/group/sub/project.json')); + $repo = $this->directory . '/repositories/github/roadrunner-server/roadrunner'; + Assert::true(\is_file($repo . '/index.json')); + Assert::true(\is_file($repo . '/releases-0001.json')); + Assert::true(\is_file($repo . '/releases-0002.json')); + Assert::true(\is_file($this->directory . '/repositories/gitlab/group/sub/project/index.json')); $loaded = $storage->load(new RepositoryId('github', 'roadrunner-server/roadrunner')); - Assert::same($loaded->releases()[0]->tag, 'v1'); + Assert::same($loaded->count(), 150); + Assert::same($loaded->releases()[0]->tag, 'v150'); + Assert::same($loaded->releases()[149]->tag, 'v1'); Assert::same($loaded->software, ['rr']); } + #[Test] + public function segmentsAreReadWhenReachedAndOnlyDirtyOnesAreWritten(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/repo', self::range(150, 1))); + $repo = $this->directory . '/repositories/github/owner/repo'; + + // A marker in the first segment shows whether the file is rewritten + $first = \file_get_contents($repo . '/releases-0001.json'); + \file_put_contents($repo . '/releases-0001.json', \str_replace('"v150"', '"v150" ', $first)); + + $loaded = $storage->load(new RepositoryId('github', 'owner/repo')); + Assert::same($loaded->count(), 150); + + // Appending to the tail opens a new segment after the full one and rewrites nothing else + $storage->save($loaded->withTail([new ReleaseRecord('v0', 'v0')])); + + Assert::string(\file_get_contents($repo . '/releases-0001.json'))->contains('"v150" '); + Assert::string(\file_get_contents($repo . '/releases-0003.json'))->contains('"v0"'); + Assert::same($storage->load(new RepositoryId('github', 'owner/repo'))->count(), 151); + } + + #[Test] + public function replacedSegmentFilesAreRemoved(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/repo', self::range(150, 1))); + $repo = $this->directory . '/repositories/github/owner/repo'; + + // The new release and the head segment of 50 are repacked into one file; the old one is dropped + $storage->save($storage->load(new RepositoryId('github', 'owner/repo'))->withHead([new ReleaseRecord('v151', 'v151'), new ReleaseRecord('v150', 'v150')])); + + Assert::false(\is_file($repo . '/releases-0001.json')); + Assert::true(\is_file($repo . '/releases-0002.json')); + Assert::true(\is_file($repo . '/releases-0003.json')); + Assert::same($storage->load(new RepositoryId('github', 'owner/repo'))->count(), 151); + } + #[Test] public function missingAndCorruptedRecordsReadAsNull(): void { @@ -47,11 +90,76 @@ public function missingAndCorruptedRecordsReadAsNull(): void Assert::null($storage->load($id)); $storage->save(self::record('github', 'owner/repo', ['v1'])); - \file_put_contents($this->directory . '/repositories/github/owner/repo.json', '{not json'); + \file_put_contents($this->directory . '/repositories/github/owner/repo/index.json', '{not json'); + + Assert::null($storage->load($id)); + } + + #[Test] + public function recordWithAMissingSegmentFileReadsAsNull(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + + \unlink($this->directory . '/repositories/github/owner/repo/releases-0001.json'); Assert::null($storage->load($id)); } + #[Test] + public function corruptedSegmentDropsTheRecordWhenReached(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + \file_put_contents($this->directory . '/repositories/github/owner/repo/releases-0001.json', '[{"name": "no tag"}]'); + + $loaded = $storage->load($id); + Assert::notNull($loaded); + + try { + $loaded->releases(); + Assert::fail('An unreadable segment must be reported.'); + } catch (\RuntimeException $e) { + Assert::string($e->getMessage())->contains('unreadable'); + Assert::false(\is_dir($this->directory . '/repositories/github/owner/repo')); + } + } + + #[Test] + public function saveReportsAnUnwritableDirectory(): void + { + $storage = $this->storage(); + // A file where the repository directory should be + \mkdir($this->directory . '/repositories/github', recursive: true); + \file_put_contents($this->directory . '/repositories/github/owner', 'not a directory'); + + try { + $storage->save(self::record('github', 'owner/repo', ['v1'])); + Assert::fail('A failed write must be reported.'); + } catch (\RuntimeException $e) { + Assert::string($e->getMessage())->contains('registry'); + } + } + + #[Test] + public function staleTemporaryFilesAreCleanedUp(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + $repo = $this->directory . '/repositories/github/owner/repo'; + + \file_put_contents($repo . '/index.json.123.tmp', '{}'); + \touch($repo . '/index.json.123.tmp', \time() - 7200); + \file_put_contents($repo . '/index.json.456.tmp', '{}'); + + $storage->save($storage->load(new RepositoryId('github', 'owner/repo'))->withSoftware('rr2')); + + Assert::false(\is_file($repo . '/index.json.123.tmp')); + Assert::true(\is_file($repo . '/index.json.456.tmp')); + } + #[Test] public function listsRemovesAndClears(): void { @@ -63,6 +171,8 @@ public function listsRemovesAndClears(): void $storage->remove(new RepositoryId('github', 'a/b')); Assert::count(\iterator_to_array($storage->all(), false), 1); + Assert::false(\is_dir($this->directory . '/repositories/github/a')); + Assert::true(\is_dir($this->directory . '/repositories/github/c/d')); $storage->clear(); Assert::count(\iterator_to_array($storage->all(), false), 0); @@ -78,14 +188,14 @@ public function unsafePathSegmentsAreSanitized(): void $storage->save(new RepositoryRecord($id)); Assert::false(\is_dir(\dirname($this->directory) . '/owner')); - Assert::true(\is_file($this->directory . '/repositories/github/_/owner/re_po_x.json')); + Assert::true(\is_file($this->directory . '/repositories/github/_/owner/re_po_x/index.json')); Assert::true($storage->load($id)?->id->equals($id) ?? false); } #[Test] - public function recordOfAnotherRepositoryInTheSameFileIsIgnored(): void + public function recordOfAnotherRepositoryInTheSameDirectoryIsIgnored(): void { - // Two identities sanitize to one file name + // Two identities sanitize to one directory name $storage = $this->storage(); $storage->save(self::record('github', 'owner/re?po', ['v1'])); @@ -101,7 +211,7 @@ public function windowsDeviceNamesAreEscaped(): void $storage->save(self::record('github', 'nul/com1', ['v1'])); - Assert::true(\is_file($this->directory . '/repositories/github/_nul/_com1.json')); + Assert::true(\is_file($this->directory . '/repositories/github/_nul/_com1/index.json')); Assert::same($storage->load($id)?->releases()[0]->tag, 'v1'); } @@ -120,20 +230,24 @@ protected function cleanup(): void /** * @param non-empty-string $type * @param non-empty-string $uri - * @param list $tags + * @param list $tags Newest first. */ private static function record(string $type, string $uri, array $tags): RepositoryRecord { - return new RepositoryRecord( - id: new RepositoryId($type, $uri), - checkedAt: 1_000, - software: ['rr'], - releases: \array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), - ); + return (new RepositoryRecord(id: new RepositoryId($type, $uri), checkedAt: 1_000, software: ['rr'])) + ->withHead(\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags)); + } + + /** + * @return list `v` down to `v`. + */ + private static function range(int $from, int $to): array + { + return \array_map(static fn(int $i): string => 'v' . $i, \range($from, $to)); } private function storage(): FileRegistryStorage { - return new FileRegistryStorage($this->directory, new Logger()); + return new FileRegistryStorage(Path::create($this->directory), new Logger()); } } diff --git a/tests/Unit/Module/Registry/PassThroughRegistryTest.php b/tests/Unit/Module/Registry/PassThroughRegistryTest.php new file mode 100644 index 0000000..3dc10a1 --- /dev/null +++ b/tests/Unit/Module/Registry/PassThroughRegistryTest.php @@ -0,0 +1,37 @@ +attach($id, 'rr'); + $first = \iterator_to_array($registry->releases($id, $source), false); + $registry->forget($id, 'v3'); + $second = \iterator_to_array($registry->releases($id, $source), false); + + Assert::same($source->served, [0, 2, 0, 2]); + Assert::same($first, $second); + Assert::same( + \array_map(static fn(ReleaseRecord $release): string => $release->tag, \array_merge(...$first)), + ['v3', 'v2', 'v1'], + ); + } +} diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php index 54d8f1c..4197b36 100644 --- a/tests/Unit/Module/Registry/RepositoryRecordTest.php +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Registry\Record\AssetRecord; use Internal\DLoad\Module\Registry\Record\ReleaseRecord; +use Internal\DLoad\Module\Registry\Record\ReleaseSegment; use Internal\DLoad\Module\Registry\Record\RepositoryRecord; use Internal\DLoad\Module\Registry\RepositoryId; use Testo\Assert; @@ -13,6 +14,7 @@ use Testo\Test; #[Covers(RepositoryRecord::class)] +#[Covers(ReleaseSegment::class)] #[Covers(ReleaseRecord::class)] #[Covers(AssetRecord::class)] #[Covers(RepositoryId::class)] @@ -21,10 +23,7 @@ final class RepositoryRecordTest #[Test] public function headReplacesKnownReleasesAndKeepsNewestFirst(): void { - $record = new RepositoryRecord(self::id(), releases: [ - new ReleaseRecord('v2', 'old v2'), - new ReleaseRecord('v1', 'v1'), - ]); + $record = self::record(['v2' => 'old v2', 'v1' => 'v1']); $updated = $record->withHead([new ReleaseRecord('v3', 'v3'), new ReleaseRecord('v2', 'new v2')]); @@ -36,14 +35,9 @@ public function headReplacesKnownReleasesAndKeepsNewestFirst(): void public function headDropsStoredReleasesMissingFromTheFetchedSpan(): void { // `v3` was deleted upstream: the fresh head reaches `v2`, and `v3` is not in it - $record = new RepositoryRecord(self::id(), releases: [ - new ReleaseRecord('v4', 'v4'), - new ReleaseRecord('v3', 'v3'), - new ReleaseRecord('v2', 'v2'), - new ReleaseRecord('v1', 'v1'), - ]); + $record = self::record(['v4', 'v3', 'v2', 'v1']); - $updated = $record->withHead([new ReleaseRecord('v5', 'v5'), new ReleaseRecord('v4', 'v4'), new ReleaseRecord('v2', 'v2')]); + $updated = $record->withHead(self::releases(['v5', 'v4', 'v2'])); Assert::same(self::tags($updated), ['v5', 'v4', 'v2', 'v1']); Assert::same($updated->count(), 4); @@ -53,9 +47,9 @@ public function headDropsStoredReleasesMissingFromTheFetchedSpan(): void public function headDropsTheStoredReleaseRightAfterTheOnlyOneItReaches(): void { // `v5` was deleted: the fresh page reads `v6, v4`, so `v5` no longer follows `v6` - $record = new RepositoryRecord(self::id(), releases: [new ReleaseRecord('v6', 'v6'), new ReleaseRecord('v5', 'v5')]); + $record = self::record(['v6', 'v5']); - $updated = $record->withHead([new ReleaseRecord('v6', 'v6'), new ReleaseRecord('v4', 'v4')]); + $updated = $record->withHead(self::releases(['v6', 'v4'])); Assert::same(self::tags($updated), ['v6', 'v4']); } @@ -63,46 +57,108 @@ public function headDropsTheStoredReleaseRightAfterTheOnlyOneItReaches(): void #[Test] public function headReachingNoStoredReleaseIsTheWholeListing(): void { - $record = new RepositoryRecord(self::id(), releases: [new ReleaseRecord('v1', 'v1')]); - - $updated = $record->withHead([new ReleaseRecord('v2', 'v2')]); + $updated = self::record(['v1'])->withHead(self::releases(['v2'])); Assert::same(self::tags($updated), ['v2']); } #[Test] - public function repositoryIdIsNormalized(): void + public function headKeepsUntouchedSegmentsAsTheyAre(): void { - $id = new RepositoryId('GitHub', '/Owner/Repo/'); + // Three full segments; the head reaches the first one only + $record = self::record(self::range(300, 1)); + $untouched = \array_slice($record->segments, 1); + + $updated = $record->withHead(self::releases(['v301', 'v300', 'v299'])); + + Assert::same($updated->count(), 301); + Assert::same(self::sizes($updated), [1, 100, 100, 100]); + Assert::same(\array_slice($updated->segments, -2), $untouched); + Assert::false($updated->segments[3]->dirty); + Assert::true($updated->segments[0]->dirty); + Assert::same($updated->segments[0]->tags, ['v301']); + } - Assert::same((string) $id, 'github:owner/repo'); - Assert::true($id->equals(new RepositoryId('github', 'owner/repo'))); + #[Test] + public function partialSegmentSitsAtTheHeadAndGrowsWithEveryCheck(): void + { + $record = self::record(self::range(150, 1)); + Assert::same(self::sizes($record), [50, 100]); + + // New releases join the head segment; the full one behind it is untouched + $updated = $record->withHead(self::releases(['v152', 'v151', 'v150'])); + Assert::same(self::sizes($updated), [52, 100]); + Assert::false($updated->segments[1]->dirty); + Assert::same(\array_slice(self::tags($updated), 0, 4), ['v152', 'v151', 'v150', 'v149']); + } - try { - new RepositoryId('github', '/'); - Assert::fail('A URI without a path must be rejected.'); - } catch (\InvalidArgumentException) { - } + #[Test] + public function shortHeadJoinsTheNextSegmentWhileBothFitIntoOne(): void + { + $record = self::record(self::range(101, 1)); + Assert::same(self::sizes($record), [1, 100]); + + // `v101` opens its segment, so nothing is split; the single new release joins it + $updated = $record->withHead(self::releases(['v102', 'v101'])); + + Assert::same(self::sizes($updated), [2, 100]); + Assert::false($updated->segments[1]->dirty); + Assert::same($updated->segments[0]->tags, ['v102', 'v101']); + } + + #[Test] + public function tailFillsTheLastSegmentBeforeStartingANewOne(): void + { + $record = self::record(self::range(150, 1)); + $older = \array_map(static fn(int $i): string => 'o' . $i, \range(1, 70)); + + // The last segment is full: the older releases open a new one + $extended = $record->withTail(self::releases(\array_slice($older, 0, 62)))->persisted(); + Assert::same(self::sizes($extended), [50, 100, 62]); + Assert::same(self::keys($extended), ['0001', '0002', '0003']); + + // The next page fills that segment up under the same key and starts another + $more = $extended->withTail(self::releases(\array_slice($older, 62)))->withTail(self::releases(self::range(0, -35))); + Assert::same(self::sizes($more), [50, 100, 100, 6]); + Assert::same(self::keys($more), ['0001', '0002', '0003', '0004']); + Assert::false($more->segments[1]->dirty); + Assert::true($more->segments[2]->dirty); + Assert::same($more->segments[2]->tags[99], 'v-29'); } #[Test] public function tailIgnoresKnownReleases(): void { - $record = new RepositoryRecord(self::id(), releases: [new ReleaseRecord('v2', 'v2')]); + $record = self::record(['v2']); $updated = $record->withTail([new ReleaseRecord('v2', 'dup'), new ReleaseRecord('v1', 'v1')]); Assert::same(self::tags($updated), ['v2', 'v1']); Assert::same($updated->releases()[0]->name, 'v2'); + Assert::same($record->withTail([new ReleaseRecord('v2', 'dup')]), $record); + } + + #[Test] + public function segmentKeysAreNeverReused(): void + { + $record = self::record(self::range(200, 1)); + Assert::same(self::keys($record), ['0001', '0002']); + + // The head repacks the first segment under fresh keys; the old key is gone + // `v200` was deleted: the first segment is repacked under a fresh key, the old key is gone + $updated = $record->withHead(self::releases(['v201', 'v199'])); + Assert::same(self::keys($updated), ['0003', '0002']); + Assert::same(self::sizes($updated), [100, 100]); + + // The tail appends after the greatest key ever used + $extended = $updated->withTail(self::releases(['v0'])); + Assert::same(self::keys($extended), ['0003', '0002', '0004']); } #[Test] public function releaseCanBeDroppedAndTheCheckForgotten(): void { - $record = new RepositoryRecord(self::id(), checkedAt: 1_000, releases: [ - new ReleaseRecord('v2', 'v2'), - new ReleaseRecord('v1', 'v1'), - ]); + $record = (new RepositoryRecord(self::id(), checkedAt: 1_000))->withHead(self::releases(['v2', 'v1'])); $dropped = $record->withoutRelease('v2'); @@ -116,6 +172,19 @@ public function releaseCanBeDroppedAndTheCheckForgotten(): void Assert::true($stale->isStale(1_000, 600)); } + #[Test] + public function droppingTheLastReleaseOfASegmentDropsTheSegment(): void + { + $record = self::record(self::range(101, 1)); + Assert::same(self::sizes($record), [1, 100]); + + $updated = $record->withoutRelease('v101'); + + Assert::same(self::sizes($updated), [100]); + Assert::false($updated->has('v101')); + Assert::false($updated->segments[0]->dirty); + } + #[Test] public function stalenessDependsOnTheLastCheck(): void { @@ -139,12 +208,8 @@ public function softwareIsAttachedOnce(): void #[Test] public function survivesTheArrayRoundTrip(): void { - $record = new RepositoryRecord( - id: self::id(), - checkedAt: 1_000, - complete: true, - software: ['rr'], - releases: [ + $record = (new RepositoryRecord(id: self::id(), checkedAt: 1_000, complete: true, software: ['rr'])) + ->withHead([ new ReleaseRecord( tag: 'v2.0.0', name: 'Release 2', @@ -153,25 +218,82 @@ public function survivesTheArrayRoundTrip(): void assets: [new AssetRecord('rr-linux-amd64.tar.gz', 'https://x/rr.tar.gz', 42, 'application/gzip', 'sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08')], ), new ReleaseRecord('v1.0.0', 'v1.0.0'), - ], + ]); + + // The index travels as JSON; the segments are handed back through the loader + $segments = []; + foreach ($record->segments as $segment) { + $segments[$segment->key] = \json_decode(\json_encode(\array_map( + static fn(ReleaseRecord $release): array => $release->toArray(), + $segment->releases(), + )), true); + } + $restored = RepositoryRecord::fromArray( + \json_decode(\json_encode($record->toArray()), true), + static fn(string $key): array => \array_map(ReleaseRecord::fromArray(...), $segments[$key]), ); - $restored = RepositoryRecord::fromArray(\json_decode(\json_encode($record->toArray()), true)); - Assert::same($restored->toArray(), $record->toArray()); Assert::true($restored->id->equals(self::id())); + Assert::same($restored->count(), 2); + Assert::false($restored->segments[0]->dirty); Assert::same($restored->releases()[0]->assets[0]->size, 42); Assert::same($restored->releases()[0]->assets[0]->digest, 'sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'); - Assert::null(AssetRecord::fromArray(['name' => 'x', 'uri' => 'https://x', 'digest' => ''])->digest); Assert::same($restored->releases()[0]->publishedAt?->format(\DATE_ATOM), '2024-01-02T03:04:05+00:00'); Assert::null($restored->releases()[1]->publishedAt); + Assert::null(AssetRecord::fromArray(['name' => 'x', 'uri' => 'https://x', 'digest' => ''])->digest); + } + + #[Test] + public function segmentsAreReadOnlyWhenReached(): void + { + $loaded = []; + $old = self::range(100, 1); + $record = RepositoryRecord::fromArray( + [ + 'version' => RepositoryRecord::FORMAT_VERSION, + 'repository' => ['type' => 'github', 'uri' => 'owner/repo'], + 'segments' => [['key' => 'a', 'tags' => ['v102', 'v101']], ['key' => 'b', 'tags' => $old]], + ], + static function (string $key) use (&$loaded, $old): array { + $loaded[] = $key; + + return self::releases($key === 'a' ? ['v102', 'v101'] : $old); + }, + ); + + Assert::same($record->count(), 102); + Assert::true($record->has('v1')); + Assert::same($loaded, []); + + $record->pages()->current(); + Assert::same($loaded, ['a']); + + // The head touches the first segment only; the full one behind it is neither read nor rewritten + $updated = $record->withHead(self::releases(['v103', 'v102'])); + Assert::same($loaded, ['a']); + Assert::same(self::sizes($updated), [3, 100]); + Assert::false($updated->segments[1]->dirty); + } + + #[Test] + public function persistedRecordHasNoDirtySegments(): void + { + $record = RepositoryRecord::empty(self::id())->withHead(self::releases(['v1'])); + Assert::true($record->segments[0]->dirty); + + $persisted = $record->persisted(); + + Assert::false($persisted->segments[0]->dirty); + Assert::same(self::tags($persisted), ['v1']); + Assert::same($persisted->persisted()->segments, $persisted->segments); } #[Test] public function rejectsUnknownFormatVersion(): void { try { - RepositoryRecord::fromArray(['version' => 99, 'repository' => ['type' => 'github', 'uri' => 'a/b']]); + RepositoryRecord::fromArray(['version' => 99, 'repository' => ['type' => 'github', 'uri' => 'a/b']], static fn(): array => []); Assert::fail('An unknown format version must be rejected.'); } catch (\InvalidArgumentException $e) { Assert::same($e->getMessage(), 'Unsupported repository record format.'); @@ -200,11 +322,58 @@ public function skipsBrokenReleasesButRequiresATag(): void Assert::count($release->assets, 1); } + #[Test] + public function repositoryIdIsNormalized(): void + { + $id = new RepositoryId('GitHub', '/Owner/Repo/'); + + Assert::same((string) $id, 'github:owner/repo'); + Assert::true($id->equals(new RepositoryId('github', 'owner/repo'))); + + try { + new RepositoryId('github', '/'); + Assert::fail('A URI without a path must be rejected.'); + } catch (\InvalidArgumentException) { + } + } + private static function id(): RepositoryId { return new RepositoryId('github', 'owner/repo'); } + /** + * Record holding the given releases as freshly packed segments. + * + * @param array $tags Newest first; a string key is the tag and the value its name. + */ + private static function record(array $tags): RepositoryRecord + { + $releases = []; + foreach ($tags as $key => $value) { + $releases[] = \is_string($key) ? new ReleaseRecord($key, $value) : new ReleaseRecord($value, $value); + } + + return RepositoryRecord::empty(self::id())->withHead($releases)->persisted(); + } + + /** + * @param list $tags + * @return list + */ + private static function releases(array $tags): array + { + return \array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags); + } + + /** + * @return list `v` down to `v`. + */ + private static function range(int $from, int $to): array + { + return \array_map(static fn(int $i): string => 'v' . $i, \range($from, $to)); + } + /** * @return list */ @@ -212,4 +381,20 @@ private static function tags(RepositoryRecord $record): array { return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $record->releases()); } + + /** + * @return list + */ + private static function sizes(RepositoryRecord $record): array + { + return \array_map(static fn(ReleaseSegment $segment): int => $segment->count(), $record->segments); + } + + /** + * @return list + */ + private static function keys(RepositoryRecord $record): array + { + return \array_map(static fn(ReleaseSegment $segment): string => $segment->key, $record->segments); + } } diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php index b92967c..c2f6d54 100644 --- a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -12,6 +12,7 @@ use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Registry\Stub\ArrayReleaseSource; use Internal\DLoad\Tests\Unit\Module\Registry\Stub\InMemoryRegistryStorage; +use Symfony\Component\Console\Output\BufferedOutput; use Testo\Assert; use Testo\Codecov\Covers; use Testo\Lifecycle\BeforeTest; @@ -143,15 +144,37 @@ public function failedTailLoadingStillServesTheStoredReleases(): void } #[Test] - public function rateLimitOnCheckFallsBackToStoredReleases(): void + public function rateLimitOnCheckFallsBackToStoredReleasesAndIsReportedOnce(): void { + $other = new RepositoryId('github', 'other/repo'); $source = ArrayReleaseSource::ofTags(['v2', 'v1']); self::flatten($this->registry()->releases($this->id, $source)); + self::flatten($this->registry()->releases($other, $source)); $source->fail(new GitHubRateLimitException('Rate limit exceeded.', 'owner/repo', null)); $this->now += 601; + $output = new BufferedOutput(); + $registry = $this->registry(logger: new Logger($output)); - Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); + Assert::same(self::flatten($registry->releases($this->id, $source)), ['v2', 'v1']); + Assert::same(self::flatten($registry->releases($other, $source)), ['v2', 'v1']); + + // Shown without any verbosity flag, but not for every repository of the run + Assert::same(\substr_count($output->fetch(), 'rate limit prevents checking'), 1); + } + + #[Test] + public function ordinaryCheckFailureStaysOutOfTheDefaultOutput(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + self::flatten($this->registry()->releases($this->id, $source)); + + $source->fail(); + $this->now += 601; + $output = new BufferedOutput(); + self::flatten($this->registry(logger: new Logger($output))->releases($this->id, $source)); + + Assert::same($output->fetch(), ''); } #[Test] @@ -260,9 +283,9 @@ public function attachRecordsTheSoftwareOnce(): void { $registry = $this->registry(); - $registry->attach('rr', $this->id); - $registry->attach('rr', $this->id); - $registry->attach('roadrunner', $this->id); + $registry->attach($this->id, 'rr'); + $registry->attach($this->id, 'rr'); + $registry->attach($this->id, 'roadrunner'); Assert::same($this->storage->load($this->id)->software, ['rr', 'roadrunner']); Assert::same($this->storage->saves, 2); @@ -299,8 +322,8 @@ private static function tagsOf(array $releases): array return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $releases); } - private function registry(bool $refresh = false): StoredVersionRegistry + private function registry(bool $refresh = false, ?Logger $logger = null): StoredVersionRegistry { - return new StoredVersionRegistry($this->storage, 600, new Logger(), $refresh, fn(): int => $this->now); + return new StoredVersionRegistry($this->storage, 600, $logger ?? new Logger(), $refresh, fn(): int => $this->now); } } diff --git a/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php index 6a4a76b..252ae9d 100644 --- a/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php +++ b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php @@ -32,7 +32,7 @@ public function save(RepositoryRecord $record): void $this->failOnSave and throw new \RuntimeException('Storage is read-only.'); ++$this->saves; - $this->records[(string) $record->id] = $record; + $this->records[(string) $record->id] = $record->persisted(); } public function all(): iterable diff --git a/tests/Unit/Module/Registry/Stub/RecordingRegistry.php b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php index 95a7603..158028f 100644 --- a/tests/Unit/Module/Registry/Stub/RecordingRegistry.php +++ b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php @@ -27,7 +27,7 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator } } - public function attach(string $software, RepositoryId $id): void + public function attach(RepositoryId $id, string $software): void { $this->attached[] = [$software, (string) $id]; } diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php index 0e41a81..6072e52 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -120,6 +120,18 @@ public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void Assert::same(\count($all), 300); } + #[Test] + public function draftReleasesAreNeitherServedNorStored(): void + { + $storage = new InMemoryRegistryStorage(); + + $names = self::names(self::createRepository(new PagedClientStub(pages: 1, drafts: 2), self::registry($storage))); + + Assert::same(\count($names), 100); + Assert::false(\in_array('draft-1', $names, true)); + Assert::false($storage->load(new RepositoryId(GitHubRepository::TYPE, 'owner/repo'))?->has('draft-1') ?? true); + } + #[Test] public function assetDigestReportedByTheApiIsStored(): void { @@ -136,14 +148,10 @@ public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): { // A fresh record holds the first 50 releases: the tail starts in the middle of API page 1 $storage = new InMemoryRegistryStorage(); - $storage->save(new RepositoryRecord( - id: new RepositoryId(GitHubRepository::TYPE, 'owner/repo'), - checkedAt: \time(), - releases: \array_map( - static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), - \range(1, 50), - ), - )); + $storage->save((new RepositoryRecord(new RepositoryId(GitHubRepository::TYPE, 'owner/repo'), checkedAt: \time()))->withHead(\array_map( + static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), + \range(1, 50), + ))); $client = new PagedClientStub(pages: 2); $all = self::names(self::createRepository($client, self::registry($storage))); diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php index 55982e6..d3d47df 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -28,10 +28,12 @@ final class PagedClientStub implements ClientInterface /** * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. * @param int<1, max> $releasesPerPage Number of releases on every such page. + * @param int<0, max> $drafts Draft releases listed on top, as the API shows them to the token holder. */ public function __construct( private readonly int $pages = 1, private readonly int $releasesPerPage = 100, + private readonly int $drafts = 0, ) {} public function sendRequest(RequestInterface $request): ResponseInterface @@ -88,8 +90,8 @@ private function allReleases(): array $releases = []; $total = $this->pages * $this->releasesPerPage; - for ($i = 1; $i <= $total; $i++) { - $tag = \sprintf('v1.0.%d', $i); + for ($i = 1 - $this->drafts; $i <= $total; $i++) { + $tag = $i < 1 ? \sprintf('draft-%d', 1 - $i) : \sprintf('v1.0.%d', $i); $releases[] = [ 'name' => $tag, 'tag_name' => $tag, @@ -102,7 +104,7 @@ private function allReleases(): array 'digest' => 'sha256:' . \hash('sha256', $tag), ]], 'prerelease' => false, - 'draft' => false, + 'draft' => $i < 1, ]; } diff --git a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php index 9f036fe..0ee8d82 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php @@ -119,14 +119,10 @@ public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): void { $storage = new InMemoryRegistryStorage(); - $storage->save(new RepositoryRecord( - id: new RepositoryId(GitLabRepository::TYPE, 'group/project'), - checkedAt: \time(), - releases: \array_map( - static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), - \range(1, 50), - ), - )); + $storage->save((new RepositoryRecord(new RepositoryId(GitLabRepository::TYPE, 'group/project'), checkedAt: \time()))->withHead(\array_map( + static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), + \range(1, 50), + ))); $client = new PagedClientStub(pages: 2); $all = self::names(self::createRepository($client, self::registry($storage))); From 90ffe6ee9f7c7a4c9ebdd05232bebea00a158a1e Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Mon, 14 Sep 2026 20:25:24 +0400 Subject: [PATCH 09/17] fix(config): let the environment override `dload.xml` and the command line override both The inflector took the first attribute in declaration order, so `cache-ttl` in a committed `dload.xml` silenced `DLOAD_CACHE_TTL=0` in CI. Sources are now consulted by a fixed rank: command line, environment, php.ini, then the configuration file. Assisted-By: Claude Fable 5.1 --- README-es.md | 2 +- README-ru.md | 2 +- README-zh.md | 2 +- README.md | 2 +- dload.xsd | 4 +-- .../Internal/Injection/ConfigInflector.php | 28 +++++++++++++++++-- .../Registry/VersionRegistryBindingTest.php | 14 ++++++++++ 7 files changed, 45 insertions(+), 9 deletions(-) diff --git a/README-es.md b/README-es.md index 46f2fb0..8075fb5 100644 --- a/README-es.md +++ b/README-es.md @@ -378,7 +378,7 @@ El registro está activado por defecto y vive en el directorio de caché del usu | `cache-dir` | `DLOAD_CACHE_DIR` | directorio de caché del usuario | Directorio del registro de versiones. | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Segundos que sigue siendo válida la última comprobación. `0` desactiva el registro. | -Un atributo definido en `dload.xml` tiene prioridad sobre la variable de entorno. +La variable de entorno tiene prioridad sobre el atributo de `dload.xml`, y la opción de línea de comandos sobre ambos. ```bash # Comprobar si hay nuevos releases aunque la última comprobación siga vigente diff --git a/README-ru.md b/README-ru.md index 3ce56f0..106e430 100644 --- a/README-ru.md +++ b/README-ru.md @@ -379,7 +379,7 @@ DLoad поддерживает три типа загрузки, которые | `cache-dir` | `DLOAD_CACHE_DIR` | каталог кэша пользователя | Каталог реестра версий. | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Сколько секунд действует последняя проверка репозитория. `0` отключает реестр. | -Атрибут в `dload.xml` имеет приоритет над переменной окружения. +Переменная окружения имеет приоритет над атрибутом в `dload.xml`, а опция командной строки над обоими. ```bash # Проверить репозитории на новые релизы, даже если последняя проверка ещё свежая diff --git a/README-zh.md b/README-zh.md index 5beae95..5a7ef98 100644 --- a/README-zh.md +++ b/README-zh.md @@ -375,7 +375,7 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： | `cache-dir` | `DLOAD_CACHE_DIR` | 用户缓存目录 | 版本注册表所在目录。 | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | 最近一次检查保持有效的秒数。`0` 表示禁用注册表。 | -`dload.xml` 中的属性优先于环境变量。 +环境变量优先于 `dload.xml` 中的属性，命令行选项优先于两者。 ```bash # 即使最近一次检查仍然有效，也强制检查仓库是否有新发布 diff --git a/README.md b/README.md index 8f35d97..4763594 100644 --- a/README.md +++ b/README.md @@ -380,7 +380,7 @@ The registry is on by default and lives in the per-user cache directory (`$XDG_C | `cache-dir` | `DLOAD_CACHE_DIR` | user cache directory | Directory of the version registry. | | `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Seconds the last check of a repository stays valid. `0` disables the registry. | -An attribute set in `dload.xml` takes precedence over the environment variable. +The environment variable takes precedence over the attribute in `dload.xml`, and the command line option over both. ```bash # Check the repositories for new releases even if the last check is still fresh diff --git a/dload.xsd b/dload.xsd index a6e146a..b737373 100644 --- a/dload.xsd +++ b/dload.xsd @@ -251,12 +251,12 @@ - Directory of the version registry (release lists database); the per-user cache directory when not set. Takes precedence over DLOAD_CACHE_DIR + Directory of the version registry (release lists database); the per-user cache directory when not set. The DLOAD_CACHE_DIR environment variable takes precedence - Number of seconds the last check of a repository for new releases stays valid; 0 disables the version registry. Takes precedence over DLOAD_CACHE_TTL + Number of seconds the last check of a repository for new releases stays valid; 0 disables the version registry. The DLOAD_CACHE_TTL environment variable takes precedence diff --git a/src/Module/Common/Internal/Injection/ConfigInflector.php b/src/Module/Common/Internal/Injection/ConfigInflector.php index 036c5ee..0f02f03 100644 --- a/src/Module/Common/Internal/Injection/ConfigInflector.php +++ b/src/Module/Common/Internal/Injection/ConfigInflector.php @@ -70,9 +70,27 @@ public function inflect(object $object, Container $container): object return $object; } + /** + * Rank of a configuration source; a lower value is consulted first. + */ + private static function priority(ConfigAttribute $attribute): int + { + return match (true) { + $attribute instanceof InputArgument, $attribute instanceof InputOption => 0, + $attribute instanceof Env => 1, + $attribute instanceof PhpIni => 2, + default => 3, + }; + } + /** * Injects values into a property based on its configuration attributes. * + * The first source that has a value wins. Sources are consulted in a fixed order, whatever + * the order of the attributes on the property: what was typed on the command line beats the + * environment, and the environment beats the configuration file, so a CI job can override a + * committed `dload.xml` without editing it. + * * @param list<\ReflectionAttribute> $attributes */ private function injectValue( @@ -81,10 +99,14 @@ private function injectValue( \ReflectionProperty $property, array $attributes, ): void { - foreach ($attributes as $attribute) { - try { - $attribute = $attribute->newInstance(); + $instances = \array_map( + static fn(\ReflectionAttribute $attribute): ConfigAttribute => $attribute->newInstance(), + $attributes, + ); + \usort($instances, static fn(ConfigAttribute $a, ConfigAttribute $b): int => self::priority($a) <=> self::priority($b)); + foreach ($instances as $attribute) { + try { /** @var mixed $value */ $value = match (true) { $attribute instanceof XPath => $this->getXPath($attribute), diff --git a/tests/Integration/Module/Registry/VersionRegistryBindingTest.php b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php index 1b8a4ba..9726c3f 100644 --- a/tests/Integration/Module/Registry/VersionRegistryBindingTest.php +++ b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php @@ -60,6 +60,20 @@ public function xmlAttributeSetsTheDirectory(): void Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); } + #[Test] + public function environmentOverridesTheXmlAttribute(): void + { + $container = self::bootstrap( + xml: '', + environment: ['DLOAD_CACHE_DIR' => $this->directory, 'DLOAD_CACHE_TTL' => '0'], + ); + + Assert::instanceOf($container->get(VersionRegistry::class), PassThroughRegistry::class); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); + } + #[Test] public function zeroTtlDisablesTheRegistry(): void { From 752d3ad9ba49a19595ab51abf4712b4f687ef297 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Tue, 15 Sep 2026 10:32:42 +0400 Subject: [PATCH 10/17] fix(registry): key the registry by the path the repository reports, not the configured URI Both factories accept a full URL and reduce it to the path, so the release records were stored under the reduced path while `attach()` and `forget()` used the raw URI: `cache:clear ` missed the record and a deleted release was never forgotten. The identifier is now built from the repository name inside the downloader; `RepositoryId::fromConfig()` is gone so the trap cannot return. Assisted-By: Claude Fable 5.1 --- psalm-baseline.xml | 1 + src/Module/Downloader/Downloader.php | 9 ++++++--- .../Downloader/Internal/DownloadContext.php | 4 ++++ src/Module/Registry/RepositoryId.php | 16 +++++---------- .../Unit/Module/Downloader/DownloaderTest.php | 20 +++++++++++++++++-- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/psalm-baseline.xml b/psalm-baseline.xml index a80494d..c884d1c 100644 --- a/psalm-baseline.xml +++ b/psalm-baseline.xml @@ -174,6 +174,7 @@ + diff --git a/src/Module/Downloader/Downloader.php b/src/Module/Downloader/Downloader.php index 7d84e80..b009427 100644 --- a/src/Module/Downloader/Downloader.php +++ b/src/Module/Downloader/Downloader.php @@ -116,8 +116,11 @@ public function download( $context->repoConfig = \array_shift($repositories); $repository = $this->repositoryProvider->getByConfig($context->repoConfig); - // The registry keeps track of which software is served from which repository - $this->registry->attach(RepositoryId::fromConfig($context->repoConfig), $context->software->getId()); + // The registry keeps track of which software is served from which repository. The + // identity comes from the repository, not the config: the factory may have reduced + // a full URL to the path the repository stores its releases under. + $context->repositoryId = new RepositoryId($context->repoConfig->type, $repository->getName()); + $this->registry->attach($context->repositoryId, $context->software->getId()); $context->repositoryAttempt = $context->diagnostics->addRepository( type: $context->repoConfig->type, name: $repository->getName(), @@ -232,7 +235,7 @@ private function processRepository(Repository $repository, DownloadContext $cont } catch (ReleaseGone $e) { // The registry must not offer this release again, and the list needs a fresh check $tag = $context->release->getVersion()->string; - $tag === '' or $this->registry->forget(RepositoryId::fromConfig($context->repoConfig), $tag); + $tag === '' or $this->registry->forget($context->repositoryId, $tag); $forgotten = true; $context->releaseAttempt->reason ??= $e->getMessage(); diff --git a/src/Module/Downloader/Internal/DownloadContext.php b/src/Module/Downloader/Internal/DownloadContext.php index 9db0fc3..e224dcc 100644 --- a/src/Module/Downloader/Internal/DownloadContext.php +++ b/src/Module/Downloader/Internal/DownloadContext.php @@ -10,6 +10,7 @@ use Internal\DLoad\Module\Downloader\Internal\Diagnostics\DownloadDiagnostics; use Internal\DLoad\Module\Downloader\Internal\Diagnostics\ReleaseAttempt; use Internal\DLoad\Module\Downloader\Internal\Diagnostics\RepositoryAttempt; +use Internal\DLoad\Module\Registry\RepositoryId; use Internal\DLoad\Module\Repository\AssetInterface; use Internal\DLoad\Module\Repository\ReleaseInterface; use Internal\DLoad\Module\Task\Progress; @@ -37,6 +38,9 @@ final class DownloadContext /** @var ReleaseInterface Current release being processed */ public ReleaseInterface $release; + /** @var RepositoryId Identity of the current repository in the version registry */ + public RepositoryId $repositoryId; + /** @var RepositoryAttempt Diagnostics of the repository being processed */ public RepositoryAttempt $repositoryAttempt; diff --git a/src/Module/Registry/RepositoryId.php b/src/Module/Registry/RepositoryId.php index 1edb936..f6e410b 100644 --- a/src/Module/Registry/RepositoryId.php +++ b/src/Module/Registry/RepositoryId.php @@ -4,18 +4,17 @@ namespace Internal\DLoad\Module\Registry; -use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; - /** * Identity of a software repository in the version registry. * * The same repository may be referenced by several software packages and by several configs, - * so the registry keys its records by the repository type and URI rather than by software name. - * GitHub and GitLab resolve paths case-insensitively, so the identity is normalized: lower case, - * no surrounding slashes. + * so the registry keys its records by the repository type and path rather than by software name. + * The path is the one the repository reports, not the configured URI: a factory may accept a full + * URL and reduce it. GitHub and GitLab resolve paths case-insensitively, so the identity is + * normalized: lower case, no surrounding slashes. * * ```php - * $id = RepositoryId::fromConfig($repositoryConfig); + * $id = new RepositoryId('github', $repository->getName()); * echo $id; // github:roadrunner-server/roadrunner * ``` * @@ -43,11 +42,6 @@ public function __construct(string $type, string $uri) $this->uri = $normalized; } - public static function fromConfig(RepositoryConfig $config): self - { - return new self($config->type, $config->uri); - } - public function equals(self $other): bool { return $this->type === $other->type && $this->uri === $other->uri; diff --git a/tests/Unit/Module/Downloader/DownloaderTest.php b/tests/Unit/Module/Downloader/DownloaderTest.php index 17e7d93..1d9e00a 100644 --- a/tests/Unit/Module/Downloader/DownloaderTest.php +++ b/tests/Unit/Module/Downloader/DownloaderTest.php @@ -60,6 +60,21 @@ public function deletedReleaseIsForgottenAndTheNextOneIsUsed(): void Assert::same($this->registry->attached, [['rr', 'github:owner/repo']]); } + #[Test] + public function registryIdComesFromTheRepositoryNotTheConfiguredUri(): void + { + // The factory reduces a full URL to the path; the registry must key the record the same way + $repository = new RepositoryStub('owner/repo'); + $gone = self::release($repository, 'v2.0.0', assets: false); + $alive = self::release($repository, 'v1.9.0', assets: true); + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create([$gone, $alive])); + + $this->download([$repository], uri: 'https://github.com/Owner/Repo'); + + Assert::same($this->registry->attached, [['rr', 'github:owner/repo']]); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + } + #[Test] public function outdatedListIsFetchedAgainWhenNothingIsLeft(): void { @@ -171,8 +186,9 @@ private static function release(RepositoryStub $repository, string $tag, bool $a /** * @param list|SequenceRepositoryFactoryStub $repositories + * @param non-empty-string $uri Repository URI as written in the config. */ - private function download(array|SequenceRepositoryFactoryStub $repositories): DownloadResult + private function download(array|SequenceRepositoryFactoryStub $repositories, string $uri = 'owner/repo'): DownloadResult { $factory = $repositories instanceof SequenceRepositoryFactoryStub ? $repositories @@ -194,7 +210,7 @@ private function download(array|SequenceRepositoryFactoryStub $repositories): Do $software = Software::fromArray([ 'name' => 'rr', - 'repositories' => [['type' => 'github', 'uri' => 'owner/repo']], + 'repositories' => [['type' => 'github', 'uri' => $uri]], ]); $task = $downloader->download($software, DownloadConfig::fromSoftwareId('rr'), static fn(): null => null); From 70c433ccadfd85e7cd431f5f9ff4914808627463 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Tue, 15 Sep 2026 10:37:57 +0400 Subject: [PATCH 11/17] fix(registry): keep the stored list when the source cannot read a release The API adapters skip a release they cannot decode, so the fetched head lacked it exactly like a deleted one and the check dropped it from the record until the next TTL expiry. The page now carries the number of skipped releases; a check that meets one leaves the stored list as it is and is repeated on the next run. Without a stored list the readable releases are still served, as before. Assisted-By: Claude Fable 5.1 --- .../Internal/StoredVersionRegistry.php | 14 ++++++++ src/Module/Registry/Record/ReleasePage.php | 3 ++ .../Internal/GitHub/Api/RepositoryApi.php | 4 ++- .../Internal/GitHub/GitHubReleaseSource.php | 2 +- .../Internal/GitLab/Api/RepositoryApi.php | 4 ++- .../Internal/GitLab/GitLabReleaseSource.php | 2 +- .../Registry/StoredVersionRegistryTest.php | 32 +++++++++++++++++++ .../Registry/Stub/ArrayReleaseSource.php | 13 +++++++- .../Internal/GitHub/GitHubRepositoryTest.php | 19 +++++++++-- .../Internal/GitHub/Stub/PagedClientStub.php | 5 ++- 10 files changed, 89 insertions(+), 9 deletions(-) diff --git a/src/Module/Registry/Internal/StoredVersionRegistry.php b/src/Module/Registry/Internal/StoredVersionRegistry.php index 356689d..c9bbaf1 100644 --- a/src/Module/Registry/Internal/StoredVersionRegistry.php +++ b/src/Module/Registry/Internal/StoredVersionRegistry.php @@ -123,6 +123,20 @@ private function check(RepositoryRecord $record, ReleaseSource $source): Reposit $complete = $record->complete; foreach ($source->pages() as $page) { + // A release the source could not read is missing from the page exactly like a + // deleted one, and `withHead()` would drop it; the stored list stays untouched + // and the next run checks again + if ($page->skipped > 0 && $record->count() > 0) { + $this->logger->debug( + 'The listing of `%s` has %d unreadable release(s); the stored %d are kept as they are.', + (string) $record->id, + $page->skipped, + $record->count(), + ); + + return $record; + } + $fetched = [...$fetched, ...$page->releases]; // The listing ended during the check: everything is known now diff --git a/src/Module/Registry/Record/ReleasePage.php b/src/Module/Registry/Record/ReleasePage.php index 93f25b1..4a19216 100644 --- a/src/Module/Registry/Record/ReleasePage.php +++ b/src/Module/Registry/Record/ReleasePage.php @@ -17,9 +17,12 @@ final class ReleasePage /** * @param list $releases Releases of the page, newest first. * @param bool $last Whether the listing has no page after this one. + * @param int<0, max> $skipped Number of releases of the page the source could not read. They + * are absent from `$releases` although they exist upstream. */ public function __construct( public readonly array $releases, public readonly bool $last, + public readonly int $skipped = 0, ) {} } diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index baf4b89..426d967 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -124,6 +124,7 @@ public function releasePages(int $page = 1): \Generator $releases = []; $failure = null; + $skipped = 0; foreach ($data as $releaseData) { try { $release = ReleaseInfo::fromApiResponse($releaseData); @@ -131,6 +132,7 @@ public function releasePages(int $page = 1): \Generator $release->draft or $releases[] = $release->toRecord(); } catch (\Throwable $e) { $failure ??= $e; + ++$skipped; $this->logger->exception($e, important: false); // Skip invalid releases continue; @@ -153,7 +155,7 @@ public function releasePages(int $page = 1): \Generator $hasMorePages = $this->hasNextPage($response); - yield new ReleasePage($releases, !$hasMorePages); + yield new ReleasePage($releases, !$hasMorePages, $skipped); $currentPage++; } while ($hasMorePages); diff --git a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php index b6ab6c5..6d334c3 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php +++ b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php @@ -29,7 +29,7 @@ public function pages(int $offset = 0): \Generator $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; foreach ($this->api->releasePages($first) as $page) { - yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last, $page->skipped); $skip = 0; } } diff --git a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php index 8f28209..5dec664 100644 --- a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php @@ -135,11 +135,13 @@ public function releasePages(int $page = 1): \Generator $releases = []; $failure = null; + $skipped = 0; foreach ($data as $releaseData) { try { $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); } catch (\Throwable $e) { $failure ??= $e; + ++$skipped; // Skip invalid releases continue; } @@ -161,7 +163,7 @@ public function releasePages(int $page = 1): \Generator $hasMorePages = $this->hasNextPage($response); - yield new ReleasePage($releases, !$hasMorePages); + yield new ReleasePage($releases, !$hasMorePages, $skipped); $currentPage++; } while ($hasMorePages); diff --git a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php index da7ea74..cb0e22d 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php +++ b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php @@ -29,7 +29,7 @@ public function pages(int $offset = 0): \Generator $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; foreach ($this->api->releasePages($first) as $page) { - yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last, $page->skipped); $skip = 0; } } diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php index c2f6d54..a5d1191 100644 --- a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -124,6 +124,38 @@ public function releaseDeletedUpstreamDoesNotShiftTheTail(): void Assert::same($source->served, [0, 2, 4]); } + #[Test] + public function unreadableReleaseIsNotTakenForADeletedOne(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1'], perPage: 3); + self::flatten($this->registry()->releases($this->id, $source)); + + // The source fails to read `v2` this time; the stored list must keep it + $source->unreadable = ['v2']; + $source->publish('v4'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + + // Nothing was stored, so the next run checks again instead of waiting for the TTL + $source->unreadable = []; + $source->served = []; + $fixed = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($fixed, ['v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0]); + } + + #[Test] + public function unreadableReleaseIsSkippedWhenNothingIsStored(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1'], perPage: 3); + $source->unreadable = ['v2']; + + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v3', 'v1']); + } + #[Test] public function failedTailLoadingStillServesTheStoredReleases(): void { diff --git a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php index 4332ee8..d1e1921 100644 --- a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php +++ b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php @@ -27,6 +27,13 @@ final class ArrayReleaseSource implements ReleaseSource /** When set, every page request fails with this exception. */ public ?\Throwable $failure = null; + /** + * Tags of releases the source cannot read: they are left out of the pages and counted as skipped. + * + * @var list + */ + public array $unreadable = []; + /** * @param list $releases Newest first. * @param int<1, max> $perPage @@ -84,8 +91,12 @@ public function pages(int $offset = 0): \Generator $this->served[] = $page * $this->perPage; $items = \array_slice($this->releases, $page * $this->perPage, $this->perPage); $last = ($page + 1) * $this->perPage >= \count($this->releases); + $readable = \array_values(\array_filter( + $items, + fn(ReleaseRecord $release): bool => !\in_array($release->tag, $this->unreadable, true), + )); - yield new ReleasePage(\array_slice($items, $skip), $last); + yield new ReleasePage(\array_slice($readable, $skip), $last, \count($items) - \count($readable)); $skip = 0; ++$page; diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php index 6072e52..20e58f7 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -132,6 +132,15 @@ public function draftReleasesAreNeitherServedNorStored(): void Assert::false($storage->load(new RepositoryId(GitHubRepository::TYPE, 'owner/repo'))?->has('draft-1') ?? true); } + #[Test] + public function unreadableReleasesAreCountedOnThePage(): void + { + $page = (new GitHubReleaseSource(self::api(new PagedClientStub(pages: 1, broken: 2))))->pages()->current(); + + Assert::same(\count($page->releases), 98); + Assert::same($page->skipped, 2); + } + #[Test] public function assetDigestReportedByTheApiIsStored(): void { @@ -166,17 +175,21 @@ private static function createRepository( PagedClientStub $client, VersionRegistry $registry = new PassThroughRegistry(), ): GitHubRepository { + return new GitHubRepository(self::api($client), 'owner', 'repo', new Logger(), $registry); + } + + private static function api(PagedClientStub $client): RepositoryApi + { $logger = new Logger(); $httpFactory = new NyholmFactoryImpl($logger); - $api = new RepositoryApi( + + return new RepositoryApi( new Client($httpFactory, $client, new GitHubConfig()), $httpFactory, 'owner', 'repo', $logger, ); - - return new GitHubRepository($api, 'owner', 'repo', $logger, $registry); } private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php index d3d47df..25a98d5 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -29,11 +29,13 @@ final class PagedClientStub implements ClientInterface * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. * @param int<1, max> $releasesPerPage Number of releases on every such page. * @param int<0, max> $drafts Draft releases listed on top, as the API shows them to the token holder. + * @param int<0, max> $broken Number of the newest published releases that cannot be decoded. */ public function __construct( private readonly int $pages = 1, private readonly int $releasesPerPage = 100, private readonly int $drafts = 0, + private readonly int $broken = 0, ) {} public function sendRequest(RequestInterface $request): ResponseInterface @@ -94,7 +96,8 @@ private function allReleases(): array $tag = $i < 1 ? \sprintf('draft-%d', 1 - $i) : \sprintf('v1.0.%d', $i); $releases[] = [ 'name' => $tag, - 'tag_name' => $tag, + // A number where a string belongs fails the strict constructor of the response object + 'tag_name' => $i >= 1 && $i <= $this->broken ? $i : $tag, 'published_at' => '2024-01-01T00:00:00Z', 'assets' => [[ 'name' => 'rr-linux-amd64.tar.gz', From 0df4e1cd5ecb0b4e6856b630a0377d33d8201627 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Tue, 15 Sep 2026 10:41:48 +0400 Subject: [PATCH 12/17] fix(github): send the API token to GitHub hosts only Asset URLs are read back from the version registry on disk, so a tampered file could point a download at any host, and the client used to attach the bearer token to every request. The token now follows requests to `github.com` and `githubusercontent.com` and their subdomains only. Assisted-By: Claude Fable 5.1 --- .../Repository/Internal/GitHub/Api/Client.php | 35 +++++++++++++++---- .../Internal/GitHub/Api/ClientTest.php | 23 ++++++++++++ .../Internal/GitHub/Stub/ClientStub.php | 4 +++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/Module/Repository/Internal/GitHub/Api/Client.php b/src/Module/Repository/Internal/GitHub/Api/Client.php index 0cf602f..aca156e 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Client.php +++ b/src/Module/Repository/Internal/GitHub/Api/Client.php @@ -18,13 +18,20 @@ * HTTP client wrapper with GitHub-specific error handling and authentication. * * Converts unsuccessful responses (rate limits, invalid token, missing repository, etc.) - * into exceptions with actionable messages. Adds GitHub API token authentication when available. + * into exceptions with actionable messages. Adds the GitHub API token to requests bound for GitHub hosts. * * @internal * @psalm-internal Internal\DLoad\Module\Repository\Internal\GitHub */ final class Client { + /** + * Hosts the token may be sent to. Asset URLs come from the API response and from the version + * registry on disk, so a tampered file must not be able to point a request with the token at + * a host of its choosing. + */ + private const TRUSTED_HOSTS = ['github.com', 'githubusercontent.com']; + /** * @var array */ @@ -39,9 +46,6 @@ public function __construct( private readonly ClientInterface $client, private readonly GitHub $gitHubConfig, ) { - // Add authorization header if token is available - $this->gitHubConfig->token !== null and $this->defaultHeaders['authorization'] = 'Bearer ' . $this->gitHubConfig->token; - $this->validator = new ResponseValidator(authenticated: $this->gitHubConfig->token !== null); } @@ -52,9 +56,11 @@ public function __construct( */ public function request(Method|string $method, string|UriInterface $uri, array $headers = []): ResponseInterface { - $request = $this->httpFactory->request($method, $uri, $headers + $this->defaultHeaders); + $headers += $this->defaultHeaders; + $this->gitHubConfig->token !== null && self::isTrusted($uri) + and $headers += ['authorization' => 'Bearer ' . $this->gitHubConfig->token]; - return $this->sendRequest($request); + return $this->sendRequest($this->httpFactory->request($method, $uri, $headers)); } /** @@ -72,4 +78,21 @@ public function sendRequest(RequestInterface $request): ResponseInterface return $response; } + + private static function isTrusted(string|UriInterface $uri): bool + { + $host = $uri instanceof UriInterface ? $uri->getHost() : \parse_url($uri, \PHP_URL_HOST); + if (!\is_string($host) || $host === '') { + return false; + } + + $host = \strtolower($host); + foreach (self::TRUSTED_HOSTS as $trusted) { + if ($host === $trusted || \str_ends_with($host, '.' . $trusted)) { + return true; + } + } + + return false; + } } diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php b/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php index b27ad59..cd7b165 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php @@ -5,12 +5,14 @@ namespace Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Api; use Internal\DLoad\Module\Config\Schema\GitHub; +use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; use Internal\DLoad\Module\Repository\Exception\AccessDeniedException; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\AuthenticationException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Client; +use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Stub\ClientExceptionStub; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Stub\ClientStub; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Stub\GitHubConfigStub; @@ -170,6 +172,27 @@ public function requestWithAuthTokenAddsAuthorizationHeader(): void Assert::equals($result, $response); } + #[Test] + public function tokenIsSentToGitHubHostsOnly(): void + { + // Asset URLs may come from a registry file on disk, so the token must not follow them anywhere + $http = new ClientStub(); + $client = new Client(new NyholmFactoryImpl(new Logger()), $http, GitHubConfigStub::withToken('secret')); + + $client->request('GET', 'https://api.github.com/repos/owner/repo/releases'); + $client->request('GET', 'https://objects.githubusercontent.com/asset'); + $client->request('GET', 'https://GitHub.com/owner/repo/releases/download/v1/rr.tar.gz'); + $client->request('GET', 'https://evil.example/github.com/asset'); + $client->request('GET', 'https://notgithub.com/asset'); + + Assert::same($http->sent[0]->getHeaderLine('authorization'), 'Bearer secret'); + Assert::same($http->sent[1]->getHeaderLine('authorization'), 'Bearer secret'); + Assert::same($http->sent[2]->getHeaderLine('authorization'), 'Bearer secret'); + Assert::same($http->sent[3]->getHeaderLine('authorization'), ''); + Assert::same($http->sent[4]->getHeaderLine('authorization'), ''); + Assert::same($http->sent[3]->getHeaderLine('accept'), 'application/vnd.github.v3+json'); + } + #[Test] public function requestWithoutTokenDoesNotAddAuthorizationHeader(): void { diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php index 86270cc..d452ec8 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php @@ -17,6 +17,9 @@ */ final class ClientStub implements ClientInterface { + /** @var list Requests in the order they were sent. */ + public array $sent = []; + /** * @var array */ @@ -43,6 +46,7 @@ public function withException(RequestInterface $request, ClientExceptionInterfac public function sendRequest(RequestInterface $request): ResponseInterface { + $this->sent[] = $request; $requestKey = $this->createRequestKey($request); if (isset($this->exceptions[$requestKey])) { From aaa260464ce7a7f1259dc8cdc96b3904b76293b5 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Tue, 15 Sep 2026 10:48:29 +0400 Subject: [PATCH 13/17] fix(registry): keep GitHub drafts as hidden placeholders so the stored count matches the API paging fix(github): accept the `null` publication date of a draft The tail is loaded from the page the stored count points at, while GitHub counts drafts in its pages: with drafts on top the count fell short, and every extension re-requested a page it already had. A draft is now stored as a hidden record with nothing but its tag, so it holds its position and is never served. Assisted-By: Claude Fable 5.1 --- README-es.md | 2 +- README-ru.md | 2 +- README-zh.md | 2 +- README.md | 2 +- .../Registry/Internal/PassThroughRegistry.php | 3 ++- .../Internal/StoredVersionRegistry.php | 7 ++++-- src/Module/Registry/Record/ReleaseRecord.php | 22 ++++++++++++++++++- .../Internal/GitHub/Api/RepositoryApi.php | 4 +--- .../GitHub/Api/Response/ReleaseInfo.php | 14 +++++++++--- .../Registry/FileRegistryStorageTest.php | 16 ++++++++++++++ .../Registry/StoredVersionRegistryTest.php | 17 ++++++++++++++ .../Internal/GitHub/GitHubRepositoryTest.php | 12 ++++++++-- .../Internal/GitHub/Stub/PagedClientStub.php | 2 +- 13 files changed, 88 insertions(+), 17 deletions(-) diff --git a/README-es.md b/README-es.md index 8075fb5..a1e0402 100644 --- a/README-es.md +++ b/README-es.md @@ -396,7 +396,7 @@ La variable de entorno tiene prioridad sobre el atributo de `dload.xml`, y la op > usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. Un > release almacenado cuyos assets desaparecieron del origen se elimina del registro en cuanto falla > su descarga, y la lista de releases se vuelve a obtener antes de que la ejecución se dé por vencida. -> Los borradores de releases de GitHub nunca se almacenan. Un release que el proveedor inserta por debajo del inicio de su lista, como un release de GitLab con un `released_at` retroactivo, no lo detecta la comprobación; ejecute `cache:clear` para ese software para incorporarlo. +> Los borradores de releases de GitHub nunca se entregan; el registro guarda solo sus etiquetas como marcadores ocultos que conservan la posición en la lista. Un release que el proveedor inserta por debajo del inicio de su lista, como un release de GitLab con un `released_at` retroactivo, no lo detecta la comprobación; ejecute `cache:clear` para ese software para incorporarlo. En GitHub Actions el directorio puede conservarse entre ejecuciones del workflow, de modo que cada ejecución gasta el límite de la API solo en los releases publicados desde la anterior: diff --git a/README-ru.md b/README-ru.md index 106e430..82b3191 100644 --- a/README-ru.md +++ b/README-ru.md @@ -398,7 +398,7 @@ DLoad поддерживает три типа загрузки, которые > завершится ошибкой. Сохранённый релиз, ассеты которого исчезли из источника, удаляется из > реестра сразу после неудачной загрузки, а список релизов запрашивается заново, прежде чем > запуск завершится ошибкой. -> Черновики релизов GitHub в реестр не попадают. Релиз, который провайдер вставляет не в начало списка, например релиз GitLab с задним числом в `released_at`, проверка не замечает; чтобы его подхватить, выполните `cache:clear` для этой программы. +> Черновики релизов GitHub никогда не выдаются; реестр хранит только их теги как скрытые заглушки, занимающие позицию в списке. Релиз, который провайдер вставляет не в начало списка, например релиз GitLab с задним числом в `released_at`, проверка не замечает; чтобы его подхватить, выполните `cache:clear` для этой программы. В GitHub Actions каталог можно переносить между запусками workflow, тогда запуск тратит лимит API только на релизы, вышедшие после предыдущего: diff --git a/README-zh.md b/README-zh.md index 5a7ef98..6d954d2 100644 --- a/README-zh.md +++ b/README-zh.md @@ -391,7 +391,7 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： > 因此该目录可以自由共享或放入 CI 缓存。若因网络错误或 API 速率限制导致检查失败，会使用已保存的发布； > 从未见过的仓库仍会明确报错。若某个已保存发布的资产在上游已被删除，下载失败后它会立即从注册表中移除， > 并在本次运行放弃之前重新获取发布列表。 -> GitHub 的草稿发布不会被保存。若提供方将某个发布插入到列表开头以下的位置，例如 GitLab 中 `released_at` 被回填的发布，检查不会发现它；请对该软件运行 `cache:clear` 以获取它。 +> GitHub 的草稿发布永远不会被提供；注册表只以隐藏占位符的形式保存其标签，用于占据列表中的位置。若提供方将某个发布插入到列表开头以下的位置，例如 GitLab 中 `released_at` 被回填的发布，检查不会发现它；请对该软件运行 `cache:clear` 以获取它。 在 GitHub Actions 中可以在多次工作流运行之间保留该目录，这样每次运行只为上次运行之后发布的版本消耗速率限制： diff --git a/README.md b/README.md index 4763594..8647ea5 100644 --- a/README.md +++ b/README.md @@ -398,7 +398,7 @@ The environment variable takes precedence over the attribute in `dload.xml`, and > releases are used instead, and a repository that was never seen before still fails loudly. > A stored release whose assets have disappeared upstream is dropped from the registry as soon as > its download fails, and the release list is fetched again before the run gives up. -> GitHub draft releases are never stored. A release that a provider inserts below the top of its listing, such as a GitLab release with a backdated `released_at`, is not noticed by a check; run `cache:clear` for that software to pick it up. +> GitHub draft releases are never served; the registry keeps only their tags, as hidden placeholders that hold the position in the listing. A release that a provider inserts below the top of its listing, such as a GitLab release with a backdated `released_at`, is not noticed by a check; run `cache:clear` for that software to pick it up. In GitHub Actions the directory can be carried between workflow runs, so a run spends the rate limit only on releases published since the previous one: diff --git a/src/Module/Registry/Internal/PassThroughRegistry.php b/src/Module/Registry/Internal/PassThroughRegistry.php index a1326da..c1fee56 100644 --- a/src/Module/Registry/Internal/PassThroughRegistry.php +++ b/src/Module/Registry/Internal/PassThroughRegistry.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Module\Registry\Internal; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Registry\ReleaseSource; use Internal\DLoad\Module\Registry\RepositoryId; use Internal\DLoad\Module\Registry\VersionRegistry; @@ -21,7 +22,7 @@ final class PassThroughRegistry implements VersionRegistry public function releases(RepositoryId $id, ReleaseSource $source): \Generator { foreach ($source->pages() as $page) { - yield $page->releases; + yield ReleaseRecord::visible($page->releases); } } diff --git a/src/Module/Registry/Internal/StoredVersionRegistry.php b/src/Module/Registry/Internal/StoredVersionRegistry.php index c9bbaf1..578b75b 100644 --- a/src/Module/Registry/Internal/StoredVersionRegistry.php +++ b/src/Module/Registry/Internal/StoredVersionRegistry.php @@ -66,7 +66,9 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator $this->logger->debug('Releases of `%s` are served from the version registry.', (string) $id); } - yield from $record->pages(); + foreach ($record->pages() as $page) { + yield ReleaseRecord::visible($page); + } if ($record->complete) { return; @@ -180,7 +182,8 @@ private function extend(RepositoryRecord $record, ReleaseSource $source): \Gener $record = $this->persist($record->withTail($new)->withComplete($page->last)); - $new === [] or yield $new; + $visible = ReleaseRecord::visible($new); + $visible === [] or yield $visible; } $record->complete or $this->persist($record->withComplete(true)); diff --git a/src/Module/Registry/Record/ReleaseRecord.php b/src/Module/Registry/Record/ReleaseRecord.php index ab82a37..4b064bb 100644 --- a/src/Module/Registry/Record/ReleaseRecord.php +++ b/src/Module/Registry/Record/ReleaseRecord.php @@ -17,6 +17,7 @@ * published_at?: string|null, * prerelease?: bool, * assets?: list, + * hidden?: bool, * } * * @internal @@ -27,6 +28,9 @@ final class ReleaseRecord * @param non-empty-string $tag Tag the release was made from; identifies the release within a repository. * @param non-empty-string $name Human-readable release name. * @param list $assets + * @param bool $hidden Whether the release is listed by the source but must not be served, as a + * GitHub draft is. It keeps its position, so the stored count still maps onto the paging + * of the source. */ public function __construct( public readonly string $tag, @@ -34,8 +38,20 @@ public function __construct( public readonly ?\DateTimeImmutable $publishedAt = null, public readonly bool $prerelease = false, public readonly array $assets = [], + public readonly bool $hidden = false, ) {} + /** + * Leaves out the releases that must not be served. + * + * @param list $releases + * @return list + */ + public static function visible(array $releases): array + { + return \array_values(\array_filter($releases, static fn(ReleaseRecord $release): bool => !$release->hidden)); + } + /** * @param array $data * @throws \InvalidArgumentException When the array does not describe a release. @@ -64,6 +80,7 @@ public static function fromArray(array $data): self publishedAt: \is_string($publishedAt) && $publishedAt !== '' ? new \DateTimeImmutable($publishedAt) : null, prerelease: (bool) ($data['prerelease'] ?? false), assets: $assets, + hidden: (bool) ($data['hidden'] ?? false), ); } @@ -72,12 +89,15 @@ public static function fromArray(array $data): self */ public function toArray(): array { - return [ + $data = [ 'tag' => $this->tag, 'name' => $this->name, 'published_at' => $this->publishedAt?->format(\DateTimeInterface::ATOM), 'prerelease' => $this->prerelease, 'assets' => \array_map(static fn(AssetRecord $asset): array => $asset->toArray(), $this->assets), ]; + $this->hidden and $data['hidden'] = true; + + return $data; } } diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index 426d967..bc8899f 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -127,9 +127,7 @@ public function releasePages(int $page = 1): \Generator $skipped = 0; foreach ($data as $releaseData) { try { - $release = ReleaseInfo::fromApiResponse($releaseData); - // A draft is visible to the token holder only; the registry may be shared - $release->draft or $releases[] = $release->toRecord(); + $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); } catch (\Throwable $e) { $failure ??= $e; ++$skipped; diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php index e06e58d..0462043 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php @@ -23,7 +23,7 @@ final class ReleaseInfo public function __construct( public readonly string $name, public readonly string $tagName, - public readonly \DateTimeImmutable $publishedAt, + public readonly ?\DateTimeImmutable $publishedAt, public readonly array $assets, public readonly bool $prerelease, public readonly bool $draft, @@ -33,7 +33,7 @@ public function __construct( * @param array{ * name: string|null, * tag_name: string, - * published_at: string, + * published_at: string|null, * assets: array