From 937bb56d382410e4a4ffe8d09c44986290797660 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 23 Nov 2025 22:03:31 +0000 Subject: [PATCH 1/4] Bump pip --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7ff821eb7..b3dbacaef 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -22,7 +22,7 @@ ENV PYTHONUSERBASE="/home/dev/.local" COPY --chown=dev:dev . . -RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.2 \ +RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.3 \ && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts,build] \ && pre-commit install --install-hooks From b887aeff110f925399f585856e3c6bfeb4a9de16 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 23 Nov 2025 22:04:00 +0000 Subject: [PATCH 2/4] Create installer packages --- .devcontainer/Dockerfile | 12 ++ .github/workflows/build.yml | 73 +++++++--- README.md | 9 ++ dfetch/resources/__init__.py | 7 +- pyproject.toml | 13 +- script/package.py | 254 +++++++++++++++++++++++++++++++++++ 6 files changed, 339 insertions(+), 29 deletions(-) create mode 100644 script/package.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b3dbacaef..7b4dbffb5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,6 +9,18 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ subversion=1.14.1-3+deb11u2 && \ rm -rf /var/lib/apt/lists/* +# Install ruby gem FPM for packaging +RUN apt-get update && apt-get install --no-install-recommends -y \ + ruby=1:2.7+2 \ + ruby-dev=1:2.7+2 \ + build-essential=12.9 \ + rpm=4.16.1.2+dfsg1-3 \ + git=1:2.30.2-1+deb11u5 \ + curl=7.74.0-1.3+deb11u15 \ + ca-certificates=20210119 && \ + gem install --no-document fpm --version 1.17.0 && \ + rm -rf /var/lib/apt/lists/* + WORKDIR /workspaces/dfetch # Add a non-root user (dev) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb3ec3683..87551cb90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,28 @@ jobs: with: python-version: '3.13' + - name: Set up Ruby + if: ${{ matrix.platform != 'windows-latest' }} + uses: ruby/setup-ruby@d354de180d0c9e813cfddfcbdc079945d4be589b #v1.275.0 + with: + ruby-version: 2.7 # See also devcontainer + + - name: Install fpm + if: ${{ matrix.platform != 'windows-latest' }} + run: | + gem install --no-document fpm --version 1.17.0 + fpm --version + + - name: Add msbuild to PATH + if: ${{ matrix.platform == 'windows-latest' }} + uses: microsoft/setup-msbuild@767f00a3f09872d96a0cb9fcd5e6a4ff33311330 # v2 + - name: Install WiX + if: ${{ matrix.platform == 'windows-latest' }} + run: | + dotnet tool install --global wix --version 6.0.2 + echo "$env:USERPROFILE\.dotnet\tools" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + wix --version + - name: ccache uses: hendrikmuhs/ccache-action@5ebbd400eff9e74630f759d94ddd7b6c26299639 # v1.2 if: ${{ matrix.platform != 'windows-latest' }} @@ -59,12 +81,17 @@ jobs: run: | pip install .[build] python script/build.py + python script/package.py - name: Store the distribution packages uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: binary-distribution-${{ matrix.platform }} - path: build/dfetch-* + path: | + build/dfetch-package/*.deb + build/dfetch-package/*.rpm + build/dfetch-package/*.pkg + build/dfetch-package/*.msi test-binary: name: test binary @@ -82,36 +109,38 @@ jobs: name: binary-distribution-${{ matrix.platform }} path: . - - name: Prepare binary + - name: Install binary if: matrix.platform == 'ubuntu-latest' run: | - binary=$(ls dfetch-*-x86_64) - ln -sf "$binary" dfetch - chmod +x dfetch - ls -la . + sudo dpkg -i dfetch*.deb + sudo dpkg -L dfetch + echo "/opt/dfetch" >> $GITHUB_PATH shell: bash - - name: Prepare binary + - name: Install binary if: matrix.platform == 'macos-latest' run: | - binary=$(ls dfetch-*-osx) - ln -sf "$binary" dfetch - chmod +x dfetch - ls -la . + sudo installer -pkg dfetch*.pkg -target / -verboseR + find /opt/dfetch -type f + echo "/opt/dfetch/opt/dfetch" >> $GITHUB_PATH shell: bash - - name: Prepare binary on Windows + - name: Install binary if: matrix.platform == 'windows-latest' run: | - $binary = Get-ChildItem dfetch-*.exe | Select-Object -First 1 - Copy-Item $binary -Destination dfetch.exe -Force - Get-ChildItem + $file = "dfetch.msi" + $log = "install.log" + $procMain = Start-Process "msiexec" "/i `"$file`" /qn /l*! `"$log`"" -NoNewWindow -PassThru + $procLog = Start-Process "powershell" "Get-Content -Path `"$log`" -Wait" -NoNewWindow -PassThru + $procMain.WaitForExit() + $procLog.Kill() + echo "C:\Program Files (x86)\dfetch" >> $env:GITHUB_PATH shell: pwsh - - run: ./dfetch init - - run: ./dfetch environment - - run: ./dfetch validate - - run: ./dfetch check - - run: ./dfetch update - - run: ./dfetch update - - run: ./dfetch report -t sbom + - run: dfetch init + - run: dfetch environment + - run: dfetch validate + - run: dfetch check + - run: dfetch update + - run: dfetch update + - run: dfetch report -t sbom diff --git a/README.md b/README.md index f8b0d5049..127b7886f 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,15 @@ pip install dfetch pip install git+https://github.com/dfetch-org/dfetch.git#egg=dfetch ``` +### Binary distributions + +The [build.yml](https://github.com/dfetch-org/dfetch/actions/workflows/build.yml) produces installers for all major platforms. +See the artifacts in the run. + +- Linux `.deb` & `.rpm` +- macOS `.pkg` +- Windows `.msi` + ## Github Action You can use DFetch in your Github Actions workflow to check your dependencies. diff --git a/dfetch/resources/__init__.py b/dfetch/resources/__init__.py index 5e7548c1a..bc2f611f2 100644 --- a/dfetch/resources/__init__.py +++ b/dfetch/resources/__init__.py @@ -9,7 +9,12 @@ def _resource_path(filename: str) -> ContextManager[Path]: """Get the path to the resource.""" - return importlib_resources.as_file(importlib_resources.files(resources) / filename) + return importlib_resources.as_file( + importlib_resources.files( + "resources" if "__compiled__" in globals() else resources + ) + / filename + ) def schema_path() -> ContextManager[Path]: diff --git a/pyproject.toml b/pyproject.toml index 46627ad0e..9e7665fbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "dfetch" authors = [{ name = "Ben Spoor", email = "dfetch@spoor.cc" }] -description = "Dependency fetcher" +description = "A vendoring tool for fetching and managing external dependencies." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9" keywords = [ @@ -190,21 +190,22 @@ reportMissingModuleSource = false pythonVersion = "3.9" [tool.nuitka] -mode = "onefile" # Switch this between standalone and onefile as needed +mode = "standalone" # jobs = "4" # Can be used to reduce memory usage, in case of compilation issues # Enable below for debugging # show-progress = true assume-yes-for-downloads = true -include-package-data="dfetch,infer_license" +include-data-dir="dfetch/resources=resources" +include-package-data="infer_license" include-module="infer_license.licenses" # python-flag = ["-OO"] # Cannot optimize (yet) commands rely on __doc__ being present output-dir = "build" -output-filename-win = "dfetch-{VERSION}.exe" -output-filename-linux = "dfetch-{VERSION}-x86_64" -output-filename-macos = "dfetch-{VERSION}-osx" +output-filename-win = "dfetch.exe" +output-filename-linux = "dfetch" +output-filename-macos = "dfetch" # windows-icon-from-ico = "static/favicon.ico" # windows-company-name = "dfetch-org" diff --git a/script/package.py b/script/package.py new file mode 100644 index 000000000..885da3ab7 --- /dev/null +++ b/script/package.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""This script packages the dfetch build directory into OS-specific installers using fpm & wix.""" +import shutil +import subprocess # nosec +import sys +import tomllib as toml +import xml.etree.ElementTree as ET # nosec (only used for XML generation, not parsing untrusted input) +from pathlib import Path + +from dfetch import __version__ + +# Configuration loading +with open("pyproject.toml", "rb") as pyproject_file: + pyproject = toml.load(pyproject_file) +project_info = pyproject.get("project", {}) + +PACKAGE_NAME = project_info.get("name") + +MAINTAINER = project_info.get("authors", [{}])[0].get("name", "") +DESCRIPTION = project_info.get("description", "") +URL = project_info.get("urls", {}).get("Homepage", "") +LICENSE = project_info.get("license", "") + +INSTALL_PREFIX = ( + f"/opt/{PACKAGE_NAME}" # Where the files will be installed on Linux/macOS +) +BUILD_DIR = Path("build", f"{PACKAGE_NAME}.dist") +OUTPUT_DIR = Path("build", f"{PACKAGE_NAME}-package") +UPGRADE_CODE = "BDC7DA0D-70C1-4189-BE88-F1BD2DEE3E33" #: Fixed UUID for WiX installer, must be unique and stable + + +def run_command(command: list[str]) -> None: + """Run a system command and handle errors.""" + resolved_cmd = shutil.which(command[0]) + if not resolved_cmd: + raise FileNotFoundError(f"Command not found: {command[0]}") + + command[0] = resolved_cmd # On windows .bat files need full path + + print("Running:", " ".join(command)) + subprocess.check_call(command) # nosec + + +def package_linux() -> None: + """Package the build directory into .deb and .rpm installers.""" + for target in ("deb", "rpm"): + output = f"{OUTPUT_DIR}/{PACKAGE_NAME}_{__version__}.{target}" + cmd = [ + "fpm", + "-s", + "dir", + "-t", + target, + "-n", + PACKAGE_NAME, + "-v", + __version__, + "-C", + str(BUILD_DIR), + "--prefix", + INSTALL_PREFIX, + "--description", + DESCRIPTION, + "--maintainer", + MAINTAINER, + "--url", + URL, + "--license", + LICENSE, + "-p", + output, + ".", + ] + run_command(cmd) + + +def package_macos() -> None: + """Package the build directory into a .pkg installer for macOS.""" + cmd = [ + "fpm", + "-s", + "dir", + "-t", + "osxpkg", + "-n", + PACKAGE_NAME, + "-v", + __version__, + "-C", + str(BUILD_DIR), + "--prefix", + INSTALL_PREFIX, + "--description", + DESCRIPTION, + "--maintainer", + MAINTAINER, + "--url", + URL, + "--license", + LICENSE, + "-p", + f"{OUTPUT_DIR}/{PACKAGE_NAME}_{__version__}.pkg", + ".", + ] + run_command(cmd) + + +def check_wix_installed() -> None: + """Check if WiX Toolset v4 is installed (candle.exe & light.exe).""" + wix = shutil.which("wix.exe") + if not wix: + print( + "Error: WiX Toolset v4 is required but not found.\n" + "Please install it from https://wixtoolset.org/releases/\n" + "and ensure 'wix.exe' is in your PATH." + ) + sys.exit(1) + + +def generate_wix_xml(build_dir: Path, output_wxs: Path) -> None: + """Generate a minimal WiX v4 XML including all files in build_dir.""" + wix = ET.Element("Wix", xmlns="http://wixtoolset.org/schemas/v4/wxs") + package = ET.SubElement( + wix, + "Package", + Name=PACKAGE_NAME, + Manufacturer=MAINTAINER, + Version=__version__, + UpgradeCode=UPGRADE_CODE, + ) + + ET.SubElement(package, "MediaTemplate", EmbedCab="yes") + + standard_dir = ET.SubElement( + package, + "StandardDirectory", + Id="ProgramFilesFolder", + ) + install_dir = ET.SubElement( + standard_dir, "Directory", Id="INSTALLFOLDER", Name=PACKAGE_NAME + ) + + component = ET.SubElement( + install_dir, + "Component", + Id="MainComponent", + Guid="*", + Directory="INSTALLFOLDER", + ) + + ET.SubElement( + component, + "Environment", + Id="AddToPath", + Name="PATH", + Value="[INSTALLFOLDER]", + Action="set", + Part="last", + System="yes", + ) + + # Registry key so InstallLocation is discoverable + ET.SubElement( + component, + "RegistryValue", + Root="HKLM", + Key=f"Software\\{PACKAGE_NAME}", + Name="InstallLocation", + Value="[INSTALLFOLDER]", + Type="string", + KeyPath="yes", + ) + + # Feature + feature = ET.SubElement( + package, + "Feature", + Id="MainFeature", + Title=PACKAGE_NAME, + Level="1", + ) + + ET.SubElement( + feature, + "ComponentRef", + Id="MainComponent", + ) + ET.SubElement(feature, "Files", Include=str(build_dir.resolve() / "**")) + + ET.SubElement(package, "Property", Id="ARPCOMMENTS", Value=DESCRIPTION) + + tree = ET.ElementTree(wix) + tree.write(output_wxs, encoding="utf-8", xml_declaration=True) + + +def generate_wix_proj(output_proj: Path, wix_file: Path) -> None: + """Generate a minimal WiX SDK project referencing the .wxs file.""" + project = ET.Element("Project", Sdk="WixToolset.Sdk/6.0.2") + + item_group = ET.SubElement(project, "ItemGroup") + ET.SubElement(item_group, "Wix", Include=str(wix_file)) + + tree = ET.ElementTree(project) + tree.write(output_proj, encoding="utf-8", xml_declaration=True) + + +def package_windows() -> None: + """Package the build directory into a .msi installer for Windows using WiX v4.""" + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + wix_file = OUTPUT_DIR / f"{PACKAGE_NAME}.wxs" + wix_proj = OUTPUT_DIR / f"{PACKAGE_NAME}.wixproj" + msi_file = OUTPUT_DIR / f"{PACKAGE_NAME}_{__version__}.msi" + + generate_wix_xml(BUILD_DIR, wix_file) + generate_wix_proj(wix_proj, wix_file) + + check_wix_installed() + + run_command( + ["dotnet", "build", str(wix_proj), "-c", "Release", "-o", str(OUTPUT_DIR)] + ) + + print(f"MSI generated at {msi_file}") + + +def list_files(path: Path) -> None: + """List all files in the given path.""" + for file_path in path.rglob("*"): + if file_path.is_file(): + print(file_path.relative_to(path)) + + +def main() -> None: + """Main packaging function.""" + if not BUILD_DIR.exists(): + print(f"Error: Build directory {BUILD_DIR} does not exist. Run build.py first.") + sys.exit(1) + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + list_files(BUILD_DIR) + + if sys.platform.startswith("linux"): + package_linux() + elif sys.platform.startswith("darwin"): + package_macos() + elif sys.platform.startswith("win"): + package_windows() + else: + print(f"Unsupported platform: {sys.platform}") + sys.exit(1) + + +if __name__ == "__main__": + main() From 78d0c20ff2037cbee41299ed0e3f3968bbc1c057 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 25 Dec 2025 22:04:07 +0000 Subject: [PATCH 3/4] Install prefix leads to /opt/dfetch/opt/dfetch https://github.com/jordansissel/fpm/issues/1996 --- .github/workflows/build.yml | 1 - script/package.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87551cb90..af8c137d6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,6 @@ jobs: if: matrix.platform == 'macos-latest' run: | sudo installer -pkg dfetch*.pkg -target / -verboseR - find /opt/dfetch -type f echo "/opt/dfetch/opt/dfetch" >> $GITHUB_PATH shell: bash diff --git a/script/package.py b/script/package.py index 885da3ab7..1ca2aa517 100644 --- a/script/package.py +++ b/script/package.py @@ -88,6 +88,7 @@ def package_macos() -> None: __version__, "-C", str(BUILD_DIR), + # https://github.com/jordansissel/fpm/issues/1996 This prefix results in /opt/dfetch/opt/dfetch "--prefix", INSTALL_PREFIX, "--description", From 91ad3830a40d6e3e1114e48c6318ffe0ee10fd6d Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 25 Dec 2025 23:42:16 +0000 Subject: [PATCH 4/4] Bump devcontainer to trixie & ruby 3.3 --- .devcontainer/Dockerfile | 24 ++++++++++++------------ .github/workflows/build.yml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7b4dbffb5..5b0443708 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,23 +1,23 @@ -FROM mcr.microsoft.com/devcontainers/python:3.13-bullseye@sha256:735ecc489de65d36b6ac4e8e5e37287cb30711bac0dd292770bf5347228c438e +FROM mcr.microsoft.com/devcontainers/python:3.13-trixie@sha256:c3a082525f51bb81e3d952d5ec12c2bed3525179a45ef3a09d48993ebc871133 # Install dependencies # pv is required for asciicasts RUN apt-get update && apt-get install --no-install-recommends -y \ - ccache=4.2-1 \ - pv=1.6.6-1+b1 \ - patchelf=0.12-1 \ - subversion=1.14.1-3+deb11u2 && \ + ccache=4.11.2-2 \ + pv=1.9.31-1 \ + patchelf=0.18.0-1.4 \ + subversion=1.14.5-3 && \ rm -rf /var/lib/apt/lists/* # Install ruby gem FPM for packaging RUN apt-get update && apt-get install --no-install-recommends -y \ - ruby=1:2.7+2 \ - ruby-dev=1:2.7+2 \ - build-essential=12.9 \ - rpm=4.16.1.2+dfsg1-3 \ - git=1:2.30.2-1+deb11u5 \ - curl=7.74.0-1.3+deb11u15 \ - ca-certificates=20210119 && \ + ruby=1:3.3+b1 \ + ruby-dev=1:3.3+b1 \ + build-essential=12.12 \ + rpm=4.20.1+dfsg-3 \ + git=1:2.47.3-0+deb13u1 \ + curl=8.14.1-2+deb13u2 \ + ca-certificates=20250419 && \ gem install --no-document fpm --version 1.17.0 && \ rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af8c137d6..e7a46b5f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: if: ${{ matrix.platform != 'windows-latest' }} uses: ruby/setup-ruby@d354de180d0c9e813cfddfcbdc079945d4be589b #v1.275.0 with: - ruby-version: 2.7 # See also devcontainer + ruby-version: 3.3 # See also devcontainer - name: Install fpm if: ${{ matrix.platform != 'windows-latest' }}