diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d2ce07c..6dbe224b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - feat(maven): Add maven target to deploy to Maven Central (#258) +- feat(symbol-collector): Add symbol-collector target (#266) ## 0.24.4 diff --git a/Dockerfile b/Dockerfile index e18679c88..b01346926 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,8 @@ RUN apt-get -qq update \ git \ ruby-full \ twine \ + jq \ + unzip \ && curl -fsSL https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -o /tmp/packages-microsoft-prod.deb \ && dpkg -i /tmp/packages-microsoft-prod.deb \ && rm /tmp/packages-microsoft-prod.deb \ @@ -23,8 +25,8 @@ RUN apt-get -qq update \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get update -qq \ && apt-get install -y --no-install-recommends \ - dotnet-sdk-3.1 \ - docker-ce-cli \ + dotnet-sdk-3.1 \ + docker-ce-cli \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile minimal -y \ @@ -32,7 +34,13 @@ RUN apt-get -qq update \ && cargo install cargo-hack \ # Stick with 3.1.x as 3.2.x doesn't install on Debian Buster for some reason && gem update --no-document --system 3.1.5 \ - && gem install cocoapods + && gem install cocoapods \ + # Install https://github.com/getsentry/symbol-collector + && symbol_collector_url=$(curl -s https://api.github.com/repos/getsentry/symbol-collector/releases/tags/1.2.1 | \ + jq -r '.assets[].browser_download_url | select(endswith("symbolcollector-console-linux-x64.zip"))') \ + && curl -sL $symbol_collector_url -o "/tmp/sym-collector.zip" \ + && unzip /tmp/sym-collector.zip -d /usr/local/bin/ \ + && chmod +x /usr/local/bin/SymbolCollector.Console COPY dist/craft /usr/local/bin/craft RUN chmod +x /usr/local/bin/craft diff --git a/README.md b/README.md index f72e4e7f0..3811a2934 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ then enforces a specific workflow for managing release branches, changelogs, art - [AWS Lambda Layer (`aws-lambda-layer`)](#aws-lambda-layer-aws-lambda-layer) - [Unity Package Manager (`upm`)](#unity-package-manager-upm) - [Maven central (`maven`)](#maven-central-maven) + - [Symbol Collector (`symbol-collector`)](#symbol-collector-symbol-collector) - [Integrating Your Project with `craft`](#integrating-your-project-with-craft) - [Pre-release (Version-bumping) Script: Conventions](#pre-release-version-bumping-script-conventions) - [Post-release Script: Conventions](#post-release-script-conventions) @@ -1033,6 +1034,29 @@ targets: fileReplacerStr: release.aar ``` +### Symbol Collector (`symbol-collector`) + +Using the [`symbol-collector`](https://github.com/getsentry/symbol-collector) client, uploads native symbols. +The `symbol-collector` needs to be available in the path. + +**Configuration** + +| Option | Description | +| ---------------- | -------------------------------------------------------------------------------------------- | +| `serverEndpoint` | **optional** The server endpoint. Defaults to `https://symbol-collector.services.sentry.io`. | +| `batchType` | The batch type of the symbols to be uploaded. I.e: `Android`, `macOS`, `iOS`. | +| `bundleIdPrefix` | The prefix of the bundle ID. The new version will be appended to the end of this prefix. | + +**Example** + +```yaml +targets: + - name: symbol-collector + includeNames: /libsentry(-android)?\.so/ + batchType: Android + bundleIdPrefix: android-ndk- +``` + ## Integrating Your Project with `craft` Here is how you can integrate your GitHub project with `craft`: diff --git a/src/artifact_providers/__tests__/base.test.ts b/src/artifact_providers/__tests__/base.test.ts new file mode 100644 index 000000000..17ceddc6a --- /dev/null +++ b/src/artifact_providers/__tests__/base.test.ts @@ -0,0 +1,41 @@ +import { stringToRegexp } from '../../utils/filters'; +import { parseFilterOptions, RawFilterOptions } from '../base'; + +describe('parseFilterOptions', () => { + test('empty object', () => { + const rawFilters: RawFilterOptions = {}; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters).not.toHaveProperty('includeNames'); + expect(parsedFilters).not.toHaveProperty('excludeNames'); + }); + + test('undefined properties', () => { + const rawFilters: RawFilterOptions = { + includeNames: undefined, + excludeNames: undefined, + }; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters).not.toHaveProperty('includeNames'); + expect(parsedFilters).not.toHaveProperty('excludeNames'); + }); + + test('string properties', () => { + const stringFilter = '/testFilter/'; + const rawFilters: RawFilterOptions = { + includeNames: stringFilter, + }; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters.includeNames).toStrictEqual( + stringToRegexp(stringFilter) + ); + }); + + test('regex properties', () => { + const regexFilter = stringToRegexp('/testFilter/'); + const rawFilters: RawFilterOptions = { + includeNames: regexFilter, + }; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters.includeNames).toStrictEqual(regexFilter); + }); +}); diff --git a/src/artifact_providers/base.ts b/src/artifact_providers/base.ts index 8c9366d74..155f5eccb 100644 --- a/src/artifact_providers/base.ts +++ b/src/artifact_providers/base.ts @@ -6,6 +6,7 @@ import { import { clearObjectProperties } from '../utils/objects'; import { ConfigurationError } from '../utils/errors'; import { logger as loggerRaw } from '../logger'; +import { stringToRegexp } from '../utils/filters'; /** Maximum concurrency for downloads */ export const MAX_DOWNLOAD_CONCURRENCY = 5; @@ -61,14 +62,48 @@ export interface LocalArtifact extends AbstractArtifact { } /** - * Fitlering options for artifacts + * Raw filtering options for artifacts. + * They should be parsed by `parseFilterOptions`. */ -export interface FilterOptions { +export interface RawFilterOptions { + /** Include files that match this pattern */ + includeNames?: RegExp | string; + /** Exclude files that match this pattern */ + excludeNames?: RegExp | string; +} + +/** + * Parsed filtering options for artifacts + */ +export interface ParsedFilterOptions { /** Include files that match this regexp */ includeNames?: RegExp; /** Exclude files that match this regexp */ excludeNames?: RegExp; } + +/** + * Returns parsed the given raw filters. + */ +export function parseFilterOptions( + rawFilters: RawFilterOptions +): ParsedFilterOptions { + const parsedFilters: ParsedFilterOptions = {}; + if (rawFilters.includeNames) { + parsedFilters.includeNames = + typeof rawFilters.includeNames === 'string' + ? stringToRegexp(rawFilters.includeNames) + : rawFilters.includeNames; + } + if (rawFilters.excludeNames) { + parsedFilters.excludeNames = + typeof rawFilters.excludeNames === 'string' + ? stringToRegexp(rawFilters.excludeNames) + : rawFilters.excludeNames; + } + return parsedFilters; +} + /** * Configuration options needed for all artifact providers */ @@ -321,7 +356,7 @@ export abstract class BaseArtifactProvider { */ public async filterArtifactsForRevision( revision: string, - filterOptions?: FilterOptions + filterOptions?: ParsedFilterOptions ): Promise { let filteredArtifacts = await this.listArtifactsForRevision(revision); if (!filterOptions || filteredArtifacts.length === 0) { diff --git a/src/targets/__tests__/symbolCollector.test.ts b/src/targets/__tests__/symbolCollector.test.ts new file mode 100644 index 000000000..b36274d90 --- /dev/null +++ b/src/targets/__tests__/symbolCollector.test.ts @@ -0,0 +1,138 @@ +import { withTempDir } from '../../utils/files'; +import { NoneArtifactProvider } from '../../artifact_providers/none'; +import { checkExecutableIsPresent, spawnProcess } from '../../utils/system'; +import { SymbolCollector, SYM_COLLECTOR_BIN_NAME } from '../symbolCollector'; + +jest.mock('../../utils/files'); +jest.mock('../../utils/system'); +jest.mock('fs', () => { + const original = jest.requireActual('fs'); + return { + ...original, + promises: { + mkdir: jest.fn(() => { + /** do nothing */ + }), + }, + }; +}); + +const customConfig = { + batchType: 'batchType', + bundleIdPrefix: 'bundleIdPrefix-', +}; + +function getSymbolCollectorInstance( + config: Record = { testKey: 'testVal' } +): SymbolCollector { + return new SymbolCollector( + { + name: 'symbol-collector', + ...config, + }, + new NoneArtifactProvider() + ); +} + +describe('target config', () => { + test('symbol collector not present in path', () => { + (checkExecutableIsPresent as jest.MockedFunction< + typeof checkExecutableIsPresent + >).mockImplementationOnce(() => { + throw new Error('Checked for executable'); + }); + + expect(getSymbolCollectorInstance).toThrowErrorMatchingInlineSnapshot( + `"Checked for executable"` + ); + expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); + expect(checkExecutableIsPresent).toHaveBeenCalledWith( + SYM_COLLECTOR_BIN_NAME + ); + }); + + test('config missing', () => { + (checkExecutableIsPresent as jest.MockedFunction< + typeof checkExecutableIsPresent + >) = jest.fn(); + + expect(getSymbolCollectorInstance).toThrowErrorMatchingInlineSnapshot( + '"The required `batchType` parameter is missing in the configuration file. ' + + 'See the documentation for more details."' + ); + expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); + expect(checkExecutableIsPresent).toHaveBeenCalledWith( + SYM_COLLECTOR_BIN_NAME + ); + }); + + test('symbol collector present and config ok', () => { + (checkExecutableIsPresent as jest.MockedFunction< + typeof checkExecutableIsPresent + >) = jest.fn(); + + const symCollector = getSymbolCollectorInstance(customConfig); + const actualConfig = symCollector.symbolCollectorConfig; + expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); + expect(checkExecutableIsPresent).toHaveBeenLastCalledWith( + SYM_COLLECTOR_BIN_NAME + ); + expect(actualConfig).toHaveProperty('serverEndpoint'); + expect(actualConfig).toHaveProperty('batchType'); + expect(actualConfig).toHaveProperty('bundleIdPrefix'); + }); +}); + +describe('publish', () => { + test('no artifacts found', () => { + const symCollector = getSymbolCollectorInstance(customConfig); + symCollector.getArtifactsForRevision = jest + .fn() + .mockReturnValueOnce(() => []); + expect(spawnProcess).not.toHaveBeenCalled(); + }); + + test('with artifacts', async () => { + (withTempDir as jest.MockedFunction).mockImplementation( + async cb => await cb('tmpDir') + ); + (spawnProcess as jest.MockedFunction< + typeof spawnProcess + >).mockImplementation(() => Promise.resolve(undefined)); + + const mockedArtifacts = ['artifact1', 'artifact2', 'artifact3']; + + const symCollector = getSymbolCollectorInstance(customConfig); + symCollector.getArtifactsForRevision = jest + .fn() + .mockReturnValueOnce(mockedArtifacts); + symCollector.artifactProvider.downloadArtifact = jest.fn(); + + await symCollector.publish('version', 'revision'); + + expect(symCollector.getArtifactsForRevision).toHaveBeenCalledTimes(1); + expect( + symCollector.artifactProvider.downloadArtifact + ).toHaveBeenCalledTimes(mockedArtifacts.length); + + expect(spawnProcess).toHaveBeenCalledTimes(1); + const [cmd, args] = (spawnProcess as jest.MockedFunction< + typeof spawnProcess + >).mock.calls[0] as string[]; + expect(cmd).toBe(SYM_COLLECTOR_BIN_NAME); + expect(args).toMatchInlineSnapshot(` + Array [ + "--upload", + "directory", + "--path", + "tmpDir", + "--batch-type", + "batchType", + "--bundle-id", + "bundleIdPrefix-version", + "--server-endpoint", + "https://symbol-collector.services.sentry.io/", + ] + `); + }); +}); diff --git a/src/targets/base.ts b/src/targets/base.ts index 9eba8b182..b8b2d64da 100644 --- a/src/targets/base.ts +++ b/src/targets/base.ts @@ -1,6 +1,10 @@ import { logger as loggerRaw } from '../logger'; import { GithubGlobalConfig, TargetConfig } from '../schemas/project_config'; -import { FilterOptions } from '../artifact_providers/base'; +import { + parseFilterOptions, + RawFilterOptions, + ParsedFilterOptions, +} from '../artifact_providers/base'; import { stringToRegexp } from '../utils/filters'; import { BaseArtifactProvider, @@ -18,7 +22,7 @@ export class BaseTarget { /** Unparsed target configuration */ public readonly config: TargetConfig; /** Artifact filtering options for the target */ - public readonly filterOptions: FilterOptions; + public readonly filterOptions: ParsedFilterOptions; /** Github repo configuration */ public readonly githubRepo?: GithubGlobalConfig; @@ -76,10 +80,10 @@ export class BaseTarget { */ public async getArtifactsForRevision( revision: string, - defaultFilterOptions: FilterOptions = {} + defaultFilterOptions: RawFilterOptions = {} ): Promise { const filterOptions = { - ...defaultFilterOptions, + ...parseFilterOptions(defaultFilterOptions), ...this.filterOptions, }; this.logger.debug( diff --git a/src/targets/index.ts b/src/targets/index.ts index 431c70f6e..ea8548c24 100644 --- a/src/targets/index.ts +++ b/src/targets/index.ts @@ -14,6 +14,7 @@ import { RegistryTarget } from './registry'; import { AwsLambdaLayerTarget } from './awsLambdaLayer'; import { UpmTarget } from './upm'; import { MavenTarget } from './maven'; +import { SymbolCollector } from './symbolCollector'; export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { brew: BrewTarget, @@ -31,6 +32,7 @@ export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { 'aws-lambda-layer': AwsLambdaLayerTarget, upm: UpmTarget, maven: MavenTarget, + 'symbol-collector': SymbolCollector, }; /** Targets that are treated specially */ diff --git a/src/targets/maven.ts b/src/targets/maven.ts index 67c8aa018..fcc6e8098 100644 --- a/src/targets/maven.ts +++ b/src/targets/maven.ts @@ -218,10 +218,7 @@ export class MavenTarget extends BaseTarget { */ public async upload(revision: string): Promise { const artifacts = await this.getArtifactsForRevision(revision, { - includeNames: - this.config.includeNames === undefined - ? undefined - : stringToRegexp(this.config.includeNames), + includeNames: this.config.includeNames, }); // We don't want to do this in parallel but in serial, because the gpg-agent diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts new file mode 100644 index 000000000..e03365984 --- /dev/null +++ b/src/targets/symbolCollector.ts @@ -0,0 +1,108 @@ +import { BaseArtifactProvider } from '../artifact_providers/base'; +import { TargetConfig } from '../schemas/project_config'; +import { ConfigurationError } from '../utils/errors'; +import { BaseTarget } from './base'; +import { withTempDir } from '../utils/files'; +import { promises as fsPromises } from 'fs'; +import { checkExecutableIsPresent, spawnProcess } from '../utils/system'; +import { join } from 'path'; + +const DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT = + 'https://symbol-collector.services.sentry.io/'; +/** + * Name of the binary of the symbol collector. + * Must be available in the path. + */ +export const SYM_COLLECTOR_BIN_NAME = 'SymbolCollector.Console'; + +/** Config options for the "symbol-collector" target. */ +interface SymbolCollectorTargetConfig { + /** Server endpoint to upload symbols. */ + serverEndpoint: string; + /** batch-type of the symbols to be uploaded. */ + batchType: string; + /** Prefix of the bundle ID to be uploaded. */ + bundleIdPrefix: string; +} + +export class SymbolCollector extends BaseTarget { + /** Target name */ + public readonly name: string = 'symbol-collector'; + /** Target options */ + public readonly symbolCollectorConfig: SymbolCollectorTargetConfig; + + public constructor( + config: TargetConfig, + artifactProvider: BaseArtifactProvider + ) { + super(config, artifactProvider); + this.symbolCollectorConfig = this.getSymbolCollectorConfig(); + } + + private getSymbolCollectorConfig(): SymbolCollectorTargetConfig { + // The Symbol Collector should be available in the path + checkExecutableIsPresent(SYM_COLLECTOR_BIN_NAME); + + if (!this.config.batchType) { + throw new ConfigurationError( + 'The required `batchType` parameter is missing in the configuration file. ' + + 'See the documentation for more details.' + ); + } + if (!this.config.bundleIdPrefix) { + throw new ConfigurationError( + 'The required `bundleIdPrefix` parameter is missing in the configuration file. ' + + 'See the documentation for more details.' + ); + } + + return { + serverEndpoint: + this.config.serverEndpoint || DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT, + batchType: this.config.batchType, + bundleIdPrefix: this.config.bundleIdPrefix, + }; + } + + public async publish(version: string, revision: string): Promise { + const bundleId = this.symbolCollectorConfig.bundleIdPrefix + version; + const artifacts = await this.getArtifactsForRevision(revision, { + includeNames: this.config.includeNames, + }); + + if (artifacts.length === 0) { + this.logger.warn(`Didn't found any artifacts after filtering`); + return; + } + + this.logger.debug(`Found ${artifacts.length} symbol artifacts.`); + + await withTempDir(async dir => { + // Download all artifacts in the same parent directory, where the symbol + // collector will recursively look for and deal with them. + // Since there are files with the same name, download them in different + // directories. + this.logger.debug('Downloading artifacts...'); + await Promise.all( + artifacts.map(async (artifact, index) => { + const subdirPath = join(dir, String(index)); + await fsPromises.mkdir(subdirPath); + await this.artifactProvider.downloadArtifact(artifact, subdirPath); + }) + ); + + await spawnProcess(SYM_COLLECTOR_BIN_NAME, [ + '--upload', + 'directory', + '--path', + dir, + '--batch-type', + this.symbolCollectorConfig.batchType, + '--bundle-id', + bundleId, + '--server-endpoint', + this.symbolCollectorConfig.serverEndpoint, + ]); + }); + } +}