From 2cb1b1557860ff9e8ecd089a29488087fa96f482 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 5 Jul 2021 11:17:28 +0200 Subject: [PATCH 01/36] Initial target code --- src/targets/index.ts | 2 ++ src/targets/javaSymbols.ts | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/targets/javaSymbols.ts diff --git a/src/targets/index.ts b/src/targets/index.ts index 442843a82..6dd19fb7a 100644 --- a/src/targets/index.ts +++ b/src/targets/index.ts @@ -13,6 +13,7 @@ import { PypiTarget } from './pypi'; import { RegistryTarget } from './registry'; import { AwsLambdaLayerTarget } from './awsLambdaLayer'; import { UpmTarget } from './upm'; +import { JavaSymbols } from './javaSymbols'; export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { brew: BrewTarget, @@ -29,6 +30,7 @@ export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { registry: RegistryTarget, 'aws-lambda-layer': AwsLambdaLayerTarget, upm: UpmTarget, + 'java-symbols': JavaSymbols, }; /** Targets that are treated specially */ diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts new file mode 100644 index 000000000..6730f3c28 --- /dev/null +++ b/src/targets/javaSymbols.ts @@ -0,0 +1,43 @@ +import { BaseArtifactProvider } from '../artifact_providers/base'; +import { TargetConfig } from '../schemas/project_config'; +import { ConfigurationError } from '../utils/errors'; +import { BaseTarget } from './base'; + +/** Config options for the "java-symbols" target. */ +interface JavaSymbolsTargetConfig { + serverEndpoint: string; + batchType: string; +} + +export class JavaSymbols extends BaseTarget { + /** Target name */ + public readonly name: string = 'java-symbols'; + /** Target options */ + public readonly javaSymbolsConfig: JavaSymbolsTargetConfig; + + public constructor( + config: TargetConfig, + artifactProvider: BaseArtifactProvider + ) { + super(config, artifactProvider); + this.javaSymbolsConfig = this.getJavaSymbolsConfig(); + } + + private getJavaSymbolsConfig(): JavaSymbolsTargetConfig { + if (!this.config.serverEndpoint || !this.config.batchType) { + throw new ConfigurationError( + 'Required configuration not found in configuration file. ' + + 'See the documentation for more details.' + ); + } + return { + serverEndpoint: this.config.serverEndpoint, + batchType: this.config.batchType, + }; + } + + public async publish(version: string, revision: string): Promise { + console.log(version); + console.log(revision); + } +} From 208575c67f23a8589803b5f732c2ca1587636f34 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:51:32 +0200 Subject: [PATCH 02/36] Download artifacts in subdirs of tmp dir --- src/targets/javaSymbols.ts | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index 6730f3c28..84b3cedf3 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -1,10 +1,15 @@ +import { checkEnvForPrerequisite } from '../utils/env'; +import { stringToRegexp } from '../utils/filters'; import { BaseArtifactProvider } from '../artifact_providers/base'; import { TargetConfig } from '../schemas/project_config'; import { ConfigurationError } from '../utils/errors'; import { BaseTarget } from './base'; +import { withTempDir } from '../utils/files'; +import { promises as fsPromises } from 'fs'; /** Config options for the "java-symbols" target. */ interface JavaSymbolsTargetConfig { + symbolCollectorPath: string; serverEndpoint: string; batchType: string; } @@ -24,6 +29,8 @@ export class JavaSymbols extends BaseTarget { } private getJavaSymbolsConfig(): JavaSymbolsTargetConfig { + checkEnvForPrerequisite({ name: 'SYMBOL_COLLECTOR_PATH' }); + if (!this.config.serverEndpoint || !this.config.batchType) { throw new ConfigurationError( 'Required configuration not found in configuration file. ' + @@ -31,13 +38,30 @@ export class JavaSymbols extends BaseTarget { ); } return { + symbolCollectorPath: process.env.SYMBOL_COLLECTOR_PATH || '', // || to make TS happy serverEndpoint: this.config.serverEndpoint, batchType: this.config.batchType, }; } - public async publish(version: string, revision: string): Promise { - console.log(version); - console.log(revision); + public async publish(_version: string, revision: string): Promise { + const artifacts = await this.getArtifactsForRevision(revision, { + includeNames: + this.config.includeNames === undefined + ? undefined + : stringToRegexp(this.config.includeNames), + }); + + await withTempDir(async dir => { + // Download all artifacts in the same parent directory, where the symbol + // collector will look for and deal with them. + // Do it in different subdirectories, since some files have the same name. + artifacts.map(async (artifact, index) => { + const subdirPath = dir + '/' + index; + await fsPromises.mkdir(subdirPath); + this.artifactProvider.downloadArtifact(artifact, subdirPath); + }); + // TODO: run command to upload + }); } } From 205b7950b1310b510fa0e4cf7900dc44d5f7bbf6 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 5 Jul 2021 15:58:12 +0200 Subject: [PATCH 03/36] Add `bundleIdPrefix` to config --- src/targets/javaSymbols.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index 84b3cedf3..5e621693a 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -12,6 +12,7 @@ interface JavaSymbolsTargetConfig { symbolCollectorPath: string; serverEndpoint: string; batchType: string; + bundleIdPrefix: string; } export class JavaSymbols extends BaseTarget { @@ -31,16 +32,22 @@ export class JavaSymbols extends BaseTarget { private getJavaSymbolsConfig(): JavaSymbolsTargetConfig { checkEnvForPrerequisite({ name: 'SYMBOL_COLLECTOR_PATH' }); - if (!this.config.serverEndpoint || !this.config.batchType) { + if ( + !this.config.serverEndpoint || + !this.config.batchType || + !this.config.bundleIdPrefix + ) { throw new ConfigurationError( 'Required configuration not found in configuration file. ' + 'See the documentation for more details.' ); } + return { symbolCollectorPath: process.env.SYMBOL_COLLECTOR_PATH || '', // || to make TS happy serverEndpoint: this.config.serverEndpoint, batchType: this.config.batchType, + bundleIdPrefix: this.config.bundleIdPrefix, }; } From d401e2f9a6defbfcd268574abf3afb390ec15c39 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 5 Jul 2021 15:58:54 +0200 Subject: [PATCH 04/36] Spawn process to call the symbol collector --- src/targets/javaSymbols.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index 5e621693a..4122bb0b8 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -6,6 +6,7 @@ import { ConfigurationError } from '../utils/errors'; import { BaseTarget } from './base'; import { withTempDir } from '../utils/files'; import { promises as fsPromises } from 'fs'; +import { spawnProcess } from '../utils/system'; /** Config options for the "java-symbols" target. */ interface JavaSymbolsTargetConfig { @@ -51,7 +52,9 @@ export class JavaSymbols extends BaseTarget { }; } - public async publish(_version: string, revision: string): Promise { + public async publish(version: string, revision: string): Promise { + const bundleId = this.javaSymbolsConfig.bundleIdPrefix + `${version}`; + const artifacts = await this.getArtifactsForRevision(revision, { includeNames: this.config.includeNames === undefined @@ -68,7 +71,18 @@ export class JavaSymbols extends BaseTarget { await fsPromises.mkdir(subdirPath); this.artifactProvider.downloadArtifact(artifact, subdirPath); }); - // TODO: run command to upload + spawnProcess(this.javaSymbolsConfig.symbolCollectorPath, [ + '--upload', + 'directory', + '--path', + dir, + '--batch-type', + this.javaSymbolsConfig.batchType, + '--bundle-id', + bundleId, + '--server-endpoint', + this.javaSymbolsConfig.serverEndpoint, + ]); }); } } From 7eaa941951fd67c65ea93721d05f4cde3dc44fd1 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:28:20 +0200 Subject: [PATCH 05/36] Add GitHub API release-related helper methods --- src/utils/githubApi.ts | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/utils/githubApi.ts b/src/utils/githubApi.ts index 19c283d14..82c245591 100644 --- a/src/utils/githubApi.ts +++ b/src/utils/githubApi.ts @@ -23,6 +23,8 @@ export class GithubRemote { /** Url in the form of /OWNER/REPO/ */ protected readonly url: string; + protected readonly github: Github; + public constructor( owner: string, repo: string, @@ -35,6 +37,7 @@ export class GithubRemote { this.setAuth(username, apiToken); } this.url = `/${this.owner}/${this.repo}/`; + this.github = new Github(); } /** @@ -69,6 +72,44 @@ export class GithubRemote { : ''; return this.PROTOCOL_PREFIX + authData + this.GITHUB_HOSTNAME + this.url; } + + public async getLatestRelease(): Promise { + const release = await this.github.repos.getLatestRelease({ + owner: this.owner, + repo: this.repo, + }); + return release.data; + } + + public async getReleaseByTag(tag: string): Promise { + const release = await this.github.repos.getReleaseByTag({ + owner: this.owner, + repo: this.repo, + tag: tag, + }); + return release.data; + } + + public async listReleaseAssets(releaseId: number): Promise { + const releaseAssets = await this.github.repos.listAssetsForRelease({ + owner: this.owner, + repo: this.repo, + release_id: releaseId, + }); + return releaseAssets.data; + } + + public async getAsset(assetId: number): Promise { + const asset = await this.github.repos.getReleaseAsset({ + owner: this.owner, + repo: this.repo, + asset_id: assetId, + headers: { + accept: 'application/octet-stream', + }, + }); + return asset.data; + } } /** From 01c98dff4748438b6bbcb5b2688b2d63330ae68e Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:29:01 +0200 Subject: [PATCH 06/36] Add system helper method to make files executable --- src/utils/system.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/utils/system.ts b/src/utils/system.ts index d8904a587..1df02bd41 100644 --- a/src/utils/system.ts +++ b/src/utils/system.ts @@ -273,6 +273,18 @@ function isExecutable(filePath: string): boolean { } } +export function makeExecutable(filePath: string): boolean { + if (isExecutable(filePath)) { + return true; + } + try { + fs.chmodSync(filePath, fs.constants.F_OK | fs.constants.X_OK); + } catch (e) { + return false; + } + return true; +} + /** * Checks if the provided executable is available * From 6e780f37e055ed0f69686e838d3377bcb384996f Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:58:00 +0200 Subject: [PATCH 07/36] Update target config params --- src/targets/javaSymbols.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index 4122bb0b8..a19c60c8f 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -10,10 +10,13 @@ import { spawnProcess } from '../utils/system'; /** Config options for the "java-symbols" target. */ interface JavaSymbolsTargetConfig { - symbolCollectorPath: string; serverEndpoint: string; batchType: string; bundleIdPrefix: string; + useLatestSymCollectorRelease: boolean; + releaseTag: string; + symCollectorAssetName: string; + binaryName: string; } export class JavaSymbols extends BaseTarget { @@ -45,10 +48,14 @@ export class JavaSymbols extends BaseTarget { } return { - symbolCollectorPath: process.env.SYMBOL_COLLECTOR_PATH || '', // || to make TS happy serverEndpoint: this.config.serverEndpoint, batchType: this.config.batchType, bundleIdPrefix: this.config.bundleIdPrefix, + // TODO: read config params below from the config file + useLatestSymCollectorRelease: true, + releaseTag: '1.3.1', + symCollectorAssetName: 'symbolcollector-console-linux-x64.zip', + binaryName: 'SymbolCollector.Console', }; } From e3f4d6d64e88d9b2ffb280375146bb8c8349cedc Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Tue, 6 Jul 2021 22:19:11 +0200 Subject: [PATCH 08/36] Add GitHub client --- src/targets/javaSymbols.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index a19c60c8f..b579853c2 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -7,6 +7,7 @@ import { BaseTarget } from './base'; import { withTempDir } from '../utils/files'; import { promises as fsPromises } from 'fs'; import { spawnProcess } from '../utils/system'; +import { GithubRemote } from '../utils/githubApi'; /** Config options for the "java-symbols" target. */ interface JavaSymbolsTargetConfig { @@ -25,11 +26,15 @@ export class JavaSymbols extends BaseTarget { /** Target options */ public readonly javaSymbolsConfig: JavaSymbolsTargetConfig; + public readonly github: GithubRemote; + public constructor( config: TargetConfig, artifactProvider: BaseArtifactProvider ) { super(config, artifactProvider); + // TODO: don't hardcode repo's data + this.github = new GithubRemote('getsentry', 'symbol-collector'); this.javaSymbolsConfig = this.getJavaSymbolsConfig(); } From 72488d6f0dffbab1e57b91fd619b03ce42693dac Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Tue, 6 Jul 2021 22:27:10 +0200 Subject: [PATCH 09/36] Download and use symbol collector Instead of requiring having it installed locally, download the symbol collector and use the binary. --- src/targets/javaSymbols.ts | 75 +++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index b579853c2..79907890f 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -2,11 +2,16 @@ import { checkEnvForPrerequisite } from '../utils/env'; import { stringToRegexp } from '../utils/filters'; import { BaseArtifactProvider } from '../artifact_providers/base'; import { TargetConfig } from '../schemas/project_config'; -import { ConfigurationError } from '../utils/errors'; +import { ConfigurationError, reportError } from '../utils/errors'; import { BaseTarget } from './base'; import { withTempDir } from '../utils/files'; import { promises as fsPromises } from 'fs'; -import { spawnProcess } from '../utils/system'; +import { + extractZipArchive, + makeExecutable, + spawnProcess, +} from '../utils/system'; +import { join } from 'path'; import { GithubRemote } from '../utils/githubApi'; /** Config options for the "java-symbols" target. */ @@ -75,6 +80,12 @@ export class JavaSymbols extends BaseTarget { }); await withTempDir(async dir => { + const collectorDir = join(dir, 'collector'); + await fsPromises.mkdir(collectorDir); + const symbolCollectorPath = await this.downloadSymbolCollector( + collectorDir + ); + // Download all artifacts in the same parent directory, where the symbol // collector will look for and deal with them. // Do it in different subdirectories, since some files have the same name. @@ -97,4 +108,64 @@ export class JavaSymbols extends BaseTarget { ]); }); } + + private async downloadSymbolCollector(dir: string): Promise { + // Currently, GitHub doesn't offer an API to download the asset of a + // release by its name, and the asset ID must be provided. The workaround + // is to get the release ID where the assets are and look for all the assets + // until there's one matching the name to get its ID + const assetDownloadId = await this.getAssetDownloadId(); + const assetDstPath = await this.downloadAsset(assetDownloadId, dir); + this.logger.debug('Extracting asset...'); + await extractZipArchive(assetDstPath, dir); + + const binaryPath = join(dir, this.javaSymbolsConfig.binaryName); + this.makeBinaryExecutable(binaryPath); + return binaryPath; + } + + private async getAssetDownloadId(): Promise { + const releaseId = await this.getReleaseId(); + this.logger.debug('Fetching release assets...'); + const releaseAssets = await this.github.listReleaseAssets(releaseId); + const matchingAssets = releaseAssets.filter( + asset => asset.name === this.javaSymbolsConfig.symCollectorAssetName + ); + if (matchingAssets.length != 1) { + reportError(`Found ${matchingAssets.length} assets, 1 expected.`); + } + const assetId = matchingAssets[0].id; + this.logger.debug('Found asset to download: ', assetId); + return assetId; + } + + private async getReleaseId(): Promise { + this.logger.debug('Fetching the release...'); + const targetRelease = this.javaSymbolsConfig.useLatestSymCollectorRelease + ? await this.github.getLatestRelease() + : await this.github.getReleaseByTag(this.javaSymbolsConfig.releaseTag); + this.logger.debug('Fetched release: ', targetRelease.id); + return targetRelease.id; + } + + private async downloadAsset(assetId: number, dir: string): Promise { + this.logger.debug('Fetching the asset to download...'); + const assetDataBuffer = await this.github.getAsset(assetId); + const assetDstPath = join( + dir, + this.javaSymbolsConfig.symCollectorAssetName + ); + this.logger.debug('Downloading asset to: ', assetDstPath); + await fsPromises.appendFile(assetDstPath, Buffer.from(assetDataBuffer)); + return assetDstPath; + } + + private makeBinaryExecutable(binaryPath: string): void { + const isExecutablePresent = makeExecutable(binaryPath); + if (!isExecutablePresent) { + throw new ConfigurationError( + 'Cannot access to the binary declared in the config file: ' + binaryPath + ); + } + } } From 5f3997bd09dfeb2cc026c7374c60fb943e10adff Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Tue, 6 Jul 2021 22:29:23 +0200 Subject: [PATCH 10/36] Parallelize symbol download and place them in a subdir --- src/targets/javaSymbols.ts | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index 79907890f..e53c20f38 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -86,19 +86,27 @@ export class JavaSymbols extends BaseTarget { collectorDir ); + const symbolsPath = join(dir, 'symbols'); + await fsPromises.mkdir(symbolsPath); + // Download all artifacts in the same parent directory, where the symbol - // collector will look for and deal with them. - // Do it in different subdirectories, since some files have the same name. - artifacts.map(async (artifact, index) => { - const subdirPath = dir + '/' + index; - await fsPromises.mkdir(subdirPath); - this.artifactProvider.downloadArtifact(artifact, subdirPath); - }); - spawnProcess(this.javaSymbolsConfig.symbolCollectorPath, [ + // collector will recursively look for and deal with them. + // Since there are files with the same name, download them in different + // directories. + this.logger.debug('Downloading artifacts...'); + await Promise.all( + artifacts.map(async (artifact, index) => { + const subdirPath = join(symbolsPath, index + ''); + await fsPromises.mkdir(subdirPath); + await this.artifactProvider.downloadArtifact(artifact, subdirPath); + }) + ); + + await spawnProcess(symbolCollectorPath, [ '--upload', 'directory', '--path', - dir, + symbolsPath, '--batch-type', this.javaSymbolsConfig.batchType, '--bundle-id', From 2414f34bee276452a8f7d978837c0bfb10fa92fb Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Tue, 6 Jul 2021 22:30:27 +0200 Subject: [PATCH 11/36] Add debug logs --- src/targets/javaSymbols.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index e53c20f38..085894294 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -71,13 +71,14 @@ export class JavaSymbols extends BaseTarget { public async publish(version: string, revision: string): Promise { const bundleId = this.javaSymbolsConfig.bundleIdPrefix + `${version}`; - + this.logger.debug('Fetching artifacts...'); const artifacts = await this.getArtifactsForRevision(revision, { includeNames: this.config.includeNames === undefined ? undefined : stringToRegexp(this.config.includeNames), }); + this.logger.debug(`Found ${artifacts.length} symbol artifacts.`); await withTempDir(async dir => { const collectorDir = join(dir, 'collector'); From fbd8fba92bb0ed3a40de2abb0933c03671ac69ea Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Wed, 7 Jul 2021 11:51:19 +0200 Subject: [PATCH 12/36] Add docs to the README --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 4dc05da90..5d8c54b6e 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ then enforces a specific workflow for managing release branches, changelogs, art - [Ruby Gems Index (`gem`)](#ruby-gems-index-gem) - [AWS Lambda Layer (`aws-lambda-layer`)](#aws-lambda-layer-aws-lambda-layer) - [Unity Package Manager (`upm`)](#unity-package-manager-upm) + - [Java Symbols (`java-symbols`)](#java-symbols-java-symbols) - [Integrating Your Project with `craft`](#integrating-your-project-with-craft) - [Pre-release (Version-bumping) Script: Conventions](#pre-release-version-bumping-script-conventions) - [Post-release Script: Conventions](#post-release-script-conventions) @@ -988,6 +989,29 @@ targets: releaseRepoName: 'unity' ``` +### Java Symbols (`java-symbols`) + +Using the [`symbol-collector`](https://github.com/getsentry/symbol-collector) client, uploads native symbols. + +**Configuration** + +| Option | Description | +| ---------------- | ---------------------------------------------------------------------------------------- | +| `serverEndpoint` | The server endpoint. | +| `batchType` | The batch type. | +| `bundleIdPrefix` | The prefix of the bundle ID. The new version will be appended to the end of this prefix. | + +**Example** + +```yaml +targets: + - name: java-symbols + includeNames: /libsentry(-android)?\.so/ + serverEndpoint: my-server.com + batchType: Android + bundleIdPrefix: android-ndk- +``` + ## Integrating Your Project with `craft` Here is how you can integrate your GitHub project with `craft`: From 4bf5addaeae3e9a4fba4c6b7569dc80c08746433 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Thu, 8 Jul 2021 11:02:55 +0200 Subject: [PATCH 13/36] Update target config requirements --- src/targets/javaSymbols.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index 085894294..7d1d6e29c 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -47,9 +47,9 @@ export class JavaSymbols extends BaseTarget { checkEnvForPrerequisite({ name: 'SYMBOL_COLLECTOR_PATH' }); if ( - !this.config.serverEndpoint || !this.config.batchType || - !this.config.bundleIdPrefix + !this.config.bundleIdPrefix || + !(this.config.useLatestSymCollectorRelease || this.config.releaseTag) ) { throw new ConfigurationError( 'Required configuration not found in configuration file. ' + @@ -61,11 +61,9 @@ export class JavaSymbols extends BaseTarget { serverEndpoint: this.config.serverEndpoint, batchType: this.config.batchType, bundleIdPrefix: this.config.bundleIdPrefix, - // TODO: read config params below from the config file - useLatestSymCollectorRelease: true, - releaseTag: '1.3.1', - symCollectorAssetName: 'symbolcollector-console-linux-x64.zip', - binaryName: 'SymbolCollector.Console', + useLatestSymCollectorRelease: this.config.useLatestSymCollectorRelease, + releaseTag: this.config.releaseTag, + symCollectorAssetName: 'symbolcollector-console-linux-x64.zip', // TODO: set default }; } From c71ed5aa0d609e8476318c5ca4f810fe340407db Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Thu, 8 Jul 2021 11:06:03 +0200 Subject: [PATCH 14/36] Set default values for some target config params --- src/targets/javaSymbols.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/targets/javaSymbols.ts b/src/targets/javaSymbols.ts index 7d1d6e29c..abc8905cc 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/javaSymbols.ts @@ -14,6 +14,12 @@ import { import { join } from 'path'; import { GithubRemote } from '../utils/githubApi'; +const DEFAULT_SYM_COLLECTOR_ENDPOINT = + 'https://symbol-collector.services.sentry.io/'; +const DEFAULT_SYM_COLLECTOR_ASSET_NAME = + 'symbolcollector-console-linux-x64.zip'; +const DEFAULT_SYM_COLLECTOR_FILENAME = 'SymbolCollector.Console'; + /** Config options for the "java-symbols" target. */ interface JavaSymbolsTargetConfig { serverEndpoint: string; @@ -58,12 +64,14 @@ export class JavaSymbols extends BaseTarget { } return { - serverEndpoint: this.config.serverEndpoint, + serverEndpoint: + this.config.serverEndpoint || DEFAULT_SYM_COLLECTOR_ENDPOINT, batchType: this.config.batchType, bundleIdPrefix: this.config.bundleIdPrefix, useLatestSymCollectorRelease: this.config.useLatestSymCollectorRelease, releaseTag: this.config.releaseTag, - symCollectorAssetName: 'symbolcollector-console-linux-x64.zip', // TODO: set default + symCollectorAssetName: DEFAULT_SYM_COLLECTOR_ASSET_NAME, + binaryName: this.config.binaryName || DEFAULT_SYM_COLLECTOR_FILENAME, }; } From 5a56941b0be9f17792aa1ba7f1f818e51e512e4f Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Thu, 8 Jul 2021 11:13:31 +0200 Subject: [PATCH 15/36] Rename target from `java-symbols` to `native-symbols` --- src/targets/index.ts | 4 ++-- src/targets/{javaSymbols.ts => nativeSymbols.ts} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename src/targets/{javaSymbols.ts => nativeSymbols.ts} (97%) diff --git a/src/targets/index.ts b/src/targets/index.ts index 6dd19fb7a..75bf1f3b4 100644 --- a/src/targets/index.ts +++ b/src/targets/index.ts @@ -13,7 +13,7 @@ import { PypiTarget } from './pypi'; import { RegistryTarget } from './registry'; import { AwsLambdaLayerTarget } from './awsLambdaLayer'; import { UpmTarget } from './upm'; -import { JavaSymbols } from './javaSymbols'; +import { NativeSymbols } from './nativeSymbols'; export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { brew: BrewTarget, @@ -30,7 +30,7 @@ export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { registry: RegistryTarget, 'aws-lambda-layer': AwsLambdaLayerTarget, upm: UpmTarget, - 'java-symbols': JavaSymbols, + 'native-symbols': NativeSymbols, }; /** Targets that are treated specially */ diff --git a/src/targets/javaSymbols.ts b/src/targets/nativeSymbols.ts similarity index 97% rename from src/targets/javaSymbols.ts rename to src/targets/nativeSymbols.ts index abc8905cc..048e1b293 100644 --- a/src/targets/javaSymbols.ts +++ b/src/targets/nativeSymbols.ts @@ -20,7 +20,7 @@ const DEFAULT_SYM_COLLECTOR_ASSET_NAME = 'symbolcollector-console-linux-x64.zip'; const DEFAULT_SYM_COLLECTOR_FILENAME = 'SymbolCollector.Console'; -/** Config options for the "java-symbols" target. */ +/** Config options for the "native-symbols" target. */ interface JavaSymbolsTargetConfig { serverEndpoint: string; batchType: string; @@ -31,9 +31,9 @@ interface JavaSymbolsTargetConfig { binaryName: string; } -export class JavaSymbols extends BaseTarget { +export class NativeSymbols extends BaseTarget { /** Target name */ - public readonly name: string = 'java-symbols'; + public readonly name: string = 'native-symbols'; /** Target options */ public readonly javaSymbolsConfig: JavaSymbolsTargetConfig; From 553309f66efed201f629a6d380a8e0a199c06f06 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Thu, 8 Jul 2021 11:14:17 +0200 Subject: [PATCH 16/36] Remove old requirement of old env var --- src/targets/nativeSymbols.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/targets/nativeSymbols.ts b/src/targets/nativeSymbols.ts index 048e1b293..2693a9c89 100644 --- a/src/targets/nativeSymbols.ts +++ b/src/targets/nativeSymbols.ts @@ -1,4 +1,3 @@ -import { checkEnvForPrerequisite } from '../utils/env'; import { stringToRegexp } from '../utils/filters'; import { BaseArtifactProvider } from '../artifact_providers/base'; import { TargetConfig } from '../schemas/project_config'; @@ -50,8 +49,6 @@ export class NativeSymbols extends BaseTarget { } private getJavaSymbolsConfig(): JavaSymbolsTargetConfig { - checkEnvForPrerequisite({ name: 'SYMBOL_COLLECTOR_PATH' }); - if ( !this.config.batchType || !this.config.bundleIdPrefix || From f6faf72129541271ab85d3f79238c5c8d51e8087 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Thu, 8 Jul 2021 11:32:09 +0200 Subject: [PATCH 17/36] Rename target config from `java*` to `native*` --- src/targets/nativeSymbols.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/targets/nativeSymbols.ts b/src/targets/nativeSymbols.ts index 2693a9c89..3da27bb2f 100644 --- a/src/targets/nativeSymbols.ts +++ b/src/targets/nativeSymbols.ts @@ -20,7 +20,7 @@ const DEFAULT_SYM_COLLECTOR_ASSET_NAME = const DEFAULT_SYM_COLLECTOR_FILENAME = 'SymbolCollector.Console'; /** Config options for the "native-symbols" target. */ -interface JavaSymbolsTargetConfig { +interface NativeSymbolsTargetConfig { serverEndpoint: string; batchType: string; bundleIdPrefix: string; @@ -34,7 +34,7 @@ export class NativeSymbols extends BaseTarget { /** Target name */ public readonly name: string = 'native-symbols'; /** Target options */ - public readonly javaSymbolsConfig: JavaSymbolsTargetConfig; + public readonly nativeSymbolsConfig: NativeSymbolsTargetConfig; public readonly github: GithubRemote; @@ -45,10 +45,10 @@ export class NativeSymbols extends BaseTarget { super(config, artifactProvider); // TODO: don't hardcode repo's data this.github = new GithubRemote('getsentry', 'symbol-collector'); - this.javaSymbolsConfig = this.getJavaSymbolsConfig(); + this.nativeSymbolsConfig = this.getNativeSymbolsConfig(); } - private getJavaSymbolsConfig(): JavaSymbolsTargetConfig { + private getNativeSymbolsConfig(): NativeSymbolsTargetConfig { if ( !this.config.batchType || !this.config.bundleIdPrefix || @@ -73,7 +73,7 @@ export class NativeSymbols extends BaseTarget { } public async publish(version: string, revision: string): Promise { - const bundleId = this.javaSymbolsConfig.bundleIdPrefix + `${version}`; + const bundleId = this.nativeSymbolsConfig.bundleIdPrefix + `${version}`; this.logger.debug('Fetching artifacts...'); const artifacts = await this.getArtifactsForRevision(revision, { includeNames: @@ -112,11 +112,11 @@ export class NativeSymbols extends BaseTarget { '--path', symbolsPath, '--batch-type', - this.javaSymbolsConfig.batchType, + this.nativeSymbolsConfig.batchType, '--bundle-id', bundleId, '--server-endpoint', - this.javaSymbolsConfig.serverEndpoint, + this.nativeSymbolsConfig.serverEndpoint, ]); }); } @@ -131,7 +131,7 @@ export class NativeSymbols extends BaseTarget { this.logger.debug('Extracting asset...'); await extractZipArchive(assetDstPath, dir); - const binaryPath = join(dir, this.javaSymbolsConfig.binaryName); + const binaryPath = join(dir, this.nativeSymbolsConfig.binaryName); this.makeBinaryExecutable(binaryPath); return binaryPath; } @@ -141,7 +141,7 @@ export class NativeSymbols extends BaseTarget { this.logger.debug('Fetching release assets...'); const releaseAssets = await this.github.listReleaseAssets(releaseId); const matchingAssets = releaseAssets.filter( - asset => asset.name === this.javaSymbolsConfig.symCollectorAssetName + asset => asset.name === this.nativeSymbolsConfig.symCollectorAssetName ); if (matchingAssets.length != 1) { reportError(`Found ${matchingAssets.length} assets, 1 expected.`); @@ -153,9 +153,9 @@ export class NativeSymbols extends BaseTarget { private async getReleaseId(): Promise { this.logger.debug('Fetching the release...'); - const targetRelease = this.javaSymbolsConfig.useLatestSymCollectorRelease + const targetRelease = this.nativeSymbolsConfig.useLatestSymCollectorRelease ? await this.github.getLatestRelease() - : await this.github.getReleaseByTag(this.javaSymbolsConfig.releaseTag); + : await this.github.getReleaseByTag(this.nativeSymbolsConfig.releaseTag); this.logger.debug('Fetched release: ', targetRelease.id); return targetRelease.id; } @@ -165,7 +165,7 @@ export class NativeSymbols extends BaseTarget { const assetDataBuffer = await this.github.getAsset(assetId); const assetDstPath = join( dir, - this.javaSymbolsConfig.symCollectorAssetName + this.nativeSymbolsConfig.symCollectorAssetName ); this.logger.debug('Downloading asset to: ', assetDstPath); await fsPromises.appendFile(assetDstPath, Buffer.from(assetDataBuffer)); From 186f00b8b3c6ed43b9e7c6c9ec1fc58911a067b6 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Thu, 8 Jul 2021 11:38:12 +0200 Subject: [PATCH 18/36] Add docstrings --- src/targets/nativeSymbols.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/targets/nativeSymbols.ts b/src/targets/nativeSymbols.ts index 3da27bb2f..2e1018348 100644 --- a/src/targets/nativeSymbols.ts +++ b/src/targets/nativeSymbols.ts @@ -13,7 +13,7 @@ import { import { join } from 'path'; import { GithubRemote } from '../utils/githubApi'; -const DEFAULT_SYM_COLLECTOR_ENDPOINT = +const DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT = 'https://symbol-collector.services.sentry.io/'; const DEFAULT_SYM_COLLECTOR_ASSET_NAME = 'symbolcollector-console-linux-x64.zip'; @@ -21,12 +21,22 @@ const DEFAULT_SYM_COLLECTOR_FILENAME = 'SymbolCollector.Console'; /** Config options for the "native-symbols" target. */ interface NativeSymbolsTargetConfig { + /** Server endpoint to upload symbols. */ serverEndpoint: string; + /** batch-type of the symbols to be uploaded. */ batchType: string; + /** Prefix of the bundle ID to be uploaded. */ bundleIdPrefix: string; + /** Whether to use the latest Symbol Collector release. */ useLatestSymCollectorRelease: boolean; + /** Tag of the release of the Symbol Collector that should be used. */ releaseTag: string; + /** + * Name of the asset in the release where the Symbol Collector + * binary should be found. + */ symCollectorAssetName: string; + /** Name of the Symbol Collector binary, inside the asset. */ binaryName: string; } @@ -62,7 +72,7 @@ export class NativeSymbols extends BaseTarget { return { serverEndpoint: - this.config.serverEndpoint || DEFAULT_SYM_COLLECTOR_ENDPOINT, + this.config.serverEndpoint || DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT, batchType: this.config.batchType, bundleIdPrefix: this.config.bundleIdPrefix, useLatestSymCollectorRelease: this.config.useLatestSymCollectorRelease, From 6d44b00e4f4568699663fe78536fe9eed3c56056 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 09:55:07 +0200 Subject: [PATCH 19/36] Rename target to `symbol-collector` --- src/targets/index.ts | 4 +-- .../{nativeSymbols.ts => symbolCollector.ts} | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) rename src/targets/{nativeSymbols.ts => symbolCollector.ts} (86%) diff --git a/src/targets/index.ts b/src/targets/index.ts index 75bf1f3b4..e5d993282 100644 --- a/src/targets/index.ts +++ b/src/targets/index.ts @@ -13,7 +13,7 @@ import { PypiTarget } from './pypi'; import { RegistryTarget } from './registry'; import { AwsLambdaLayerTarget } from './awsLambdaLayer'; import { UpmTarget } from './upm'; -import { NativeSymbols } from './nativeSymbols'; +import { SymbolCollector } from './symbolCollector'; export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { brew: BrewTarget, @@ -30,7 +30,7 @@ export const TARGET_MAP: { [key: string]: typeof BaseTarget } = { registry: RegistryTarget, 'aws-lambda-layer': AwsLambdaLayerTarget, upm: UpmTarget, - 'native-symbols': NativeSymbols, + 'symbol-collector': SymbolCollector, }; /** Targets that are treated specially */ diff --git a/src/targets/nativeSymbols.ts b/src/targets/symbolCollector.ts similarity index 86% rename from src/targets/nativeSymbols.ts rename to src/targets/symbolCollector.ts index 2e1018348..2170290d5 100644 --- a/src/targets/nativeSymbols.ts +++ b/src/targets/symbolCollector.ts @@ -19,8 +19,8 @@ const DEFAULT_SYM_COLLECTOR_ASSET_NAME = 'symbolcollector-console-linux-x64.zip'; const DEFAULT_SYM_COLLECTOR_FILENAME = 'SymbolCollector.Console'; -/** Config options for the "native-symbols" target. */ -interface NativeSymbolsTargetConfig { +/** Config options for the "symbol-collector" target. */ +interface SymbolCollectorTargetConfig { /** Server endpoint to upload symbols. */ serverEndpoint: string; /** batch-type of the symbols to be uploaded. */ @@ -40,11 +40,11 @@ interface NativeSymbolsTargetConfig { binaryName: string; } -export class NativeSymbols extends BaseTarget { +export class SymbolCollector extends BaseTarget { /** Target name */ - public readonly name: string = 'native-symbols'; + public readonly name: string = 'symbol-collector'; /** Target options */ - public readonly nativeSymbolsConfig: NativeSymbolsTargetConfig; + public readonly symbolCollectorConfig: SymbolCollectorTargetConfig; public readonly github: GithubRemote; @@ -55,10 +55,10 @@ export class NativeSymbols extends BaseTarget { super(config, artifactProvider); // TODO: don't hardcode repo's data this.github = new GithubRemote('getsentry', 'symbol-collector'); - this.nativeSymbolsConfig = this.getNativeSymbolsConfig(); + this.symbolCollectorConfig = this.getSymbolCollectorConfig(); } - private getNativeSymbolsConfig(): NativeSymbolsTargetConfig { + private getSymbolCollectorConfig(): SymbolCollectorTargetConfig { if ( !this.config.batchType || !this.config.bundleIdPrefix || @@ -83,7 +83,7 @@ export class NativeSymbols extends BaseTarget { } public async publish(version: string, revision: string): Promise { - const bundleId = this.nativeSymbolsConfig.bundleIdPrefix + `${version}`; + const bundleId = this.symbolCollectorConfig.bundleIdPrefix + `${version}`; this.logger.debug('Fetching artifacts...'); const artifacts = await this.getArtifactsForRevision(revision, { includeNames: @@ -122,11 +122,11 @@ export class NativeSymbols extends BaseTarget { '--path', symbolsPath, '--batch-type', - this.nativeSymbolsConfig.batchType, + this.symbolCollectorConfig.batchType, '--bundle-id', bundleId, '--server-endpoint', - this.nativeSymbolsConfig.serverEndpoint, + this.symbolCollectorConfig.serverEndpoint, ]); }); } @@ -141,7 +141,7 @@ export class NativeSymbols extends BaseTarget { this.logger.debug('Extracting asset...'); await extractZipArchive(assetDstPath, dir); - const binaryPath = join(dir, this.nativeSymbolsConfig.binaryName); + const binaryPath = join(dir, this.symbolCollectorConfig.binaryName); this.makeBinaryExecutable(binaryPath); return binaryPath; } @@ -151,7 +151,7 @@ export class NativeSymbols extends BaseTarget { this.logger.debug('Fetching release assets...'); const releaseAssets = await this.github.listReleaseAssets(releaseId); const matchingAssets = releaseAssets.filter( - asset => asset.name === this.nativeSymbolsConfig.symCollectorAssetName + asset => asset.name === this.symbolCollectorConfig.symCollectorAssetName ); if (matchingAssets.length != 1) { reportError(`Found ${matchingAssets.length} assets, 1 expected.`); @@ -163,9 +163,12 @@ export class NativeSymbols extends BaseTarget { private async getReleaseId(): Promise { this.logger.debug('Fetching the release...'); - const targetRelease = this.nativeSymbolsConfig.useLatestSymCollectorRelease + const targetRelease = this.symbolCollectorConfig + .useLatestSymCollectorRelease ? await this.github.getLatestRelease() - : await this.github.getReleaseByTag(this.nativeSymbolsConfig.releaseTag); + : await this.github.getReleaseByTag( + this.symbolCollectorConfig.releaseTag + ); this.logger.debug('Fetched release: ', targetRelease.id); return targetRelease.id; } @@ -175,7 +178,7 @@ export class NativeSymbols extends BaseTarget { const assetDataBuffer = await this.github.getAsset(assetId); const assetDstPath = join( dir, - this.nativeSymbolsConfig.symCollectorAssetName + this.symbolCollectorConfig.symCollectorAssetName ); this.logger.debug('Downloading asset to: ', assetDstPath); await fsPromises.appendFile(assetDstPath, Buffer.from(assetDataBuffer)); From 280bfcb80a36f87b98fed3b674a5a12952134190 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 10:19:47 +0200 Subject: [PATCH 20/36] Use symbol collector available in the path, instead of downloading it --- src/targets/symbolCollector.ts | 107 +++------------------------------ 1 file changed, 9 insertions(+), 98 deletions(-) diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index 2170290d5..2a7fb6943 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -15,9 +15,11 @@ import { GithubRemote } from '../utils/githubApi'; const DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT = 'https://symbol-collector.services.sentry.io/'; -const DEFAULT_SYM_COLLECTOR_ASSET_NAME = - 'symbolcollector-console-linux-x64.zip'; -const DEFAULT_SYM_COLLECTOR_FILENAME = 'SymbolCollector.Console'; +/** + * Name of the binary of the symbol collector. + * Must be available in the path. + */ +const SYM_COLLECTOR_BIN_NAME = 'SymbolCollector.Console'; /** Config options for the "symbol-collector" target. */ interface SymbolCollectorTargetConfig { @@ -27,17 +29,6 @@ interface SymbolCollectorTargetConfig { batchType: string; /** Prefix of the bundle ID to be uploaded. */ bundleIdPrefix: string; - /** Whether to use the latest Symbol Collector release. */ - useLatestSymCollectorRelease: boolean; - /** Tag of the release of the Symbol Collector that should be used. */ - releaseTag: string; - /** - * Name of the asset in the release where the Symbol Collector - * binary should be found. - */ - symCollectorAssetName: string; - /** Name of the Symbol Collector binary, inside the asset. */ - binaryName: string; } export class SymbolCollector extends BaseTarget { @@ -59,11 +50,7 @@ export class SymbolCollector extends BaseTarget { } private getSymbolCollectorConfig(): SymbolCollectorTargetConfig { - if ( - !this.config.batchType || - !this.config.bundleIdPrefix || - !(this.config.useLatestSymCollectorRelease || this.config.releaseTag) - ) { + if (!this.config.batchType || !this.config.bundleIdPrefix) { throw new ConfigurationError( 'Required configuration not found in configuration file. ' + 'See the documentation for more details.' @@ -75,10 +62,6 @@ export class SymbolCollector extends BaseTarget { this.config.serverEndpoint || DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT, batchType: this.config.batchType, bundleIdPrefix: this.config.bundleIdPrefix, - useLatestSymCollectorRelease: this.config.useLatestSymCollectorRelease, - releaseTag: this.config.releaseTag, - symCollectorAssetName: DEFAULT_SYM_COLLECTOR_ASSET_NAME, - binaryName: this.config.binaryName || DEFAULT_SYM_COLLECTOR_FILENAME, }; } @@ -94,15 +77,6 @@ export class SymbolCollector extends BaseTarget { this.logger.debug(`Found ${artifacts.length} symbol artifacts.`); await withTempDir(async dir => { - const collectorDir = join(dir, 'collector'); - await fsPromises.mkdir(collectorDir); - const symbolCollectorPath = await this.downloadSymbolCollector( - collectorDir - ); - - const symbolsPath = join(dir, 'symbols'); - await fsPromises.mkdir(symbolsPath); - // Download all artifacts in the same parent directory, where the symbol // collector will recursively look for and deal with them. // Since there are files with the same name, download them in different @@ -110,17 +84,17 @@ export class SymbolCollector extends BaseTarget { this.logger.debug('Downloading artifacts...'); await Promise.all( artifacts.map(async (artifact, index) => { - const subdirPath = join(symbolsPath, index + ''); + const subdirPath = join(dir, index + ''); await fsPromises.mkdir(subdirPath); await this.artifactProvider.downloadArtifact(artifact, subdirPath); }) ); - await spawnProcess(symbolCollectorPath, [ + await spawnProcess(SYM_COLLECTOR_BIN_NAME, [ '--upload', 'directory', '--path', - symbolsPath, + dir, '--batch-type', this.symbolCollectorConfig.batchType, '--bundle-id', @@ -130,67 +104,4 @@ export class SymbolCollector extends BaseTarget { ]); }); } - - private async downloadSymbolCollector(dir: string): Promise { - // Currently, GitHub doesn't offer an API to download the asset of a - // release by its name, and the asset ID must be provided. The workaround - // is to get the release ID where the assets are and look for all the assets - // until there's one matching the name to get its ID - const assetDownloadId = await this.getAssetDownloadId(); - const assetDstPath = await this.downloadAsset(assetDownloadId, dir); - this.logger.debug('Extracting asset...'); - await extractZipArchive(assetDstPath, dir); - - const binaryPath = join(dir, this.symbolCollectorConfig.binaryName); - this.makeBinaryExecutable(binaryPath); - return binaryPath; - } - - private async getAssetDownloadId(): Promise { - const releaseId = await this.getReleaseId(); - this.logger.debug('Fetching release assets...'); - const releaseAssets = await this.github.listReleaseAssets(releaseId); - const matchingAssets = releaseAssets.filter( - asset => asset.name === this.symbolCollectorConfig.symCollectorAssetName - ); - if (matchingAssets.length != 1) { - reportError(`Found ${matchingAssets.length} assets, 1 expected.`); - } - const assetId = matchingAssets[0].id; - this.logger.debug('Found asset to download: ', assetId); - return assetId; - } - - private async getReleaseId(): Promise { - this.logger.debug('Fetching the release...'); - const targetRelease = this.symbolCollectorConfig - .useLatestSymCollectorRelease - ? await this.github.getLatestRelease() - : await this.github.getReleaseByTag( - this.symbolCollectorConfig.releaseTag - ); - this.logger.debug('Fetched release: ', targetRelease.id); - return targetRelease.id; - } - - private async downloadAsset(assetId: number, dir: string): Promise { - this.logger.debug('Fetching the asset to download...'); - const assetDataBuffer = await this.github.getAsset(assetId); - const assetDstPath = join( - dir, - this.symbolCollectorConfig.symCollectorAssetName - ); - this.logger.debug('Downloading asset to: ', assetDstPath); - await fsPromises.appendFile(assetDstPath, Buffer.from(assetDataBuffer)); - return assetDstPath; - } - - private makeBinaryExecutable(binaryPath: string): void { - const isExecutablePresent = makeExecutable(binaryPath); - if (!isExecutablePresent) { - throw new ConfigurationError( - 'Cannot access to the binary declared in the config file: ' + binaryPath - ); - } - } } From dd2f49d4135734c7139f9a160e5f69b3c4ef7909 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 11:00:27 +0200 Subject: [PATCH 21/36] Remove unused imports --- src/targets/symbolCollector.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index 2a7fb6943..09bf6e056 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -1,17 +1,12 @@ import { stringToRegexp } from '../utils/filters'; import { BaseArtifactProvider } from '../artifact_providers/base'; import { TargetConfig } from '../schemas/project_config'; -import { ConfigurationError, reportError } from '../utils/errors'; +import { ConfigurationError } from '../utils/errors'; import { BaseTarget } from './base'; import { withTempDir } from '../utils/files'; import { promises as fsPromises } from 'fs'; -import { - extractZipArchive, - makeExecutable, - spawnProcess, -} from '../utils/system'; +import { spawnProcess } from '../utils/system'; import { join } from 'path'; -import { GithubRemote } from '../utils/githubApi'; const DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT = 'https://symbol-collector.services.sentry.io/'; @@ -37,15 +32,11 @@ export class SymbolCollector extends BaseTarget { /** Target options */ public readonly symbolCollectorConfig: SymbolCollectorTargetConfig; - public readonly github: GithubRemote; - public constructor( config: TargetConfig, artifactProvider: BaseArtifactProvider ) { super(config, artifactProvider); - // TODO: don't hardcode repo's data - this.github = new GithubRemote('getsentry', 'symbol-collector'); this.symbolCollectorConfig = this.getSymbolCollectorConfig(); } From 9125c7025a60b4448f72e019f94a291c1ed40efc Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 11:02:37 +0200 Subject: [PATCH 22/36] Require the symbol collector to be in the path --- src/targets/symbolCollector.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index 09bf6e056..368e8e035 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -5,7 +5,7 @@ import { ConfigurationError } from '../utils/errors'; import { BaseTarget } from './base'; import { withTempDir } from '../utils/files'; import { promises as fsPromises } from 'fs'; -import { spawnProcess } from '../utils/system'; +import { checkExecutableIsPresent, spawnProcess } from '../utils/system'; import { join } from 'path'; const DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT = @@ -41,6 +41,9 @@ export class SymbolCollector extends BaseTarget { } private getSymbolCollectorConfig(): SymbolCollectorTargetConfig { + // The Symbol Collector should be available in the path + checkExecutableIsPresent(SYM_COLLECTOR_BIN_NAME); + if (!this.config.batchType || !this.config.bundleIdPrefix) { throw new ConfigurationError( 'Required configuration not found in configuration file. ' + From 70b5a6250b79f5c280c1f779d7811abb11a293e6 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 11:24:15 +0200 Subject: [PATCH 23/36] Update README --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5d8c54b6e..4c67ba109 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ then enforces a specific workflow for managing release branches, changelogs, art - [Ruby Gems Index (`gem`)](#ruby-gems-index-gem) - [AWS Lambda Layer (`aws-lambda-layer`)](#aws-lambda-layer-aws-lambda-layer) - [Unity Package Manager (`upm`)](#unity-package-manager-upm) - - [Java Symbols (`java-symbols`)](#java-symbols-java-symbols) + - [Symbol Collector (`symbol-collector`)](#symbol-collector-symbol-collector) - [Integrating Your Project with `craft`](#integrating-your-project-with-craft) - [Pre-release (Version-bumping) Script: Conventions](#pre-release-version-bumping-script-conventions) - [Post-release Script: Conventions](#post-release-script-conventions) @@ -989,25 +989,25 @@ targets: releaseRepoName: 'unity' ``` -### Java Symbols (`java-symbols`) +### Symbol Collector (`symbol-collector`) Using the [`symbol-collector`](https://github.com/getsentry/symbol-collector) client, uploads native symbols. +The client needs to be available in the path. **Configuration** -| Option | Description | -| ---------------- | ---------------------------------------------------------------------------------------- | -| `serverEndpoint` | The server endpoint. | -| `batchType` | The batch type. | -| `bundleIdPrefix` | The prefix of the bundle ID. The new version will be appended to the end of this prefix. | +| Option | Description | +| ---------------- | -------------------------------------------------------------------------------------------- | +| `serverEndpoint` | **optional** The server endpoint. Defaults to `https://symbol-collector.services.sentry.io`. | +| `batchType` | The batch type of the symbols to be uploaded. | +| `bundleIdPrefix` | The prefix of the bundle ID. The new version will be appended to the end of this prefix. | **Example** ```yaml targets: - - name: java-symbols + - name: symbol-collector includeNames: /libsentry(-android)?\.so/ - serverEndpoint: my-server.com batchType: Android bundleIdPrefix: android-ndk- ``` From 83b3d1b5c9a2039bfe76ea97bcb6fdd4631850e5 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 13:35:11 +0200 Subject: [PATCH 24/36] Install `symbol-collector` in the docker image --- Dockerfile | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index e18679c88..f3c24ff91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,14 +8,16 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get -qq update \ && apt-get install -y --no-install-recommends \ - apt-transport-https \ - build-essential \ - curl \ - dirmngr \ - gnupg \ - git \ - ruby-full \ - twine \ + apt-transport-https \ + build-essential \ + curl \ + dirmngr \ + gnupg \ + git \ + ruby-full \ + twine \ + jq \ + unzip \ && curl -fsSL https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -o /tmp/packages-microsoft-prod.deb \ && dpkg -i /tmp/packages-microsoft-prod.deb \ && rm /tmp/packages-microsoft-prod.deb \ @@ -23,8 +25,8 @@ RUN apt-get -qq update \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get update -qq \ && apt-get install -y --no-install-recommends \ - dotnet-sdk-3.1 \ - docker-ce-cli \ + dotnet-sdk-3.1 \ + docker-ce-cli \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile minimal -y \ @@ -32,7 +34,13 @@ RUN apt-get -qq update \ && cargo install cargo-hack \ # Stick with 3.1.x as 3.2.x doesn't install on Debian Buster for some reason && gem update --no-document --system 3.1.5 \ - && gem install cocoapods + && gem install cocoapods \ + # Install https://github.com/getsentry/symbol-collector + && symbol_collector_url=$(curl -s https://api.github.com/repos/getsentry/symbol-collector/releases/tags/1.3.1 | \ + jq -r '.assets[].browser_download_url | select(endswith("symbolcollector-console-linux-x64.zip"))') \ + && curl -sL $symbol_collector_url -o "/tmp/sym-collector.zip" \ + && unzip /tmp/sym-collector.zip -d /usr/local/bin/ \ + && chmod +x /usr/local/bin/SymbolCollector.Console COPY dist/craft /usr/local/bin/craft RUN chmod +x /usr/local/bin/craft From 78bf8d669251dfcdbbff421e64cf0c9a88ba5734 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 14:25:42 +0200 Subject: [PATCH 25/36] Add tests for target config --- src/targets/__tests__/symbolCollector.test.ts | 75 +++++++++++++++++++ src/targets/symbolCollector.ts | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 src/targets/__tests__/symbolCollector.test.ts diff --git a/src/targets/__tests__/symbolCollector.test.ts b/src/targets/__tests__/symbolCollector.test.ts new file mode 100644 index 000000000..d2b9224db --- /dev/null +++ b/src/targets/__tests__/symbolCollector.test.ts @@ -0,0 +1,75 @@ +import { NoneArtifactProvider } from '../../artifact_providers/none'; +import { checkExecutableIsPresent } from '../../utils/system'; +import { SymbolCollector, SYM_COLLECTOR_BIN_NAME } from '../symbolCollector'; + +jest.mock('../../utils/system'); + +function getSymbolCollectorInstance( + customConfig?: Record +): SymbolCollector { + const config = customConfig + ? customConfig + : { + ['testKey']: 'testVal', + }; + return new SymbolCollector( + { + name: 'aws-lambda-layer', + ...config, + }, + new NoneArtifactProvider() + ); +} + +describe('target config', () => { + test('symbol collector not present in path', () => { + (checkExecutableIsPresent as jest.MockedFunction< + typeof checkExecutableIsPresent + >).mockImplementationOnce(() => { + throw new Error('Checked for executable'); + }); + + expect(getSymbolCollectorInstance).toThrowErrorMatchingInlineSnapshot( + `"Checked for executable"` + ); + expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); + expect(checkExecutableIsPresent).toHaveBeenCalledWith( + SYM_COLLECTOR_BIN_NAME + ); + }); + + test('config missing', () => { + (checkExecutableIsPresent as jest.MockedFunction< + typeof checkExecutableIsPresent + >) = jest.fn(); + + expect(getSymbolCollectorInstance).toThrowErrorMatchingInlineSnapshot( + `"Required configuration not found in configuration file. See the documentation for more details."` + ); + expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); + expect(checkExecutableIsPresent).toHaveBeenCalledWith( + SYM_COLLECTOR_BIN_NAME + ); + }); + + test('symbol collector present and config ok', () => { + (checkExecutableIsPresent as jest.MockedFunction< + typeof checkExecutableIsPresent + >) = jest.fn(); + + const customConfig = { + batchType: 'batch type', + bundleIdPrefix: 'bundle id prefix', + }; + + const symCollector = getSymbolCollectorInstance(customConfig); + const actualConfig = symCollector.symbolCollectorConfig; + expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); + expect(checkExecutableIsPresent).toHaveBeenLastCalledWith( + SYM_COLLECTOR_BIN_NAME + ); + expect(actualConfig).toHaveProperty('serverEndpoint'); + expect(actualConfig).toHaveProperty('batchType'); + expect(actualConfig).toHaveProperty('bundleIdPrefix'); + }); +}); diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index 368e8e035..a07bcb384 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -14,7 +14,7 @@ const DEFAULT_SYM_COLLECTOR_SERVER_ENDPOINT = * Name of the binary of the symbol collector. * Must be available in the path. */ -const SYM_COLLECTOR_BIN_NAME = 'SymbolCollector.Console'; +export const SYM_COLLECTOR_BIN_NAME = 'SymbolCollector.Console'; /** Config options for the "symbol-collector" target. */ interface SymbolCollectorTargetConfig { From 4c02ab0b854f47a265b52f320d45a74eeb13bbda Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 17:12:45 +0200 Subject: [PATCH 26/36] Stop and log when no artifacts were found --- src/targets/symbolCollector.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index a07bcb384..cc74ae3b5 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -68,6 +68,12 @@ export class SymbolCollector extends BaseTarget { ? undefined : stringToRegexp(this.config.includeNames), }); + + if (artifacts.length == 0) { + this.logger.info(`Didn't found any artifacts after filtering`); + return; + } + this.logger.debug(`Found ${artifacts.length} symbol artifacts.`); await withTempDir(async dir => { From 02b125f8008a4221368268ab13c81cefef3ead8e Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 17:13:21 +0200 Subject: [PATCH 27/36] Log output of calling the symbol collector --- src/targets/symbolCollector.ts | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index cc74ae3b5..3ea2ae0fc 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -85,23 +85,26 @@ export class SymbolCollector extends BaseTarget { await Promise.all( artifacts.map(async (artifact, index) => { const subdirPath = join(dir, index + ''); - await fsPromises.mkdir(subdirPath); + await fsPromises.mkdir(subdirPath); // FIXME await this.artifactProvider.downloadArtifact(artifact, subdirPath); }) ); - await spawnProcess(SYM_COLLECTOR_BIN_NAME, [ - '--upload', - 'directory', - '--path', - dir, - '--batch-type', - this.symbolCollectorConfig.batchType, - '--bundle-id', - bundleId, - '--server-endpoint', - this.symbolCollectorConfig.serverEndpoint, - ]); + const processOutput = ( + await spawnProcess(SYM_COLLECTOR_BIN_NAME, [ + '--upload', + 'directory', + '--path', + dir, + '--batch-type', + this.symbolCollectorConfig.batchType, + '--bundle-id', + bundleId, + '--server-endpoint', + this.symbolCollectorConfig.serverEndpoint, + ]) + )?.toString(); + this.logger.debug('Process output: ', processOutput); }); } } From 831abd029e6a3b8f7b42caad1db71a920237ef0a Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 17:15:10 +0200 Subject: [PATCH 28/36] Add publish test --- src/targets/__tests__/symbolCollector.test.ts | 83 +++++++++++++++++-- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/src/targets/__tests__/symbolCollector.test.ts b/src/targets/__tests__/symbolCollector.test.ts index d2b9224db..639ea663b 100644 --- a/src/targets/__tests__/symbolCollector.test.ts +++ b/src/targets/__tests__/symbolCollector.test.ts @@ -1,8 +1,26 @@ +import { withTempDir } from '../../utils/files'; import { NoneArtifactProvider } from '../../artifact_providers/none'; -import { checkExecutableIsPresent } from '../../utils/system'; +import { checkExecutableIsPresent, spawnProcess } from '../../utils/system'; import { SymbolCollector, SYM_COLLECTOR_BIN_NAME } from '../symbolCollector'; +jest.mock('../../utils/files'); jest.mock('../../utils/system'); +jest.mock('fs', () => { + const original = jest.requireActual('fs'); + return { + ...original, + promises: { + mkdir: jest.fn(() => { + /** do nothing */ + }), + }, + }; +}); + +const customConfig = { + batchType: 'batchType', + bundleIdPrefix: 'bundleIdPrefix-', +}; function getSymbolCollectorInstance( customConfig?: Record @@ -57,11 +75,6 @@ describe('target config', () => { typeof checkExecutableIsPresent >) = jest.fn(); - const customConfig = { - batchType: 'batch type', - bundleIdPrefix: 'bundle id prefix', - }; - const symCollector = getSymbolCollectorInstance(customConfig); const actualConfig = symCollector.symbolCollectorConfig; expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); @@ -73,3 +86,61 @@ describe('target config', () => { expect(actualConfig).toHaveProperty('bundleIdPrefix'); }); }); + +describe('publish', () => { + test('no artifacts found', () => { + const symCollector = getSymbolCollectorInstance(customConfig); + symCollector.getArtifactsForRevision = jest + .fn() + .mockReturnValueOnce(() => []); + expect(spawnProcess).not.toHaveBeenCalled(); + }); + + test('with artifacts', async () => { + (withTempDir as jest.MockedFunction).mockImplementation( + async cb => await cb('tmpDir') + ); + (spawnProcess as jest.MockedFunction< + typeof spawnProcess + >).mockImplementation(() => Promise.resolve(undefined)); + + const mockedArtifacts = [ + { filename: 'artifact1', storedFile: { lastUpdated: 'tomorrow' } }, + { filename: 'artifact2', storedFile: { lastUpdated: 'next week' } }, + { filename: 'artifact2', storedFile: { lastUpdated: 'in 5 years' } }, + ]; + + const symCollector = getSymbolCollectorInstance(customConfig); + symCollector.getArtifactsForRevision = jest + .fn() + .mockReturnValueOnce(mockedArtifacts); + symCollector.artifactProvider.downloadArtifact = jest.fn(); + + await symCollector.publish('version', 'revision'); + + expect(symCollector.getArtifactsForRevision).toHaveBeenCalledTimes(1); + expect( + symCollector.artifactProvider.downloadArtifact + ).toHaveBeenCalledTimes(mockedArtifacts.length); + + expect(spawnProcess).toHaveBeenCalledTimes(1); + const [cmd, args] = (spawnProcess as jest.MockedFunction< + typeof spawnProcess + >).mock.calls[0] as string[]; + expect(cmd).toBe(SYM_COLLECTOR_BIN_NAME); + expect(args).toMatchInlineSnapshot(` + Array [ + "--upload", + "directory", + "--path", + "tmpDir", + "--batch-type", + "batchType", + "--bundle-id", + "bundleIdPrefix-version", + "--server-endpoint", + "https://symbol-collector.services.sentry.io/", + ] + `); + }); +}); From 69d379c2042c2a0566e81d62f6cfcdfecebceeed Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 17:26:38 +0200 Subject: [PATCH 29/36] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d2ce07c..6dbe224b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - feat(maven): Add maven target to deploy to Maven Central (#258) +- feat(symbol-collector): Add symbol-collector target (#266) ## 0.24.4 From d177cdd078eae794b0fd185aff0fa31e2555bb77 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Fri, 9 Jul 2021 18:14:10 +0200 Subject: [PATCH 30/36] Remove unused methods These were used in the past, but are no longer required. --- src/utils/githubApi.ts | 41 ----------------------------------------- src/utils/system.ts | 12 ------------ 2 files changed, 53 deletions(-) diff --git a/src/utils/githubApi.ts b/src/utils/githubApi.ts index 82c245591..19c283d14 100644 --- a/src/utils/githubApi.ts +++ b/src/utils/githubApi.ts @@ -23,8 +23,6 @@ export class GithubRemote { /** Url in the form of /OWNER/REPO/ */ protected readonly url: string; - protected readonly github: Github; - public constructor( owner: string, repo: string, @@ -37,7 +35,6 @@ export class GithubRemote { this.setAuth(username, apiToken); } this.url = `/${this.owner}/${this.repo}/`; - this.github = new Github(); } /** @@ -72,44 +69,6 @@ export class GithubRemote { : ''; return this.PROTOCOL_PREFIX + authData + this.GITHUB_HOSTNAME + this.url; } - - public async getLatestRelease(): Promise { - const release = await this.github.repos.getLatestRelease({ - owner: this.owner, - repo: this.repo, - }); - return release.data; - } - - public async getReleaseByTag(tag: string): Promise { - const release = await this.github.repos.getReleaseByTag({ - owner: this.owner, - repo: this.repo, - tag: tag, - }); - return release.data; - } - - public async listReleaseAssets(releaseId: number): Promise { - const releaseAssets = await this.github.repos.listAssetsForRelease({ - owner: this.owner, - repo: this.repo, - release_id: releaseId, - }); - return releaseAssets.data; - } - - public async getAsset(assetId: number): Promise { - const asset = await this.github.repos.getReleaseAsset({ - owner: this.owner, - repo: this.repo, - asset_id: assetId, - headers: { - accept: 'application/octet-stream', - }, - }); - return asset.data; - } } /** diff --git a/src/utils/system.ts b/src/utils/system.ts index 1df02bd41..d8904a587 100644 --- a/src/utils/system.ts +++ b/src/utils/system.ts @@ -273,18 +273,6 @@ function isExecutable(filePath: string): boolean { } } -export function makeExecutable(filePath: string): boolean { - if (isExecutable(filePath)) { - return true; - } - try { - fs.chmodSync(filePath, fs.constants.F_OK | fs.constants.X_OK); - } catch (e) { - return false; - } - return true; -} - /** * Checks if the provided executable is available * From 751deba98a851769a5c9a32cc96fa6f831153fc7 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 12 Jul 2021 11:03:01 +0200 Subject: [PATCH 31/36] Add `target` entry before maven examples This line was missed in the merge --- README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 23c3b8df0..ecb6ac91d 100644 --- a/README.md +++ b/README.md @@ -1021,16 +1021,17 @@ The `android` structure contains the following options: **Example** ```yaml -- name: maven - gradleCliPath: ./gradlew - mavenCliPath: scripts/mvnw.cmd - mavenSettingsPath: scripts/settings.xml - mavenRepoId: ossrh - mavenRepoUrl: https://oss.sonatype.org/service/local/staging/deploy/maven2/ - android: - distDirRegex: /^sentry-android-.*$/ - fileReplaceeRegex: /\d\.\d\.\d(-SNAPSHOT)?/ - fileReplacerStr: release.aar +targets: + - name: maven + gradleCliPath: ./gradlew + mavenCliPath: scripts/mvnw.cmd + mavenSettingsPath: scripts/settings.xml + mavenRepoId: ossrh + mavenRepoUrl: https://oss.sonatype.org/service/local/staging/deploy/maven2/ + android: + distDirRegex: /^sentry-android-.*$/ + fileReplaceeRegex: /\d\.\d\.\d(-SNAPSHOT)?/ + fileReplacerStr: release.aar ``` ### Symbol Collector (`symbol-collector`) From f6b4789e8e12c702ebe48bad2c75f1ad2ec23282 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 12 Jul 2021 12:17:11 +0200 Subject: [PATCH 32/36] Address feedback --- Dockerfile | 22 ++++----- README.md | 4 +- src/targets/__tests__/symbolCollector.test.ts | 12 ++--- src/targets/symbolCollector.ts | 48 ++++++++++--------- 4 files changed, 42 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index f3c24ff91..b01346926 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,16 +8,16 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get -qq update \ && apt-get install -y --no-install-recommends \ - apt-transport-https \ - build-essential \ - curl \ - dirmngr \ - gnupg \ - git \ - ruby-full \ - twine \ - jq \ - unzip \ + apt-transport-https \ + build-essential \ + curl \ + dirmngr \ + gnupg \ + git \ + ruby-full \ + twine \ + jq \ + unzip \ && curl -fsSL https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -o /tmp/packages-microsoft-prod.deb \ && dpkg -i /tmp/packages-microsoft-prod.deb \ && rm /tmp/packages-microsoft-prod.deb \ @@ -36,7 +36,7 @@ RUN apt-get -qq update \ && gem update --no-document --system 3.1.5 \ && gem install cocoapods \ # Install https://github.com/getsentry/symbol-collector - && symbol_collector_url=$(curl -s https://api.github.com/repos/getsentry/symbol-collector/releases/tags/1.3.1 | \ + && symbol_collector_url=$(curl -s https://api.github.com/repos/getsentry/symbol-collector/releases/tags/1.2.1 | \ jq -r '.assets[].browser_download_url | select(endswith("symbolcollector-console-linux-x64.zip"))') \ && curl -sL $symbol_collector_url -o "/tmp/sym-collector.zip" \ && unzip /tmp/sym-collector.zip -d /usr/local/bin/ \ diff --git a/README.md b/README.md index ecb6ac91d..3811a2934 100644 --- a/README.md +++ b/README.md @@ -1037,14 +1037,14 @@ targets: ### Symbol Collector (`symbol-collector`) Using the [`symbol-collector`](https://github.com/getsentry/symbol-collector) client, uploads native symbols. -The client needs to be available in the path. +The `symbol-collector` needs to be available in the path. **Configuration** | Option | Description | | ---------------- | -------------------------------------------------------------------------------------------- | | `serverEndpoint` | **optional** The server endpoint. Defaults to `https://symbol-collector.services.sentry.io`. | -| `batchType` | The batch type of the symbols to be uploaded. | +| `batchType` | The batch type of the symbols to be uploaded. I.e: `Android`, `macOS`, `iOS`. | | `bundleIdPrefix` | The prefix of the bundle ID. The new version will be appended to the end of this prefix. | **Example** diff --git a/src/targets/__tests__/symbolCollector.test.ts b/src/targets/__tests__/symbolCollector.test.ts index 639ea663b..e38d05dd1 100644 --- a/src/targets/__tests__/symbolCollector.test.ts +++ b/src/targets/__tests__/symbolCollector.test.ts @@ -23,16 +23,11 @@ const customConfig = { }; function getSymbolCollectorInstance( - customConfig?: Record + config: Record = { ['testKey']: 'testVal' } ): SymbolCollector { - const config = customConfig - ? customConfig - : { - ['testKey']: 'testVal', - }; return new SymbolCollector( { - name: 'aws-lambda-layer', + name: 'symbol-collector', ...config, }, new NoneArtifactProvider() @@ -62,7 +57,8 @@ describe('target config', () => { >) = jest.fn(); expect(getSymbolCollectorInstance).toThrowErrorMatchingInlineSnapshot( - `"Required configuration not found in configuration file. See the documentation for more details."` + '"The required `batchType` parameter is missing in the configuration file. ' + + 'See the documentation for more details."' ); expect(checkExecutableIsPresent).toHaveBeenCalledTimes(1); expect(checkExecutableIsPresent).toHaveBeenCalledWith( diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index 3ea2ae0fc..550e1a65b 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -44,9 +44,15 @@ export class SymbolCollector extends BaseTarget { // The Symbol Collector should be available in the path checkExecutableIsPresent(SYM_COLLECTOR_BIN_NAME); - if (!this.config.batchType || !this.config.bundleIdPrefix) { + if (!this.config.batchType) { throw new ConfigurationError( - 'Required configuration not found in configuration file. ' + + 'The required `batchType` parameter is missing in the configuration file. ' + + 'See the documentation for more details.' + ); + } + if (!this.config.bundleIdPrefix) { + throw new ConfigurationError( + 'The required `bundleIdPrefix` parameter is missing in the configuration file. ' + 'See the documentation for more details.' ); } @@ -60,8 +66,7 @@ export class SymbolCollector extends BaseTarget { } public async publish(version: string, revision: string): Promise { - const bundleId = this.symbolCollectorConfig.bundleIdPrefix + `${version}`; - this.logger.debug('Fetching artifacts...'); + const bundleId = this.symbolCollectorConfig.bundleIdPrefix + version; const artifacts = await this.getArtifactsForRevision(revision, { includeNames: this.config.includeNames === undefined @@ -69,8 +74,8 @@ export class SymbolCollector extends BaseTarget { : stringToRegexp(this.config.includeNames), }); - if (artifacts.length == 0) { - this.logger.info(`Didn't found any artifacts after filtering`); + if (artifacts.length === 0) { + this.logger.warn(`Didn't found any artifacts after filtering`); return; } @@ -84,27 +89,24 @@ export class SymbolCollector extends BaseTarget { this.logger.debug('Downloading artifacts...'); await Promise.all( artifacts.map(async (artifact, index) => { - const subdirPath = join(dir, index + ''); - await fsPromises.mkdir(subdirPath); // FIXME + const subdirPath = join(dir, String(index)); + await fsPromises.mkdir(subdirPath); await this.artifactProvider.downloadArtifact(artifact, subdirPath); }) ); - const processOutput = ( - await spawnProcess(SYM_COLLECTOR_BIN_NAME, [ - '--upload', - 'directory', - '--path', - dir, - '--batch-type', - this.symbolCollectorConfig.batchType, - '--bundle-id', - bundleId, - '--server-endpoint', - this.symbolCollectorConfig.serverEndpoint, - ]) - )?.toString(); - this.logger.debug('Process output: ', processOutput); + await spawnProcess(SYM_COLLECTOR_BIN_NAME, [ + '--upload', + 'directory', + '--path', + dir, + '--batch-type', + this.symbolCollectorConfig.batchType, + '--bundle-id', + bundleId, + '--server-endpoint', + this.symbolCollectorConfig.serverEndpoint, + ]); }); } } From e0bbb33ef00f7a488e8aa83f983ef3d3c6d3a549 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 12 Jul 2021 12:35:57 +0200 Subject: [PATCH 33/36] Simplify test artifacts to string array --- src/targets/__tests__/symbolCollector.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/targets/__tests__/symbolCollector.test.ts b/src/targets/__tests__/symbolCollector.test.ts index e38d05dd1..29e226252 100644 --- a/src/targets/__tests__/symbolCollector.test.ts +++ b/src/targets/__tests__/symbolCollector.test.ts @@ -100,11 +100,7 @@ describe('publish', () => { typeof spawnProcess >).mockImplementation(() => Promise.resolve(undefined)); - const mockedArtifacts = [ - { filename: 'artifact1', storedFile: { lastUpdated: 'tomorrow' } }, - { filename: 'artifact2', storedFile: { lastUpdated: 'next week' } }, - { filename: 'artifact2', storedFile: { lastUpdated: 'in 5 years' } }, - ]; + const mockedArtifacts = ['artifact1', 'artifact2', 'artifact3']; const symCollector = getSymbolCollectorInstance(customConfig); symCollector.getArtifactsForRevision = jest From 872a7d5ec93da1ee884d3a41cf4a6a82b4371e94 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 12 Jul 2021 13:35:37 +0200 Subject: [PATCH 34/36] Move artifact filtering option parsing to the artifact provider --- src/artifact_providers/base.ts | 34 +++++++++++++++++++++++++++++++--- src/targets/base.ts | 12 ++++++++---- src/targets/maven.ts | 5 +---- src/targets/symbolCollector.ts | 6 +----- 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/artifact_providers/base.ts b/src/artifact_providers/base.ts index 8c9366d74..31467571d 100644 --- a/src/artifact_providers/base.ts +++ b/src/artifact_providers/base.ts @@ -6,6 +6,7 @@ import { import { clearObjectProperties } from '../utils/objects'; import { ConfigurationError } from '../utils/errors'; import { logger as loggerRaw } from '../logger'; +import { stringToRegexp } from '../utils/filters'; /** Maximum concurrency for downloads */ export const MAX_DOWNLOAD_CONCURRENCY = 5; @@ -61,14 +62,41 @@ export interface LocalArtifact extends AbstractArtifact { } /** - * Fitlering options for artifacts + * Raw filtering options for artifacts */ -export interface FilterOptions { +export interface RawFilterOptions { + /** Include files that match this pattern */ + includeNames?: RegExp | string; + /** Exclude files that match this pattern */ + excludeNames?: RegExp | string; +} + +/** + * Parsed filtering options for artifacts + */ +export interface ParsedFilterOptions { /** Include files that match this regexp */ includeNames?: RegExp; /** Exclude files that match this regexp */ excludeNames?: RegExp; } + +/** + * Returns parsed the given raw filters. + */ +export function parseFilterOptions( + rawFilters: RawFilterOptions +): ParsedFilterOptions { + const parsedFilters: ParsedFilterOptions = {}; + if (typeof rawFilters.includeNames === 'string') { + parsedFilters.includeNames = stringToRegexp(rawFilters.includeNames); + } + if (typeof rawFilters.excludeNames === 'string') { + parsedFilters.excludeNames = stringToRegexp(rawFilters.excludeNames); + } + return parsedFilters; +} + /** * Configuration options needed for all artifact providers */ @@ -321,7 +349,7 @@ export abstract class BaseArtifactProvider { */ public async filterArtifactsForRevision( revision: string, - filterOptions?: FilterOptions + filterOptions?: ParsedFilterOptions ): Promise { let filteredArtifacts = await this.listArtifactsForRevision(revision); if (!filterOptions || filteredArtifacts.length === 0) { diff --git a/src/targets/base.ts b/src/targets/base.ts index 9eba8b182..b8b2d64da 100644 --- a/src/targets/base.ts +++ b/src/targets/base.ts @@ -1,6 +1,10 @@ import { logger as loggerRaw } from '../logger'; import { GithubGlobalConfig, TargetConfig } from '../schemas/project_config'; -import { FilterOptions } from '../artifact_providers/base'; +import { + parseFilterOptions, + RawFilterOptions, + ParsedFilterOptions, +} from '../artifact_providers/base'; import { stringToRegexp } from '../utils/filters'; import { BaseArtifactProvider, @@ -18,7 +22,7 @@ export class BaseTarget { /** Unparsed target configuration */ public readonly config: TargetConfig; /** Artifact filtering options for the target */ - public readonly filterOptions: FilterOptions; + public readonly filterOptions: ParsedFilterOptions; /** Github repo configuration */ public readonly githubRepo?: GithubGlobalConfig; @@ -76,10 +80,10 @@ export class BaseTarget { */ public async getArtifactsForRevision( revision: string, - defaultFilterOptions: FilterOptions = {} + defaultFilterOptions: RawFilterOptions = {} ): Promise { const filterOptions = { - ...defaultFilterOptions, + ...parseFilterOptions(defaultFilterOptions), ...this.filterOptions, }; this.logger.debug( diff --git a/src/targets/maven.ts b/src/targets/maven.ts index 67c8aa018..fcc6e8098 100644 --- a/src/targets/maven.ts +++ b/src/targets/maven.ts @@ -218,10 +218,7 @@ export class MavenTarget extends BaseTarget { */ public async upload(revision: string): Promise { const artifacts = await this.getArtifactsForRevision(revision, { - includeNames: - this.config.includeNames === undefined - ? undefined - : stringToRegexp(this.config.includeNames), + includeNames: this.config.includeNames, }); // We don't want to do this in parallel but in serial, because the gpg-agent diff --git a/src/targets/symbolCollector.ts b/src/targets/symbolCollector.ts index 550e1a65b..e03365984 100644 --- a/src/targets/symbolCollector.ts +++ b/src/targets/symbolCollector.ts @@ -1,4 +1,3 @@ -import { stringToRegexp } from '../utils/filters'; import { BaseArtifactProvider } from '../artifact_providers/base'; import { TargetConfig } from '../schemas/project_config'; import { ConfigurationError } from '../utils/errors'; @@ -68,10 +67,7 @@ export class SymbolCollector extends BaseTarget { public async publish(version: string, revision: string): Promise { const bundleId = this.symbolCollectorConfig.bundleIdPrefix + version; const artifacts = await this.getArtifactsForRevision(revision, { - includeNames: - this.config.includeNames === undefined - ? undefined - : stringToRegexp(this.config.includeNames), + includeNames: this.config.includeNames, }); if (artifacts.length === 0) { From 06f53f033a8f97df6ce93ee7e6ab7eda97730050 Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 12 Jul 2021 14:12:43 +0200 Subject: [PATCH 35/36] Update and add tests for parsing the filter options --- src/artifact_providers/__tests__/base.test.ts | 41 +++++++++++++++++++ src/artifact_providers/base.ts | 14 +++++-- 2 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 src/artifact_providers/__tests__/base.test.ts diff --git a/src/artifact_providers/__tests__/base.test.ts b/src/artifact_providers/__tests__/base.test.ts new file mode 100644 index 000000000..17ceddc6a --- /dev/null +++ b/src/artifact_providers/__tests__/base.test.ts @@ -0,0 +1,41 @@ +import { stringToRegexp } from '../../utils/filters'; +import { parseFilterOptions, RawFilterOptions } from '../base'; + +describe('parseFilterOptions', () => { + test('empty object', () => { + const rawFilters: RawFilterOptions = {}; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters).not.toHaveProperty('includeNames'); + expect(parsedFilters).not.toHaveProperty('excludeNames'); + }); + + test('undefined properties', () => { + const rawFilters: RawFilterOptions = { + includeNames: undefined, + excludeNames: undefined, + }; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters).not.toHaveProperty('includeNames'); + expect(parsedFilters).not.toHaveProperty('excludeNames'); + }); + + test('string properties', () => { + const stringFilter = '/testFilter/'; + const rawFilters: RawFilterOptions = { + includeNames: stringFilter, + }; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters.includeNames).toStrictEqual( + stringToRegexp(stringFilter) + ); + }); + + test('regex properties', () => { + const regexFilter = stringToRegexp('/testFilter/'); + const rawFilters: RawFilterOptions = { + includeNames: regexFilter, + }; + const parsedFilters = parseFilterOptions(rawFilters); + expect(parsedFilters.includeNames).toStrictEqual(regexFilter); + }); +}); diff --git a/src/artifact_providers/base.ts b/src/artifact_providers/base.ts index 31467571d..3c02ff042 100644 --- a/src/artifact_providers/base.ts +++ b/src/artifact_providers/base.ts @@ -88,11 +88,17 @@ export function parseFilterOptions( rawFilters: RawFilterOptions ): ParsedFilterOptions { const parsedFilters: ParsedFilterOptions = {}; - if (typeof rawFilters.includeNames === 'string') { - parsedFilters.includeNames = stringToRegexp(rawFilters.includeNames); + if (typeof rawFilters.includeNames !== 'undefined') { + parsedFilters.includeNames = + typeof rawFilters.includeNames === 'string' + ? stringToRegexp(rawFilters.includeNames) + : rawFilters.includeNames; } - if (typeof rawFilters.excludeNames === 'string') { - parsedFilters.excludeNames = stringToRegexp(rawFilters.excludeNames); + if (typeof rawFilters.excludeNames !== 'undefined') { + parsedFilters.excludeNames = + typeof rawFilters.excludeNames === 'string' + ? stringToRegexp(rawFilters.excludeNames) + : rawFilters.excludeNames; } return parsedFilters; } From 58baf81ac18781cc58540e02533a01f9a56581ed Mon Sep 17 00:00:00 2001 From: iker barriocanal <32816711+iker-barriocanal@users.noreply.github.com> Date: Mon, 12 Jul 2021 14:55:16 +0200 Subject: [PATCH 36/36] Address feedback --- src/artifact_providers/base.ts | 7 ++++--- src/targets/__tests__/symbolCollector.test.ts | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/artifact_providers/base.ts b/src/artifact_providers/base.ts index 3c02ff042..155f5eccb 100644 --- a/src/artifact_providers/base.ts +++ b/src/artifact_providers/base.ts @@ -62,7 +62,8 @@ export interface LocalArtifact extends AbstractArtifact { } /** - * Raw filtering options for artifacts + * Raw filtering options for artifacts. + * They should be parsed by `parseFilterOptions`. */ export interface RawFilterOptions { /** Include files that match this pattern */ @@ -88,13 +89,13 @@ export function parseFilterOptions( rawFilters: RawFilterOptions ): ParsedFilterOptions { const parsedFilters: ParsedFilterOptions = {}; - if (typeof rawFilters.includeNames !== 'undefined') { + if (rawFilters.includeNames) { parsedFilters.includeNames = typeof rawFilters.includeNames === 'string' ? stringToRegexp(rawFilters.includeNames) : rawFilters.includeNames; } - if (typeof rawFilters.excludeNames !== 'undefined') { + if (rawFilters.excludeNames) { parsedFilters.excludeNames = typeof rawFilters.excludeNames === 'string' ? stringToRegexp(rawFilters.excludeNames) diff --git a/src/targets/__tests__/symbolCollector.test.ts b/src/targets/__tests__/symbolCollector.test.ts index 29e226252..b36274d90 100644 --- a/src/targets/__tests__/symbolCollector.test.ts +++ b/src/targets/__tests__/symbolCollector.test.ts @@ -23,7 +23,7 @@ const customConfig = { }; function getSymbolCollectorInstance( - config: Record = { ['testKey']: 'testVal' } + config: Record = { testKey: 'testVal' } ): SymbolCollector { return new SymbolCollector( {