From 267a035b35a859156dcafd337abcd36547e001eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Mon, 11 Oct 2021 18:58:02 +0200 Subject: [PATCH 1/6] Add runtime-community.yml pipeline and add s390x job (#60255) (cherry picked from commit 5f3062d81a5caa444308937539ad136b72218c29) --- eng/pipelines/common/platform-matrix.yml | 26 ++++++++ .../libraries/helix-queues-setup.yml | 4 ++ eng/pipelines/runtime-community.yml | 59 +++++++++++++++++++ src/libraries/tests.proj | 4 ++ 4 files changed, 93 insertions(+) create mode 100644 eng/pipelines/runtime-community.yml diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 15fe948e92967e..2bd8b6d21ea402 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -205,6 +205,32 @@ jobs: ${{ insert }}: ${{ parameters.jobParameters }} buildingOnSourceBuildImage: true +# Linux s390x + +- ${{ if or(containsValue(parameters.platforms, 'Linux_s390x'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: s390x + targetRid: linux-s390x + platform: Linux_s390x + container: + image: ubuntu-18.04-cross-s390x-20201102145728-d6e0352 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/s390x' + ${{ insert }}: ${{ parameters.jobParameters }} + # WebAssembly - ${{ if containsValue(parameters.platforms, 'Browser_wasm') }}: diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 880aa8939932c0..3dc5d4dc2ee604 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -85,6 +85,10 @@ jobs: # Limiting interp runs as we don't need as much coverage. - Debian.9.Amd64.Open + # Linux s390x + - ${{ if eq(parameters.platform, 'Linux_s390x') }}: + - Ubuntu.2004.S390X.Experimental.Open + # OSX arm64 - ${{ if eq(parameters.platform, 'OSX_arm64') }}: - OSX.1100.ARM64.Open diff --git a/eng/pipelines/runtime-community.yml b/eng/pipelines/runtime-community.yml new file mode 100644 index 00000000000000..5bebcc00a2663e --- /dev/null +++ b/eng/pipelines/runtime-community.yml @@ -0,0 +1,59 @@ +trigger: none + +schedules: + - cron: "0 7,19 * * *" # run at 7:00 and 19:00 (UTC) which is 23:00 and 11:00 (PST). + displayName: Runtime-community default schedule + branches: + include: + - main + always: false # run only if there were changes since the last successful scheduled run. + +variables: + - template: /eng/pipelines/common/variables.yml + +jobs: +# +# Evaluate paths +# +- ${{ if eq(variables.dependOnEvaluatePaths, true) }}: + - template: /eng/pipelines/common/evaluate-default-paths.yml + +# +# s390x +# Build the whole product using Mono and run libraries tests +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: Release + runtimeFlavor: mono + platforms: + - Linux_s390x + variables: + # map dependencies variables to local variables + - name: librariesContainsChange + value: $[ dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'] ] + - name: monoContainsChange + value: $[ dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'] ] + jobParameters: + testGroup: innerloop + nameSuffix: AllSubsets_Mono + buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 180 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), + eq(variables['isFullMatrix'], true)) + # extra steps, run tests + extraStepsTemplate: /eng/pipelines/libraries/helix.yml + extraStepsParameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['monoContainsChange'], true), + eq(variables['isFullMatrix'], true)) diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index 8c0fc9060c28ac..c85b97c3c94b3d 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -258,6 +258,10 @@ + + + + From d229187782452c2ca06a765f408ce2eb1d2520ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Wed, 13 Oct 2021 19:23:00 +0200 Subject: [PATCH 2/6] Don't include s390x in coreclr outerloop runs (#60351) This was a copy-paste mistake, see https://github.com/dotnet/runtime/pull/60255#issuecomment-942539639 (cherry picked from commit 49cf05cdea3baa22592d08d74ce2f0e2c32786a7) --- eng/pipelines/common/platform-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 2bd8b6d21ea402..244240a68f32fb 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -207,7 +207,7 @@ jobs: # Linux s390x -- ${{ if or(containsValue(parameters.platforms, 'Linux_s390x'), in(parameters.platformGroup, 'all', 'gcstress')) }}: +- ${{ if containsValue(parameters.platforms, 'Linux_s390x') }}: - template: xplat-setup.yml parameters: jobTemplate: ${{ parameters.jobTemplate }} From d446ab0a09b2df178cd033b40490115c87c89755 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 14 Dec 2021 15:17:31 -0500 Subject: [PATCH 3/6] Enable for release/*.* branch, not just main --- eng/pipelines/runtime-community.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/runtime-community.yml b/eng/pipelines/runtime-community.yml index 5bebcc00a2663e..9ecb1f2677ae2d 100644 --- a/eng/pipelines/runtime-community.yml +++ b/eng/pipelines/runtime-community.yml @@ -6,6 +6,7 @@ schedules: branches: include: - main + - release/*.* always: false # run only if there were changes since the last successful scheduled run. variables: From 92148560271d2515fc6ef130608e8357931b5ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Thu, 16 Dec 2021 11:54:15 +0100 Subject: [PATCH 4/6] Fix condition for dependOnEvaluatePaths for runtime-community pipeline (#60552) It was missed where we set a default value for that variable. (cherry picked from commit 157a8b5f72f69e18fbd5ec1f549954e3a28f8f71) --- eng/pipelines/common/variables.yml | 5 +++-- eng/pipelines/common/xplat-setup.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index c724f4beb367f9..e2ed01f836cf55 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -14,9 +14,10 @@ variables: - name: isFullMatrix value: ${{ and(eq(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }} -# We only run evaluate paths on runtime and runtime-staging pipelines on PRs +# We only run evaluate paths on runtime, runtime-staging and runtime-community pipelines on PRs +# keep in sync with /eng/pipelines/common/xplat-setup.yml - name: dependOnEvaluatePaths - value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }} + value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community')) }} - name: debugOnPrReleaseOnRolling ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: value: Release diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 2010c653f89cc2..9c9861e3a963f1 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -22,7 +22,7 @@ jobs: shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }} # keep in sync with /eng/pipelines/common/variables.yml - dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }} + dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community')) }} variables: # Disable component governance in our CI builds. These builds are not shipping nor From 2f946e340c40254e4598edc361886191a3fb517e Mon Sep 17 00:00:00 2001 From: stefan-sf-ibm <73470131+stefan-sf-ibm@users.noreply.github.com> Date: Wed, 18 Aug 2021 01:39:03 +0200 Subject: [PATCH 5/6] [mono] For cross builds targeting s390x use proper objcopy (#57566) Co-authored-by: Stefan Schulze Frielinghaus (cherry picked from commit 616ace39bb8d9431097b433c31caca6dd934d349) --- src/mono/mono.proj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mono/mono.proj b/src/mono/mono.proj index dd8140fb81e8be..fb98ffc1896906 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -506,6 +506,7 @@ <_Objcopy>objcopy <_Objcopy Condition="'$(Platform)' == 'arm'">arm-linux-$(_LinuxAbi)eabi$(_LinuxFloatAbi)-$(_Objcopy) <_Objcopy Condition="'$(Platform)' == 'arm64'">aarch64-linux-$(_LinuxAbi)-$(_Objcopy) + <_Objcopy Condition="'$(Platform)' == 's390x'">s390x-linux-$(_LinuxAbi)-$(_Objcopy) <_Objcopy Condition="'$(Platform)' == 'x64'">x86_64-linux-$(_LinuxAbi)-$(_Objcopy) <_Objcopy Condition="'$(Platform)' == 'x86'">i686-linux-$(_LinuxAbi)-$(_Objcopy) <_Objcopy Condition="'$(TargetsAndroid)' == 'true'">$(ANDROID_NDK_ROOT)/toolchains/llvm/prebuilt/$(MonoToolchainPrebuiltOS)/bin/$(_Objcopy) From b2b08f39c43a4eb2cdec1bfc72026c88e6d18fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Tue, 5 Oct 2021 17:58:40 +0200 Subject: [PATCH 6/6] Use live-built corehost on s390x for creating testhost (#58952) This allows consuming the dotnet host that was just built instead of relying on a downloaded one. We need to move the `host` subset build before the `libs` subset so that the artifacts are available before externals.csproj runs. (cherry picked from commit cab3a63fe3b9810b801ea6a7d9fb958d1af5e380) --- Directory.Build.props | 4 ++ eng/Subsets.props | 49 +++++++++++++------------ src/installer/pkg/Directory.Build.props | 1 - src/libraries/externals.csproj | 19 +++++++++- 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 5cb87bb7c25725..54ae373485c0fa 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -210,6 +210,10 @@ $([MSBuild]::NormalizeDirectory('$(MicrosoftNetCoreAppRuntimePackRidDir)', 'native')) + + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', '$(OutputRid).$(Configuration)', 'corehost')) + + true diff --git a/eng/Subsets.props b/eng/Subsets.props index b2292ac98e61c8..be4c6a8862480b 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -120,7 +120,14 @@ - + + + + + + + + @@ -130,13 +137,6 @@ - - - - - - - @@ -263,6 +263,22 @@ + + + + + + + + + + + + + + + + @@ -288,22 +304,7 @@ - - - - - - - - - - - - - - - - + diff --git a/src/installer/pkg/Directory.Build.props b/src/installer/pkg/Directory.Build.props index 10f36b399c73b6..f51718676368a2 100644 --- a/src/installer/pkg/Directory.Build.props +++ b/src/installer/pkg/Directory.Build.props @@ -3,7 +3,6 @@ $(TargetArchitecture) - $(ArtifactsBinDir)$(OutputRid).$(Configuration)\corehost diff --git a/src/libraries/externals.csproj b/src/libraries/externals.csproj index 6ef3ddc0f19015..969ed9603a2cda 100644 --- a/src/libraries/externals.csproj +++ b/src/libraries/externals.csproj @@ -9,6 +9,7 @@ true false true + true @@ -16,7 +17,7 @@ Version="$(MicrosoftDiaSymReaderNativeVersion)" /> - + - + + + + + + + + + +