From f113bc8a77ca93531c7ba4b0a5835ca337bc3696 Mon Sep 17 00:00:00 2001 From: Clint Rutkas Date: Fri, 11 Dec 2020 12:45:52 -0800 Subject: [PATCH] Improving CI build times with new build agent (#8516) --- .github/actions/spell-check/expect.txt | 10 + .../ci/templates/build-powertoys-ci.yml | 4 +- .../ci/templates/build-powertoys-steps.yml | 2 + .pipelines/restore-dependencies.ps1 | 32 ++ .pipelines/restore-dependencies.yml | 15 + .pipelines/restore-windowsSdk17134.ps1 | 286 ++++++++++++++++++ 6 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 .pipelines/restore-dependencies.ps1 create mode 100644 .pipelines/restore-dependencies.yml create mode 100644 .pipelines/restore-windowsSdk17134.ps1 diff --git a/.github/actions/spell-check/expect.txt b/.github/actions/spell-check/expect.txt index 0a6cbd3c43..424276b0d4 100644 --- a/.github/actions/spell-check/expect.txt +++ b/.github/actions/spell-check/expect.txt @@ -2490,3 +2490,13 @@ ZONESETCHANGE Zoneszonabletester Zoomusingmagnifier zzz +buildtools +CPPARM +CPPx +LASTEXITCODE +linkid +nocache +retval +umd +webclient +windevbuildagents diff --git a/.pipelines/ci/templates/build-powertoys-ci.yml b/.pipelines/ci/templates/build-powertoys-ci.yml index 1500c43185..6f5b6231b4 100644 --- a/.pipelines/ci/templates/build-powertoys-ci.yml +++ b/.pipelines/ci/templates/build-powertoys-ci.yml @@ -1,7 +1,7 @@ parameters: configuration: 'Release' platform: '' - additionalBuildArguments: '/p:CIBuild=true' + additionalBuildArguments: '-m' jobs: - job: Build${{ parameters.platform }}${{ parameters.configuration }} @@ -9,7 +9,7 @@ jobs: variables: BuildConfiguration: ${{ parameters.configuration }} BuildPlatform: ${{ parameters.platform }} - pool: { vmImage: windows-2019 } + pool: "windevbuildagents" timeoutInMinutes: 120 strategy: maxParallel: 10 diff --git a/.pipelines/ci/templates/build-powertoys-steps.yml b/.pipelines/ci/templates/build-powertoys-steps.yml index a90befc692..7499207d8c 100644 --- a/.pipelines/ci/templates/build-powertoys-steps.yml +++ b/.pipelines/ci/templates/build-powertoys-steps.yml @@ -12,6 +12,8 @@ steps: inputs: versionSpec: 5.8.0 +#- template: .\..\..\..\restore-dependencies.yml + - task: VisualStudioTestPlatformInstaller@1 displayName: Ensure VSTest Platform diff --git a/.pipelines/restore-dependencies.ps1 b/.pipelines/restore-dependencies.ps1 new file mode 100644 index 0000000000..4e30a55099 --- /dev/null +++ b/.pipelines/restore-dependencies.ps1 @@ -0,0 +1,32 @@ +# not using this but keeping around in case we need it in the future. +# good use case here could be to set up a new machine, we just point people at it. +# https://github.com/microsoft/PowerToys/tree/master/doc/devdocs#prerequisites-for-compiling-powertoys + +# improvements if this script is used to replace the snippet +# Add in a param for passive versus quiet. Could be a IsSettingUpDevComputer true/false flag + # default it to true which would be passive flag for normal people, false would set to quiet + +$VS_DOWNLOAD_LINK = "https://aka.ms/vs/16/release/vs_buildtools.exe" +$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", + "--add Microsoft.VisualStudio.Workload.NativeDesktop", + "--add Microsoft.VisualStudio.Workload.ManagedDesktop", + "--add Microsoft.VisualStudio.Workload.Universal", + "--add Microsoft.VisualStudio.ComponentGroup.UWP.VC", + "--add Microsoft.VisualStudio.Component.Windows10SDK.17134", + "--add Microsoft.VisualStudio.Component.VC.Runtimes.x86.x64.Spectre", + "--add Microsoft.NetCore.Component.Runtime.3.1", + "--add Microsoft.VisualStudio.Component.VC.ATL.Spectre") + +curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe +if ($LASTEXITCODE -ne 0) { + echo "Download of the VS 2019 installer failed" + exit 1 +} + +$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru +Remove-Item -Path vs_installer.exe -Force +$exitCode = $process.ExitCode +if (($exitCode -ne 0) -and ($exitCode -ne 3010)) { + echo "VS 2019 installer exited with code $exitCode, which should be one of [0, 3010]." + exit 1 +} \ No newline at end of file diff --git a/.pipelines/restore-dependencies.yml b/.pipelines/restore-dependencies.yml new file mode 100644 index 0000000000..2874416916 --- /dev/null +++ b/.pipelines/restore-dependencies.yml @@ -0,0 +1,15 @@ +parameters: + sdkVersion: 17134 + +steps: + - task: powershell@2 + inputs: + targetType: filePath + filePath: .\.pipelines\restore-dependencies.ps1 + displayName: 'Install VS dependencies' + - task: powershell@2 + inputs: + targetType: filePath + filePath: .\.pipelines\restore-windowsSdk17134.ps1 + arguments: ${{ parameters.sdkVersion }} + displayName: 'Install Windows SDK 17134' \ No newline at end of file diff --git a/.pipelines/restore-windowsSdk17134.ps1 b/.pipelines/restore-windowsSdk17134.ps1 new file mode 100644 index 0000000000..3286cab675 --- /dev/null +++ b/.pipelines/restore-windowsSdk17134.ps1 @@ -0,0 +1,286 @@ +# Not using this but keeping around in case we need it in the future. +# It will install 17134 and can be modified to support iso's. + +[CmdletBinding()] +param([Parameter(Mandatory=$true, Position=0)] + [string]$buildNumber) + +# Ensure the error action preference is set to the default for PowerShell3, 'Stop' +$ErrorActionPreference = 'Stop' + +# Constants +$WindowsSDKOptions = @("OptionId.UWPCpp", "OptionId.DesktopCPPx64", "OptionId.DesktopCPPx86", "OptionId.DesktopCPPARM64", "OptionId.DesktopCPPARM", "OptionId.WindowsDesktopDebuggers") +$WindowsSDKRegPath = "HKLM:\Software\WOW6432Node\Microsoft\Windows Kits\Installed Roots" +$WindowsSDKRegRootKey = "KitsRoot10" +$WindowsSDKVersion = "10.0.$buildNumber.0" +$WindowsSDKInstalledRegPath = "$WindowsSDKRegPath\$WindowsSDKVersion\Installed Options" +$StrongNameRegPath = "HKLM:\SOFTWARE\Microsoft\StrongName\Verification" +$PublicKeyTokens = @("31bf3856ad364e35") + +if ($buildNumber -notmatch "^\d{5,}$") +{ + Write-Host "ERROR: '$buildNumber' doesn't look like a windows build number" + Write-Host + Exit 1 +} + +function Download-File +{ + param ([string] $outDir, + [string] $downloadUrl, + [string] $downloadName) + + $downloadPath = Join-Path $outDir "$downloadName.download" + $downloadDest = Join-Path $outDir $downloadName + $downloadDestTemp = Join-Path $outDir "$downloadName.tmp" + + Write-Host -NoNewline "Downloading $downloadName..." + + $retries = 10 + $downloaded = $false + while (-not $downloaded) + { + try + { + $webclient = new-object System.Net.WebClient + $webclient.DownloadFile($downloadUrl, $downloadPath) + $downloaded = $true + } + catch [System.Net.WebException] + { + Write-Host + Write-Warning "Failed to fetch updated file from $downloadUrl : $($error[0])" + if (!(Test-Path $downloadDest)) + { + if ($retries -gt 0) + { + Write-Host "$retries retries left, trying download again" + $retries-- + start-sleep -Seconds 10 + } + else + { + throw "$downloadName was not found at $downloadDest" + } + } + else + { + Write-Warning "$downloadName may be out of date" + } + } + } + + Unblock-File $downloadPath + + $downloadDestTemp = $downloadPath; + + # Delete and rename to final dest + Write-Host "testing $downloadDest" + if (Test-Path $downloadDest) + { + Write-Host "Deleting: $downloadDest" + Remove-Item $downloadDest -Force + } + + Move-Item -Force $downloadDestTemp $downloadDest + Write-Host "Done" + + return $downloadDest +} + +function Disable-StrongName +{ + param ([string] $publicKeyToken = "*") + + reg ADD "HKLM\SOFTWARE\Microsoft\StrongName\Verification\*,$publicKeyToken" /f | Out-Null + if ($env:PROCESSOR_ARCHITECTURE -eq "AMD64") + { + reg ADD "HKLM\SOFTWARE\Wow6432Node\Microsoft\StrongName\Verification\*,$publicKeyToken" /f | Out-Null + } +} + +function Test-Admin +{ + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = New-Object Security.Principal.WindowsPrincipal $identity + $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) +} + +function Test-RegistryPathAndValue +{ + param ( + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] $path, + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] $value) + + try + { + if (Test-Path $path) + { + Get-ItemProperty -Path $path | Select-Object -ExpandProperty $value -ErrorAction Stop | Out-Null + return $true + } + } + catch + { + } + + return $false +} + +function Test-InstallWindowsSDK +{ + $retval = $true + + if (Test-RegistryPathAndValue -Path $WindowsSDKRegPath -Value $WindowsSDKRegRootKey) + { + # A Windows SDK is installed + # Is an SDK of our version installed with the options we need? + $allRequiredSdkOptionsInstalled = $true + foreach($sdkOption in $WindowsSDKOptions) + { + if (!(Test-RegistryPathAndValue -Path $WindowsSDKInstalledRegPath -Value $sdkOption)) + { + $allRequiredSdkOptionsInstalled = $false + } + } + + if($allRequiredSdkOptionsInstalled) + { + # It appears we have what we need. Double check the disk + $sdkRoot = Get-ItemProperty -Path $WindowsSDKRegPath | Select-Object -ExpandProperty $WindowsSDKRegRootKey + if ($sdkRoot) + { + if (Test-Path $sdkRoot) + { + $refPath = Join-Path $sdkRoot "References\$WindowsSDKVersion" + if (Test-Path $refPath) + { + $umdPath = Join-Path $sdkRoot "UnionMetadata\$WindowsSDKVersion" + if (Test-Path $umdPath) + { + # Pretty sure we have what we need + $retval = $false + } + } + } + } + } + } + + return $retval +} + +function Test-InstallStrongNameHijack +{ + foreach($publicKeyToken in $PublicKeyTokens) + { + $key = "$StrongNameRegPath\*,$publicKeyToken" + if (!(Test-Path $key)) + { + return $true + } + } + + return $false +} + +Write-Host -NoNewline "Checking for installed Windows SDK $WindowsSDKVersion..." +$InstallWindowsSDK = Test-InstallWindowsSDK +if ($InstallWindowsSDK) +{ + Write-Host "Installation required" +} +else +{ + Write-Host "INSTALLED" +} + +$StrongNameHijack = Test-InstallStrongNameHijack +Write-Host -NoNewline "Checking if StrongName bypass required..." + +if ($StrongNameHijack) +{ + Write-Host "REQUIRED" +} +else +{ + Write-Host "Done" +} + +if ($StrongNameHijack -or $InstallWindowsSDK) +{ + if (!(Test-Admin)) + { + Write-Host + throw "ERROR: Elevation required" + } +} + +if ($InstallWindowsSDK) +{ + # Static(ish) link for Windows SDK + # Note: there is a delay from Windows SDK announcements to availability via the static link + # $uri = "https://software-download.microsoft.com/download/sg/Windows_InsiderPreview_SDK_en-us_$($buildNumber)_1.iso"; + + # https://developer.microsoft.com/en-us/windows/downloads/sdk-archive/ + $uri = "https://go.microsoft.com/fwlink/p/?linkid=870807" + + if ($env:TEMP -eq $null) + { + $env:TEMP = Join-Path $env:SystemDrive 'temp' + } + + $winsdkTempDir = Join-Path (Join-Path $env:TEMP ([System.IO.Path]::GetRandomFileName())) "WindowsSDK" + + if (![System.IO.Directory]::Exists($winsdkTempDir)) + { + [void][System.IO.Directory]::CreateDirectory($winsdkTempDir) + } + + # $file = "winsdk_$buildNumber.iso" + $file = "winsdk_$buildNumber.exe" + + Write-Host -NoNewline "Getting WinSDK from $uri" + $downloadFile = Download-File $winsdkTempDir $uri $file + + Write-Host -NoNewline "File is at $downloadFile" + $downloadFileItem = Get-Item $downloadFile + + # Check to make sure the file is at least 10 MB. + # if ($downloadFileItem.Length -lt 10*1024*1024) + # { + # Write-Host + # Write-Host "ERROR: Downloaded file doesn't look large enough to be an ISO. The requested version may not be on microsoft.com yet." + # Write-Host + # Exit 1 + # } + + # TODO Check if zip, exe, iso, etc. + try + { + Write-Host -NoNewLine "Installing WinSDK..." + + Start-Process -Wait $downloadFileItem "/features $WindowsSDKOptions /q" + Write-Host "Done installing" + } + finally + { + Write-Host "Done" + } +} + +if ($StrongNameHijack) +{ + Write-Host -NoNewline "Disabling StrongName for Windows SDK..." + + foreach($key in $PublicKeyTokens) + { + Disable-StrongName $key + } + + Write-Host "Done" +} \ No newline at end of file