early-access version 2853

externals/vcpkg/scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -0,0 +1,78 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: poolName
+    type: string
+
+jobs:
+- job: x64_linux
+  pool:
+    name: ${{ parameters.poolName }}
+  workspace:
+    clean: resources
+  timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: /mnt/vcpkg-ci
+  - name: VCPKG_DOWNLOADS
+    value: /mnt/vcpkg-ci/downloads
+  - group: vcpkg-asset-caching-credentials
+  - name: X_VCPKG_ASSET_SOURCES
+    value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite"
+  - group: vcpkg-binary-caching-credentials
+  - name: X_VCPKG_BINARY_SOURCE_STUB
+    value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"
+
+  steps:
+    # Note: /mnt is the Azure machines' temporary disk.
+  - bash: |
+      sudo mkdir /home/agent -m=777
+      sudo chown `id -u` /home/agent
+      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
+      exit 0
+    displayName: 'Create working directories'
+  - bash: ./bootstrap-vcpkg.sh
+    displayName: 'Bootstrap vcpkg'
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
+  - bash: ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
+  - task: PowerShell@2
+    displayName: '*** Test Modified Ports'
+    inputs:
+      failOnStderr: true
+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      pwsh: true
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: failure logs for x64-linux'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)/failure-logs'
+      ArtifactName: 'failure logs for x64-linux'
+    condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
+  - bash: |
+      python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/
+    displayName: 'Build a file list for all packages'
+    condition: always()
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: file lists for x64-linux'
+    condition: always()
+    inputs:
+      PathtoPublish: scripts/list_files
+      ArtifactName: 'file lists for x64-linux'
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results'
+    condition: ne(variables['XML_RESULTS_FILE'], '')
+    inputs:
+      testRunTitle: x64-linux
+      testResultsFormat: xUnit
+      testResultsFiles: $(XML_RESULTS_FILE)
+      platform: x64-linux
+      configuration: static

externals/vcpkg/scripts/azure-pipelines/linux/create-image.ps1

@@ -0,0 +1,177 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+<#
+.SYNOPSIS
+Creates a Linux virtual machine image, set up for vcpkg's CI.
+
+.DESCRIPTION
+create-image.ps1 creates an Azure Linux VM image, set up for vcpkg's CI system.
+This script assumes you have installed Azure tools into PowerShell by following the instructions
+at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
+or are running from Azure Cloud Shell.
+
+This script assumes you have installed the OpenSSH Client optional Windows component.
+#>
+
+$Location = 'eastasia'
+$Prefix = 'Lin-'
+$Prefix += (Get-Date -Format 'yyyy-MM-dd')
+$VMSize = 'Standard_D8a_v4'
+$ProtoVMName = 'PROTOTYPE'
+$ErrorActionPreference = 'Stop'
+
+$ProgressActivity = 'Creating Linux Image'
+$TotalProgress = 11
+$CurrentProgress = 1
+
+Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating SSH key' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$sshDir = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName()
+mkdir $sshDir
+try {
+  ssh-keygen.exe -q -b 2048 -t rsa -f "$sshDir/key" -P [string]::Empty
+  $sshPublicKey = Get-Content "$sshDir/key.pub"
+} finally {
+  Remove-Item $sshDir -Recurse -Force
+}
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating resource group' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$ResourceGroupName = Find-ResourceGroupName $Prefix
+$AdminPW = New-Password
+New-AzResourceGroup -Name $ResourceGroupName -Location $Location
+$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
+$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating virtual network' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating prototype VM' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$NicName = $ResourceGroupName + 'NIC'
+$Nic = New-AzNetworkInterface `
+  -Name $NicName `
+  -ResourceGroupName $ResourceGroupName `
+  -Location $Location `
+  -Subnet $VirtualNetwork.Subnets[0]
+
+$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
+$VM = Set-AzVMOperatingSystem `
+  -VM $VM `
+  -Linux `
+  -ComputerName $ProtoVMName `
+  -Credential $Credential `
+  -DisablePasswordAuthentication
+
+$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
+$VM = Set-AzVMSourceImage `
+  -VM $VM `
+  -PublisherName 'Canonical' `
+  -Offer '0001-com-ubuntu-server-focal' `
+  -Skus '20_04-lts' `
+  -Version latest
+
+$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
+
+$VM = Add-AzVMSshPublicKey `
+  -VM $VM `
+  -KeyData $sshPublicKey `
+  -Path "/home/AdminUser/.ssh/authorized_keys"
+
+New-AzVm `
+  -ResourceGroupName $ResourceGroupName `
+  -Location $Location `
+  -VM $VM
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Waiting 1 minute to let Azure VM customizations be applied' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Start-Sleep -Seconds 60
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Restarting VM' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Restart-AzVm -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Running provisioning script provision-image.sh in VM' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
+  -ResourceGroupName $ResourceGroupName `
+  -VMName $ProtoVMName `
+  -CommandId 'RunShellScript' `
+  -ScriptPath "$PSScriptRoot\provision-image.sh"
+
+Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Restarting VM' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Converting VM to Image' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Stop-AzVM `
+  -ResourceGroupName $ResourceGroupName `
+  -Name $ProtoVMName `
+  -Force
+
+Set-AzVM `
+  -ResourceGroupName $ResourceGroupName `
+  -Name $ProtoVMName `
+  -Generalized
+
+$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID
+$ImageName = Find-ImageName -ResourceGroupName 'vcpkg-image-minting' -Prefix $Prefix
+New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName 'vcpkg-image-minting'
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Deleting unused temporary resources' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Remove-AzResourceGroup $ResourceGroupName -Force
+
+####################################################################################################
+Write-Progress -Activity $ProgressActivity -Completed
+Write-Host "Generated Image:  $ImageName"
+Write-Host 'Finished!'

externals/vcpkg/scripts/azure-pipelines/linux/create-vmss.ps1

@@ -0,0 +1,108 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+<#
+.SYNOPSIS
+Creates a Linux virtual machine scale set, set up for vcpkg's CI.
+
+.DESCRIPTION
+create-vmss.ps1 creates an Azure Linux VM scale set, set up for vcpkg's CI
+system. See https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/overview
+for more information.
+
+This script assumes you have installed Azure tools into PowerShell by following the instructions
+at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
+or are running from Azure Cloud Shell.
+
+This script assumes you have installed the OpenSSH Client optional Windows component.
+
+
+.PARAMETER ImageName
+The name of the image to deploy into the scale set.
+#>
+
+[CmdLetBinding()]
+Param(
+  [parameter(Mandatory=$true)]
+  [string]$ImageName
+)
+
+$Location = 'eastasia'
+$Prefix = 'PrLin-'
+$Prefix += (Get-Date -Format 'yyyy-MM-dd')
+$VMSize = 'Standard_D32a_v4'
+$LiveVMPrefix = 'BUILD'
+$ErrorActionPreference = 'Stop'
+
+Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
+
+$sshDir = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName()
+mkdir $sshDir
+try {
+  ssh-keygen.exe -q -b 2048 -t rsa -f "$sshDir/key" -P [string]::Empty
+  $sshPublicKey = Get-Content "$sshDir/key.pub"
+} finally {
+  Remove-Item $sshDir -Recurse -Force
+}
+$ResourceGroupName = Find-ResourceGroupName $Prefix
+$AdminPW = New-Password
+$Image = Get-AzImage -ResourceGroupName 'vcpkg-image-minting' -ImageName $ImageName
+
+New-AzResourceGroup -Name $ResourceGroupName -Location $Location
+
+$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
+$VmssIpConfigName = $ResourceGroupName + 'VmssIpConfig'
+$VmssIpConfig = New-AzVmssIpConfig -SubnetId $VirtualNetwork.Subnets[0].Id -Primary -Name $VmssIpConfigName
+$VmssName = $ResourceGroupName + 'Vmss'
+$Vmss = New-AzVmssConfig `
+  -Location $Location `
+  -SkuCapacity 0 `
+  -SkuName $VMSize `
+  -SkuTier 'Standard' `
+  -Overprovision $false `
+  -UpgradePolicyMode Manual `
+  -EvictionPolicy Delete `
+  -Priority Spot `
+  -MaxPrice -1
+
+$NicName = $ResourceGroupName + 'NIC'
+New-AzNetworkInterface `
+  -Name $NicName `
+  -ResourceGroupName $ResourceGroupName `
+  -Location $Location `
+  -Subnet $VirtualNetwork.Subnets[0]
+
+$Vmss = Add-AzVmssNetworkInterfaceConfiguration `
+  -VirtualMachineScaleSet $Vmss `
+  -Primary $true `
+  -IpConfiguration $VmssIpConfig `
+  -NetworkSecurityGroupId $VirtualNetwork.Subnets[0].NetworkSecurityGroup.Id `
+  -Name $NicName
+
+$VmssPublicKey = New-Object -TypeName 'Microsoft.Azure.Management.Compute.Models.SshPublicKey' `
+  -ArgumentList @('/home/AdminUser/.ssh/authorized_keys', $sshPublicKey)
+
+$Vmss = Set-AzVmssOsProfile `
+  -VirtualMachineScaleSet $Vmss `
+  -ComputerNamePrefix $LiveVMPrefix `
+  -AdminUsername AdminUser `
+  -AdminPassword $AdminPW `
+  -LinuxConfigurationDisablePasswordAuthentication $true `
+  -PublicKey @($VmssPublicKey)
+
+$Vmss = Set-AzVmssStorageProfile `
+  -VirtualMachineScaleSet $Vmss `
+  -OsDiskCreateOption 'FromImage' `
+  -OsDiskCaching ReadOnly `
+  -DiffDiskSetting Local `
+  -ImageReferenceId $Image.Id
+
+New-AzVmss `
+  -ResourceGroupName $ResourceGroupName `
+  -Name $VmssName `
+  -VirtualMachineScaleSet $Vmss
+
+Write-Host "Location: $Location"
+Write-Host "Resource group name: $ResourceGroupName"
+Write-Host 'Finished!'

externals/vcpkg/scripts/azure-pipelines/linux/provision-image.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get -y update
+apt-get -y dist-upgrade
+# Install common build dependencies
+APT_PACKAGES="at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils \
+  build-essential g++ gfortran zip libx11-dev libxkbcommon-x11-dev libxi-dev \
+  libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev \
+  libxcursor-dev yasm libnuma1 libnuma-dev python-six python3-six python-yaml \
+  flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \
+  xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full \
+  pkg-config meson nasm cmake ninja-build"
+
+# Additionally required by qt5-base
+APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \
+  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev"
+
+# Additionally required by qt5-base for qt5-x11extras
+APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \
+  libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync0-dev \
+  libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev \
+  libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev"
+
+# Additionally required by libhdfs3
+APT_PACKAGES="$APT_PACKAGES libkrb5-dev"
+
+# Additionally required by kf5windowsystem
+APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev"
+
+# Additionally required by mesa
+APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako"
+
+# Additionally required by some packages to install additional python packages
+APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv"
+
+# Additionally required by qtwebengine
+APT_PACKAGES="$APT_PACKAGES nodejs"
+
+# Additionally required by qtwayland
+APT_PACKAGES="$APT_PACKAGES libwayland-dev"
+
+# Additionally required by all GN projects
+APT_PACKAGES="$APT_PACKAGES python2 python-is-python3"
+
+# Additionally required by libctl
+APT_PACKAGES="$APT_PACKAGES guile-2.2-dev"
+
+# Additionally required by gtk
+APT_PACKAGES="$APT_PACKAGES libxdamage-dev"
+
+# Additionally required/installed by Azure DevOps Scale Set Agents
+APT_PACKAGES="$APT_PACKAGES liblttng-ust0 libkrb5-3 zlib1g libicu66"
+
+apt-get -y install $APT_PACKAGES
+
+# Install the latest Haskell stack for bond
+curl -sSL https://get.haskellstack.org/ | sh
+
+# Install CUDA
+wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin
+mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600
+apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
+add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
+apt-get -y update
+apt-get install -y --no-install-recommends cuda-compiler-11-6 cuda-libraries-dev-11-6 cuda-driver-dev-11-6 \
+  cuda-cudart-dev-11-6 libcublas-11-6 libcurand-dev-11-6 cuda-nvml-dev-11-6 libcudnn8-dev libnccl2 libnccl-dev
+
+# Install PowerShell
+wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb
+dpkg -i packages-microsoft-prod.deb
+apt-get update
+add-apt-repository universe
+apt-get install -y powershell