build: split big arch patch (wip)

package_linux_bin.sh

@@ -16,23 +16,26 @@ cd vscode || { echo "'vscode' dir not found"; exit 1; }
 export VSCODE_SKIP_NODE_VERSION_CHECK=1
 export VSCODE_SYSROOT_PREFIX='-glibc-2.17'
 
-if [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
-  export VSCODE_SYSROOT_REPO='VSCodium/vscode-linux-build-agent'
+if [[ "${VSCODE_ARCH}" == "arm64" || "${VSCODE_ARCH}" == "armhf" ]]; then
+  export VSCODE_SKIP_SYSROOT=1
+  export USE_GNUPP2A=1
+elif [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
+  export VSCODE_SYSROOT_REPOSITORY='VSCodium/vscode-linux-build-agent'
   export VSCODE_SYSROOT_VERSION='20240129-253798'
   export VSCODE_SYSROOT_PREFIX='-glibc-2.28'
-fi
-
-if [[ "${VSCODE_ARCH}" == "riscv64" ]]; then
-  export VSCODE_ELECTRON_REPO='riscv-forks/electron-riscv-releases'
+  export USE_GNUPP2A=1
+elif [[ "${VSCODE_ARCH}" == "riscv64" ]]; then
+  export VSCODE_ELECTRON_REPOSITORY='riscv-forks/electron-riscv-releases'
   export ELECTRON_SKIP_BINARY_DOWNLOAD=1
   export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
+  export VSCODE_SKIP_SETUPENV=1
 
   source ../electron.riscv64.sh
 
   if [[ "${ELECTRON_VERSION}" != "$(yarn config get target)" ]]; then
     # Fail the pipeline if electron target doesn't match what is used.
     echo "Electron RISC-V binary version doesn't match target electron version!"
-    echo "Releases available at: https://github.com/${VSCODE_ELECTRON_REPO}/releases"
+    echo "Releases available at: https://github.com/${VSCODE_ELECTRON_REPOSITORY}/releases"
     exit 1
   fi
 fi
@@ -49,6 +52,28 @@ if [[ -d "../patches/linux/client/" ]]; then
   done
 fi
 
+if [[ -n "${USE_GNUPP2A}" ]]; then
+  INCLUDES=$(cat <<EOF
+{
+  "target_defaults": {
+    "conditions": [
+      ["OS=='linux'", {
+        'cflags_cc!': [ '-std=gnu++20' ],
+        'cflags_cc': [ '-std=gnu++2a' ],
+      }]
+    ]
+  }
+}
+EOF
+)
+
+  if [ ! -d "$HOME/.gyp" ]; then
+    mkdir -p "$HOME/.gyp"
+  fi
+
+  echo "${INCLUDES}" > "$HOME/.gyp/include.gypi"
+fi
+
 for i in {1..5}; do # try 5 times
   npm ci --prefix build && break
   if [[ $i == 3 ]]; then
@@ -58,10 +83,12 @@ for i in {1..5}; do # try 5 times
   echo "Npm install failed $i, trying again..."
 done
 
-if [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
-  source ./build/azure-pipelines/linux/setup-env.sh
-else
-  ./build/azure-pipelines/linux/setup-env.sh
+if [[ -z "${VSCODE_SKIP_SETUPENV}" ]]; then
+  if [[ -n "${VSCODE_SKIP_SYSROOT}" ]]; then
+    source ./build/azure-pipelines/linux/setup-env.sh --skip-sysroot
+  else
+    source ./build/azure-pipelines/linux/setup-env.sh
+  fi
 fi
 
 for i in {1..5}; do # try 5 times

package_linux_reh.sh

@@ -19,38 +19,48 @@ GLIBC_VERSION="2.17"
 GLIBCXX_VERSION="3.4.22"
 NODE_VERSION="16.20.2"
 
-if [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
-  GLIBC_VERSION="2.28"
-elif [[ "${VSCODE_ARCH}" == "riscv64" ]]; then
-  # Unofficial RISC-V nodejs builds doesn't provide v16.x
-  # Node 18 is buggy so use 20 here for now: https://github.com/VSCodium/vscodium/issues/2060
-  NODE_VERSION="20.16.0"
-fi
-
-export VSCODE_PLATFORM='linux'
-export VSCODE_SKIP_NODE_VERSION_CHECK=1
-export VSCODE_SYSROOT_PREFIX="-glibc-${GLIBC_VERSION}"
-
-VSCODE_HOST_MOUNT="$( pwd )"
-
-export VSCODE_HOST_MOUNT
-
-if [[ "${VSCODE_ARCH}" == "x64" || "${VSCODE_ARCH}" == "arm64" ]]; then
+if [[ "${VSCODE_ARCH}" == "x64" ]]; then
+  VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:centos7-devtoolset8-${VSCODE_ARCH}"
+elif [[ "${VSCODE_ARCH}" == "arm64" ]]; then
   VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:centos7-devtoolset8-${VSCODE_ARCH}"
+
+  export VSCODE_SKIP_SYSROOT=1
+  export USE_GNUPP2A=1
 elif [[ "${VSCODE_ARCH}" == "armhf" ]]; then
   VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:bionic-devtoolset-arm32v7"
+
+  export VSCODE_SKIP_SYSROOT=1
+  export USE_GNUPP2A=1
 elif [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
+  GLIBC_VERSION="2.28"
   VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:bionic-devtoolset-ppc64le"
+
   export ELECTRON_SKIP_BINARY_DOWNLOAD=1
   export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
-  export VSCODE_SYSROOT_REPO='VSCodium/vscode-linux-build-agent'
+  export VSCODE_SYSROOT_REPOSITORY='VSCodium/vscode-linux-build-agent'
   export VSCODE_SYSROOT_VERSION='20240129-253798'
+  export USE_GNUPP2A=1
 elif [[ "${VSCODE_ARCH}" == "riscv64" ]]; then
+  # Unofficial RISC-V nodejs builds doesn't provide v16.x
+  # Node 18 is buggy so use 20 here for now: https://github.com/VSCodium/vscodium/issues/2060
+  NODE_VERSION="20.16.0"
   VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:focal-devtoolset-riscv64"
+
   export ELECTRON_SKIP_BINARY_DOWNLOAD=1
   export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
+  export VSCODE_SKIP_SETUPENV=1
+  export VSCODE_NODEJS_SITE='https://unofficial-builds.nodejs.org'
+  # part of the url before '/v${nodeVersion}/node-v${nodeVersion}-${platform}-${arch}.tar.gz'
+  export VSCODE_NODEJS_URLROOT='/download/release'
 fi
 
+export VSCODE_PLATFORM='linux'
+export VSCODE_SKIP_NODE_VERSION_CHECK=1
+export VSCODE_SYSROOT_PREFIX="-glibc-${GLIBC_VERSION}"
+
+VSCODE_HOST_MOUNT="$( pwd )"
+
+export VSCODE_HOST_MOUNT
 export VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME
 
 sed -i "/target/s/\"20.*\"/\"${NODE_VERSION}\"/" remote/.npmrc
@@ -73,7 +83,7 @@ if [[ -d "../patches/linux/reh/" ]]; then
   done
 fi
 
-if [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
+if [[ -n "${USE_GNUPP2A}" ]]; then
   INCLUDES=$(cat <<EOF
 {
   "target_defaults": {
@@ -104,10 +114,12 @@ for i in {1..5}; do # try 5 times
   echo "Npm install failed $i, trying again..."
 done
 
-if [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
-  source ./build/azure-pipelines/linux/setup-env.sh
-else
-  ./build/azure-pipelines/linux/setup-env.sh --only-remote
+if [[ -z "${VSCODE_SKIP_SETUPENV}" ]]; then
+  if [[ -n "${VSCODE_SKIP_SYSROOT}" ]]; then
+    source ./build/azure-pipelines/linux/setup-env.sh --skip-sysroot
+  else
+    source ./build/azure-pipelines/linux/setup-env.sh
+  fi
 fi
 
 for i in {1..5}; do # try 5 times

patch.sh

@@ -1,28 +1,49 @@
 #!/usr/bin/env bash
 
-if [[ "${1}" == *patch ]]; then
-  FILE="../patches/${1}"
-else
-  FILE="../patches/${1}.patch"
-fi
+echo "$#"
 
 cd vscode || { echo "'vscode' dir not found"; exit 1; }
 
 git add .
 git reset -q --hard HEAD
 
-if [[ -f "${FILE}" ]]; then
+while [[ -n "$( git log -1 | grep "VSCODIUM HELPER" )" ]]; do
+  git reset -q --hard HEAD~
+done
+
+git apply --reject "../patches/helper/settings.patch"
+
+while [ $# -gt 1 ]; do
+  echo "Parameter: $1"
+  if [[ "${1}" == *patch ]]; then
+    FILE="../patches/${1}"
+  else
+    FILE="../patches/${1}.patch"
+  fi
+
   git apply --reject "${FILE}"
+
+  shift
+done
+
+git add .
+git commit -q -m "VSCODIUM HELPER"
+
+if [[ "${1}" == *patch ]]; then
+  FILE="../patches/${1}"
+else
+  FILE="../patches/${1}.patch"
 fi
 
-git apply --reject "../patches/helper/settings.patch"
+if [[ -f "${FILE}" ]]; then
+  git apply --reject "${FILE}"
+fi
 
 read -rp "Press any key when the conflict have been resolved..." -n1 -s
 
-git restore .vscode/settings.json
-
 git add .
 git diff --staged -U1 > "${FILE}"
+git reset -q --hard HEAD~
 
 cd ..
 

patches/linux/arch-0-support.patch

@@ -0,0 +1,116 @@
+diff --git a/build/azure-pipelines/linux/setup-env.sh b/build/azure-pipelines/linux/setup-env.sh
+index d836fb7..fbe67b0 100755
+--- a/build/azure-pipelines/linux/setup-env.sh
++++ b/build/azure-pipelines/linux/setup-env.sh
+@@ -2,3 +2,3 @@
+
+-set -e
++set -ex
+
+diff --git a/build/gulpfile.reh.js b/build/gulpfile.reh.js
+index 53ef6f3..d0770d6 100644
+--- a/build/gulpfile.reh.js
++++ b/build/gulpfile.reh.js
+@@ -229,9 +229,23 @@ function nodejs(platform, arch) {
+ 		case 'linux':
+-			return (product.nodejsRepository !== 'https://nodejs.org' ?
+-				fetchGithub(product.nodejsRepository, { version: `${nodeVersion}-${internalNodeVersion}`, name: expectedName, checksumSha256 }) :
+-				fetchUrls(`/dist/v${nodeVersion}/node-v${nodeVersion}-${platform}-${arch}.tar.gz`, { base: 'https://nodejs.org', checksumSha256 })
+-			).pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
+-				.pipe(filter('**/node'))
+-				.pipe(util.setExecutableBit('**'))
+-				.pipe(rename('node'));
++			if (process.env.VSCODE_NODEJS_SITE && process.env.VSCODE_NODEJS_URLROOT) {
++				return fetchUrls(`${process.env.VSCODE_NODEJS_URLROOT}/v${nodeVersion}/node-v${nodeVersion}-${platform}-${arch}.tar.gz`, { base: process.env.VSCODE_NODEJS_SITE, checksumSha256 })
++					.pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
++					.pipe(filter('**/node'))
++					.pipe(util.setExecutableBit('**'))
++					.pipe(rename('node'));
++			}
++			if (product.nodejsRepository !== 'https://nodejs.org') {
++				return fetchGithub(product.nodejsRepository, { version: `${nodeVersion}-${internalNodeVersion}`, name: expectedName, checksumSha256 })
++					.pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
++					.pipe(filter('**/node'))
++					.pipe(util.setExecutableBit('**'))
++					.pipe(rename('node'));
++			}
++			else {
++				return fetchUrls(`/dist/v${nodeVersion}/node-v${nodeVersion}-${platform}-${arch}.tar.gz`, { base: 'https://nodejs.org', checksumSha256 })
++					.pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
++					.pipe(filter('**/node'))
++					.pipe(util.setExecutableBit('**'))
++					.pipe(rename('node'));
++			}
+ 		case 'alpine':
+diff --git a/build/gulpfile.vscode.js b/build/gulpfile.vscode.js
+index da753e9..95f2cf0 100644
+--- a/build/gulpfile.vscode.js
++++ b/build/gulpfile.vscode.js
+@@ -385,2 +385,12 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
+
++		const electronOverride = {};
++		if (process.env.VSCODE_ELECTRON_REPOSITORY) {
++			// official electron doesn't support all arch, override the repo with `VSCODE_ELECTRON_REPOSITORY`.
++			electronOverride.repo = process.env.VSCODE_ELECTRON_REPOSITORY;
++		}
++
++		if (process.env.VSCODE_ELECTRON_TAG) {
++			electronOverride.tag = process.env.VSCODE_ELECTRON_TAG;
++		}
++
+ 		let result = all
+@@ -389,3 +399,3 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
+ 			.pipe(filter(['**', '!**/.github/**'], { dot: true })) // https://github.com/microsoft/vscode/issues/116523
+-			.pipe(electron({ ...config, platform, arch: arch === 'armhf' ? 'arm' : arch, ffmpegChromium: false }))
++			.pipe(electron({ ...config, ...electronOverride, platform, arch: arch === 'armhf' ? 'arm' : arch, ffmpegChromium: false }))
+ 			.pipe(filter(['**', '!LICENSE', '!version'], { dot: true }));
+diff --git a/build/linux/debian/dep-lists.js b/build/linux/debian/dep-lists.js
+index 3bb58fb..306d91e 100644
+--- a/build/linux/debian/dep-lists.js
++++ b/build/linux/debian/dep-lists.js
+@@ -138,3 +138,3 @@ exports.referenceGeneratedDepsByArch = {
+         'xdg-utils (>= 1.0.2)'
+-    ]
++    ],
+ };
+diff --git a/build/linux/debian/dep-lists.ts b/build/linux/debian/dep-lists.ts
+index e3d78d1..9f29943 100644
+--- a/build/linux/debian/dep-lists.ts
++++ b/build/linux/debian/dep-lists.ts
+@@ -138,3 +138,3 @@ export const referenceGeneratedDepsByArch = {
+ 		'xdg-utils (>= 1.0.2)'
+-	]
++	],
+ };
+diff --git a/build/linux/debian/install-sysroot.js b/build/linux/debian/install-sysroot.js
+index 354c67a..2cd8f2d 100644
+--- a/build/linux/debian/install-sysroot.js
++++ b/build/linux/debian/install-sysroot.js
+@@ -71,3 +71,5 @@ async function fetchUrl(options, retries = 10, retryDelay = 1000) {
+         try {
+-            const response = await fetch(`https://api.github.com/repos/Microsoft/vscode-linux-build-agent/releases/tags/v${version}`, {
++            const repository = process.env['VSCODE_SYSROOT_REPOSITORY'] ?? 'Microsoft/vscode-linux-build-agent';
++            const actualVersion = process.env['VSCODE_SYSROOT_VERSION'] ?? version;
++            const response = await fetch(`https://api.github.com/repos/${repository}/releases/tags/v${actualVersion}`, {
+                 headers: ghApiHeaders,
+@@ -80,3 +82,3 @@ async function fetchUrl(options, retries = 10, retryDelay = 1000) {
+                 if (!asset) {
+-                    throw new Error(`Could not find asset in release of Microsoft/vscode-linux-build-agent @ ${version}`);
++                    throw new Error(`Could not find asset in release of ${repository} @ ${actualVersion}`);
+                 }
+diff --git a/build/linux/debian/install-sysroot.ts b/build/linux/debian/install-sysroot.ts
+index 8ea43a5..d8de38e 100644
+--- a/build/linux/debian/install-sysroot.ts
++++ b/build/linux/debian/install-sysroot.ts
+@@ -83,3 +83,5 @@ async function fetchUrl(options: IFetchOptions, retries = 10, retryDelay = 1000)
+ 		try {
+-			const response = await fetch(`https://api.github.com/repos/Microsoft/vscode-linux-build-agent/releases/tags/v${version}`, {
++			const repository = process.env['VSCODE_SYSROOT_REPOSITORY'] ?? 'Microsoft/vscode-linux-build-agent';
++			const actualVersion = process.env['VSCODE_SYSROOT_VERSION'] ?? version;
++			const response = await fetch(`https://api.github.com/repos/${repository}/releases/tags/v${actualVersion}`, {
+ 				headers: ghApiHeaders,
+@@ -92,3 +94,3 @@ async function fetchUrl(options: IFetchOptions, retries = 10, retryDelay = 1000)
+ 				if (!asset) {
+-					throw new Error(`Could not find asset in release of Microsoft/vscode-linux-build-agent @ ${version}`);
++					throw new Error(`Could not find asset in release of ${repository} @ ${actualVersion}`);
+ 				}