From ad7dbe18da68aff54de4315274a7ca8442d08ed8 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Sun, 1 May 2022 10:43:03 -0500 Subject: [PATCH] Use curl by default --- src/bootstrap/bootstrap.py | 53 +++++++++++++++++++++----------- src/bootstrap/lib.rs | 14 ++++++++- src/bootstrap/native.rs | 63 +++++++++++++++++++------------------- src/bootstrap/util.rs | 18 +++++++++++ 4 files changed, 98 insertions(+), 50 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index e38a574ca2310..f55040671a7cf 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -110,29 +110,42 @@ def download(path, url, probably_big, verbose, help_on_error=None): def _download(path, url, probably_big, verbose, exception, help_on_error=None): + # Try to use curl (potentially available on win32 + # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/) + # If an error occurs: + # - If we are on win32 fallback to powershell + # - Otherwise raise the error if appropriate if probably_big or verbose: print("downloading {}".format(url)) - # see https://serverfault.com/questions/301128/how-to-download - if sys.platform == 'win32': - run(["PowerShell.exe", "/nologo", "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], - verbose=verbose, - exception=exception) - else: + + platform_is_win32 = sys.platform == 'win32' + try: if probably_big or verbose: option = "-#" else: option = "-s" - require(["curl", "--version"]) + # If curl is not present on Win32, we shoud not sys.exit + # but raise `CalledProcessError` or `OSError` instead + require(["curl", "--version"], exception=platform_is_win32) run(["curl", option, "-L", # Follow redirect. "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds "--connect-timeout", "30", # timeout if cannot connect within 30 seconds "--retry", "3", "-Sf", "-o", path, url], verbose=verbose, - exception=exception, + exception=True, # Will raise RuntimeError on failure help_on_error=help_on_error) + except (subprocess.CalledProcessError, OSError, RuntimeError): + # see http://serverfault.com/questions/301128/how-to-download + if platform_is_win32: + run(["PowerShell.exe", "/nologo", "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], + verbose=verbose, + exception=exception) + # Check if the RuntimeError raised by run(curl) should be silenced + elif verbose or exception: + raise def verify(path, expected, verbose): @@ -198,19 +211,23 @@ def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error= sys.exit(err) -def require(cmd, exit=True): +def require(cmd, exit=True, exception=False): '''Run a command, returning its output. On error, - If `exit` is `True`, exit the process. - Otherwise, return None.''' + If `exception` is `True`, raise the error + Otherwise If `exit` is `True`, exit the process + Else return None.''' try: return subprocess.check_output(cmd).strip() except (subprocess.CalledProcessError, OSError) as exc: - if not exit: - return None - print("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) - print("Please make sure it's installed and in the path.") - sys.exit(1) + if exception: + raise + elif exit: + print("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) + print("Please make sure it's installed and in the path.") + sys.exit(1) + return None + def format_build_time(duration): diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index b4b973b42479e..0c0bc1aee3aa2 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -122,7 +122,8 @@ use once_cell::sync::OnceCell; use crate::builder::Kind; use crate::config::{LlvmLibunwind, TargetSelection}; use crate::util::{ - exe, libdir, mtime, output, run, run_suppressed, t, try_run, try_run_suppressed, CiEnv, + check_run, exe, libdir, mtime, output, run, run_suppressed, t, try_run, try_run_suppressed, + CiEnv, }; mod builder; @@ -956,6 +957,17 @@ impl Build { try_run_suppressed(cmd) } + /// Runs a command, printing out nice contextual information if it fails. + /// Returns false if do not execute at all, otherwise returns its + /// `status.success()`. + fn check_run(&self, cmd: &mut Command) -> bool { + if self.config.dry_run { + return true; + } + self.verbose(&format!("running: {:?}", cmd)); + check_run(cmd, self.is_verbose()) + } + pub fn is_verbose(&self) -> bool { self.verbosity > 0 } diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 64e25f803b27f..8245bebfe3118 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -306,39 +306,40 @@ fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: & fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) { println!("downloading {}", url); - - // FIXME: check if curl is installed instead of skipping straight to powershell - if builder.build.build.contains("windows-msvc") { - for _ in 0..3 { - if builder.try_run(Command::new("PowerShell.exe").args(&[ - "/nologo", - "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile - ), - ])) { - return; + // Try curl. If that fails and we are on windows, fallback to PowerShell. + if !builder.check_run(Command::new("curl").args(&[ + "-#", + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "--retry", + "3", + "-Sf", + "-o", + tempfile, + url, + ])) { + if builder.build.build.contains("windows-msvc") { + println!("Fallback to PowerShell"); + for _ in 0..3 { + if builder.try_run(Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile + ), + ])) { + return; + } + println!("\nspurious failure, trying again"); } - println!("\nspurious failure, trying again"); } - } else { - builder.run(Command::new("curl").args(&[ - "-#", - "-y", - "30", - "-Y", - "10", // timeout if speed is < 10 bytes/sec for > 30 seconds - "--connect-timeout", - "30", // timeout if cannot connect within 30 seconds - "--retry", - "3", - "-Sf", - "-o", - tempfile, - url, - ])); + std::process::exit(1); } } diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index b78ca3712bd45..710b3588b845b 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -346,6 +346,24 @@ pub fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { status.success() } +pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { + let status = match cmd.status() { + Ok(status) => status, + Err(e) => { + println!("failed to execute command: {:?}\nerror: {}", cmd, e); + return false; + } + }; + if !status.success() && print_cmd_on_fail { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n", + cmd, status + ); + } + status.success() +} + pub fn run_suppressed(cmd: &mut Command) { if !try_run_suppressed(cmd) { std::process::exit(1);