From fa6d54f5c4373c19f0f63b64df483a5537b42c30 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 20 Aug 2024 15:59:35 +0300 Subject: [PATCH 01/10] don't copy `.rustc-dev-contents` from CI rustc Since https://github.com/rust-lang/rust/pull/127188, copying files from `.rustc-dev-contents` regressed https://github.com/rust-lang/rust/issues/108767 again. Since `rustc-src` is already included in the CI rustc sysroot, we don't need to copy these files to have `rustc-src` component. Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/compile.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 4353cfadd8d35..e669f10643882 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1672,16 +1672,8 @@ impl Step for Sysroot { build_helper::exit!(1); } - // Unlike rust-src component, we have to handle rustc-src a bit differently. - // When using CI rustc, we copy rustc-src component from its sysroot, - // otherwise we handle it in a similar way what we do for rust-src above. - if builder.download_rustc() { - cp_rustc_component_to_ci_sysroot( - builder, - &sysroot, - builder.config.ci_rustc_dev_contents(), - ); - } else { + // rustc-src component is already part of CI rustc's sysroot + if !builder.download_rustc() { let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); From 9d694b583e3c4e899f61cb2086e2ed9627ec3619 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Mon, 26 Aug 2024 08:28:26 +0300 Subject: [PATCH 02/10] support custom clippy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similar to cargo, rustc, and rustfmt, this adds the support of using custom clippy on bootstrap. It’s designed for those who want to test their own clippy builds or avoid downloading the stage0 clippy. Signed-off-by: onur-ozkan --- src/bootstrap/src/core/builder.rs | 8 ++++++-- src/bootstrap/src/core/config/config.rs | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index ff0d1f3a725d2..d7398b76cc91e 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -1298,8 +1298,12 @@ impl<'a> Builder<'a> { pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { if run_compiler.stage == 0 { - // `ensure(Clippy { stage: 0 })` *builds* clippy with stage0, it doesn't use the beta clippy. - let cargo_clippy = self.build.config.download_clippy(); + let cargo_clippy = self + .config + .initial_cargo_clippy + .clone() + .unwrap_or_else(|| self.build.config.download_clippy()); + let mut cmd = command(cargo_clippy); cmd.env("CARGO", &self.initial_cargo); return cmd; diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index ce23b7735f8bd..cf87f1ff9470e 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -344,6 +344,7 @@ pub struct Config { // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, + pub initial_cargo_clippy: Option, #[cfg(not(test))] initial_rustfmt: RefCell, @@ -834,6 +835,7 @@ define_config! { cargo: Option = "cargo", rustc: Option = "rustc", rustfmt: Option = "rustfmt", + cargo_clippy: Option = "cargo-clippy", docs: Option = "docs", compiler_docs: Option = "compiler-docs", library_docs_private_items: Option = "library-docs-private-items", @@ -1439,6 +1441,7 @@ impl Config { cargo, rustc, rustfmt, + cargo_clippy, docs, compiler_docs, library_docs_private_items, @@ -1491,6 +1494,14 @@ impl Config { config.out = absolute(&config.out).expect("can't make empty path absolute"); } + if cargo_clippy.is_some() && rustc.is_none() { + println!( + "WARNING: Using `build.cargo-clippy` without `build.rustc` usually fails due to toolchain conflict." + ); + } + + config.initial_cargo_clippy = cargo_clippy; + config.initial_rustc = if let Some(rustc) = rustc { if !flags.skip_stage0_validation { config.check_stage0_version(&rustc, "rustc"); From 9dcc65600e63d23a727190ca1fd91ab645adefb8 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Mon, 26 Aug 2024 08:28:54 +0300 Subject: [PATCH 03/10] document `build.cargo-clippy` option Signed-off-by: onur-ozkan --- config.example.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config.example.toml b/config.example.toml index f1dc32234ccf2..b967d5d9fe8c2 100644 --- a/config.example.toml +++ b/config.example.toml @@ -230,6 +230,13 @@ # use this rustfmt binary instead as the stage0 snapshot rustfmt. #rustfmt = "/path/to/rustfmt" +# Instead of downloading the src/stage0 version of cargo-clippy specified, +# use this cargo-clippy binary instead as the stage0 snapshot cargo-clippy. +# +# Note that this option should be used with the same toolchain as the `rustc` option above. +# Otherwise, clippy is likely to fail due to a toolchain conflict. +#cargo-clippy = "/path/to/cargo-clippy" + # Whether to build documentation by default. If false, rustdoc and # friends will still be compiled but they will not be used to generate any # documentation. From 1a991e5b805832aca83862458864a7caf57d6725 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Mon, 26 Aug 2024 08:29:17 +0300 Subject: [PATCH 04/10] add change entry for custom clippy support Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 51a25104e4cfb..3ab14aade9f81 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -235,4 +235,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "The `build.profiler` option now tries to use source code from `download-ci-llvm` if possible, instead of checking out the `src/llvm-project` submodule.", }, + ChangeInfo { + change_id: 129152, + severity: ChangeSeverity::Info, + summary: "New option `build.cargo-clippy` added for supporting the use of custom/external clippy.", + }, ]; From bb9d5c4658218f6e1c477d8c3e0224c1d708e42b Mon Sep 17 00:00:00 2001 From: Chris Denton Date: Sat, 31 Aug 2024 02:40:35 +0000 Subject: [PATCH 05/10] Move remove_dir_all impl into a module --- library/std/src/sys/pal/windows/api.rs | 5 +- .../std/src/sys/pal/windows/c/bindings.txt | 5 + .../std/src/sys/pal/windows/c/windows_sys.rs | 5 + library/std/src/sys/pal/windows/fs.rs | 211 ++++-------------- .../src/sys/pal/windows/fs/remove_dir_all.rs | 196 ++++++++++++++++ 5 files changed, 253 insertions(+), 169 deletions(-) create mode 100644 library/std/src/sys/pal/windows/fs/remove_dir_all.rs diff --git a/library/std/src/sys/pal/windows/api.rs b/library/std/src/sys/pal/windows/api.rs index 00c816a6c09b8..9e336ff2d473d 100644 --- a/library/std/src/sys/pal/windows/api.rs +++ b/library/std/src/sys/pal/windows/api.rs @@ -254,7 +254,7 @@ pub struct WinError { pub code: u32, } impl WinError { - const fn new(code: u32) -> Self { + pub const fn new(code: u32) -> Self { Self { code } } } @@ -272,8 +272,11 @@ impl WinError { // tidy-alphabetical-start pub const ACCESS_DENIED: Self = Self::new(c::ERROR_ACCESS_DENIED); pub const ALREADY_EXISTS: Self = Self::new(c::ERROR_ALREADY_EXISTS); + pub const BAD_NET_NAME: Self = Self::new(c::ERROR_BAD_NET_NAME); + pub const BAD_NETPATH: Self = Self::new(c::ERROR_BAD_NETPATH); pub const CANT_ACCESS_FILE: Self = Self::new(c::ERROR_CANT_ACCESS_FILE); pub const DELETE_PENDING: Self = Self::new(c::ERROR_DELETE_PENDING); + pub const DIR_NOT_EMPTY: Self = Self::new(c::ERROR_DIR_NOT_EMPTY); pub const DIRECTORY: Self = Self::new(c::ERROR_DIRECTORY); pub const FILE_NOT_FOUND: Self = Self::new(c::ERROR_FILE_NOT_FOUND); pub const INSUFFICIENT_BUFFER: Self = Self::new(c::ERROR_INSUFFICIENT_BUFFER); diff --git a/library/std/src/sys/pal/windows/c/bindings.txt b/library/std/src/sys/pal/windows/c/bindings.txt index afacc370c3420..9c2e4500da068 100644 --- a/library/std/src/sys/pal/windows/c/bindings.txt +++ b/library/std/src/sys/pal/windows/c/bindings.txt @@ -34,6 +34,7 @@ Windows.Wdk.Storage.FileSystem.FILE_WRITE_THROUGH Windows.Wdk.Storage.FileSystem.NtCreateFile Windows.Wdk.Storage.FileSystem.NTCREATEFILE_CREATE_DISPOSITION Windows.Wdk.Storage.FileSystem.NTCREATEFILE_CREATE_OPTIONS +Windows.Wdk.Storage.FileSystem.NtOpenFile Windows.Wdk.Storage.FileSystem.NtReadFile Windows.Wdk.Storage.FileSystem.NtWriteFile Windows.Wdk.Storage.FileSystem.SYMLINK_FLAG_RELATIVE @@ -1931,10 +1932,14 @@ Windows.Win32.Foundation.RtlNtStatusToDosError Windows.Win32.Foundation.SetHandleInformation Windows.Win32.Foundation.SetLastError Windows.Win32.Foundation.STATUS_DELETE_PENDING +Windows.Win32.Foundation.STATUS_DIRECTORY_NOT_EMPTY Windows.Win32.Foundation.STATUS_END_OF_FILE +Windows.Win32.Foundation.STATUS_FILE_DELETED +Windows.Win32.Foundation.STATUS_INVALID_HANDLE Windows.Win32.Foundation.STATUS_INVALID_PARAMETER Windows.Win32.Foundation.STATUS_NOT_IMPLEMENTED Windows.Win32.Foundation.STATUS_PENDING +Windows.Win32.Foundation.STATUS_SHARING_VIOLATION Windows.Win32.Foundation.STATUS_SUCCESS Windows.Win32.Foundation.TRUE Windows.Win32.Foundation.UNICODE_STRING diff --git a/library/std/src/sys/pal/windows/c/windows_sys.rs b/library/std/src/sys/pal/windows/c/windows_sys.rs index 529c96a0e1e6b..ab5f8919d7af6 100644 --- a/library/std/src/sys/pal/windows/c/windows_sys.rs +++ b/library/std/src/sys/pal/windows/c/windows_sys.rs @@ -105,6 +105,7 @@ windows_targets::link!("kernel32.dll" "system" fn WideCharToMultiByte(codepage : windows_targets::link!("kernel32.dll" "system" fn WriteConsoleW(hconsoleoutput : HANDLE, lpbuffer : PCWSTR, nnumberofcharstowrite : u32, lpnumberofcharswritten : *mut u32, lpreserved : *const core::ffi::c_void) -> BOOL); windows_targets::link!("kernel32.dll" "system" fn WriteFileEx(hfile : HANDLE, lpbuffer : *const u8, nnumberofbytestowrite : u32, lpoverlapped : *mut OVERLAPPED, lpcompletionroutine : LPOVERLAPPED_COMPLETION_ROUTINE) -> BOOL); windows_targets::link!("ntdll.dll" "system" fn NtCreateFile(filehandle : *mut HANDLE, desiredaccess : FILE_ACCESS_RIGHTS, objectattributes : *const OBJECT_ATTRIBUTES, iostatusblock : *mut IO_STATUS_BLOCK, allocationsize : *const i64, fileattributes : FILE_FLAGS_AND_ATTRIBUTES, shareaccess : FILE_SHARE_MODE, createdisposition : NTCREATEFILE_CREATE_DISPOSITION, createoptions : NTCREATEFILE_CREATE_OPTIONS, eabuffer : *const core::ffi::c_void, ealength : u32) -> NTSTATUS); +windows_targets::link!("ntdll.dll" "system" fn NtOpenFile(filehandle : *mut HANDLE, desiredaccess : u32, objectattributes : *const OBJECT_ATTRIBUTES, iostatusblock : *mut IO_STATUS_BLOCK, shareaccess : u32, openoptions : u32) -> NTSTATUS); windows_targets::link!("ntdll.dll" "system" fn NtReadFile(filehandle : HANDLE, event : HANDLE, apcroutine : PIO_APC_ROUTINE, apccontext : *const core::ffi::c_void, iostatusblock : *mut IO_STATUS_BLOCK, buffer : *mut core::ffi::c_void, length : u32, byteoffset : *const i64, key : *const u32) -> NTSTATUS); windows_targets::link!("ntdll.dll" "system" fn NtWriteFile(filehandle : HANDLE, event : HANDLE, apcroutine : PIO_APC_ROUTINE, apccontext : *const core::ffi::c_void, iostatusblock : *mut IO_STATUS_BLOCK, buffer : *const core::ffi::c_void, length : u32, byteoffset : *const i64, key : *const u32) -> NTSTATUS); windows_targets::link!("ntdll.dll" "system" fn RtlNtStatusToDosError(status : NTSTATUS) -> u32); @@ -2982,10 +2983,14 @@ pub struct STARTUPINFOW { } pub type STARTUPINFOW_FLAGS = u32; pub const STATUS_DELETE_PENDING: NTSTATUS = 0xC0000056_u32 as _; +pub const STATUS_DIRECTORY_NOT_EMPTY: NTSTATUS = 0xC0000101_u32 as _; pub const STATUS_END_OF_FILE: NTSTATUS = 0xC0000011_u32 as _; +pub const STATUS_FILE_DELETED: NTSTATUS = 0xC0000123_u32 as _; +pub const STATUS_INVALID_HANDLE: NTSTATUS = 0xC0000008_u32 as _; pub const STATUS_INVALID_PARAMETER: NTSTATUS = 0xC000000D_u32 as _; pub const STATUS_NOT_IMPLEMENTED: NTSTATUS = 0xC0000002_u32 as _; pub const STATUS_PENDING: NTSTATUS = 0x103_u32 as _; +pub const STATUS_SHARING_VIOLATION: NTSTATUS = 0xC0000043_u32 as _; pub const STATUS_SUCCESS: NTSTATUS = 0x0_u32 as _; pub const STD_ERROR_HANDLE: STD_HANDLE = 4294967284u32; pub type STD_HANDLE = u32; diff --git a/library/std/src/sys/pal/windows/fs.rs b/library/std/src/sys/pal/windows/fs.rs index 2134152ea93f1..5b360640c4e67 100644 --- a/library/std/src/sys/pal/windows/fs.rs +++ b/library/std/src/sys/pal/windows/fs.rs @@ -14,8 +14,11 @@ use crate::sys::handle::Handle; use crate::sys::path::maybe_verbatim; use crate::sys::time::SystemTime; use crate::sys::{c, cvt, Align8}; -use crate::sys_common::{ignore_notfound, AsInner, FromInner, IntoInner}; -use crate::{fmt, ptr, slice, thread}; +use crate::sys_common::{AsInner, FromInner, IntoInner}; +use crate::{fmt, ptr, slice}; + +mod remove_dir_all; +use remove_dir_all::remove_dir_all_iterative; pub struct File { handle: Handle, @@ -646,6 +649,22 @@ impl File { Ok(info) } } + + /// Deletes the file, consuming the file handle to ensure the delete occurs + /// as immediately as possible. + /// This attempts to use `posix_delete` but falls back to `win32_delete` + /// if that is not supported by the filesystem. + #[allow(unused)] + fn delete(self) -> Result<(), WinError> { + // If POSIX delete is not supported for this filesystem then fallback to win32 delete. + match self.posix_delete() { + Err(WinError::INVALID_PARAMETER) + | Err(WinError::NOT_SUPPORTED) + | Err(WinError::INVALID_FUNCTION) => self.win32_delete(), + result => result, + } + } + /// Delete using POSIX semantics. /// /// Files will be deleted as soon as the handle is closed. This is supported @@ -654,21 +673,23 @@ impl File { /// /// If the operation is not supported for this filesystem or OS version /// then errors will be `ERROR_NOT_SUPPORTED` or `ERROR_INVALID_PARAMETER`. - fn posix_delete(&self) -> io::Result<()> { + #[allow(unused)] + fn posix_delete(&self) -> Result<(), WinError> { let info = c::FILE_DISPOSITION_INFO_EX { Flags: c::FILE_DISPOSITION_FLAG_DELETE | c::FILE_DISPOSITION_FLAG_POSIX_SEMANTICS | c::FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE, }; - api::set_file_information_by_handle(self.handle.as_raw_handle(), &info).io_result() + api::set_file_information_by_handle(self.handle.as_raw_handle(), &info) } /// Delete a file using win32 semantics. The file won't actually be deleted /// until all file handles are closed. However, marking a file for deletion /// will prevent anyone from opening a new handle to the file. - fn win32_delete(&self) -> io::Result<()> { + #[allow(unused)] + fn win32_delete(&self) -> Result<(), WinError> { let info = c::FILE_DISPOSITION_INFO { DeleteFile: c::TRUE as _ }; - api::set_file_information_by_handle(self.handle.as_raw_handle(), &info).io_result() + api::set_file_information_by_handle(self.handle.as_raw_handle(), &info) } /// Fill the given buffer with as many directory entries as will fit. @@ -684,21 +705,23 @@ impl File { /// A symlink directory is simply an empty directory with some "reparse" metadata attached. /// So if you open a link (not its target) and iterate the directory, /// you will always iterate an empty directory regardless of the target. - fn fill_dir_buff(&self, buffer: &mut DirBuff, restart: bool) -> io::Result { + #[allow(unused)] + fn fill_dir_buff(&self, buffer: &mut DirBuff, restart: bool) -> Result { let class = if restart { c::FileIdBothDirectoryRestartInfo } else { c::FileIdBothDirectoryInfo }; unsafe { - let result = cvt(c::GetFileInformationByHandleEx( - self.handle.as_raw_handle(), + let result = c::GetFileInformationByHandleEx( + self.as_raw_handle(), class, buffer.as_mut_ptr().cast(), buffer.capacity() as _, - )); - match result { - Ok(_) => Ok(true), - Err(e) if e.raw_os_error() == Some(c::ERROR_NO_MORE_FILES as _) => Ok(false), - Err(e) => Err(e), + ); + if result == 0 { + let err = api::get_last_error(); + if err.code == c::ERROR_NO_MORE_FILES { Ok(false) } else { Err(err) } + } else { + Ok(true) } } } @@ -804,62 +827,6 @@ unsafe fn from_maybe_unaligned<'a>(p: *const u16, len: usize) -> Cow<'a, [u16]> } } -/// Open a link relative to the parent directory, ensure no symlinks are followed. -fn open_link_no_reparse(parent: &File, name: &[u16], access: u32) -> io::Result { - // This is implemented using the lower level `NtCreateFile` function as - // unfortunately opening a file relative to a parent is not supported by - // win32 functions. It is however a fundamental feature of the NT kernel. - // - // See https://docs.microsoft.com/en-us/windows/win32/api/winternl/nf-winternl-ntcreatefile - unsafe { - let mut handle = ptr::null_mut(); - let mut io_status = c::IO_STATUS_BLOCK::PENDING; - let mut name_str = c::UNICODE_STRING::from_ref(name); - use crate::sync::atomic::{AtomicU32, Ordering}; - // The `OBJ_DONT_REPARSE` attribute ensures that we haven't been - // tricked into following a symlink. However, it may not be available in - // earlier versions of Windows. - static ATTRIBUTES: AtomicU32 = AtomicU32::new(c::OBJ_DONT_REPARSE); - let object = c::OBJECT_ATTRIBUTES { - ObjectName: &mut name_str, - RootDirectory: parent.as_raw_handle(), - Attributes: ATTRIBUTES.load(Ordering::Relaxed), - ..c::OBJECT_ATTRIBUTES::default() - }; - let status = c::NtCreateFile( - &mut handle, - access, - &object, - &mut io_status, - crate::ptr::null_mut(), - 0, - c::FILE_SHARE_DELETE | c::FILE_SHARE_READ | c::FILE_SHARE_WRITE, - c::FILE_OPEN, - // If `name` is a symlink then open the link rather than the target. - c::FILE_OPEN_REPARSE_POINT, - crate::ptr::null_mut(), - 0, - ); - // Convert an NTSTATUS to the more familiar Win32 error codes (aka "DosError") - if c::nt_success(status) { - Ok(File::from_raw_handle(handle)) - } else if status == c::STATUS_DELETE_PENDING { - // We make a special exception for `STATUS_DELETE_PENDING` because - // otherwise this will be mapped to `ERROR_ACCESS_DENIED` which is - // very unhelpful. - Err(io::Error::from_raw_os_error(c::ERROR_DELETE_PENDING as i32)) - } else if status == c::STATUS_INVALID_PARAMETER - && ATTRIBUTES.load(Ordering::Relaxed) == c::OBJ_DONT_REPARSE - { - // Try without `OBJ_DONT_REPARSE`. See above. - ATTRIBUTES.store(0, Ordering::Relaxed); - open_link_no_reparse(parent, name, access) - } else { - Err(io::Error::from_raw_os_error(c::RtlNtStatusToDosError(status) as _)) - } - } -} - impl AsInner for File { #[inline] fn as_inner(&self) -> &Handle { @@ -1142,114 +1109,22 @@ pub fn rmdir(p: &Path) -> io::Result<()> { Ok(()) } -/// Open a file or directory without following symlinks. -fn open_link(path: &Path, access_mode: u32) -> io::Result { +pub fn remove_dir_all(path: &Path) -> io::Result<()> { + // Open a file or directory without following symlinks. let mut opts = OpenOptions::new(); - opts.access_mode(access_mode); + opts.access_mode(c::FILE_LIST_DIRECTORY); // `FILE_FLAG_BACKUP_SEMANTICS` allows opening directories. // `FILE_FLAG_OPEN_REPARSE_POINT` opens a link instead of its target. opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS | c::FILE_FLAG_OPEN_REPARSE_POINT); - File::open(path, &opts) -} - -pub fn remove_dir_all(path: &Path) -> io::Result<()> { - let file = open_link(path, c::DELETE | c::FILE_LIST_DIRECTORY)?; + let file = File::open(path, &opts)?; // Test if the file is not a directory or a symlink to a directory. if (file.basic_info()?.FileAttributes & c::FILE_ATTRIBUTE_DIRECTORY) == 0 { return Err(io::Error::from_raw_os_error(c::ERROR_DIRECTORY as _)); } - match ignore_notfound(remove_dir_all_iterative(&file, File::posix_delete)) { - Err(e) => { - if let Some(code) = e.raw_os_error() { - match code as u32 { - // If POSIX delete is not supported for this filesystem then fallback to win32 delete. - c::ERROR_NOT_SUPPORTED - | c::ERROR_INVALID_FUNCTION - | c::ERROR_INVALID_PARAMETER => { - remove_dir_all_iterative(&file, File::win32_delete) - } - _ => Err(e), - } - } else { - Err(e) - } - } - ok => ok, - } -} - -fn remove_dir_all_iterative(f: &File, delete: fn(&File) -> io::Result<()>) -> io::Result<()> { - // When deleting files we may loop this many times when certain error conditions occur. - // This allows remove_dir_all to succeed when the error is temporary. - const MAX_RETRIES: u32 = 10; - - let mut buffer = DirBuff::new(); - let mut dirlist = vec![f.duplicate()?]; - - // FIXME: This is a hack so we can push to the dirlist vec after borrowing from it. - fn copy_handle(f: &File) -> mem::ManuallyDrop { - unsafe { mem::ManuallyDrop::new(File::from_raw_handle(f.as_raw_handle())) } - } - - let mut restart = true; - while let Some(dir) = dirlist.last() { - let dir = copy_handle(dir); - - // Fill the buffer and iterate the entries. - let more_data = dir.fill_dir_buff(&mut buffer, restart)?; - restart = false; - for (name, is_directory) in buffer.iter() { - if is_directory { - let child_dir = open_link_no_reparse( - &dir, - &name, - c::SYNCHRONIZE | c::DELETE | c::FILE_LIST_DIRECTORY, - ); - // On success, add the handle to the queue. - // If opening the directory fails we treat it the same as a file - if let Ok(child_dir) = child_dir { - dirlist.push(child_dir); - continue; - } - } - for i in 1..=MAX_RETRIES { - let result = open_link_no_reparse(&dir, &name, c::SYNCHRONIZE | c::DELETE); - match result { - Ok(f) => delete(&f)?, - // Already deleted, so skip. - Err(e) if e.kind() == io::ErrorKind::NotFound => break, - // Retry a few times if the file is locked or a delete is already in progress. - Err(e) - if i < MAX_RETRIES - && (e.raw_os_error() == Some(c::ERROR_DELETE_PENDING as _) - || e.raw_os_error() == Some(c::ERROR_SHARING_VIOLATION as _)) => {} - // Otherwise return the error. - Err(e) => return Err(e), - } - thread::yield_now(); - } - } - // If there were no more files then delete the directory. - if !more_data { - if let Some(dir) = dirlist.pop() { - // Retry deleting a few times in case we need to wait for a file to be deleted. - for i in 1..=MAX_RETRIES { - let result = delete(&dir); - if let Err(e) = result { - if i == MAX_RETRIES || e.kind() != io::ErrorKind::DirectoryNotEmpty { - return Err(e); - } - thread::yield_now(); - } else { - break; - } - } - } - } - } - Ok(()) + // Remove the directory and all its contents. + remove_dir_all_iterative(file).io_result() } pub fn readlink(path: &Path) -> io::Result { diff --git a/library/std/src/sys/pal/windows/fs/remove_dir_all.rs b/library/std/src/sys/pal/windows/fs/remove_dir_all.rs new file mode 100644 index 0000000000000..e7234ed8e5f56 --- /dev/null +++ b/library/std/src/sys/pal/windows/fs/remove_dir_all.rs @@ -0,0 +1,196 @@ +//! The Windows implementation of std::fs::remove_dir_all. +//! +//! This needs to address two issues: +//! +//! - It must not be possible to trick this into deleting files outside of +//! the parent directory (see CVE-2022-21658). +//! - It should not fail if many threads or processes call `remove_dir_all` +//! on the same path. +//! +//! The first is handled by using the low-level `NtOpenFile` API to open a file +//! relative to a parent directory. +//! +//! The second is trickier. Deleting a file works by setting its "disposition" +//! to delete. However, it isn't actually deleted until the file is closed. +//! During the gap between these two events, the file is in a kind of limbo +//! state where it still exists in the filesystem but anything trying to open +//! it fails with an error. +//! +//! The mitigations we use here are: +//! +//! - When attempting to open the file, we treat ERROR_DELETE_PENDING as a +//! successful delete. +//! - If the file still hasn't been removed from the filesystem by the time we +//! attempt to delete the parent directory, we try to wait for it to finish. +//! We can't wait indefinitely though so after some number of spins, we give +//! up and return an error. +//! +//! In short, we can't guarantee this will always succeed in the event of a +//! race but we do make a best effort such that it *should* do so. + +use core::ptr; +use core::sync::atomic::{AtomicU32, Ordering}; + +use super::{AsRawHandle, DirBuff, File, FromRawHandle}; +use crate::sys::c; +use crate::sys::pal::windows::api::WinError; +use crate::thread; + +// The maximum number of times to spin when waiting for deletes to complete. +const MAX_RETRIES: usize = 50; + +/// A wrapper around a raw NtOpenFile call. +/// +/// This isn't completely safe because `OBJECT_ATTRIBUTES` contains raw pointers. +unsafe fn nt_open_file( + access: u32, + object_attribute: &c::OBJECT_ATTRIBUTES, + share: u32, + options: u32, +) -> Result { + unsafe { + let mut handle = ptr::null_mut(); + let mut io_status = c::IO_STATUS_BLOCK::PENDING; + let status = + c::NtOpenFile(&mut handle, access, object_attribute, &mut io_status, share, options); + if c::nt_success(status) { + Ok(File::from_raw_handle(handle)) + } else { + // Convert an NTSTATUS to the more familiar Win32 error code (aka "DosError") + let win_error = if status == c::STATUS_DELETE_PENDING { + // We make a special exception for `STATUS_DELETE_PENDING` because + // otherwise this will be mapped to `ERROR_ACCESS_DENIED` which is + // very unhelpful because that can also mean a permission error. + WinError::DELETE_PENDING + } else { + WinError::new(c::RtlNtStatusToDosError(status)) + }; + Err(win_error) + } + } +} + +/// Open the file `path` in the directory `parent`, requesting the given `access` rights. +fn open_link_no_reparse( + parent: &File, + path: &[u16], + access: u32, +) -> Result, WinError> { + // This is implemented using the lower level `NtOpenFile` function as + // unfortunately opening a file relative to a parent is not supported by + // win32 functions. + // + // See https://learn.microsoft.com/windows/win32/api/winternl/nf-winternl-ntopenfile + + // The `OBJ_DONT_REPARSE` attribute ensures that we haven't been + // tricked into following a symlink. However, it may not be available in + // earlier versions of Windows. + static ATTRIBUTES: AtomicU32 = AtomicU32::new(c::OBJ_DONT_REPARSE); + + let result = unsafe { + let mut path_str = c::UNICODE_STRING::from_ref(path); + let mut object = c::OBJECT_ATTRIBUTES { + ObjectName: &mut path_str, + RootDirectory: parent.as_raw_handle(), + Attributes: ATTRIBUTES.load(Ordering::Relaxed), + ..c::OBJECT_ATTRIBUTES::default() + }; + let share = c::FILE_SHARE_DELETE | c::FILE_SHARE_READ | c::FILE_SHARE_WRITE; + let options = c::FILE_OPEN_REPARSE_POINT; + let result = nt_open_file(access, &object, share, options); + + // Retry without OBJ_DONT_REPARSE if it's not supported. + if matches!(result, Err(WinError::INVALID_PARAMETER)) + && ATTRIBUTES.load(Ordering::Relaxed) == c::OBJ_DONT_REPARSE + { + ATTRIBUTES.store(0, Ordering::Relaxed); + object.Attributes = 0; + nt_open_file(access, &object, share, options) + } else { + result + } + }; + + // Ignore not found errors + match result { + Ok(f) => Ok(Some(f)), + Err( + WinError::FILE_NOT_FOUND + | WinError::PATH_NOT_FOUND + | WinError::BAD_NETPATH + | WinError::BAD_NET_NAME + // `DELETE_PENDING` means something else is already trying to delete it + // so we assume that will eventually succeed. + | WinError::DELETE_PENDING, + ) => Ok(None), + Err(e) => Err(e), + } +} + +fn open_dir(parent: &File, name: &[u16]) -> Result, WinError> { + open_link_no_reparse(parent, name, c::SYNCHRONIZE | c::FILE_LIST_DIRECTORY) +} + +fn delete(parent: &File, name: &[u16]) -> Result<(), WinError> { + // Note that the `delete` function consumes the opened file to ensure it's + // dropped immediately. See module comments for why this is important. + match open_link_no_reparse(parent, name, c::SYNCHRONIZE | c::DELETE) { + Ok(Some(f)) => f.delete(), + Ok(None) => Ok(()), + Err(e) => Err(e), + } +} + +/// A simple retry loop that keeps running `f` while it fails with the given +/// error code or until `MAX_RETRIES` is reached. +fn retry( + mut f: impl FnMut() -> Result, + ignore: WinError, +) -> Result { + let mut i = MAX_RETRIES; + loop { + i -= 1; + if i == 0 { + return f(); + } else { + let result = f(); + if result != Err(ignore) { + return result; + } + } + thread::yield_now(); + } +} + +pub fn remove_dir_all_iterative(dir: File) -> Result<(), WinError> { + let mut buffer = DirBuff::new(); + let mut dirlist = vec![dir]; + + let mut restart = true; + 'outer: while let Some(dir) = dirlist.pop() { + let more_data = dir.fill_dir_buff(&mut buffer, restart)?; + for (name, is_directory) in buffer.iter() { + if is_directory { + let Some(subdir) = open_dir(&dir, &name)? else { continue }; + dirlist.push(dir); + dirlist.push(subdir); + continue 'outer; + } else { + // Attempt to delete, retrying on sharing violation errors as these + // can often be very temporary. E.g. if something takes just a + // bit longer than expected to release a file handle. + retry(|| delete(&dir, &name), WinError::SHARING_VIOLATION)?; + } + } + if more_data { + dirlist.push(dir); + restart = false; + } else { + // Attempt to delete, retrying on not empty errors because we may + // need to wait some time for files to be removed from the filesystem. + retry(|| delete(&dir, &[]), WinError::DIR_NOT_EMPTY)?; + restart = true; + } + } + Ok(()) +} From d30b5f00035bf55b18d576f9f2014e28e40f04bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sun, 1 Sep 2024 12:21:04 +0000 Subject: [PATCH 06/10] update `object` dependency to deduplicate `wasmparser` --- Cargo.lock | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7244ae14fb8da..ce78d921244fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,7 +199,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01667f6f40216b9a0b2945e05fed5f1ad0ab6470e69cb9378001e37b1c0668e4" dependencies = [ - "object 0.36.3", + "object 0.36.4", ] [[package]] @@ -2453,9 +2453,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "crc32fast", "flate2", @@ -2463,7 +2463,7 @@ dependencies = [ "indexmap", "memchr", "ruzstd 0.7.0", - "wasmparser 0.215.0", + "wasmparser", ] [[package]] @@ -3129,11 +3129,11 @@ dependencies = [ "build_helper", "gimli 0.31.0", "libc", - "object 0.36.3", + "object 0.36.4", "regex", "serde_json", "similar", - "wasmparser 0.216.0", + "wasmparser", ] [[package]] @@ -3408,7 +3408,7 @@ dependencies = [ "itertools", "libc", "measureme", - "object 0.36.3", + "object 0.36.4", "rustc-demangle", "rustc_ast", "rustc_attr", @@ -3447,7 +3447,7 @@ dependencies = [ "itertools", "jobserver", "libc", - "object 0.36.3", + "object 0.36.4", "pathdiff", "regex", "rustc_arena", @@ -4431,7 +4431,7 @@ name = "rustc_target" version = "0.0.0" dependencies = [ "bitflags 2.6.0", - "object 0.36.3", + "object 0.36.4", "rustc_abi", "rustc_data_structures", "rustc_feature", @@ -5849,7 +5849,7 @@ dependencies = [ "lexopt", "tempfile", "wasi-preview1-component-adapter-provider", - "wasmparser 0.216.0", + "wasmparser", "wat", "wit-component", "wit-parser", @@ -5869,7 +5869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04c23aebea22c8a75833ae08ed31ccc020835b12a41999e58c31464271b94a88" dependencies = [ "leb128", - "wasmparser 0.216.0", + "wasmparser", ] [[package]] @@ -5885,16 +5885,7 @@ dependencies = [ "serde_json", "spdx", "wasm-encoder", - "wasmparser 0.216.0", -] - -[[package]] -name = "wasmparser" -version = "0.215.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fbde0881f24199b81cf49b6ff8f9c145ac8eb1b7fc439adb5c099734f7d90e" -dependencies = [ - "bitflags 2.6.0", + "wasmparser", ] [[package]] @@ -6228,7 +6219,7 @@ dependencies = [ "serde_json", "wasm-encoder", "wasm-metadata", - "wasmparser 0.216.0", + "wasmparser", "wit-parser", ] @@ -6247,7 +6238,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser 0.216.0", + "wasmparser", ] [[package]] From 25c4aa8979a39ea7c6e7af57093be64c36936b2b Mon Sep 17 00:00:00 2001 From: cuishuang Date: Mon, 2 Sep 2024 18:14:06 +0800 Subject: [PATCH 07/10] chore: remove repetitive words Signed-off-by: cuishuang --- library/core/src/mem/transmutability.rs | 2 +- library/std/src/sync/reentrant_lock.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/core/src/mem/transmutability.rs b/library/core/src/mem/transmutability.rs index 049b32ede9c3d..cda999a7f0c91 100644 --- a/library/core/src/mem/transmutability.rs +++ b/library/core/src/mem/transmutability.rs @@ -149,7 +149,7 @@ where #[derive(PartialEq, Eq, Clone, Copy, Debug)] pub struct Assume { /// When `false`, [`TransmuteFrom`] is not implemented for transmutations - /// that might violate the the alignment requirements of references; e.g.: + /// that might violate the alignment requirements of references; e.g.: /// #[cfg_attr(bootstrap, doc = "```rust,ignore not runnable on bootstrap")] #[cfg_attr(not(bootstrap), doc = "```compile_fail,E0277")] diff --git a/library/std/src/sync/reentrant_lock.rs b/library/std/src/sync/reentrant_lock.rs index 84a0b36db1798..0b23681e90726 100644 --- a/library/std/src/sync/reentrant_lock.rs +++ b/library/std/src/sync/reentrant_lock.rs @@ -136,7 +136,7 @@ cfg_if!( // match do we read out the actual TID. // Note also that we can use relaxed atomic operations here, because // we only ever read from the tid if `tls_addr` matches the current - // TLS address. In that case, either the the tid has been set by + // TLS address. In that case, either the tid has been set by // the current thread, or by a thread that has terminated before // the current thread was created. In either case, no further // synchronization is needed (as per ) From fcb7d3fdf3216ba50167ff8d7ae63e63932d9d06 Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Mon, 2 Sep 2024 16:14:28 -0400 Subject: [PATCH 08/10] Add missing read_buf stub for x86_64-unknown-l5re-uclibc --- library/std/src/sys/pal/unix/l4re.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/library/std/src/sys/pal/unix/l4re.rs b/library/std/src/sys/pal/unix/l4re.rs index fe9559f2a569f..52d39dcfb16fb 100644 --- a/library/std/src/sys/pal/unix/l4re.rs +++ b/library/std/src/sys/pal/unix/l4re.rs @@ -54,6 +54,10 @@ pub mod net { unimpl!(); } + pub fn read_buf(&self, _: BorrowedCursor<'_>) -> io::Result<()> { + unimpl!(); + } + pub fn read_vectored(&self, _: &mut [IoSliceMut<'_>]) -> io::Result { unimpl!(); } From 0a89f72065445fff15678f66fc6c7eb0d40375e5 Mon Sep 17 00:00:00 2001 From: Tshepang Mbambo Date: Mon, 2 Sep 2024 22:36:25 +0200 Subject: [PATCH 09/10] process.rs: remove "Basic usage" text where not useful Is not useful because just a single example is given. --- library/std/src/process.rs | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/library/std/src/process.rs b/library/std/src/process.rs index bbea27ebc1056..a155855029e70 100644 --- a/library/std/src/process.rs +++ b/library/std/src/process.rs @@ -617,8 +617,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -699,8 +697,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -748,8 +744,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -786,8 +780,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -822,8 +814,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::{Command, Stdio}; /// use std::env; @@ -870,8 +860,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -900,8 +888,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -928,8 +914,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -959,8 +943,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::{Command, Stdio}; /// @@ -988,8 +970,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::{Command, Stdio}; /// @@ -1017,8 +997,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::{Command, Stdio}; /// @@ -1039,8 +1017,6 @@ impl Command { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -2105,8 +2081,6 @@ impl Child { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -2129,8 +2103,6 @@ impl Child { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -2158,8 +2130,6 @@ impl Child { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -2194,8 +2164,6 @@ impl Child { /// /// # Examples /// - /// Basic usage: - /// /// ```no_run /// use std::process::Command; /// @@ -2398,15 +2366,11 @@ pub fn abort() -> ! { /// /// # Examples /// -/// Basic usage: -/// /// ```no_run /// use std::process; /// /// println!("My pid is {}", process::id()); /// ``` -/// -/// #[must_use] #[stable(feature = "getpid", since = "1.26.0")] pub fn id() -> u32 { From f1e5191807dfa68b3cc477fe4b9eb4bb928b800d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 2 Sep 2024 23:16:15 +0200 Subject: [PATCH 10/10] Fix parsing of beta version in dry-run mode --- src/bootstrap/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 268392c5fb118..82b640f54234d 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -1501,6 +1501,7 @@ Executed at: {executed_at}"#, "refs/remotes/origin/{}..HEAD", self.config.stage0_metadata.config.nightly_branch )) + .run_always() .run_capture(self) .stdout() });