Skip to content

Commit

Permalink
Auto merge of rust-lang#131972 - klensy:FindFirstFileExW, r=ChrisDenton
Browse files Browse the repository at this point in the history 
speedup directory traversal on windows

Optimizes walking over dirs on windows by replacing `FindFirstFileW` with `FindFirstFileExW` with `FindExInfoBasic` option, that allows skipping filling unused struct field which should be faster.

Also adds the same change for fallback call of `FindFirstFileExW` in metadata call.

Locally shows small speedup, but bench results from other users are welcome.
  • Loading branch information
@bors
bors committed Oct 21, 2024
2 parents 3e33bda + 2920ed0 commit 3ec4308
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 14 deletions.
2 changes: 1 addition & 1 deletion library/std/src/fs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2618,7 +2618,7 @@ pub fn remove_dir_all<P: AsRef<Path>>(path: P) -> io::Result<()> {
/// # Platform-specific behavior
///
/// This function currently corresponds to the `opendir` function on Unix
/// and the `FindFirstFile` function on Windows. Advancing the iterator
/// and the `FindFirstFileEx` function on Windows. Advancing the iterator
/// currently corresponds to `readdir` on Unix and `FindNextFile` on Windows.
/// Note that, this [may change in the future][changes].
///
Expand Down
4 changes: 3 additions & 1 deletion library/std/src/sys/pal/windows/c/bindings.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2337,7 +2337,9 @@ Windows.Win32.Storage.FileSystem.FileStandardInfo
Windows.Win32.Storage.FileSystem.FileStorageInfo
Windows.Win32.Storage.FileSystem.FileStreamInfo
Windows.Win32.Storage.FileSystem.FindClose
Windows.Win32.Storage.FileSystem.FindFirstFileW
Windows.Win32.Storage.FileSystem.FindExInfoBasic
Windows.Win32.Storage.FileSystem.FindExSearchNameMatch
Windows.Win32.Storage.FileSystem.FindFirstFileExW
Windows.Win32.Storage.FileSystem.FindNextFileW
Windows.Win32.Storage.FileSystem.FlushFileBuffers
Windows.Win32.Storage.FileSystem.GetFileAttributesW
Expand Down
8 changes: 6 additions & 2 deletions library/std/src/sys/pal/windows/c/windows_sys.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ windows_targets::link!("kernel32.dll" "system" fn DeviceIoControl(hdevice : HAND
windows_targets::link!("kernel32.dll" "system" fn DuplicateHandle(hsourceprocesshandle : HANDLE, hsourcehandle : HANDLE, htargetprocesshandle : HANDLE, lptargethandle : *mut HANDLE, dwdesiredaccess : u32, binherithandle : BOOL, dwoptions : DUPLICATE_HANDLE_OPTIONS) -> BOOL);
windows_targets::link!("kernel32.dll" "system" fn ExitProcess(uexitcode : u32) -> !);
windows_targets::link!("kernel32.dll" "system" fn FindClose(hfindfile : HANDLE) -> BOOL);
windows_targets::link!("kernel32.dll" "system" fn FindFirstFileW(lpfilename : PCWSTR, lpfindfiledata : *mut WIN32_FIND_DATAW) -> HANDLE);
windows_targets::link!("kernel32.dll" "system" fn FindFirstFileExW(lpfilename : PCWSTR, finfolevelid : FINDEX_INFO_LEVELS, lpfindfiledata : *mut core::ffi::c_void, fsearchop : FINDEX_SEARCH_OPS, lpsearchfilter : *const core::ffi::c_void, dwadditionalflags : FIND_FIRST_EX_FLAGS) -> HANDLE);
windows_targets::link!("kernel32.dll" "system" fn FindNextFileW(hfindfile : HANDLE, lpfindfiledata : *mut WIN32_FIND_DATAW) -> BOOL);
windows_targets::link!("kernel32.dll" "system" fn FlushFileBuffers(hfile : HANDLE) -> BOOL);
windows_targets::link!("kernel32.dll" "system" fn FormatMessageW(dwflags : FORMAT_MESSAGE_OPTIONS, lpsource : *const core::ffi::c_void, dwmessageid : u32, dwlanguageid : u32, lpbuffer : PWSTR, nsize : u32, arguments : *const *const i8) -> u32);
Expand Down Expand Up @@ -2501,6 +2501,9 @@ pub const FILE_WRITE_ATTRIBUTES: FILE_ACCESS_RIGHTS = 256u32;
pub const FILE_WRITE_DATA: FILE_ACCESS_RIGHTS = 2u32;
pub const FILE_WRITE_EA: FILE_ACCESS_RIGHTS = 16u32;
pub const FILE_WRITE_THROUGH: NTCREATEFILE_CREATE_OPTIONS = 2u32;
pub type FINDEX_INFO_LEVELS = i32;
pub type FINDEX_SEARCH_OPS = i32;
pub type FIND_FIRST_EX_FLAGS = u32;
pub const FIONBIO: i32 = -2147195266i32;
#[repr(C)]
#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))]
Expand Down Expand Up @@ -2565,6 +2568,8 @@ pub const FileRenameInfoEx: FILE_INFO_BY_HANDLE_CLASS = 22i32;
pub const FileStandardInfo: FILE_INFO_BY_HANDLE_CLASS = 1i32;
pub const FileStorageInfo: FILE_INFO_BY_HANDLE_CLASS = 16i32;
pub const FileStreamInfo: FILE_INFO_BY_HANDLE_CLASS = 7i32;
pub const FindExInfoBasic: FINDEX_INFO_LEVELS = 1i32;
pub const FindExSearchNameMatch: FINDEX_SEARCH_OPS = 0i32;
pub type GENERIC_ACCESS_RIGHTS = u32;
pub const GENERIC_ALL: GENERIC_ACCESS_RIGHTS = 268435456u32;
pub const GENERIC_EXECUTE: GENERIC_ACCESS_RIGHTS = 536870912u32;
Expand Down Expand Up @@ -3307,7 +3312,6 @@ pub struct XSAVE_FORMAT {
pub XmmRegisters: [M128A; 8],
pub Reserved4: [u8; 224],
}

#[cfg(target_arch = "arm")]
#[repr(C)]
pub struct WSADATA {
Expand Down
41 changes: 31 additions & 10 deletions library/std/src/sys/pal/windows/fs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ impl Iterator for ReadDir {
fn next(&mut self) -> Option<io::Result<DirEntry>> {
if self.handle.0 == c::INVALID_HANDLE_VALUE {
// This iterator was initialized with an `INVALID_HANDLE_VALUE` as its handle.
// Simply return `None` because this is only the case when `FindFirstFileW` in
// Simply return `None` because this is only the case when `FindFirstFileExW` in
// the construction of this iterator returns `ERROR_FILE_NOT_FOUND` which means
// no matchhing files can be found.
return None;
Expand Down Expand Up @@ -1047,8 +1047,22 @@ pub fn readdir(p: &Path) -> io::Result<ReadDir> {
let path = maybe_verbatim(&star)?;

unsafe {
let mut wfd = mem::zeroed();
let find_handle = c::FindFirstFileW(path.as_ptr(), &mut wfd);
let mut wfd: c::WIN32_FIND_DATAW = mem::zeroed();
// this is like FindFirstFileW (see https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-findfirstfileexw),
// but with FindExInfoBasic it should skip filling WIN32_FIND_DATAW.cAlternateFileName
// (see https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-win32_find_dataw)
// (which will be always null string value and currently unused) and should be faster.
//
// We can pass FIND_FIRST_EX_LARGE_FETCH to dwAdditionalFlags to speed up things more,
// but as we don't know user's use profile of this function, lets be conservative.
let find_handle = c::FindFirstFileExW(
path.as_ptr(),
c::FindExInfoBasic,
&mut wfd as *mut _ as _,
c::FindExSearchNameMatch,
ptr::null(),
0,
);

if find_handle != c::INVALID_HANDLE_VALUE {
Ok(ReadDir {
Expand All @@ -1057,7 +1071,7 @@ pub fn readdir(p: &Path) -> io::Result<ReadDir> {
first: Some(wfd),
})
} else {
// The status `ERROR_FILE_NOT_FOUND` is returned by the `FindFirstFileW` function
// The status `ERROR_FILE_NOT_FOUND` is returned by the `FindFirstFileExW` function
// if no matching files can be found, but not necessarily that the path to find the
// files in does not exist.
//
Expand All @@ -1079,7 +1093,7 @@ pub fn readdir(p: &Path) -> io::Result<ReadDir> {

// Just return the error constructed from the raw OS error if the above is not the case.
//
// Note: `ERROR_PATH_NOT_FOUND` would have been returned by the `FindFirstFileW` function
// Note: `ERROR_PATH_NOT_FOUND` would have been returned by the `FindFirstFileExW` function
// when the path to search in does not exist in the first place.
Err(Error::from_raw_os_error(last_error.code as i32))
}
Expand Down Expand Up @@ -1220,7 +1234,7 @@ fn metadata(path: &Path, reparse: ReparsePoint) -> io::Result<FileAttr> {
opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS | reparse.as_flag());

// Attempt to open the file normally.
// If that fails with `ERROR_SHARING_VIOLATION` then retry using `FindFirstFileW`.
// If that fails with `ERROR_SHARING_VIOLATION` then retry using `FindFirstFileExW`.
// If the fallback fails for any reason we return the original error.
match File::open(path, &opts) {
Ok(file) => file.file_attr(),
Expand All @@ -1237,13 +1251,20 @@ fn metadata(path: &Path, reparse: ReparsePoint) -> io::Result<FileAttr> {
unsafe {
let path = maybe_verbatim(path)?;

// `FindFirstFileW` accepts wildcard file names.
// `FindFirstFileExW` accepts wildcard file names.
// Fortunately wildcards are not valid file names and
// `ERROR_SHARING_VIOLATION` means the file exists (but is locked)
// therefore it's safe to assume the file name given does not
// include wildcards.
let mut wfd = mem::zeroed();
let handle = c::FindFirstFileW(path.as_ptr(), &mut wfd);
let mut wfd: c::WIN32_FIND_DATAW = mem::zeroed();
let handle = c::FindFirstFileExW(
path.as_ptr(),
c::FindExInfoBasic,
&mut wfd as *mut _ as _,
c::FindExSearchNameMatch,
ptr::null(),
0,
);

if handle == c::INVALID_HANDLE_VALUE {
// This can fail if the user does not have read access to the
Expand All @@ -1253,7 +1274,7 @@ fn metadata(path: &Path, reparse: ReparsePoint) -> io::Result<FileAttr> {
// We no longer need the find handle.
c::FindClose(handle);

// `FindFirstFileW` reads the cached file information from the
// `FindFirstFileExW` reads the cached file information from the
// directory. The downside is that this metadata may be outdated.
let attrs = FileAttr::from(wfd);
if reparse == ReparsePoint::Follow && attrs.file_type().is_symlink() {
Expand Down

0 comments on commit 3ec4308

Please sign in to comment.