diff --git a/std/src/sys/pal/unix/kernel_copy.rs b/std/src/sys/pal/unix/kernel_copy.rs index 18acd5ecccd5c..cd38b7c07e2b1 100644 --- a/std/src/sys/pal/unix/kernel_copy.rs +++ b/std/src/sys/pal/unix/kernel_copy.rs @@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> // We store the availability in a global to avoid unnecessary syscalls static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED); + let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) { + NOT_PROBED => false, + UNAVAILABLE => return CopyResult::Fallback(0), + _ => true, + }; + syscall! { fn copy_file_range( fd_in: libc::c_int, @@ -571,25 +577,22 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> ) -> libc::ssize_t } - match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) { - NOT_PROBED => { - // EPERM can indicate seccomp filters or an immutable file. - // To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported - // and some other error (ENOSYS or EPERM) if it's not available - let result = unsafe { - cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0)) - }; - - if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) { - HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed); - } else { - HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed); - return CopyResult::Fallback(0); - } + fn probe_copy_file_range_support() -> u8 { + // In some cases, we cannot determine availability from the first + // `copy_file_range` call. In this case, we probe with an invalid file + // descriptor so that the results are easily interpretable. + match unsafe { + cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0)) + .map_err(|e| e.raw_os_error()) + } { + Err(Some(EPERM | ENOSYS)) => UNAVAILABLE, + Err(Some(EBADF)) => AVAILABLE, + Ok(_) => panic!("unexpected copy_file_range probe success"), + // Treat other errors as the syscall + // being unavailable. + Err(_) => UNAVAILABLE, } - UNAVAILABLE => return CopyResult::Fallback(0), - _ => {} - }; + } let mut written = 0u64; while written < max_len { @@ -604,6 +607,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0)) }; + if !have_probed && copy_result.is_ok() { + have_probed = true; + HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed); + } + match copy_result { Ok(0) if written == 0 => { // fallback to work around several kernel bugs where copy_file_range will fail to @@ -619,7 +627,28 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> return match err.raw_os_error() { // when file offset + max_length > u64::MAX Some(EOVERFLOW) => CopyResult::Fallback(written), - Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => { + Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF)) + if written == 0 => + { + if !have_probed { + let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) { + // EPERM can indicate seccomp filters or an + // immutable file. To distinguish these + // cases we probe with invalid file + // descriptors which should result in EBADF + // if the syscall is supported and EPERM or + // ENOSYS if it's not available. + // + // For EOPNOTSUPP, see below. In the case of + // ENOSYS, we try to cover for faulty FUSE + // drivers. + probe_copy_file_range_support() + } else { + AVAILABLE + }; + HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed); + } + // Try fallback io::copy if either: // - Kernel version is < 4.5 (ENOSYS¹) // - Files are mounted on different fs (EXDEV)