diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a2e142..d4d0ab8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,10 +39,10 @@ jobs: include: - build: pinned os: ubuntu-latest - rust: 1.60.0 + rust: 1.71.0 - build: pinned-win os: windows-latest - rust: 1.60.0 + rust: 1.71.0 - build: stable os: ubuntu-latest rust: stable diff --git a/.gitignore b/.gitignore index d63756d..ba188fc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,5 @@ examples/ss10pusa.csv build target Cargo.lock -scratch* bench_large/huge tmp diff --git a/Cargo.toml b/Cargo.toml index e9df650..a0ca4a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ categories = ["filesystem"] license = "Unlicense/MIT" exclude = ["/ci/*", "/.travis.yml", "/appveyor.yml"] edition = "2018" +rust-version = "1.71" [badges] travis-ci = { repository = "BurntSushi/walkdir" } @@ -23,8 +24,28 @@ members = ["walkdir-list"] [dependencies] same-file = "1.0.1" +[target.'cfg(unix)'.dependencies.libc] +version = "0.2.186" + +[build-dependencies] +cc = "1.0.36" + +[target.'cfg(windows)'.dependencies.windows-sys] +version = "0.61" +features = [ + "Wdk_Foundation", + "Wdk_Storage_FileSystem", + "Win32_Foundation", + "Win32_Security", + "Win32_Storage_FileSystem", + "Win32_System_IO", +] + [target.'cfg(windows)'.dependencies.winapi-util] -version = "0.1.1" +version = "0.1.11" [dev-dependencies] doc-comment = "0.3" + +[profile.release] +debug = true diff --git a/README.md b/README.md index 61390c7..668879d 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ for entry in walker.filter_entry(|e| !is_hidden(e)) { ### Minimum Rust version policy -This crate's minimum supported `rustc` version is `1.60.0`. +This crate's minimum supported `rustc` version is `1.71.0`. The current policy is that the minimum Rust version required to use this crate can be increased in minor version updates. For example, if `crate 1.0` requires diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..c391406 --- /dev/null +++ b/build.rs @@ -0,0 +1,58 @@ +fn main() { + if check_cfg_supported() { + println!("cargo:rustc-check-cfg=cfg(walkdir_getdents)"); + println!("cargo:rustc-check-cfg=cfg(walkdir_unix)"); + } + + let target_os = match std::env::var_os("CARGO_CFG_TARGET_OS") { + Some(target_os) => target_os, + None => return, + }; + let target_os = target_os.to_string_lossy(); + if supports_unix_backend(&target_os) { + println!("cargo:rustc-cfg=walkdir_unix"); + } + if target_os == "dragonfly" { + cc::Build::new() + .file("src/os/unix/errno-dragonfly.c") + .compile("errno-dragonfly"); + } + if target_os == "linux" + && std::env::var_os("CARGO_CFG_WALKDIR_DISABLE_GETDENTS").is_none() + { + println!("cargo:rustc-cfg=walkdir_getdents"); + } +} + +fn check_cfg_supported() -> bool { + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let output = + match std::process::Command::new(rustc).arg("--version").output() { + Ok(output) => output, + Err(_) => return false, + }; + let version = String::from_utf8_lossy(&output.stdout); + let minor = version + .split_whitespace() + .nth(1) + .and_then(|version| version.split('.').nth(1)) + .and_then(|minor| minor.parse::().ok()); + matches!(minor, Some(minor) if minor >= 80) +} + +fn supports_unix_backend(target_os: &str) -> bool { + matches!( + target_os, + "android" + | "dragonfly" + | "emscripten" + | "freebsd" + | "fuchsia" + | "haiku" + | "linux" + | "macos" + | "netbsd" + | "openbsd" + | "solaris" + ) +} diff --git a/src/dent.rs b/src/dent.rs index adf54f7..5fcf706 100644 --- a/src/dent.rs +++ b/src/dent.rs @@ -1,10 +1,10 @@ use std::ffi::OsStr; use std::fmt; -use std::fs::{self, FileType}; +use std::fs; use std::path::{Path, PathBuf}; use crate::error::Error; -use crate::Result; +use crate::{FileType, Result}; /// A directory entry. /// @@ -48,14 +48,6 @@ pub struct DirEntry { /// The underlying inode number (Unix only). #[cfg(unix)] ino: u64, - /// The underlying metadata (Windows only). We store this on Windows - /// because this comes for free while reading a directory. - /// - /// We use this to determine whether an entry is a directory or not, which - /// works around a bug in Rust's standard library: - /// https://github.com/rust-lang/rust/issues/46484 - #[cfg(windows)] - metadata: fs::Metadata, } impl DirEntry { @@ -124,21 +116,6 @@ impl DirEntry { /// [`std::fs::metadata`]: https://doc.rust-lang.org/std/fs/fn.metadata.html /// [`std::fs::symlink_metadata`]: https://doc.rust-lang.org/stable/std/fs/fn.symlink_metadata.html pub fn metadata(&self) -> Result { - self.metadata_internal() - } - - #[cfg(windows)] - fn metadata_internal(&self) -> Result { - if self.follow_link { - fs::metadata(&self.path) - } else { - Ok(self.metadata.clone()) - } - .map_err(|err| Error::from_entry(self, err)) - } - - #[cfg(not(windows))] - fn metadata_internal(&self) -> Result { if self.follow_link { fs::metadata(&self.path) } else { @@ -155,7 +132,7 @@ impl DirEntry { /// This never makes any system calls. /// /// [`follow_links`]: struct.WalkDir.html#method.follow_links - pub fn file_type(&self) -> fs::FileType { + pub fn file_type(&self) -> FileType { self.ty } @@ -182,21 +159,45 @@ impl DirEntry { } #[cfg(windows)] - pub(crate) fn from_entry( + pub(crate) fn from_os_entry( depth: usize, - ent: &fs::DirEntry, + parent_path: &Path, + ent: &crate::os::windows::DirEntry, ) -> Result { - let path = ent.path(); - let ty = ent - .file_type() - .map_err(|err| Error::from_path(depth, path.clone(), err))?; - let md = ent - .metadata() - .map_err(|err| Error::from_path(depth, path.clone(), err))?; - Ok(DirEntry { path, ty, follow_link: false, depth, metadata: md }) + let path = parent_path.join(ent.file_name_os()); + Ok(DirEntry { + path, + ty: ent.file_type().into(), + follow_link: false, + depth, + }) } - #[cfg(unix)] + #[cfg(walkdir_unix)] + pub(crate) fn from_os_entry( + depth: usize, + parent_path: &Path, + parent_fd: Option, + ent: &crate::os::unix::DirEntry, + ) -> Result { + let path = parent_path.join(ent.file_name_os()); + let ty = match ent.file_type() { + Some(file_type) => file_type.into(), + None => { + let metadata = match parent_fd { + Some(fd) => { + crate::os::unix::lstatat_c(fd, ent.file_name()) + } + None => crate::os::unix::lstat(path.clone()), + } + .map_err(|err| Error::from_path(depth, path.clone(), err))?; + metadata.file_type().into() + } + }; + Ok(DirEntry { path, ty, follow_link: false, depth, ino: ent.ino() }) + } + + #[cfg(all(unix, not(walkdir_unix)))] pub(crate) fn from_entry( depth: usize, ent: &fs::DirEntry, @@ -208,7 +209,7 @@ impl DirEntry { .map_err(|err| Error::from_path(depth, ent.path(), err))?; Ok(DirEntry { path: ent.path(), - ty, + ty: ty.into(), follow_link: false, depth, ino: ent.ino(), @@ -223,7 +224,12 @@ impl DirEntry { let ty = ent .file_type() .map_err(|err| Error::from_path(depth, ent.path(), err))?; - Ok(DirEntry { path: ent.path(), ty, follow_link: false, depth }) + Ok(DirEntry { + path: ent.path(), + ty: ty.into(), + follow_link: false, + depth, + }) } #[cfg(windows)] @@ -232,23 +238,42 @@ impl DirEntry { pb: PathBuf, follow: bool, ) -> Result { - let md = if follow { - fs::metadata(&pb) - .map_err(|err| Error::from_path(depth, pb.clone(), err))? + let metadata = if follow { + crate::os::windows::stat(&pb) } else { - fs::symlink_metadata(&pb) - .map_err(|err| Error::from_path(depth, pb.clone(), err))? - }; + crate::os::windows::lstat(&pb) + } + .map_err(|err| Error::from_path(depth, pb.clone(), err))?; Ok(DirEntry { path: pb, - ty: md.file_type(), + ty: metadata.file_type().into(), follow_link: follow, depth, - metadata: md, }) } - #[cfg(unix)] + #[cfg(walkdir_unix)] + pub(crate) fn from_path( + depth: usize, + pb: PathBuf, + follow: bool, + ) -> Result { + let metadata = if follow { + crate::os::unix::stat(pb.clone()) + } else { + crate::os::unix::lstat(pb.clone()) + } + .map_err(|err| Error::from_path(depth, pb.clone(), err))?; + Ok(DirEntry { + path: pb, + ty: metadata.file_type().into(), + follow_link: follow, + depth, + ino: metadata.ino(), + }) + } + + #[cfg(all(unix, not(walkdir_unix)))] pub(crate) fn from_path( depth: usize, pb: PathBuf, @@ -265,7 +290,7 @@ impl DirEntry { }; Ok(DirEntry { path: pb, - ty: md.file_type(), + ty: md.file_type().into(), follow_link: follow, depth, ino: md.ino(), @@ -287,7 +312,7 @@ impl DirEntry { }; Ok(DirEntry { path: pb, - ty: md.file_type(), + ty: md.file_type().into(), follow_link: follow, depth, }) @@ -302,7 +327,6 @@ impl Clone for DirEntry { ty: self.ty, follow_link: self.follow_link, depth: self.depth, - metadata: self.metadata.clone(), } } diff --git a/src/dir.rs b/src/dir.rs new file mode 100644 index 0000000..8304840 --- /dev/null +++ b/src/dir.rs @@ -0,0 +1,435 @@ +#[cfg(walkdir_unix)] +use std::ffi::CStr; +#[cfg(not(any(walkdir_unix, windows)))] +use std::fs; +use std::io; +#[cfg(walkdir_unix)] +use std::os::unix::io::{AsRawFd, RawFd}; +#[cfg(windows)] +use std::os::windows::io::{AsRawHandle, RawHandle}; +#[cfg(not(walkdir_unix))] +use std::path::Path; + +#[cfg(walkdir_getdents)] +use crate::os::linux; +#[cfg(walkdir_unix)] +use crate::os::unix::{Dir, DirEntry as OsDirEntry}; +#[cfg(windows)] +use crate::os::windows::{Dir as WindowsDir, DirEntry as WindowsDirEntry}; + +#[cfg(not(any(walkdir_unix, windows)))] +#[derive(Debug)] +pub struct DirList { + depth: usize, + scratch: Option, + stream: Stream, +} + +#[cfg(not(any(walkdir_unix, windows)))] +#[derive(Debug)] +enum Stream { + Open(fs::ReadDir), + Spilled(std::vec::IntoIter>), + Built(std::vec::IntoIter>), + Failed(Option), +} + +#[cfg(not(any(walkdir_unix, windows)))] +impl DirList { + pub fn open_path(depth: usize, path: &Path) -> DirList { + let stream = match fs::read_dir(path) { + Ok(iter) => Stream::Open(iter), + Err(err) => Stream::Failed(Some(err)), + }; + DirList { depth, scratch: None, stream } + } + + pub fn depth(&self) -> usize { + self.depth + } + + pub fn is_failed(&self) -> bool { + matches!(self.stream, Stream::Failed(_)) + } + + pub fn into_built( + self, + entries: Vec>, + ) -> DirList { + DirList { + depth: self.depth, + scratch: self.scratch, + stream: Stream::Built(entries.into_iter()), + } + } + + pub fn next_built(&mut self) -> Option> { + match self.stream { + Stream::Built(ref mut iter) => iter.next(), + _ => None, + } + } + + pub fn is_built(&self) -> bool { + matches!(self.stream, Stream::Built(_)) + } + + pub fn next(&mut self) -> Option> { + let result = match self.stream { + Stream::Open(ref mut iter) => iter.next()?, + Stream::Spilled(ref mut iter) => iter.next()?, + Stream::Built(_) => return None, + Stream::Failed(ref mut err) => return err.take().map(Err), + }; + match result { + Ok(entry) => { + self.scratch = Some(entry); + Some(Ok(())) + } + Err(err) => Some(Err(err)), + } + } + + pub fn entry(&self) -> &fs::DirEntry { + self.scratch.as_ref().expect("entry follows a successful read") + } + + pub fn spill(&mut self) { + let stream = std::mem::replace(&mut self.stream, Stream::Failed(None)); + self.stream = match stream { + Stream::Open(iter) => { + Stream::Spilled(iter.collect::>().into_iter()) + } + stream => stream, + }; + } +} + +#[cfg(walkdir_unix)] +#[derive(Debug)] +pub struct DirList { + depth: usize, + done: bool, + scratch: OsDirEntry, + stream: Stream, +} + +#[cfg(walkdir_unix)] +#[derive(Debug)] +enum Stream { + Open { + dir: Dir, + #[cfg(walkdir_getdents)] + cursor: linux::DirEntryCursor, + }, + Spilled(std::vec::IntoIter>), + Built(std::vec::IntoIter>), + Failed(Option), +} + +#[cfg(walkdir_unix)] +impl DirList { + fn from_open(depth: usize, result: io::Result) -> DirList { + let stream = match result { + Ok(dir) => Stream::Open { + dir, + #[cfg(walkdir_getdents)] + cursor: linux::DirEntryCursor::new(), + }, + Err(err) => Stream::Failed(Some(err)), + }; + DirList { depth, done: false, scratch: OsDirEntry::empty(), stream } + } + + pub fn open_path(depth: usize, path_c: &CStr, follow: bool) -> DirList { + DirList::from_open(depth, Dir::open_c_follow(path_c, follow)) + } + + pub fn openat( + depth: usize, + parent: RawFd, + name: &CStr, + follow: bool, + ) -> DirList { + DirList::from_open(depth, Dir::openat_c_follow(parent, name, follow)) + } + + pub fn depth(&self) -> usize { + self.depth + } + + pub fn parent_handle(&self) -> Option { + match self.stream { + Stream::Open { ref dir, .. } => Some(dir.as_raw_fd()), + _ => None, + } + } + + pub fn is_failed(&self) -> bool { + matches!(self.stream, Stream::Failed(_)) + } + + pub fn into_built( + self, + entries: Vec>, + ) -> DirList { + DirList { + depth: self.depth, + done: false, + scratch: self.scratch, + stream: Stream::Built(entries.into_iter()), + } + } + + pub fn next_built(&mut self) -> Option> { + match self.stream { + Stream::Built(ref mut iter) => iter.next(), + _ => None, + } + } + + pub fn is_built(&self) -> bool { + matches!(self.stream, Stream::Built(_)) + } + + pub fn next(&mut self) -> Option> { + match self.stream { + Stream::Failed(ref mut err) => err.take().map(Err), + Stream::Spilled(ref mut iter) => match iter.next()? { + Ok(entry) => { + self.scratch = entry; + Some(Ok(())) + } + Err(err) => Some(Err(err)), + }, + Stream::Built(_) => None, + Stream::Open { .. } => self.next_open(), + } + } + + pub fn entry(&self) -> &OsDirEntry { + &self.scratch + } + + #[cfg(walkdir_getdents)] + fn next_open(&mut self) -> Option> { + if self.done { + return None; + } + let result = match self.stream { + Stream::Open { ref mut dir, ref mut cursor } => loop { + if cursor.advance() { + cursor.current().write_to_unix(&mut self.scratch); + break Ok(true); + } + match linux::getdents(dir.as_raw_fd(), cursor) { + Ok(false) => break Ok(false), + Ok(true) => continue, + Err(ref err) + if err.kind() == io::ErrorKind::Interrupted => + { + continue; + } + Err(err) => break Err(err), + } + }, + _ => unreachable!(), + }; + self.finish_read(result) + } + + #[cfg(not(walkdir_getdents))] + fn next_open(&mut self) -> Option> { + if self.done { + return None; + } + let result = match self.stream { + Stream::Open { ref mut dir } => loop { + match dir.read_into(&mut self.scratch) { + Err(ref err) + if err.kind() == io::ErrorKind::Interrupted => + { + continue; + } + result => break result, + } + }, + _ => unreachable!(), + }; + self.finish_read(result) + } + + fn finish_read( + &mut self, + result: io::Result, + ) -> Option> { + match result { + Ok(true) => Some(Ok(())), + Ok(false) => { + self.done = true; + None + } + Err(err) => { + self.done = true; + Some(Err(err)) + } + } + } + + pub fn spill(&mut self) { + if let Stream::Open { .. } = self.stream { + let mut entries = Vec::new(); + while let Some(result) = self.next_open() { + entries.push(result.map(|()| self.scratch.clone())); + } + self.stream = Stream::Spilled(entries.into_iter()); + } + } +} + +#[cfg(windows)] +#[derive(Debug)] +pub struct DirList { + depth: usize, + done: bool, + scratch: WindowsDirEntry, + stream: Stream, +} + +#[cfg(windows)] +#[derive(Debug)] +enum Stream { + Open(WindowsDir), + Spilled(std::vec::IntoIter>), + Built(std::vec::IntoIter>), + Failed(Option), +} + +#[cfg(windows)] +impl DirList { + fn from_open(depth: usize, result: io::Result) -> DirList { + let stream = match result { + Ok(dir) => Stream::Open(dir), + Err(err) => Stream::Failed(Some(err)), + }; + DirList { + depth, + done: false, + scratch: WindowsDirEntry::empty(), + stream, + } + } + + pub fn open_path(depth: usize, path: &Path, follow: bool) -> DirList { + DirList::from_open(depth, WindowsDir::open_path_follow(path, follow)) + } + + pub fn openat( + depth: usize, + parent: RawHandle, + name: &[u16], + follow: bool, + ) -> DirList { + DirList::from_open( + depth, + WindowsDir::openat_follow(parent, name, follow), + ) + } + + pub fn depth(&self) -> usize { + self.depth + } + + pub fn parent_handle(&self) -> Option { + match self.stream { + Stream::Open(ref dir) => Some(dir.as_raw_handle()), + _ => None, + } + } + + pub fn is_failed(&self) -> bool { + matches!(self.stream, Stream::Failed(_)) + } + + pub fn into_built( + self, + entries: Vec>, + ) -> DirList { + DirList { + depth: self.depth, + done: false, + scratch: self.scratch, + stream: Stream::Built(entries.into_iter()), + } + } + + pub fn next_built(&mut self) -> Option> { + match self.stream { + Stream::Built(ref mut iter) => iter.next(), + _ => None, + } + } + + pub fn is_built(&self) -> bool { + matches!(self.stream, Stream::Built(_)) + } + + pub fn next(&mut self) -> Option> { + match self.stream { + Stream::Failed(ref mut err) => err.take().map(Err), + Stream::Spilled(ref mut iter) => match iter.next()? { + Ok(entry) => { + self.scratch = entry; + Some(Ok(())) + } + Err(err) => Some(Err(err)), + }, + Stream::Built(_) => None, + Stream::Open(_) => self.next_open(), + } + } + + pub fn entry(&self) -> &WindowsDirEntry { + &self.scratch + } + + fn next_open(&mut self) -> Option> { + if self.done { + return None; + } + let result = match self.stream { + Stream::Open(ref mut dir) => loop { + match dir.read_into(&mut self.scratch) { + Err(ref err) + if err.kind() == io::ErrorKind::Interrupted => + { + continue; + } + result => break result, + } + }, + _ => unreachable!(), + }; + match result { + Ok(true) => Some(Ok(())), + Ok(false) => { + self.done = true; + None + } + Err(err) => { + self.done = true; + Some(Err(err)) + } + } + } + + pub fn spill(&mut self) { + if let Stream::Open(_) = self.stream { + let mut entries = Vec::new(); + while let Some(result) = self.next_open() { + entries.push(result.map(|()| self.scratch.clone())); + } + self.stream = Stream::Spilled(entries.into_iter()); + } + } +} diff --git a/src/error.rs b/src/error.rs index 3342d9b..bd9cf09 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,9 +2,23 @@ use std::error; use std::fmt; use std::io; use std::path::{Path, PathBuf}; +use std::result; use crate::DirEntry; +/// A result type for walkdir operations. +/// +/// Note that this result type embeds the error type in this crate. This +/// is only useful if you care about the additional information provided by +/// the error (such as the path associated with the error or whether a loop +/// was dectected). If you want things to Just Work, then you can use +/// [`io::Result`] instead since the error type in this package will +/// automatically convert to an [`io::Result`] when using the [`try!`] macro. +/// +/// [`io::Result`]: https://doc.rust-lang.org/stable/std/io/type.Result.html +/// [`try!`]: https://doc.rust-lang.org/stable/std/macro.try.html +pub type Result = result::Result; + /// An error produced by recursively walking a directory. /// /// This error type is a light wrapper around [`std::io::Error`]. In diff --git a/src/filetype.rs b/src/filetype.rs new file mode 100644 index 0000000..aadff04 --- /dev/null +++ b/src/filetype.rs @@ -0,0 +1,45 @@ +#[cfg(walkdir_unix)] +use crate::os::unix::FileType as OsFileType; +#[cfg(windows)] +use crate::os::windows::FileType as OsFileType; +#[cfg(not(any(walkdir_unix, windows)))] +use std::fs::FileType as OsFileType; + +/// File type yielded by a [`crate::DirEntry`]. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct FileType(OsFileType); + +impl FileType { + /// Returns true if this entry is a regular file. + pub fn is_file(&self) -> bool { + self.0.is_file() + } + + /// Returns true if this entry is a directory. + pub fn is_dir(&self) -> bool { + self.0.is_dir() + } + + /// Returns true if this entry is a symbolic link. + pub fn is_symlink(&self) -> bool { + self.0.is_symlink() + } + + /// Returns true if this entry is a symbolic link to a directory. + #[cfg(windows)] + pub fn is_symlink_dir(&self) -> bool { + self.0.is_symlink_dir() + } + + /// Returns true if this entry is a symbolic link to a file. + #[cfg(windows)] + pub fn is_symlink_file(&self) -> bool { + self.0.is_symlink_file() + } +} + +impl From for FileType { + fn from(file_type: OsFileType) -> FileType { + FileType(file_type) + } +} diff --git a/src/lib.rs b/src/lib.rs index 4b96df4..a8462f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,15 @@ iteration, such as whether to follow symbolic links (default off), limit the maximum number of simultaneous open file descriptors and the ability to efficiently skip descending into directories. +On supported Unix platforms, traversal opens children with `openat`; Windows +uses `NtOpenFile` for the same parent-relative operation. This avoids platform +path-length limits and prevents symlink swaps between listing and descending +while that parent remains open. + +When `max_open` closes an older parent, or `sort_by` reads a directory into +memory, descent falls back to the full path. That fallback retains the old +path-length and race limitations. + To use this crate, add `walkdir` as a dependency to your project's `Cargo.toml`: @@ -101,1094 +110,27 @@ for entry in walker.filter_entry(|e| !is_hidden(e)) { ``` [`filter_entry`]: struct.IntoIter.html#method.filter_entry + */ #![deny(missing_docs)] -#![allow(unknown_lints)] #[cfg(doctest)] doc_comment::doctest!("../README.md"); -use std::cmp::{min, Ordering}; -use std::fmt; -use std::fs::{self, ReadDir}; -use std::io; -use std::iter; -use std::path::{Path, PathBuf}; -use std::result; -use std::vec; - -use same_file::Handle; - pub use crate::dent::DirEntry; #[cfg(unix)] pub use crate::dent::DirEntryExt; -pub use crate::error::Error; +pub use crate::error::{Error, Result}; +pub use crate::filetype::FileType; +pub use crate::walk::{FilterEntry, IntoIter, WalkDir}; mod dent; +pub(crate) mod dir; mod error; +mod filetype; +pub mod os; #[cfg(test)] mod tests; mod util; - -/// Like try, but for iterators that return [`Option>`]. -/// -/// [`Option>`]: https://doc.rust-lang.org/stable/std/option/enum.Option.html -macro_rules! itry { - ($e:expr) => { - match $e { - Ok(v) => v, - Err(err) => return Some(Err(From::from(err))), - } - }; -} - -/// A result type for walkdir operations. -/// -/// Note that this result type embeds the error type in this crate. This -/// is only useful if you care about the additional information provided by -/// the error (such as the path associated with the error or whether a loop -/// was dectected). If you want things to Just Work, then you can use -/// [`io::Result`] instead since the error type in this package will -/// automatically convert to an [`io::Result`] when using the [`try!`] macro. -/// -/// [`io::Result`]: https://doc.rust-lang.org/stable/std/io/type.Result.html -/// [`try!`]: https://doc.rust-lang.org/stable/std/macro.try.html -pub type Result = ::std::result::Result; - -/// A builder to create an iterator for recursively walking a directory. -/// -/// Results are returned in depth first fashion, with directories yielded -/// before their contents. If [`contents_first`] is true, contents are yielded -/// before their directories. The order is unspecified but if [`sort_by`] is -/// given, directory entries are sorted according to this function. Directory -/// entries `.` and `..` are always omitted. -/// -/// If an error occurs at any point during iteration, then it is returned in -/// place of its corresponding directory entry and iteration continues as -/// normal. If an error occurs while opening a directory for reading, then it -/// is not descended into (but the error is still yielded by the iterator). -/// Iteration may be stopped at any time. When the iterator is destroyed, all -/// resources associated with it are freed. -/// -/// [`contents_first`]: struct.WalkDir.html#method.contents_first -/// [`sort_by`]: struct.WalkDir.html#method.sort_by -/// -/// # Usage -/// -/// This type implements [`IntoIterator`] so that it may be used as the subject -/// of a `for` loop. You may need to call [`into_iter`] explicitly if you want -/// to use iterator adapters such as [`filter_entry`]. -/// -/// Idiomatic use of this type should use method chaining to set desired -/// options. For example, this only shows entries with a depth of `1`, `2` or -/// `3` (relative to `foo`): -/// -/// ```no_run -/// use walkdir::WalkDir; -/// # use walkdir::Error; -/// -/// # fn try_main() -> Result<(), Error> { -/// for entry in WalkDir::new("foo").min_depth(1).max_depth(3) { -/// println!("{}", entry?.path().display()); -/// } -/// # Ok(()) -/// # } -/// ``` -/// -/// [`IntoIterator`]: https://doc.rust-lang.org/stable/std/iter/trait.IntoIterator.html -/// [`into_iter`]: https://doc.rust-lang.org/nightly/core/iter/trait.IntoIterator.html#tymethod.into_iter -/// [`filter_entry`]: struct.IntoIter.html#method.filter_entry -/// -/// Note that the iterator by default includes the top-most directory. Since -/// this is the only directory yielded with depth `0`, it is easy to ignore it -/// with the [`min_depth`] setting: -/// -/// ```no_run -/// use walkdir::WalkDir; -/// # use walkdir::Error; -/// -/// # fn try_main() -> Result<(), Error> { -/// for entry in WalkDir::new("foo").min_depth(1) { -/// println!("{}", entry?.path().display()); -/// } -/// # Ok(()) -/// # } -/// ``` -/// -/// [`min_depth`]: struct.WalkDir.html#method.min_depth -/// -/// This will only return descendents of the `foo` directory and not `foo` -/// itself. -/// -/// # Loops -/// -/// This iterator (like most/all recursive directory iterators) assumes that -/// no loops can be made with *hard* links on your file system. In particular, -/// this would require creating a hard link to a directory such that it creates -/// a loop. On most platforms, this operation is illegal. -/// -/// Note that when following symbolic/soft links, loops are detected and an -/// error is reported. -#[derive(Debug)] -pub struct WalkDir { - opts: WalkDirOptions, - root: PathBuf, -} - -struct WalkDirOptions { - follow_links: bool, - follow_root_links: bool, - max_open: usize, - min_depth: usize, - max_depth: usize, - sorter: Option< - Box< - dyn FnMut(&DirEntry, &DirEntry) -> Ordering - + Send - + Sync - + 'static, - >, - >, - contents_first: bool, - same_file_system: bool, -} - -impl fmt::Debug for WalkDirOptions { - fn fmt( - &self, - f: &mut fmt::Formatter<'_>, - ) -> result::Result<(), fmt::Error> { - let sorter_str = if self.sorter.is_some() { - // FnMut isn't `Debug` - "Some(...)" - } else { - "None" - }; - f.debug_struct("WalkDirOptions") - .field("follow_links", &self.follow_links) - .field("follow_root_link", &self.follow_root_links) - .field("max_open", &self.max_open) - .field("min_depth", &self.min_depth) - .field("max_depth", &self.max_depth) - .field("sorter", &sorter_str) - .field("contents_first", &self.contents_first) - .field("same_file_system", &self.same_file_system) - .finish() - } -} - -impl WalkDir { - /// Create a builder for a recursive directory iterator starting at the - /// file path `root`. If `root` is a directory, then it is the first item - /// yielded by the iterator. If `root` is a file, then it is the first - /// and only item yielded by the iterator. If `root` is a symlink, then it - /// is always followed for the purposes of directory traversal. (A root - /// `DirEntry` still obeys its documentation with respect to symlinks and - /// the `follow_links` setting.) - pub fn new>(root: P) -> Self { - WalkDir { - opts: WalkDirOptions { - follow_links: false, - follow_root_links: true, - max_open: 10, - min_depth: 0, - max_depth: ::std::usize::MAX, - sorter: None, - contents_first: false, - same_file_system: false, - }, - root: root.as_ref().to_path_buf(), - } - } - - /// Set the minimum depth of entries yielded by the iterator. - /// - /// The smallest depth is `0` and always corresponds to the path given - /// to the `new` function on this type. Its direct descendents have depth - /// `1`, and their descendents have depth `2`, and so on. - pub fn min_depth(mut self, depth: usize) -> Self { - self.opts.min_depth = depth; - if self.opts.min_depth > self.opts.max_depth { - self.opts.min_depth = self.opts.max_depth; - } - self - } - - /// Set the maximum depth of entries yield by the iterator. - /// - /// The smallest depth is `0` and always corresponds to the path given - /// to the `new` function on this type. Its direct descendents have depth - /// `1`, and their descendents have depth `2`, and so on. - /// - /// Note that this will not simply filter the entries of the iterator, but - /// it will actually avoid descending into directories when the depth is - /// exceeded. - pub fn max_depth(mut self, depth: usize) -> Self { - self.opts.max_depth = depth; - if self.opts.max_depth < self.opts.min_depth { - self.opts.max_depth = self.opts.min_depth; - } - self - } - - /// Follow symbolic links. By default, this is disabled. - /// - /// When `yes` is `true`, symbolic links are followed as if they were - /// normal directories and files. If a symbolic link is broken or is - /// involved in a loop, an error is yielded. - /// - /// When enabled, the yielded [`DirEntry`] values represent the target of - /// the link while the path corresponds to the link. See the [`DirEntry`] - /// type for more details. - /// - /// [`DirEntry`]: struct.DirEntry.html - pub fn follow_links(mut self, yes: bool) -> Self { - self.opts.follow_links = yes; - self - } - - /// Follow symbolic links if these are the root of the traversal. - /// By default, this is enabled. - /// - /// When `yes` is `true`, symbolic links on root paths are followed - /// which is effective if the symbolic link points to a directory. - /// If a symbolic link is broken or is involved in a loop, an error is yielded - /// as the first entry of the traversal. - /// - /// When enabled, the yielded [`DirEntry`] values represent the target of - /// the link while the path corresponds to the link. See the [`DirEntry`] - /// type for more details, and all future entries will be contained within - /// the resolved directory behind the symbolic link of the root path. - /// - /// [`DirEntry`]: struct.DirEntry.html - pub fn follow_root_links(mut self, yes: bool) -> Self { - self.opts.follow_root_links = yes; - self - } - - /// Set the maximum number of simultaneously open file descriptors used - /// by the iterator. - /// - /// `n` must be greater than or equal to `1`. If `n` is `0`, then it is set - /// to `1` automatically. If this is not set, then it defaults to some - /// reasonably low number. - /// - /// This setting has no impact on the results yielded by the iterator - /// (even when `n` is `1`). Instead, this setting represents a trade off - /// between scarce resources (file descriptors) and memory. Namely, when - /// the maximum number of file descriptors is reached and a new directory - /// needs to be opened to continue iteration, then a previous directory - /// handle is closed and has its unyielded entries stored in memory. In - /// practice, this is a satisfying trade off because it scales with respect - /// to the *depth* of your file tree. Therefore, low values (even `1`) are - /// acceptable. - /// - /// Note that this value does not impact the number of system calls made by - /// an exhausted iterator. - /// - /// # Platform behavior - /// - /// On Windows, if `follow_links` is enabled, then this limit is not - /// respected. In particular, the maximum number of file descriptors opened - /// is proportional to the depth of the directory tree traversed. - pub fn max_open(mut self, mut n: usize) -> Self { - if n == 0 { - n = 1; - } - self.opts.max_open = n; - self - } - - /// Set a function for sorting directory entries with a comparator - /// function. - /// - /// If a compare function is set, the resulting iterator will return all - /// paths in sorted order. The compare function will be called to compare - /// entries from the same directory. - /// - /// ```rust,no_run - /// use std::cmp; - /// use std::ffi::OsString; - /// use walkdir::WalkDir; - /// - /// WalkDir::new("foo").sort_by(|a,b| a.file_name().cmp(b.file_name())); - /// ``` - pub fn sort_by(mut self, cmp: F) -> Self - where - F: FnMut(&DirEntry, &DirEntry) -> Ordering + Send + Sync + 'static, - { - self.opts.sorter = Some(Box::new(cmp)); - self - } - - /// Set a function for sorting directory entries with a key extraction - /// function. - /// - /// If a compare function is set, the resulting iterator will return all - /// paths in sorted order. The compare function will be called to compare - /// entries from the same directory. - /// - /// ```rust,no_run - /// use std::cmp; - /// use std::ffi::OsString; - /// use walkdir::WalkDir; - /// - /// WalkDir::new("foo").sort_by_key(|a| a.file_name().to_owned()); - /// ``` - pub fn sort_by_key(self, mut cmp: F) -> Self - where - F: FnMut(&DirEntry) -> K + Send + Sync + 'static, - K: Ord, - { - self.sort_by(move |a, b| cmp(a).cmp(&cmp(b))) - } - - /// Sort directory entries by file name, to ensure a deterministic order. - /// - /// This is a convenience function for calling `Self::sort_by()`. - /// - /// ```rust,no_run - /// use walkdir::WalkDir; - /// - /// WalkDir::new("foo").sort_by_file_name(); - /// ``` - pub fn sort_by_file_name(self) -> Self { - self.sort_by(|a, b| a.file_name().cmp(b.file_name())) - } - - /// Yield a directory's contents before the directory itself. By default, - /// this is disabled. - /// - /// When `yes` is `false` (as is the default), the directory is yielded - /// before its contents are read. This is useful when, e.g. you want to - /// skip processing of some directories. - /// - /// When `yes` is `true`, the iterator yields the contents of a directory - /// before yielding the directory itself. This is useful when, e.g. you - /// want to recursively delete a directory. - /// - /// # Example - /// - /// Assume the following directory tree: - /// - /// ```text - /// foo/ - /// abc/ - /// qrs - /// tuv - /// def/ - /// ``` - /// - /// With contents_first disabled (the default), the following code visits - /// the directory tree in depth-first order: - /// - /// ```no_run - /// use walkdir::WalkDir; - /// - /// for entry in WalkDir::new("foo") { - /// let entry = entry.unwrap(); - /// println!("{}", entry.path().display()); - /// } - /// - /// // foo - /// // foo/abc - /// // foo/abc/qrs - /// // foo/abc/tuv - /// // foo/def - /// ``` - /// - /// With contents_first enabled: - /// - /// ```no_run - /// use walkdir::WalkDir; - /// - /// for entry in WalkDir::new("foo").contents_first(true) { - /// let entry = entry.unwrap(); - /// println!("{}", entry.path().display()); - /// } - /// - /// // foo/abc/qrs - /// // foo/abc/tuv - /// // foo/abc - /// // foo/def - /// // foo - /// ``` - pub fn contents_first(mut self, yes: bool) -> Self { - self.opts.contents_first = yes; - self - } - - /// Do not cross file system boundaries. - /// - /// When this option is enabled, directory traversal will not descend into - /// directories that are on a different file system from the root path. - /// - /// Currently, this option is only supported on Unix and Windows. If this - /// option is used on an unsupported platform, then directory traversal - /// will immediately return an error and will not yield any entries. - pub fn same_file_system(mut self, yes: bool) -> Self { - self.opts.same_file_system = yes; - self - } -} - -impl IntoIterator for WalkDir { - type Item = Result; - type IntoIter = IntoIter; - - fn into_iter(self) -> IntoIter { - IntoIter { - opts: self.opts, - start: Some(self.root), - stack_list: vec![], - stack_path: vec![], - oldest_opened: 0, - depth: 0, - deferred_dirs: vec![], - root_device: None, - } - } -} - -/// An iterator for recursively descending into a directory. -/// -/// A value with this type must be constructed with the [`WalkDir`] type, which -/// uses a builder pattern to set options such as min/max depth, max open file -/// descriptors and whether the iterator should follow symbolic links. After -/// constructing a `WalkDir`, call [`.into_iter()`] at the end of the chain. -/// -/// The order of elements yielded by this iterator is unspecified. -/// -/// [`WalkDir`]: struct.WalkDir.html -/// [`.into_iter()`]: struct.WalkDir.html#into_iter.v -#[derive(Debug)] -pub struct IntoIter { - /// Options specified in the builder. Depths, max fds, etc. - opts: WalkDirOptions, - /// The start path. - /// - /// This is only `Some(...)` at the beginning. After the first iteration, - /// this is always `None`. - start: Option, - /// A stack of open (up to max fd) or closed handles to directories. - /// An open handle is a plain [`fs::ReadDir`] while a closed handle is - /// a `Vec` corresponding to the as-of-yet consumed entries. - /// - /// [`fs::ReadDir`]: https://doc.rust-lang.org/stable/std/fs/struct.ReadDir.html - stack_list: Vec, - /// A stack of file paths. - /// - /// This is *only* used when [`follow_links`] is enabled. In all other - /// cases this stack is empty. - /// - /// [`follow_links`]: struct.WalkDir.html#method.follow_links - stack_path: Vec, - /// An index into `stack_list` that points to the oldest open directory - /// handle. If the maximum fd limit is reached and a new directory needs to - /// be read, the handle at this index is closed before the new directory is - /// opened. - oldest_opened: usize, - /// The current depth of iteration (the length of the stack at the - /// beginning of each iteration). - depth: usize, - /// A list of DirEntries corresponding to directories, that are - /// yielded after their contents has been fully yielded. This is only - /// used when `contents_first` is enabled. - deferred_dirs: Vec, - /// The device of the root file path when the first call to `next` was - /// made. - /// - /// If the `same_file_system` option isn't enabled, then this is always - /// `None`. Conversely, if it is enabled, this is always `Some(...)` after - /// handling the root path. - root_device: Option, -} - -/// An ancestor is an item in the directory tree traversed by walkdir, and is -/// used to check for loops in the tree when traversing symlinks. -#[derive(Debug)] -struct Ancestor { - /// The path of this ancestor. - path: PathBuf, - /// An open file to this ancesor. This is only used on Windows where - /// opening a file handle appears to be quite expensive, so we choose to - /// cache it. This comes at the cost of not respecting the file descriptor - /// limit set by the user. - #[cfg(windows)] - handle: Handle, -} - -impl Ancestor { - /// Create a new ancestor from the given directory path. - #[cfg(windows)] - fn new(dent: &DirEntry) -> io::Result { - let handle = Handle::from_path(dent.path())?; - Ok(Ancestor { path: dent.path().to_path_buf(), handle }) - } - - /// Create a new ancestor from the given directory path. - #[cfg(not(windows))] - fn new(dent: &DirEntry) -> io::Result { - Ok(Ancestor { path: dent.path().to_path_buf() }) - } - - /// Returns true if and only if the given open file handle corresponds to - /// the same directory as this ancestor. - #[cfg(windows)] - fn is_same(&self, child: &Handle) -> io::Result { - Ok(child == &self.handle) - } - - /// Returns true if and only if the given open file handle corresponds to - /// the same directory as this ancestor. - #[cfg(not(windows))] - fn is_same(&self, child: &Handle) -> io::Result { - Ok(child == &Handle::from_path(&self.path)?) - } -} - -/// A sequence of unconsumed directory entries. -/// -/// This represents the opened or closed state of a directory handle. When -/// open, future entries are read by iterating over the raw `fs::ReadDir`. -/// When closed, all future entries are read into memory. Iteration then -/// proceeds over a [`Vec`]. -/// -/// [`fs::ReadDir`]: https://doc.rust-lang.org/stable/std/fs/struct.ReadDir.html -/// [`Vec`]: https://doc.rust-lang.org/stable/std/vec/struct.Vec.html -#[derive(Debug)] -enum DirList { - /// An opened handle. - /// - /// This includes the depth of the handle itself. - /// - /// If there was an error with the initial [`fs::read_dir`] call, then it - /// is stored here. (We use an [`Option<...>`] to make yielding the error - /// exactly once simpler.) - /// - /// [`fs::read_dir`]: https://doc.rust-lang.org/stable/std/fs/fn.read_dir.html - /// [`Option<...>`]: https://doc.rust-lang.org/stable/std/option/enum.Option.html - Opened { depth: usize, it: result::Result> }, - /// A closed handle. - /// - /// All remaining directory entries are read into memory. - Closed(vec::IntoIter>), -} - -impl Iterator for IntoIter { - type Item = Result; - /// Advances the iterator and returns the next value. - /// - /// # Errors - /// - /// If the iterator fails to retrieve the next value, this method returns - /// an error value. The error will be wrapped in an Option::Some. - fn next(&mut self) -> Option> { - if let Some(start) = self.start.take() { - if self.opts.same_file_system { - let result = util::device_num(&start) - .map_err(|e| Error::from_path(0, start.clone(), e)); - self.root_device = Some(itry!(result)); - } - let dent = itry!(DirEntry::from_path(0, start, false)); - if let Some(result) = self.handle_entry(dent) { - return Some(result); - } - } - while !self.stack_list.is_empty() { - self.depth = self.stack_list.len(); - if let Some(dentry) = self.get_deferred_dir() { - return Some(Ok(dentry)); - } - if self.depth > self.opts.max_depth { - // If we've exceeded the max depth, pop the current dir - // so that we don't descend. - self.pop(); - continue; - } - // Unwrap is safe here because we've verified above that - // `self.stack_list` is not empty - let next = self - .stack_list - .last_mut() - .expect("BUG: stack should be non-empty") - .next(); - match next { - None => self.pop(), - Some(Err(err)) => return Some(Err(err)), - Some(Ok(dent)) => { - if let Some(result) = self.handle_entry(dent) { - return Some(result); - } - } - } - } - if self.opts.contents_first { - self.depth = self.stack_list.len(); - if let Some(dentry) = self.get_deferred_dir() { - return Some(Ok(dentry)); - } - } - None - } -} - -impl IntoIter { - /// Skips the current directory. - /// - /// This causes the iterator to stop traversing the contents of the least - /// recently yielded directory. This means any remaining entries in that - /// directory will be skipped (including sub-directories). - /// - /// Note that the ergonomics of this method are questionable since it - /// borrows the iterator mutably. Namely, you must write out the looping - /// condition manually. For example, to skip hidden entries efficiently on - /// unix systems: - /// - /// ```no_run - /// use walkdir::{DirEntry, WalkDir}; - /// - /// fn is_hidden(entry: &DirEntry) -> bool { - /// entry.file_name() - /// .to_str() - /// .map(|s| s.starts_with(".")) - /// .unwrap_or(false) - /// } - /// - /// let mut it = WalkDir::new("foo").into_iter(); - /// loop { - /// let entry = match it.next() { - /// None => break, - /// Some(Err(err)) => panic!("ERROR: {}", err), - /// Some(Ok(entry)) => entry, - /// }; - /// if is_hidden(&entry) { - /// if entry.file_type().is_dir() { - /// it.skip_current_dir(); - /// } - /// continue; - /// } - /// println!("{}", entry.path().display()); - /// } - /// ``` - /// - /// You may find it more convenient to use the [`filter_entry`] iterator - /// adapter. (See its documentation for the same example functionality as - /// above.) - /// - /// [`filter_entry`]: #method.filter_entry - pub fn skip_current_dir(&mut self) { - if !self.stack_list.is_empty() { - self.pop(); - } - } - - /// Yields only entries which satisfy the given predicate and skips - /// descending into directories that do not satisfy the given predicate. - /// - /// The predicate is applied to all entries. If the predicate is - /// true, iteration carries on as normal. If the predicate is false, the - /// entry is ignored and if it is a directory, it is not descended into. - /// - /// This is often more convenient to use than [`skip_current_dir`]. For - /// example, to skip hidden files and directories efficiently on unix - /// systems: - /// - /// ```no_run - /// use walkdir::{DirEntry, WalkDir}; - /// # use walkdir::Error; - /// - /// fn is_hidden(entry: &DirEntry) -> bool { - /// entry.file_name() - /// .to_str() - /// .map(|s| s.starts_with(".")) - /// .unwrap_or(false) - /// } - /// - /// # fn try_main() -> Result<(), Error> { - /// for entry in WalkDir::new("foo") - /// .into_iter() - /// .filter_entry(|e| !is_hidden(e)) { - /// println!("{}", entry?.path().display()); - /// } - /// # Ok(()) - /// # } - /// ``` - /// - /// Note that the iterator will still yield errors for reading entries that - /// may not satisfy the predicate. - /// - /// Note that entries skipped with [`min_depth`] and [`max_depth`] are not - /// passed to this predicate. - /// - /// Note that if the iterator has `contents_first` enabled, then this - /// method is no different than calling the standard `Iterator::filter` - /// method (because directory entries are yielded after they've been - /// descended into). - /// - /// [`skip_current_dir`]: #method.skip_current_dir - /// [`min_depth`]: struct.WalkDir.html#method.min_depth - /// [`max_depth`]: struct.WalkDir.html#method.max_depth - pub fn filter_entry

(self, predicate: P) -> FilterEntry - where - P: FnMut(&DirEntry) -> bool, - { - FilterEntry { it: self, predicate } - } - - fn handle_entry( - &mut self, - mut dent: DirEntry, - ) -> Option> { - if self.opts.follow_links && dent.file_type().is_symlink() { - dent = itry!(self.follow(dent)); - } - let is_normal_dir = !dent.file_type().is_symlink() && dent.is_dir(); - if is_normal_dir { - if self.opts.same_file_system && dent.depth() > 0 { - if itry!(self.is_same_file_system(&dent)) { - itry!(self.push(&dent)); - } - } else { - itry!(self.push(&dent)); - } - } else if dent.depth() == 0 - && dent.file_type().is_symlink() - && self.opts.follow_root_links - { - // As a special case, if we are processing a root entry, then we - // always follow it even if it's a symlink and follow_links is - // false. We are careful to not let this change the semantics of - // the DirEntry however. Namely, the DirEntry should still respect - // the follow_links setting. When it's disabled, it should report - // itself as a symlink. When it's enabled, it should always report - // itself as the target. - let md = itry!(fs::metadata(dent.path()).map_err(|err| { - Error::from_path(dent.depth(), dent.path().to_path_buf(), err) - })); - if md.file_type().is_dir() { - itry!(self.push(&dent)); - } - } - if is_normal_dir && self.opts.contents_first { - self.deferred_dirs.push(dent); - None - } else if self.skippable() { - None - } else { - Some(Ok(dent)) - } - } - - fn get_deferred_dir(&mut self) -> Option { - if self.opts.contents_first { - if self.depth < self.deferred_dirs.len() { - // Unwrap is safe here because we've guaranteed that - // `self.deferred_dirs.len()` can never be less than 1 - let deferred: DirEntry = self - .deferred_dirs - .pop() - .expect("BUG: deferred_dirs should be non-empty"); - if !self.skippable() { - return Some(deferred); - } - } - } - None - } - - fn push(&mut self, dent: &DirEntry) -> Result<()> { - // Make room for another open file descriptor if we've hit the max. - let free = - self.stack_list.len().checked_sub(self.oldest_opened).unwrap(); - if free == self.opts.max_open { - self.stack_list[self.oldest_opened].close(); - } - // Open a handle to reading the directory's entries. - let rd = fs::read_dir(dent.path()).map_err(|err| { - Some(Error::from_path(self.depth, dent.path().to_path_buf(), err)) - }); - let mut list = DirList::Opened { depth: self.depth, it: rd }; - if let Some(ref mut cmp) = self.opts.sorter { - let mut entries: Vec<_> = list.collect(); - entries.sort_by(|a, b| match (a, b) { - (&Ok(ref a), &Ok(ref b)) => cmp(a, b), - (&Err(_), &Err(_)) => Ordering::Equal, - (&Ok(_), &Err(_)) => Ordering::Greater, - (&Err(_), &Ok(_)) => Ordering::Less, - }); - list = DirList::Closed(entries.into_iter()); - } - if self.opts.follow_links { - let ancestor = Ancestor::new(&dent) - .map_err(|err| Error::from_io(self.depth, err))?; - self.stack_path.push(ancestor); - } - // We push this after stack_path since creating the Ancestor can fail. - // If it fails, then we return the error and won't descend. - self.stack_list.push(list); - // If we had to close out a previous directory stream, then we need to - // increment our index the oldest still-open stream. We do this only - // after adding to our stack, in order to ensure that the oldest_opened - // index remains valid. The worst that can happen is that an already - // closed stream will be closed again, which is a no-op. - // - // We could move the close of the stream above into this if-body, but - // then we would have more than the maximum number of file descriptors - // open at a particular point in time. - if free == self.opts.max_open { - // Unwrap is safe here because self.oldest_opened is guaranteed to - // never be greater than `self.stack_list.len()`, which implies - // that the subtraction won't underflow and that adding 1 will - // never overflow. - self.oldest_opened = self.oldest_opened.checked_add(1).unwrap(); - } - Ok(()) - } - - fn pop(&mut self) { - self.stack_list.pop().expect("BUG: cannot pop from empty stack"); - if self.opts.follow_links { - self.stack_path.pop().expect("BUG: list/path stacks out of sync"); - } - // If everything in the stack is already closed, then there is - // room for at least one more open descriptor and it will - // always be at the top of the stack. - self.oldest_opened = min(self.oldest_opened, self.stack_list.len()); - } - - fn follow(&self, mut dent: DirEntry) -> Result { - dent = - DirEntry::from_path(self.depth, dent.path().to_path_buf(), true)?; - // The only way a symlink can cause a loop is if it points - // to a directory. Otherwise, it always points to a leaf - // and we can omit any loop checks. - if dent.is_dir() { - self.check_loop(dent.path())?; - } - Ok(dent) - } - - fn check_loop>(&self, child: P) -> Result<()> { - let hchild = Handle::from_path(&child) - .map_err(|err| Error::from_io(self.depth, err))?; - for ancestor in self.stack_path.iter().rev() { - let is_same = ancestor - .is_same(&hchild) - .map_err(|err| Error::from_io(self.depth, err))?; - if is_same { - return Err(Error::from_loop( - self.depth, - &ancestor.path, - child.as_ref(), - )); - } - } - Ok(()) - } - - fn is_same_file_system(&mut self, dent: &DirEntry) -> Result { - let dent_device = util::device_num(dent.path()) - .map_err(|err| Error::from_entry(dent, err))?; - Ok(self - .root_device - .map(|d| d == dent_device) - .expect("BUG: called is_same_file_system without root device")) - } - - fn skippable(&self) -> bool { - self.depth < self.opts.min_depth || self.depth > self.opts.max_depth - } -} - -impl iter::FusedIterator for IntoIter {} - -impl DirList { - fn close(&mut self) { - if let DirList::Opened { .. } = *self { - *self = DirList::Closed(self.collect::>().into_iter()); - } - } -} - -impl Iterator for DirList { - type Item = Result; - - #[inline(always)] - fn next(&mut self) -> Option> { - match *self { - DirList::Closed(ref mut it) => it.next(), - DirList::Opened { depth, ref mut it } => match *it { - Err(ref mut err) => err.take().map(Err), - Ok(ref mut rd) => rd.next().map(|r| match r { - Ok(r) => DirEntry::from_entry(depth + 1, &r), - Err(err) => Err(Error::from_io(depth + 1, err)), - }), - }, - } - } -} - -/// A recursive directory iterator that skips entries. -/// -/// Values of this type are created by calling [`.filter_entry()`] on an -/// `IntoIter`, which is formed by calling [`.into_iter()`] on a `WalkDir`. -/// -/// Directories that fail the predicate `P` are skipped. Namely, they are -/// never yielded and never descended into. -/// -/// Entries that are skipped with the [`min_depth`] and [`max_depth`] options -/// are not passed through this filter. -/// -/// If opening a handle to a directory resulted in an error, then it is yielded -/// and no corresponding call to the predicate is made. -/// -/// Type parameter `I` refers to the underlying iterator and `P` refers to the -/// predicate, which is usually `FnMut(&DirEntry) -> bool`. -/// -/// [`.filter_entry()`]: struct.IntoIter.html#method.filter_entry -/// [`.into_iter()`]: struct.WalkDir.html#into_iter.v -/// [`min_depth`]: struct.WalkDir.html#method.min_depth -/// [`max_depth`]: struct.WalkDir.html#method.max_depth -#[derive(Debug)] -pub struct FilterEntry { - it: I, - predicate: P, -} - -impl

Iterator for FilterEntry -where - P: FnMut(&DirEntry) -> bool, -{ - type Item = Result; - - /// Advances the iterator and returns the next value. - /// - /// # Errors - /// - /// If the iterator fails to retrieve the next value, this method returns - /// an error value. The error will be wrapped in an `Option::Some`. - fn next(&mut self) -> Option> { - loop { - let dent = match self.it.next() { - None => return None, - Some(result) => itry!(result), - }; - if !(self.predicate)(&dent) { - if dent.is_dir() { - self.it.skip_current_dir(); - } - continue; - } - return Some(Ok(dent)); - } - } -} - -impl

iter::FusedIterator for FilterEntry where - P: FnMut(&DirEntry) -> bool -{ -} - -impl

FilterEntry -where - P: FnMut(&DirEntry) -> bool, -{ - /// Yields only entries which satisfy the given predicate and skips - /// descending into directories that do not satisfy the given predicate. - /// - /// The predicate is applied to all entries. If the predicate is - /// true, iteration carries on as normal. If the predicate is false, the - /// entry is ignored and if it is a directory, it is not descended into. - /// - /// This is often more convenient to use than [`skip_current_dir`]. For - /// example, to skip hidden files and directories efficiently on unix - /// systems: - /// - /// ```no_run - /// use walkdir::{DirEntry, WalkDir}; - /// # use walkdir::Error; - /// - /// fn is_hidden(entry: &DirEntry) -> bool { - /// entry.file_name() - /// .to_str() - /// .map(|s| s.starts_with(".")) - /// .unwrap_or(false) - /// } - /// - /// # fn try_main() -> Result<(), Error> { - /// for entry in WalkDir::new("foo") - /// .into_iter() - /// .filter_entry(|e| !is_hidden(e)) { - /// println!("{}", entry?.path().display()); - /// } - /// # Ok(()) - /// # } - /// ``` - /// - /// Note that the iterator will still yield errors for reading entries that - /// may not satisfy the predicate. - /// - /// Note that entries skipped with [`min_depth`] and [`max_depth`] are not - /// passed to this predicate. - /// - /// Note that if the iterator has `contents_first` enabled, then this - /// method is no different than calling the standard `Iterator::filter` - /// method (because directory entries are yielded after they've been - /// descended into). - /// - /// [`skip_current_dir`]: #method.skip_current_dir - /// [`min_depth`]: struct.WalkDir.html#method.min_depth - /// [`max_depth`]: struct.WalkDir.html#method.max_depth - pub fn filter_entry(self, predicate: P) -> FilterEntry { - FilterEntry { it: self, predicate } - } - - /// Skips the current directory. - /// - /// This causes the iterator to stop traversing the contents of the least - /// recently yielded directory. This means any remaining entries in that - /// directory will be skipped (including sub-directories). - /// - /// Note that the ergonomics of this method are questionable since it - /// borrows the iterator mutably. Namely, you must write out the looping - /// condition manually. For example, to skip hidden entries efficiently on - /// unix systems: - /// - /// ```no_run - /// use walkdir::{DirEntry, WalkDir}; - /// - /// fn is_hidden(entry: &DirEntry) -> bool { - /// entry.file_name() - /// .to_str() - /// .map(|s| s.starts_with(".")) - /// .unwrap_or(false) - /// } - /// - /// let mut it = WalkDir::new("foo").into_iter(); - /// loop { - /// let entry = match it.next() { - /// None => break, - /// Some(Err(err)) => panic!("ERROR: {}", err), - /// Some(Ok(entry)) => entry, - /// }; - /// if is_hidden(&entry) { - /// if entry.file_type().is_dir() { - /// it.skip_current_dir(); - /// } - /// continue; - /// } - /// println!("{}", entry.path().display()); - /// } - /// ``` - /// - /// You may find it more convenient to use the [`filter_entry`] iterator - /// adapter. (See its documentation for the same example functionality as - /// above.) - /// - /// [`filter_entry`]: #method.filter_entry - pub fn skip_current_dir(&mut self) { - self.it.skip_current_dir(); - } -} +mod walk; diff --git a/src/os/linux/dirent.rs b/src/os/linux/dirent.rs new file mode 100644 index 0000000..18ae8ce --- /dev/null +++ b/src/os/linux/dirent.rs @@ -0,0 +1,80 @@ +use std::ffi::CStr; +use std::fmt; + +use libc::c_char; + +use crate::os::unix::FileType; + +/// A raw directory entry used to read entries from Linux's getdents64 syscall. +/// +/// Note that this type is very much not safe to use because `d_name` is a +/// flexible array member. That is, the *size* of values of this type are +/// usually larger than size_of::, since the file name will extend +/// beyond the end of the struct. Therefore, values of this type should only be +/// read when they exist in their original buffer. +/// +/// We expose this by making it not safe to ask for the name in this +/// entry, since its `NUL` terminator scan could result in a buffer overrun +/// when used incorrectly. +#[derive(Clone)] +#[repr(C)] +pub struct RawDirEntry { + d_ino: u64, + d_off: u64, + d_reclen: u16, + d_type: u8, + d_name: [u8; 0], +} + +impl fmt::Debug for RawDirEntry { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("RawDirEntry") + .field("d_ino", &self.ino()) + .field("d_off", &self.d_off) + .field("d_reclen", &self.d_reclen) + .field("d_type", &self.file_type()) + // Reading the name is not safe, and we can't guarantee its + // safety in the context of this Debug impl unfortunately. We + // *could* use a small fixed size array instead of [u8; 0] as the + // representation, which would at least let us safely read a prefix + // to show here, but it's not clear what cost that would have + // (probably none?) or whether it's worth it. + .field("d_name", &"") + .finish() + } +} + +impl RawDirEntry { + /// Return the file name in this directory entry as a C string. + /// + /// This computes the length of the name in this entry by scanning for a + /// `NUL` terminator. + /// + /// # Safety + /// + /// This is not safe because callers who call this function must guarantee + /// that the `RawDirEntry` is still within its original buffer. Otherwise, + /// it's possible for a buffer overrun to occur. + pub unsafe fn file_name(&self) -> &CStr { + CStr::from_ptr(self.d_name.as_ptr() as *const c_char) + } + + /// Return the file type of this directory entry, if one exists. + /// + /// A file type may not exist if the underlying file system reports an + /// unknown file type in the directory entry. + pub fn file_type(&self) -> Option { + FileType::from_dirent_type(self.d_type) + } + + /// Returns the underlying file serial number for this directory entry. + pub fn ino(&self) -> u64 { + self.d_ino + } + + /// Returns the total length (including padding), in bytes, of this + /// directory entry. + pub fn record_len(&self) -> usize { + self.d_reclen as usize + } +} diff --git a/src/os/linux/mod.rs b/src/os/linux/mod.rs new file mode 100644 index 0000000..1322685 --- /dev/null +++ b/src/os/linux/mod.rs @@ -0,0 +1,370 @@ +/*! +Low level Linux specific APIs for reading directory entries via `getdents64`. +*/ + +use std::alloc::{alloc_zeroed, dealloc, handle_alloc_error, Layout}; +use std::ffi::{CStr, OsStr}; +use std::fmt; +use std::io; +use std::mem; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::io::RawFd; +use std::ptr::NonNull; + +use libc::{syscall, SYS_getdents64}; + +use crate::os::linux::dirent::RawDirEntry; +use crate::os::unix::{DirEntry as UnixDirEntry, FileType}; + +mod dirent; + +/// A safe function for calling Linux's `getdents64` API. +/// +/// The basic idea of `getdents` is that it executes a single syscall but +/// returns potentially many directory entries in a single buffer. This can +/// provide a small speed boost when compared with the typical `readdir` POSIX +/// API, depending on your platform's implementation. +/// +/// This routine will read directory entries from the given file descriptor +/// into the given cursor. The cursor can then be used to cheaply and safely +/// iterate over the directory entries that were read. +/// +/// When all directory entries have been read from the given file descriptor, +/// then this function will return `false`. Otherwise, it returns `true`. +/// +/// If there was a problem calling the underlying `getdents64` syscall, then +/// an error is returned. +pub fn getdents(fd: RawFd, cursor: &mut DirEntryCursor) -> io::Result { + cursor.clear(); + let res = unsafe { + syscall( + SYS_getdents64, + fd, + cursor.raw.as_ptr() as *mut RawDirEntry, + cursor.capacity, + ) + }; + match res { + -1 => Err(io::Error::last_os_error()), + 0 => Ok(false), + nwritten => { + cursor.len = nwritten as usize; + Ok(true) + } + } +} + +/// A Linux specific directory entry. +/// +/// This directory entry is just like the Unix `DirEntry`, except its file +/// name is borrowed from a `DirEntryCursor`'s internal buffer. This makes +/// it possible to iterate over directory entries on Linux by reusing the +/// cursor's internal buffer with no additional allocations or copying. +/// +/// In practice, if one needs an owned directory entry, then convert it to a +/// Unix `DirEntry` either via the Unix methods on this `DirEntry`, or by +/// simply reading a Unix `DirEntry` directly from `DirEntryCursor`. +#[derive(Clone)] +pub struct DirEntry<'a> { + /// A borrowed version of the `d_name` field found in the raw directory + /// entry. This field is the only reason why this type exists, otherwise + /// we'd just expose `RawDirEntry` directly to users. The issue with + /// exposing the raw directory entry is that its size isn't correct (since + /// the file name may extend beyond the end of the struct). + /// + /// This borrow ties this entry to the `DirEntryBuffer`. + file_name: &'a CStr, + /// The file type, as is, from the raw dirent. + file_type: Option, + /// The file serial number, as is, from the raw dirent. + ino: u64, +} + +impl<'a> fmt::Debug for DirEntry<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use crate::os::unix::escaped_bytes; + + f.debug_struct("DirEntry") + .field("file_name", &escaped_bytes(self.file_name_bytes())) + .field("file_type", &self.file_type) + .field("ino", &self.ino) + .finish() + } +} + +impl<'a> DirEntry<'a> { + /// Return the file name in this directory entry as a C string. + #[inline] + pub fn file_name(&self) -> &CStr { + self.file_name + } + + /// Return the file name in this directory entry as raw bytes without + /// a `NUL` terminator. + #[inline] + pub fn file_name_bytes(&self) -> &[u8] { + self.file_name.to_bytes() + } + + /// Return the file name in this directory entry as an OS string without + /// a `NUL` terminator. + #[inline] + pub fn file_name_os(&self) -> &OsStr { + OsStr::from_bytes(self.file_name_bytes()) + } + + /// Return the file type of this directory entry, if one exists. + /// + /// A file type may not exist if the underlying file system reports an + /// unknown file type in the directory entry. + #[inline] + pub fn file_type(&self) -> Option { + self.file_type + } + + /// Returns the underlying file serial number for this directory entry. + #[inline] + pub fn ino(&self) -> u64 { + self.ino + } + + /// Convert this directory entry into an owned Unix `DirEntry`. If you + /// want to be able to reuse allocations, then use `write_to_unix` instead. + #[inline] + pub fn to_unix(&self) -> UnixDirEntry { + let mut ent = UnixDirEntry::empty(); + self.write_to_unix(&mut ent); + ent + } + + /// Write this directory entry into the given Unix `DirEntry`. This makes + /// it possible to amortize allocation. + #[inline] + pub fn write_to_unix(&self, unix_dirent: &mut UnixDirEntry) { + unix_dirent.from_linux_raw(self) + } +} + +/// A cursor for reading directory entries from a `getdents` buffer. +/// +/// This cursor allocates space internally for storing one or more Linux +/// directory entries, and exposes an API for cheaply iterating over those +/// directory entries. +/// +/// A cursor can and should be reused across multiple calls to `getdents`. A +/// cursor is not tied to any one particular directory. +/// +/// This is not `Clone` because shallow clones would double-free the buffer. +#[derive(Debug)] +pub struct DirEntryCursor { + /// Spiritually, this is a *mut RawDirEntry. Unfortunately, this doesn't + /// quite make sense since a value with type `RawDirEntry` does not + /// actually have a size of `size_of::()` due to the way in + /// which the entry's name is stored in a flexible array member. + /// + /// With that said, we do transmute bytes in this buffer to a + /// `RawDirEntry`, which lets us read the members of the struct (including + /// the flexible array member) correctly. However, because of that, we need + /// to make sure our memory has the correct alignment. Hence, this is why + /// we use a raw `*mut u8` created by the std::alloc APIs. If there was an + /// easy way to control alignment with a `Vec`, then we could use that + /// instead. (It is indeed possible, but seems fragile.) + /// + /// Since a `RawDirEntry` is inherently unsafe to use because of its + /// flexible array member, it is converted to a `DirEntry` (cheaply, + /// without allocation) before being exposed to the caller. + raw: NonNull, + /// The lenth, in bytes, of all valid entries in `raw`. + len: usize, + /// The lenth, in bytes, of `raw`. + capacity: usize, + /// The current position of this buffer as a pointer into `raw`. + cursor: NonNull, + /// Whether the cursor has been advanced at least once. + advanced: bool, +} + +// This is sound because the cursor uniquely owns its heap buffer (freed on drop) and +// `cursor` is an interior pointer into `raw`, so there is no shared aliasing. +unsafe impl Send for DirEntryCursor {} +unsafe impl Sync for DirEntryCursor {} + +impl Drop for DirEntryCursor { + fn drop(&mut self) { + unsafe { + dealloc(self.raw.as_ptr(), layout(self.capacity)); + } + } +} + +/// Returns the allocation layout used for constructing the getdents buffer +/// with the given capacity (in bytes). +/// +/// This panics if the given length isn't a multiple of the alignment of +/// `RawDirEntry` or is `0`. +fn layout(capacity: usize) -> Layout { + let align = mem::align_of::(); + assert!(capacity > 0, "capacity must be greater than 0"); + assert!(capacity % align == 0, "capacity must be a multiple of alignment"); + Layout::from_size_align(capacity, align).expect("failed to create Layout") +} + +impl DirEntryCursor { + /// Create a new cursor for reading directory entries. + /// + /// It is beneficial to reuse a cursor in multiple calls to `getdents`. A + /// cursor can be used with any number of directories. + pub fn new() -> DirEntryCursor { + DirEntryCursor::with_capacity(32 * (1 << 10)) + } + + /// Create a new cursor with the specified capacity. The capacity given + /// should be in bytes, and must be a multiple of the alignment of a raw + /// directory entry. + fn with_capacity(capacity: usize) -> DirEntryCursor { + // TODO: It would be nice to expose a way to control the capacity to + // the caller, but we'd really like the capacity to be a multiple of + // the alignment. (Technically, the only restriction is that + // the capacity and the alignment have a least common multiple that + // doesn't overflow `usize::MAX`. But requiring the size to be a + // multiple of alignment just seems like good sense in this case.) + // + // Anyway, exposing raw capacity to the caller is weird, because they + // shouldn't need to care about the alignment of an internal type. + // We *could* expose capacity in "units" of `RawDirEntry` itself, but + // even this is somewhat incorrect because the size of `RawDirEntry` + // is smaller than what it typically is, since the size doesn't account + // for file names. We could just pick a fixed approximate size for + // file names and add that to the size of `RawDirEntry`. But let's wait + // for a more concrete use case to emerge before exposing anything. + let lay = layout(capacity); + let raw = match NonNull::new(unsafe { alloc_zeroed(lay) }) { + Some(raw) => raw, + None => handle_alloc_error(lay), + }; + DirEntryCursor { raw, len: 0, capacity, cursor: raw, advanced: false } + } + + /// Read the next directory entry from this cursor. If the cursor has been + /// exhausted, then return `None`. + /// + /// The returned directory entry contains a file name that is borrowed from + /// this cursor's internal buffer. In particular, no allocation is + /// performed by this routine. If you need an owned directory entry, then + /// use `read_unix` or `read_unix_into`. + /// + /// Note that no filtering of entries (such as `.` and `..`) is performed. + pub fn read<'a>(&'a mut self) -> Option> { + if !self.advance() { + return None; + } + Some(self.current()) + } + + /// Advance this cursor to the next directory entry. If there are no more + /// directory entries to read, then this returns false. + /// + /// Calling `current()` after `advance` is guaranteed to panic when this + /// returns false. Conversely, calling `current()` after `advance` is + /// guaranteed not to panic when this returns true. + pub fn advance(&mut self) -> bool { + if self.is_done() { + return false; + } + if !self.advanced { + self.advanced = true; + return true; + } + // SAFETY: This is safe by the assumption that `d_reclen` on the raw + // dirent is correct. + self.cursor = unsafe { + let raw = self.current_raw(); + let next = self.cursor.as_ptr().add(raw.record_len()); + NonNull::new_unchecked(next) + }; + !self.is_done() + } + + /// Return the current directory entry in this cursor. + /// + /// This panics is the cursor has been exhausted, or if the cursor has not + /// yet had `advance` called. + /// + /// Calling `current()` after `advance` is guaranteed to panic when this + /// returns false. Conversely, calling `current()` after `advance` is + /// guaranteed not to panic when this returns true. + pub fn current<'a>(&'a self) -> DirEntry<'a> { + let raw = self.current_raw(); + DirEntry { + // SAFETY: This is safe since we are asking for the file name on a + // `RawDirEntry` that resides in its original buffer. + file_name: unsafe { raw.file_name() }, + file_type: raw.file_type(), + ino: raw.ino(), + } + } + + fn current_raw(&self) -> &RawDirEntry { + assert!(self.advanced); + assert!(!self.is_done()); + // SAFETY: This is safe by the contract of getdents64. Namely, that it + // writes structures of type `RawDirEntry` to `raw`. The lifetime of + // this raw dirent is also tied to this buffer via the type signature + // of this method, which prevents use-after-free. Moreover, our + // allocation layout guarantees that the cursor is correctly aligned + // for RawDirEntry. Finally, we assert that self.cursor has not + // reached the end yet, and since the cursor is only ever incremented + // by correct amounts, we know it points to the beginning of a valid + // directory entry. + unsafe { &*(self.cursor.as_ptr() as *const RawDirEntry) } + } + + fn is_done(&self) -> bool { + self.cursor.as_ptr() >= self.raw.as_ptr().wrapping_add(self.len) + } + + /// Read the next directory entry from this cursor as an owned Unix + /// `DirEntry`. If the cursor has been exhausted, then return `None`. + /// + /// This will allocate new space to store the file name in the directory + /// entry. To reuse a previous allocation, use `read_unix_into` instead. + /// + /// Note that no filtering of entries (such as `.` and `..`) is performed. + pub fn read_unix(&mut self) -> Option { + self.read().map(|ent| ent.to_unix()) + } + + /// Read the next directory entry from this cursor into the given Unix + /// `DirEntry`. If the cursor has been exhausted, then return `false`. + /// Otherwise return `true`. + /// + /// Note that no filtering of entries (such as `.` and `..`) is performed. + pub fn read_unix_into(&mut self, unix_dirent: &mut UnixDirEntry) -> bool { + match self.read() { + None => false, + Some(dent) => { + dent.write_to_unix(unix_dirent); + true + } + } + } + + /// Rewind this cursor such that it points to the first directory entry. + pub fn rewind(&mut self) { + self.cursor = self.raw; + self.advanced = false; + } + + /// Clear this cursor such that it has no entries. + fn clear(&mut self) { + self.cursor = self.raw; + self.len = 0; + self.advanced = false; + } +} + +impl Default for DirEntryCursor { + fn default() -> Self { + Self::new() + } +} diff --git a/src/os/mod.rs b/src/os/mod.rs new file mode 100644 index 0000000..c6c2058 --- /dev/null +++ b/src/os/mod.rs @@ -0,0 +1,10 @@ +/*! +Low level platform specific APIs for reading directory entries. +*/ + +#[cfg(target_os = "linux")] +pub mod linux; +#[cfg(walkdir_unix)] +pub mod unix; +#[cfg(windows)] +pub mod windows; diff --git a/src/os/unix/dirent.rs b/src/os/unix/dirent.rs new file mode 100644 index 0000000..5b6f6ac --- /dev/null +++ b/src/os/unix/dirent.rs @@ -0,0 +1,209 @@ +// It turns out that `dirent` is a complete mess across different platforms. +// All POSIX defines is a "struct containing the fields d_ino and d_name" where +// d_name has an unspecified size. And not even that minimal subset is actually +// portable. For example, DragonflyBSD, FreeBSD, NetBSD and OpenBSD all use +// `d_fileno` instead of `d_ino`. +// +// Some platforms (macOS) have a `d_namlen` field indicating the number of +// bytes in `d_name`, while other platforms (Linux) only have a `d_reclen` +// field indicating the total size of the entry. +// +// Finally, not every platform (Solaris) has a `d_type` field, which is insane, +// because that means you need an extra stat call for every directory entry +// in order to do recursive directory traversal. +// +// Rebuilding all this goop that's already done in std really sucks, but if we +// want to specialize even one platform (e.g., using getdents64 on Linux), then +// we wind up needing to specialize ALL of them because std::fs::FileType is +// impossible to either cheaply construct or convert, so we wind up needing to +// roll our own. +// +// So basically what we do here is define a very thin unix-specific but +// platform independent layer on top of all the different dirent formulations +// that we support. We try to avoid costs (e.g., NUL-byte scanning) wherever +// possible. + +use std::ffi::CStr; +use std::fmt; +use std::ptr::NonNull; +#[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", +))] +use std::slice; + +#[cfg(any( + target_os = "android", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "haiku", + target_os = "linux", + target_os = "solaris", +))] +use libc::c_char; +#[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "haiku", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "solaris", +))] +use libc::dirent; +#[cfg(any( + target_os = "android", + target_os = "emscripten", + target_os = "linux", +))] +use libc::dirent64 as dirent; + +use crate::os::unix::{escaped_bytes, FileType}; + +/// A low-level Unix-specific but otherwise platform independent API to the +/// underlying platform's `dirent` struct. +#[derive(Clone)] +pub struct RawDirEntry(NonNull); + +impl fmt::Debug for RawDirEntry { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("RawDirEntry") + .field("d_name", &escaped_bytes(self.file_name().to_bytes())) + .field("d_type", &self.file_type()) + .field("d_ino", &self.ino()) + .finish() + } +} + +impl RawDirEntry { + /// Create a new raw directory entry from the given dirent structure. + /// + /// If the given entry is null, then this returns `None`. + pub fn new(ent: *const dirent) -> Option { + NonNull::new(ent as *mut _).map(RawDirEntry) + } + + /// Return the underlying dirent. + pub fn dirent(&self) -> &dirent { + // SAFETY: This is safe since we tie the lifetime of the returned + // dirent to self. + unsafe { self.0.as_ref() } + } + + /// Return the file name in this directory entry as a C string. + pub fn file_name(&self) -> &CStr { + // This implementation uses namlen to determine the size of file name. + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + ))] + fn imp(ent: &RawDirEntry) -> &CStr { + // SAFETY: This is safe given the guarantee that `d_namlen` is the + // total number of bytes in the file name, minus the NUL + // terminator. This is also only safe given the guarantee that + // `d_name` contains a NUL terminator. + unsafe { + let bytes = slice::from_raw_parts( + ent.dirent().d_name.as_ptr() as *const u8, + ent.dirent().d_namlen as usize + 1, // +1 for NUL + ); + CStr::from_bytes_with_nul_unchecked(bytes) + } + } + + // This implementation uses strlen to determine the size of the file + // name, since these platforms don't have a `d_namlen` field. Some of + // them *do* have a `d_reclen` field, which seems like it could help + // us, but there is no clear documentation on how to use it properly. + // In particular, it can include the padding between the directory + // entries, and it's not clear how to account for that. + #[cfg(any( + target_os = "android", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "haiku", + target_os = "linux", + target_os = "solaris", + ))] + fn imp(ent: &RawDirEntry) -> &CStr { + // SAFETY: This is safe since `d_name` is guaranteed to be valid + // and guaranteed to contain a NUL terminator. + unsafe { + CStr::from_ptr(ent.dirent().d_name.as_ptr() as *const c_char) + } + } + + imp(self) + } + + /// Return the file type embedded in this directory entry, if one exists. + /// + /// If this platform doesn't support reporting the file type in the + /// directory entry, or if the file type was reported as unknown, then + /// this returns `None`. + pub fn file_type(&self) -> Option { + // This implementation uses the `d_type` field. + // + // Note that this can still return None if the value of this field + // is DT_UNKNOWN. + #[cfg(any( + target_os = "android", + target_os = "emscripten", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + ))] + fn imp(ent: &RawDirEntry) -> Option { + FileType::from_dirent_type(ent.dirent().d_type) + } + + // No `d_type` field is available here, so always return None. + #[cfg(any(target_os = "haiku", target_os = "solaris",))] + fn imp(_ent: &RawDirEntry) -> Option { + None + } + + imp(self) + } + + /// Return the file serial number for this directory entry. + pub fn ino(&self) -> u64 { + // This implementation uses the d_fileno field. + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + ))] + fn imp(ent: &RawDirEntry) -> u64 { + ent.dirent().d_fileno as u64 + } + + // This implementation uses the d_ino field. + #[cfg(any( + target_os = "android", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "macos", + target_os = "haiku", + target_os = "linux", + target_os = "solaris", + ))] + fn imp(ent: &RawDirEntry) -> u64 { + ent.dirent().d_ino as u64 + } + + imp(self) + } +} diff --git a/src/os/unix/errno-dragonfly.c b/src/os/unix/errno-dragonfly.c new file mode 100644 index 0000000..dfbe9e5 --- /dev/null +++ b/src/os/unix/errno-dragonfly.c @@ -0,0 +1,5 @@ +#include + +int *errno_location() { + return &errno; +} diff --git a/src/os/unix/errno.rs b/src/os/unix/errno.rs new file mode 100644 index 0000000..fbfbd83 --- /dev/null +++ b/src/os/unix/errno.rs @@ -0,0 +1,46 @@ +// This was mostly lifted from the standard library's sys module. The main +// difference is that we need to use a C shim to get access to errno on +// DragonflyBSD, since the #[thread_local] attribute isn't stable (as of Rust +// 1.34). + +use libc::c_int; + +extern "C" { + #[cfg_attr( + any( + target_os = "linux", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "l4re", + ), + link_name = "__errno_location" + )] + #[cfg_attr( + any( + target_os = "netbsd", + target_os = "openbsd", + target_os = "android", + ), + link_name = "__errno" + )] + #[cfg_attr(target_os = "dragonfly", link_name = "errno_location")] + #[cfg_attr(target_os = "solaris", link_name = "___errno")] + #[cfg_attr( + any(target_os = "macos", target_os = "ios", target_os = "freebsd",), + link_name = "__error" + )] + #[cfg_attr(target_os = "haiku", link_name = "_errnop")] + fn errno_location() -> *mut c_int; +} + +/// Returns the platform-specific value of errno. +pub fn errno() -> i32 { + unsafe { (*errno_location()) as i32 } +} + +/// Clears the platform-specific value of errno to 0. +pub fn clear() { + unsafe { + *errno_location() = 0; + } +} diff --git a/src/os/unix/mod.rs b/src/os/unix/mod.rs new file mode 100644 index 0000000..e6a57d8 --- /dev/null +++ b/src/os/unix/mod.rs @@ -0,0 +1,574 @@ +/*! +Low level Unix specific APIs for reading directory entries via `readdir`. +*/ + +use std::ffi::{CStr, CString, OsStr, OsString}; +use std::fmt; +use std::fs::File; +use std::io; +use std::mem; +use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; +use std::path::PathBuf; +use std::ptr::NonNull; + +use libc; +#[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "haiku", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "solaris", +))] +use libc::readdir; +#[cfg(any( + target_os = "android", + target_os = "emscripten", + target_os = "linux", +))] +use libc::readdir64 as readdir; + +#[cfg(target_os = "linux")] +use crate::os::linux::DirEntry as LinuxDirEntry; +use crate::os::unix::dirent::RawDirEntry; +pub use crate::os::unix::rawpath::RawPathBuf; +pub use crate::os::unix::stat::{ + lstat, lstat_c, lstatat, lstatat_c, stat, stat_c, statat, statat_c, + FileType, Metadata, +}; + +mod dirent; +pub(crate) mod errno; +mod rawpath; +mod stat; + +/// A low-level Unix specific directory entry. +/// +/// This type corresponds as closely as possible to the `dirent` structure +/// found on Unix-like platforms. It exposes the underlying file name, file +/// serial number, and, on platforms that support it, the file type. +/// +/// All methods on this directory entry have zero cost. That is, no allocations +/// or syscalls are performed. +#[derive(Clone)] +pub struct DirEntry { + /// A copy of the file name contents from the raw dirent, represented as a + /// NUL terminated C string. We use a `Vec` here instead of a `CString` + /// because it makes it easier to correctly amortize allocation, and keep + /// track of the correct length of the string without needing to recompute + /// it. + /// + /// Note that this has to be a copy since the lifetime of `d_name` from + /// `struct dirent *` is not guaranteed to last beyond the next call to + /// `readdir`. + file_name: Vec, + /// The file type, as is, from the raw dirent. + file_type: Option, + /// The file serial number, as is, from the raw dirent. + ino: u64, +} + +impl fmt::Debug for DirEntry { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("DirEntry") + .field("file_name", &escaped_bytes(self.file_name_bytes())) + .field("file_type", &self.file_type) + .field("ino", &self.ino) + .finish() + } +} + +impl DirEntry { + /// Read the contents of the given raw Unix/POSIX directory entry into this + /// entry. + #[inline] + fn from_unix_raw(&mut self, raw: &RawDirEntry) { + self.file_type = raw.file_type(); + self.ino = raw.ino(); + + let bytes = raw.file_name().to_bytes_with_nul(); + self.file_name.resize(bytes.len(), 0); + self.file_name.copy_from_slice(bytes); + } + + /// Read the contents of the given raw Unix/POSIX directory entry into this + /// entry. + #[cfg(target_os = "linux")] + #[inline] + pub(crate) fn from_linux_raw(&mut self, raw: &LinuxDirEntry) { + self.file_type = raw.file_type(); + self.ino = raw.ino(); + + let bytes = raw.file_name().to_bytes_with_nul(); + self.file_name.resize(bytes.len(), 0); + self.file_name.copy_from_slice(bytes); + } + + /// Create a new empty directory entry. + /// + /// For an empty directory entry, the file name is empty, the file type is + /// `None` and the inode number is `0`. + /// + /// This is useful for creating space for using `Dir::read_into`. + #[inline] + pub fn empty() -> DirEntry { + DirEntry { file_name: vec![0], file_type: None, ino: 0 } + } + + /// Return the file name in this directory entry as a C string. + #[inline] + pub fn file_name(&self) -> &CStr { + // SAFETY: file_name is always a normal NUL terminated C string. + // We just represent it as a Vec to make amortizing allocation + // easier. + unsafe { CStr::from_bytes_with_nul_unchecked(&self.file_name) } + } + + /// Consume this directory entry and return the underlying C string. + #[inline] + pub fn into_file_name(mut self) -> CString { + // SAFETY: file_name is always a normal NUL terminated C string. + // We just represent it as a Vec to make amortizing allocation + // easier. + unsafe { + // There's no way to build a CString from a Vec with zero overhead. + // Namely, from_vec_unchecked actually adds a NUL byte. Since we + // already have one, pop it. + // + // FIXME: CString really should have a from_vec_with_nul_unchecked + // routine like CStr. + self.file_name.pop().expect("a NUL byte"); + CString::from_vec_unchecked(self.file_name) + } + } + + /// Return the file name in this directory entry as raw bytes without + /// a `NUL` terminator. + #[inline] + pub fn file_name_bytes(&self) -> &[u8] { + &self.file_name[..self.file_name.len() - 1] + } + + /// Returns true if this entry is the `.` or `..` pseudo-entry. + #[inline] + pub fn is_dots(&self) -> bool { + matches!(self.file_name_bytes(), b"." | b"..") + } + + /// Consume this directory entry and return the underlying bytes without + /// a `NUL` terminator. + #[inline] + pub fn into_file_name_bytes(mut self) -> Vec { + self.file_name.pop().expect("a NUL terminator"); + self.file_name + } + + /// Return the file name in this directory entry as an OS string. The + /// string returned does not contain a `NUL` terminator. + #[inline] + pub fn file_name_os(&self) -> &OsStr { + OsStr::from_bytes(self.file_name_bytes()) + } + + /// Consume this directory entry and return its file name as an OS string + /// without a `NUL` terminator. + #[inline] + pub fn into_file_name_os(self) -> OsString { + OsString::from_vec(self.into_file_name_bytes()) + } + + /// Return the file type of this directory entry, if one exists. + /// + /// A file type may not exist if the underlying file system reports an + /// unknown file type in the directory entry, or if the platform does not + /// support reporting the file type in the directory entry at all. + #[inline] + pub fn file_type(&self) -> Option { + self.file_type + } + + /// Returns the underlying file serial number for this directory entry. + #[inline] + pub fn ino(&self) -> u64 { + self.ino + } +} + +/// A file descriptor opened as a directory. +/// +/// The file descriptor is automatically closed when it's dropped. +#[derive(Debug)] +pub struct DirFd(RawFd); + +unsafe impl Send for DirFd {} + +impl Drop for DirFd { + fn drop(&mut self) { + unsafe { + // Explicitly ignore the error here if one occurs. To get an error + // when closing, use DirFd::close. + libc::close(self.0); + } + } +} + +impl AsRawFd for DirFd { + fn as_raw_fd(&self) -> RawFd { + self.0 + } +} + +impl IntoRawFd for DirFd { + fn into_raw_fd(self) -> RawFd { + let fd = self.0; + mem::forget(self); + fd + } +} + +impl FromRawFd for DirFd { + unsafe fn from_raw_fd(fd: RawFd) -> DirFd { + DirFd(fd) + } +} + +impl io::Seek for DirFd { + fn seek(&mut self, pos: io::SeekFrom) -> io::Result { + let mut file = + mem::ManuallyDrop::new(unsafe { File::from_raw_fd(self.0) }); + file.seek(pos) + } +} + +impl DirFd { + /// Open a file descriptor for the given directory path. + /// + /// If there was a problem opening the directory, or if the given path + /// contains a `NUL` byte, then an error is returned. + /// + /// If possible, prefer using `openat` since it is generally faster. + pub fn open>(dir_path: P) -> io::Result { + let bytes = dir_path.into().into_os_string().into_vec(); + DirFd::open_c(&CString::new(bytes)?) + } + + /// Open a file descriptor for the given directory path. + /// + /// This is just like `DirFd::open`, except it accepts a pre-made C string. + /// As such, this only returns an error when opening the directory fails. + pub fn open_c(dir_path: &CStr) -> io::Result { + DirFd::open_c_follow(dir_path, true) + } + + /// Protect the final component with [`O_NOFOLLOW`](`libc::O_NOFOLLOW`) + /// when `follow` is false. + pub fn open_c_follow(dir_path: &CStr, follow: bool) -> io::Result { + let mut flags = libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY; + if !follow { + flags |= libc::O_NOFOLLOW; + } + // SAFETY: This is safe since we've guaranteed that cstr has no + // interior NUL bytes and is terminated by a NUL. + let fd = unsafe { libc::open(dir_path.as_ptr(), flags) }; + if fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(DirFd(fd)) + } + } + + /// Open a file descriptor for the given directory name, where the given + /// file descriptor (`parent_dirfd`) corresponds to the parent directory + /// of the given name. + /// + /// One should prefer using this in lieu of `open` when possible, since it + /// should generally be faster (but does of course require having an open + /// file descriptor to the parent directory). + /// + /// If there was a problem opening the directory, or if the given path + /// contains a `NUL` byte, then an error is returned. + pub fn openat>( + parent_dirfd: RawFd, + dir_name: D, + ) -> io::Result { + DirFd::openat_c( + parent_dirfd, + &CString::new(dir_name.into().into_vec())?, + ) + } + + /// Open a file descriptor for the given directory name, where the given + /// file descriptor (`parent_dirfd`) corresponds to the parent directory + /// of the given name. + /// + /// This is just like `DirFd::openat`, except it accepts a pre-made C + /// string. As such, this only returns an error when opening the directory + /// fails. + pub fn openat_c( + parent_dirfd: RawFd, + dir_name: &CStr, + ) -> io::Result { + DirFd::openat_c_follow(parent_dirfd, dir_name, true) + } + + /// Protect `dir_name` with [`O_NOFOLLOW`](`libc::O_NOFOLLOW`) when `follow` + /// is false. + pub fn openat_c_follow( + parent_dirfd: RawFd, + dir_name: &CStr, + follow: bool, + ) -> io::Result { + let mut flags = libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY; + if !follow { + flags |= libc::O_NOFOLLOW; + } + // SAFETY: This is safe since we've guaranteed that cstr has no + // interior NUL bytes and is terminated by a NUL. + let fd = + unsafe { libc::openat(parent_dirfd, dir_name.as_ptr(), flags) }; + if fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(DirFd(fd)) + } + } + + /// Close this directory file descriptor and return an error if closing + /// failed. + /// + /// Note that this does not need to be called explicitly. This directory + /// file descriptor will be closed automatically when it is dropped (and + /// if an error occurs, it is ignored). This routine is only useful if you + /// want to explicitly close the directory file descriptor and check the + /// error. + pub fn close(self) -> io::Result<()> { + let res = if unsafe { libc::close(self.0) } < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + }; + // Don't drop DirFd after we've explicitly closed the dir stream to + // avoid running close again. + mem::forget(self); + res + } +} + +/// A handle to a directory stream. +/// +/// The handle is automatically closed when it's dropped. +#[derive(Debug)] +pub struct Dir(NonNull); + +// This is sound because `Dir` uniquely owns its `DIR*`. +unsafe impl Send for Dir {} +unsafe impl Sync for Dir {} + +impl Drop for Dir { + fn drop(&mut self) { + unsafe { + // Explicitly ignore the error here if one occurs. To get an error + // when closing, use Dir::close. + libc::closedir(self.0.as_ptr()); + } + } +} + +impl AsRawFd for Dir { + fn as_raw_fd(&self) -> RawFd { + // It's possible for this to return an error according to POSIX, but I + // guess we just ignore it. In particular, it looks like common + // implementations (e.g., Linux) do not actually ever return an error. + unsafe { libc::dirfd(self.0.as_ptr()) } + } +} + +impl FromRawFd for Dir { + unsafe fn from_raw_fd(fd: RawFd) -> Dir { + match NonNull::new(unsafe { libc::fdopendir(fd) }) { + Some(dir) => Dir(dir), + None => panic!( + "failed to create libc::DIR from file descriptor: {}", + io::Error::last_os_error() + ), + } + } +} + +impl Dir { + /// Open a handle to a directory stream for the given directory path. + /// + /// If there was a problem opening the directory stream, or if the given + /// path contains a `NUL` byte, then an error is returned. + /// + /// If possible, prefer using `openat` since it is generally faster. + pub fn open>(dir_path: P) -> io::Result

{ + let bytes = dir_path.into().into_os_string().into_vec(); + Dir::open_c(&CString::new(bytes)?) + } + + /// Open a handle to a directory stream for the given directory path. + /// + /// This is just like `Dir::open`, except it accepts a pre-made C string. + /// As such, this only returns an error when opening the directory stream + /// fails. + pub fn open_c(dir_path: &CStr) -> io::Result { + // SAFETY: This is safe since we've guaranteed that cstr has no + // interior NUL bytes and is terminated by a NUL. + match NonNull::new(unsafe { libc::opendir(dir_path.as_ptr()) }) { + None => Err(io::Error::last_os_error()), + Some(dir) => Ok(Dir(dir)), + } + } + + /// Convert an open directory fd into a directory stream. + fn from_dirfd(dirfd: DirFd) -> io::Result { + // SAFETY: dirfd is valid. Success transfers fd ownership to DIR. + match NonNull::new(unsafe { libc::fdopendir(dirfd.as_raw_fd()) }) { + None => Err(io::Error::last_os_error()), + Some(dir) => { + let _ = dirfd.into_raw_fd(); + Ok(Dir(dir)) + } + } + } + + /// Open by full path with the requested follow mode. + pub fn open_c_follow(dir_path: &CStr, follow: bool) -> io::Result { + Dir::from_dirfd(DirFd::open_c_follow(dir_path, follow)?) + } + + /// Open a handle to a directory stream for the given directory name, where + /// the file descriptor corresponds to the parent directory of the given + /// name. + /// + /// One should prefer using this in lieu of `open` when possible, since it + /// should generally be faster (but does of course require having an open + /// file descriptor to the parent directory). + /// + /// If there was a problem opening the directory stream, or if the given + /// path contains a `NUL` byte, then an error is returned. + pub fn openat>( + parent_dirfd: RawFd, + dir_name: D, + ) -> io::Result { + Dir::openat_c(parent_dirfd, &CString::new(dir_name.into().into_vec())?) + } + + /// Open a handle to a directory stream for the given directory name, where + /// the file descriptor corresponds to the parent directory of the given + /// name. + /// + /// This is just like `Dir::openat`, except it accepts a pre-made C string + /// for the directory name. As such, this only returns an error when + /// opening the directory stream fails. + pub fn openat_c(parent_dirfd: RawFd, dir_name: &CStr) -> io::Result { + Dir::from_dirfd(DirFd::openat_c(parent_dirfd, dir_name)?) + } + + /// Open relative to `parent_dirfd` with the requested follow mode. + pub fn openat_c_follow( + parent_dirfd: RawFd, + dir_name: &CStr, + follow: bool, + ) -> io::Result { + Dir::from_dirfd(DirFd::openat_c_follow( + parent_dirfd, + dir_name, + follow, + )?) + } + + /// Read the next directory entry from this stream. + /// + /// This returns `None` when no more directory entries could be read. + /// + /// If there was a problem reading the next directory entry, then an error + /// is returned. When an error occurs, callers can still continue to read + /// subsequent directory entries. + /// + /// Note that no filtering of entries (such as `.` and `..`) is performed. + pub fn read(&mut self) -> Option> { + let mut ent = DirEntry::empty(); + match self.read_into(&mut ent) { + Ok(true) => Some(Ok(ent)), + Ok(false) => None, + Err(err) => Some(Err(err)), + } + } + + /// Read the next directory entry from this stream into the given space. + /// + /// This returns false when no more directory entries could be read. + /// + /// If there was a problem reading the next directory entry, then an error + /// is returned. When an error occurs, callers can still continue to read + /// subsequent directory entries. + /// + /// The contents of `ent` when the end of the stream has been reached or + /// when an error occurs are unspecified. + /// + /// Note that no filtering of entries (such as `.` and `..`) is performed. + pub fn read_into(&mut self, ent: &mut DirEntry) -> io::Result { + // We need to clear the errno because it's the only way to + // differentiate errors and end-of-stream. (Since both return a NULL + // dirent.) + // + // TODO: It might be worth experimenting with readdir_r, but note that + // it is deprecated on Linux, and is presumably going to be deprecated + // in POSIX. The idea is that readdir is supposed to be reentrant these + // days. readdir_r does have some of its own interesting problems + // associated with it. See readdir_r(3) on Linux. + errno::clear(); + match RawDirEntry::new(unsafe { readdir(self.0.as_ptr()) }) { + Some(rawent) => { + ent.from_unix_raw(&rawent); + Ok(true) + } + None => { + if errno::errno() != 0 { + Err(io::Error::last_os_error()) + } else { + Ok(false) + } + } + } + } + + /// Rewind this directory stream such that it restarts back at the + /// beginning of the directory. + pub fn rewind(&mut self) { + unsafe { + libc::rewinddir(self.0.as_ptr()); + } + } + + /// Close this directory stream and return an error if closing failed. + /// + /// Note that this does not need to be called explicitly. This directory + /// stream will be closed automatically when it is dropped (and if an error + /// occurs, it is ignored). This routine is only useful if you want to + /// explicitly close the directory stream and check the error. + pub fn close(self) -> io::Result<()> { + let res = if unsafe { libc::closedir(self.0.as_ptr()) } < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + }; + // Don't drop Dir after we've explicitly closed the dir stream to + // avoid running close again. + mem::forget(self); + res + } +} + +/// Return a convenience ASCII-only debug representation of the given bytes. +/// In essence, non-ASCII and non-printable bytes are escaped. +pub(crate) fn escaped_bytes(bytes: &[u8]) -> String { + use std::ascii::escape_default; + + bytes.iter().cloned().flat_map(escape_default).map(|b| b as char).collect() +} diff --git a/src/os/unix/rawpath.rs b/src/os/unix/rawpath.rs new file mode 100644 index 0000000..12fea3d --- /dev/null +++ b/src/os/unix/rawpath.rs @@ -0,0 +1,340 @@ +use std::ffi::{CStr, CString, OsStr, OsString}; +use std::fmt; +use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::path::{Path, PathBuf}; + +// Currently, these types are not exported in the public API of this crate, +// even though they (or something like them) are seemingly necessary to +// implement recursive directory traversal without superfluous allocations. +// Figuring out how to expose them is tricky, since invariably, they _aren't_ +// the same type with the same API. So they wind up being a hazard if one +// accidentally tries to treat them as a platform independent type. + +/// A platform dependent representation of a file path. +/// +/// Unlike Rust's standard library `PathBuf`, a `RawPathBuf` uses the same +/// in-memory representation of a file path as the platform itself. Moreover, +/// the APIs of each `RawPathBuf` are also platform dependent. For example, +/// on Unix, a `RawPathBuf` can be cheaply converted between types such as +/// `Vec` and `CString`. But on Windows, since its internal representation +/// is a sequence of 16-bit integers, these conversions are not available. +#[derive(Clone)] +pub struct RawPathBuf { + /// Buf always has length at least 1 and always ends with a zero byte. + /// Buf only ever contains exactly 1 zero byte. (i.e., no interior NULs.) + buf: Vec, +} + +impl fmt::Debug for RawPathBuf { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use crate::os::unix::escaped_bytes; + + f.debug_struct("RawPathBuf") + .field("buf", &escaped_bytes(self.as_code_units())) + .finish() + } +} + +impl<'a> From<&'a str> for RawPathBuf { + fn from(s: &'a str) -> RawPathBuf { + RawPathBuf::from(s.to_string()) + } +} + +impl From for RawPathBuf { + fn from(s: String) -> RawPathBuf { + let mut buf = s.into_bytes(); + buf.push(0); + RawPathBuf { buf } + } +} + +impl From for RawPathBuf { + fn from(cstr: CString) -> RawPathBuf { + RawPathBuf { buf: cstr.into_bytes_with_nul() } + } +} + +impl From for CString { + fn from(rawp: RawPathBuf) -> CString { + // SAFETY: Our internal buffer is guaranteed to end with a NUL and have + // no interior NULs. + unsafe { CString::from_vec_unchecked(rawp.buf) } + } +} + +impl From for RawPathBuf { + fn from(osstr: OsString) -> RawPathBuf { + let mut buf = osstr.into_vec(); + buf.push(0); + RawPathBuf { buf } + } +} + +impl From for OsString { + fn from(mut rawp: RawPathBuf) -> OsString { + // SAFETY: We are dropping this raw path and converting it into an + // OS string, which has no NUL terminator. + unsafe { + rawp.drop_nul(); + } + OsString::from_vec(rawp.buf) + } +} + +impl From for RawPathBuf { + fn from(path: PathBuf) -> RawPathBuf { + RawPathBuf::from(path.into_os_string()) + } +} + +impl From for PathBuf { + fn from(rawp: RawPathBuf) -> PathBuf { + PathBuf::from(OsString::from(rawp)) + } +} + +impl RawPathBuf { + /// Returns the code units (bytes) of this path without the NUL terminator. + pub fn as_code_units(&self) -> &[u8] { + &self.buf[..self.buf.len() - 1] + } + + /// Returns this raw path as a C string slice. + pub fn as_cstr(&self) -> &CStr { + // SAFETY: buf is guaranteed to have a NUL terminator with no interior + // NULs. + unsafe { CStr::from_bytes_with_nul_unchecked(&self.buf) } + } + + /// Returns this raw path as a OS string slice. + pub fn as_os_str(&self) -> &OsStr { + OsStr::from_bytes(self.as_code_units()) + } + + /// Return this raw path as a standard library path. + pub fn as_path(&self) -> &Path { + Path::new(self.as_os_str()) + } + + /// Push the given C string slice to the end of this path. + pub fn push_cstr(&mut self, slice: &CStr) { + // SAFETY: The internal buffer is guaranteed to have a NUL byte at + // this point, and we always add it back below via the CStr's NUL + // byte. + unsafe { + self.drop_nul(); + } + self.buf.extend_from_slice(slice.to_bytes_with_nul()); + } + + /// Join the given C string slice to this path in place via a path + /// separator. + /// + /// If this path ends with a `/`, and/or if name starts with a `/`, then + /// only one separator will be used to join them. This otherwise does no + /// other normalization. e.g., joining `a/b//` with `/c` will result in + /// `a/b//c`. + pub fn join(&mut self, name: &CStr) { + // SAFETY: The internal buffer is guaranteed to have a NUL byte at + // this point, and we always add it back below via the CStr's NUL + // byte. + unsafe { + self.drop_nul(); + } + if self.buf.last() != Some(&b'/') { + self.buf.push(b'/'); + } + if name.to_bytes().first() == Some(&b'/') { + debug_assert_eq!(self.buf.last(), Some(&b'/')); + self.buf.pop(); + } + self.buf.extend_from_slice(name.to_bytes_with_nul()); + } + + /// Pop the last element in this path. Return true if an element was + /// popped. An element isn't popped if the path is empty or represents + /// a root path. + pub fn pop(&mut self) -> bool { + // Move backwards through the path, finding the first location that + // ends the parent element, if one exists. Basically, we want to + // implement the following regex: + // + // ^.*?(/*[^/]+/*)$ + // + // Where everything in the capturing group is deleted. + + // First, start by skipping through all repeated separators in reverse. + let mut new_len = self.buf.len() - 1; + while new_len > 0 && self.buf[new_len - 1] == b'/' { + new_len -= 1; + } + // The path is either empty, or just made up of separators. + if new_len == 0 { + return false; + } + // Now find either the first preceding / or the beginning. + while new_len > 0 && self.buf[new_len - 1] != b'/' { + new_len -= 1; + } + // And now finally, remove all trailing separators. + // But we're careful not to remove a root slash if it's present. + while new_len > 1 && self.buf[new_len - 1] == b'/' { + new_len -= 1; + } + self.buf[new_len] = 0; + + // SAFETY: This is safe because our buffer contains Copy data and + // `new_len + 1` is guaranteed to be <= the original length of the + // buffer. Therefore, we do not need to worry about unitialized data. + unsafe { + self.buf.set_len(new_len + 1); + } + true + } + + /// Drop the trailing NUL byte from the internal buffer in place. + /// + /// # Safety + /// + /// This is unsafe to call because it removes the NUL byte from the buffer, + /// which is necessary for safety in many contexts. + /// + /// When callers use this method, they MUST ensure that a NUL byte is + /// added back to the internal buffer before its absence can be observed + /// by callers. + /// + /// Callers must also never call this method if the NUL byte has already + /// been removed. + unsafe fn drop_nul(&mut self) { + // SAFETY: This is safe since the new length is always <= than the + // old length, and thus there are no initialization worries. Moreover, + // since the buffer stores Copy data, there are no leaks. + debug_assert_eq!(*self.buf.last().unwrap(), 0); + self.buf.set_len(self.buf.len() - 1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + fn tostr(p: &RawPathBuf) -> &str { + std::str::from_utf8(p.as_code_units()).unwrap() + } + + fn cstr(s: &str) -> &CStr { + CStr::from_bytes_with_nul(s.as_bytes()).unwrap() + } + + #[test] + fn push1() { + let mut p = RawPathBuf::from("a/b"); + p.join(cstr("c\0")); + assert_eq!("a/b/c", tostr(&p)); + } + + #[test] + fn push2() { + let mut p = RawPathBuf::from("a/b/"); + p.join(cstr("c\0")); + assert_eq!("a/b/c", tostr(&p)); + } + + #[test] + fn push3() { + let mut p = RawPathBuf::from("a/b"); + p.join(cstr("/c\0")); + assert_eq!("a/b/c", tostr(&p)); + } + + #[test] + fn push4() { + let mut p = RawPathBuf::from("a/b/"); + p.join(cstr("/c\0")); + assert_eq!("a/b/c", tostr(&p)); + } + + #[test] + fn push5() { + let mut p = RawPathBuf::from("a/b//"); + p.join(cstr("/c\0")); + assert_eq!("a/b//c", tostr(&p)); + } + + #[test] + fn pop1() { + let mut p = RawPathBuf::from("/foo/bar////baz/"); + + assert!(p.pop()); + assert_eq!("/foo/bar", tostr(&p)); + + assert!(p.pop()); + assert_eq!("/foo", tostr(&p)); + + assert!(p.pop()); + assert_eq!("/", tostr(&p)); + + assert!(!p.pop()); + assert_eq!("/", tostr(&p)); + } + + #[test] + fn pop2() { + let mut p = RawPathBuf::from("////foo/"); + + assert!(p.pop()); + assert_eq!("/", tostr(&p)); + + assert!(!p.pop()); + assert_eq!("/", tostr(&p)); + } + + #[test] + fn pop3() { + let mut p = RawPathBuf::from("foo/bar/baz"); + + assert!(p.pop()); + assert_eq!("foo/bar", tostr(&p)); + + assert!(p.pop()); + assert_eq!("foo", tostr(&p)); + + assert!(p.pop()); + assert_eq!("", tostr(&p)); + + assert!(!p.pop()); + assert_eq!("", tostr(&p)); + } + + #[test] + fn pop4() { + let mut p = RawPathBuf::from("////"); + + assert!(!p.pop()); + assert_eq!("////", tostr(&p)); + } + + #[test] + fn pop5() { + let mut p = RawPathBuf::from("////a"); + + assert!(p.pop()); + assert_eq!("/", tostr(&p)); + + assert!(!p.pop()); + assert_eq!("/", tostr(&p)); + } + + #[test] + fn pop6() { + let mut p = RawPathBuf::from("foo"); + + assert!(p.pop()); + assert_eq!("", tostr(&p)); + + assert!(!p.pop()); + assert_eq!("", tostr(&p)); + } +} diff --git a/src/os/unix/stat.rs b/src/os/unix/stat.rs new file mode 100644 index 0000000..5cb9fa9 --- /dev/null +++ b/src/os/unix/stat.rs @@ -0,0 +1,329 @@ +use std::ffi::{CStr, CString, OsString}; +use std::fmt; +use std::io; +use std::mem; +use std::os::unix::ffi::OsStringExt; +use std::os::unix::io::RawFd; +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use libc; + +#[cfg(not(any(target_os = "linux", target_os = "android",)))] +use libc::{fstatat as fstatat64, lstat as lstat64, stat as stat64}; +#[cfg(any(target_os = "linux", target_os = "android",))] +use libc::{fstatat64, lstat64, stat64}; + +/// Metadata for a file as reported by stat. +pub struct Metadata { + stat: stat64, +} + +impl Metadata { + /// The file type. + pub fn file_type(&self) -> FileType { + FileType::from_stat_mode(self.stat.st_mode as u64) + } + + /// The file size in bytes. + pub fn len(&self) -> u64 { + self.stat.st_size as u64 + } + + /// The device the file resides on. + pub fn dev(&self) -> u64 { + self.stat.st_dev as u64 + } + + /// The file serial number. + pub fn ino(&self) -> u64 { + self.stat.st_ino as u64 + } + + /// The raw st_mode, including permission bits. + pub fn mode(&self) -> u64 { + self.stat.st_mode as u64 + } +} + +#[cfg(target_os = "netbsd")] +impl Metadata { + /// The last modification time. + pub fn modified(&self) -> io::Result { + let dur = Duration::new( + self.stat.st_mtime as u64, + self.stat.st_mtimensec as u32, + ); + Ok(SystemTime::UNIX_EPOCH + dur) + } + + /// The last access time. + pub fn accessed(&self) -> io::Result { + let dur = Duration::new( + self.stat.st_atime as u64, + self.stat.st_atimensec as u32, + ); + Ok(SystemTime::UNIX_EPOCH + dur) + } + + /// The creation time, when available. + pub fn created(&self) -> io::Result { + let dur = Duration::new( + self.stat.st_birthtime as u64, + self.stat.st_birthtimensec as u32, + ); + Ok(SystemTime::UNIX_EPOCH + dur) + } +} + +#[cfg(not(target_os = "netbsd"))] +impl Metadata { + /// The last modification time. + pub fn modified(&self) -> io::Result { + let dur = Duration::new( + self.stat.st_mtime as u64, + self.stat.st_mtime_nsec as u32, + ); + Ok(SystemTime::UNIX_EPOCH + dur) + } + + /// The last access time. + pub fn accessed(&self) -> io::Result { + let dur = Duration::new( + self.stat.st_atime as u64, + self.stat.st_atime_nsec as u32, + ); + Ok(SystemTime::UNIX_EPOCH + dur) + } + + #[cfg(any( + target_os = "freebsd", + target_os = "openbsd", + target_os = "macos", + target_os = "ios" + ))] + /// The creation time, when available. + pub fn created(&self) -> io::Result { + let dur = Duration::new( + self.stat.st_birthtime as u64, + self.stat.st_birthtime_nsec as u32, + ); + Ok(SystemTime::UNIX_EPOCH + dur) + } + + #[cfg(not(any( + target_os = "freebsd", + target_os = "openbsd", + target_os = "macos", + target_os = "ios" + )))] + /// The creation time, when available. + pub fn created(&self) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Other, + "creation time is not available on this platform currently", + )) + } +} + +/// One of seven possible file types on Unix. +#[derive(Clone, Copy)] +pub struct FileType(libc::mode_t); + +/// Compare only the [`S_IFMT`](`libc::S_IFMT`) bits since stat modes +/// also carry permission bits. +impl PartialEq for FileType { + fn eq(&self, other: &FileType) -> bool { + self.masked() == other.masked() + } +} + +impl Eq for FileType {} + +impl std::hash::Hash for FileType { + fn hash(&self, state: &mut H) { + self.masked().hash(state); + } +} + +impl fmt::Debug for FileType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let human = if self.is_file() { + "File" + } else if self.is_dir() { + "Directory" + } else if self.is_symlink() { + "Symbolic Link" + } else if self.is_block_device() { + "Block Device" + } else if self.is_char_device() { + "Char Device" + } else if self.is_fifo() { + "FIFO" + } else if self.is_socket() { + "Socket" + } else { + "Unknown" + }; + write!(f, "FileType({human})") + } +} + +impl FileType { + fn masked(&self) -> libc::mode_t { + self.0 & libc::S_IFMT + } + + /// Create a new file type from a directory entry's type field. + /// + /// If the given type is not recognized or is `DT_UNKNOWN`, then `None` + /// is returned. + pub fn from_dirent_type(d_type: u8) -> Option { + Some(FileType(match d_type { + libc::DT_REG => libc::S_IFREG, + libc::DT_DIR => libc::S_IFDIR, + libc::DT_LNK => libc::S_IFLNK, + libc::DT_BLK => libc::S_IFBLK, + libc::DT_CHR => libc::S_IFCHR, + libc::DT_FIFO => libc::S_IFIFO, + libc::DT_SOCK => libc::S_IFSOCK, + libc::DT_UNKNOWN => return None, + _ => return None, // wat? + })) + } + + /// Create a new file type from a stat's `st_mode` field. + pub fn from_stat_mode(st_mode: u64) -> FileType { + FileType(st_mode as libc::mode_t) + } + + /// Returns true if this file type is a regular file. + /// + /// This corresponds to the `S_IFREG` value on Unix. + pub fn is_file(&self) -> bool { + self.0 & libc::S_IFMT == libc::S_IFREG + } + + /// Returns true if this file type is a directory. + /// + /// This corresponds to the `S_IFDIR` value on Unix. + pub fn is_dir(&self) -> bool { + self.0 & libc::S_IFMT == libc::S_IFDIR + } + + /// Returns true if this file type is a symbolic link. + /// + /// This corresponds to the `S_IFLNK` value on Unix. + pub fn is_symlink(&self) -> bool { + self.0 & libc::S_IFMT == libc::S_IFLNK + } + + /// Returns true if this file type is a block device. + /// + /// This corresponds to the `S_IFBLK` value on Unix. + pub fn is_block_device(&self) -> bool { + self.0 & libc::S_IFMT == libc::S_IFBLK + } + + /// Returns true if this file type is a character device. + /// + /// This corresponds to the `S_IFCHR` value on Unix. + pub fn is_char_device(&self) -> bool { + self.0 & libc::S_IFMT == libc::S_IFCHR + } + + /// Returns true if this file type is a FIFO. + /// + /// This corresponds to the `S_IFIFO` value on Unix. + pub fn is_fifo(&self) -> bool { + self.0 & libc::S_IFMT == libc::S_IFIFO + } + + /// Returns true if this file type is a socket. + /// + /// This corresponds to the `S_IFSOCK` value on Unix. + pub fn is_socket(&self) -> bool { + self.0 & libc::S_IFMT == libc::S_IFSOCK + } +} + +/// Run stat on the given path and return the metadata. +pub fn stat>(path: P) -> io::Result { + let bytes = path.into().into_os_string().into_vec(); + stat_c(&CString::new(bytes)?) +} + +/// Like stat but accepts a pre-made C string. +pub fn stat_c(path: &CStr) -> io::Result { + let mut stat: stat64 = unsafe { mem::zeroed() }; + let res = unsafe { stat64(path.as_ptr(), &mut stat) }; + if res < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(Metadata { stat }) + } +} + +/// Like stat but does not follow a trailing symlink. +pub fn lstat>(path: P) -> io::Result { + let bytes = path.into().into_os_string().into_vec(); + lstat_c(&CString::new(bytes)?) +} + +/// Like lstat but accepts a pre-made C string. +pub fn lstat_c(path: &CStr) -> io::Result { + let mut stat: stat64 = unsafe { mem::zeroed() }; + let res = unsafe { lstat64(path.as_ptr(), &mut stat) }; + if res < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(Metadata { stat }) + } +} + +/// Run stat on `name` relative to the open directory `parent_dirfd`. +pub fn statat>( + parent_dirfd: RawFd, + name: N, +) -> io::Result { + let bytes = name.into().into_vec(); + statat_c(parent_dirfd, &CString::new(bytes)?) +} + +/// Like statat but accepts a pre-made C string. +pub fn statat_c(parent_dirfd: RawFd, name: &CStr) -> io::Result { + let mut stat: stat64 = unsafe { mem::zeroed() }; + let res = unsafe { fstatat64(parent_dirfd, name.as_ptr(), &mut stat, 0) }; + if res < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(Metadata { stat }) + } +} + +/// Like statat but does not follow a trailing symlink. +pub fn lstatat>( + parent_dirfd: RawFd, + name: N, +) -> io::Result { + let bytes = name.into().into_vec(); + lstatat_c(parent_dirfd, &CString::new(bytes)?) +} + +/// Like lstatat but accepts a pre-made C string. +pub fn lstatat_c(parent_dirfd: RawFd, name: &CStr) -> io::Result { + let mut stat: stat64 = unsafe { mem::zeroed() }; + let res = unsafe { + fstatat64( + parent_dirfd, + name.as_ptr(), + &mut stat, + libc::AT_SYMLINK_NOFOLLOW, + ) + }; + if res < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(Metadata { stat }) + } +} diff --git a/src/os/windows/mod.rs b/src/os/windows/mod.rs new file mode 100644 index 0000000..2511ae1 --- /dev/null +++ b/src/os/windows/mod.rs @@ -0,0 +1,662 @@ +/*! +Low level Windows specific APIs for reading directory entries via +`NtOpenFile` relative opens and `GetFileInformationByHandleEx`. +*/ + +use std::char; +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::fs::OpenOptions; +use std::io; +use std::mem; +use std::os::windows::ffi::OsStringExt; +use std::os::windows::fs::OpenOptionsExt; +use std::os::windows::io::{ + AsRawHandle, FromRawHandle, OwnedHandle, RawHandle, +}; +use std::path::Path; +use std::ptr; +use std::time::{self, SystemTime}; + +use windows_sys::Wdk::Foundation::OBJECT_ATTRIBUTES; +use windows_sys::Wdk::Storage::FileSystem::{ + NtOpenFile, FILE_DIRECTORY_FILE, FILE_OPEN_REPARSE_POINT, + FILE_SYNCHRONOUS_IO_NONALERT, +}; +use windows_sys::Win32::Foundation::{ + RtlNtStatusToDosError, ERROR_DIRECTORY, ERROR_NO_MORE_FILES, HANDLE, + NTSTATUS, STATUS_DELETE_PENDING, STATUS_PENDING, UNICODE_STRING, +}; +use windows_sys::Win32::Storage::FileSystem::{ + FileIdBothDirectoryInfo, FileIdBothDirectoryRestartInfo, + GetFileInformationByHandle, GetFileInformationByHandleEx, + BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, + FILE_ATTRIBUTE_HIDDEN, FILE_ATTRIBUTE_REPARSE_POINT, + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_ID_BOTH_DIR_INFO, FILE_LIST_DIRECTORY, FILE_SHARE_DELETE, + FILE_SHARE_READ, FILE_SHARE_WRITE, SYNCHRONIZE, +}; +use windows_sys::Win32::System::IO::{IO_STATUS_BLOCK, IO_STATUS_BLOCK_0}; + +pub use crate::os::windows::stat::{lstat, stat, FileType, Metadata}; + +mod stat; + +/// The Win32 error code mapped from [`STATUS_DELETE_PENDING`], so a pending +/// delete is not misreported as an access-denied error. +const ERROR_DELETE_PENDING: u32 = 303; + +/// A heap buffer aligned to 8 bytes, as [`FILE_ID_BOTH_DIR_INFO`] requires. +#[repr(C, align(8))] +struct Align8(T); + +/// The enumeration buffer size. Larger than std's 1024 to cut syscalls. +const BUF_LEN: usize = 4096; + +/// A low-level Windows specific directory entry. +/// +/// This type corresponds as closely as possible to the [`FILE_ID_BOTH_DIR_INFO`] +/// structure reported by directory enumeration on Windows platforms. It +/// exposes the underlying file name, raw file attributes, time information and +/// file size. Notably, this is quite a bit more information than Unix APIs, +/// which typically only expose the file name, file serial number, and in most +/// cases, the file type. +/// +/// All methods on this directory entry have zero cost. That is, no allocations +/// or syscalls are performed. +#[derive(Clone)] +pub struct DirEntry { + /// The file name converted to an OsString (using WTF-8 internally). + file_name: OsString, + /// The raw 16-bit code units that make up a file name in Windows. This + /// does not include a NUL terminator. + file_name_u16: Vec, + attr: u32, + reparse_tag: u32, + creation_time: u64, + last_access_time: u64, + last_write_time: u64, + file_size: u64, + file_type: FileType, +} + +impl fmt::Debug for DirEntry { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("DirEntry") + .field("file_name", &escaped_u16s(&self.file_name_u16)) + .field("attr", &self.attr) + .field("reparse_tag", &self.reparse_tag) + .field("file_type", &self.file_type) + .finish() + } +} + +impl DirEntry { + /// Create a new empty directory entry. + /// + /// For an empty directory entry, the file name is empty, the file + /// type returns `true` for `is_file` and `false` for all other public + /// predicates, and the rest of the public API methods on a [`DirEntry`] + /// return `0`. + /// + /// This is useful for creating space for using [`Dir::read_into`]. + #[inline] + pub fn empty() -> DirEntry { + DirEntry { + file_name: OsString::new(), + file_name_u16: vec![], + attr: 0, + reparse_tag: 0, + creation_time: 0, + last_access_time: 0, + last_write_time: 0, + file_size: 0, + file_type: FileType::from_attr(0, 0), + } + } + + /// Return the raw file attributes reported in this directory entry. + /// + /// The value returned directly corresponds to the `FileAttributes` member + /// of the [`FILE_ID_BOTH_DIR_INFO`] structure. + #[inline] + pub fn file_attributes(&self) -> u32 { + self.attr + } + + /// Returns true if this file is marked as hidden via the + /// [`FILE_ATTRIBUTE_HIDDEN`] marker. + #[inline] + pub fn is_hidden(&self) -> bool { + self.file_attributes() & FILE_ATTRIBUTE_HIDDEN != 0 + } + + /// Return the creation time of the underlying file as a system time. + /// + /// If the underlying file system does not support creation time, then an + /// error is returned. + #[inline] + pub fn created(&self) -> io::Result { + if self.creation_time == 0 { + Err(io::Error::new( + io::ErrorKind::Other, + "creation time is not available on this platform currently", + )) + } else { + Ok(intervals_to_system_time(self.creation_time)) + } + } + + /// Return last access time of the underlying file as a system time. + /// + /// If the underlying file system does not support creation time, then an + /// error is returned. + #[inline] + pub fn accessed(&self) -> io::Result { + if self.last_access_time == 0 { + Err(io::Error::new( + io::ErrorKind::Other, + "last access time is not available on this platform currently", + )) + } else { + Ok(intervals_to_system_time(self.last_access_time)) + } + } + + /// Return the last modified time of the underlying file as a system time. + /// + /// If the underlying file system does not support creation time, then an + /// error is returned. + #[inline] + pub fn modified(&self) -> io::Result { + if self.last_write_time == 0 { + Err(io::Error::new( + io::ErrorKind::Other, + "last write time is not available on this platform currently", + )) + } else { + Ok(intervals_to_system_time(self.last_write_time)) + } + } + + /// Return the file size, in bytes, of the corresponding file. + /// + /// This value has no meaning if this entry corresponds to a directory. + #[inline] + pub fn len(&self) -> u64 { + self.file_size + } + + /// Return the file type of this directory entry. + #[inline] + pub fn file_type(&self) -> FileType { + self.file_type + } + + /// Return the file name in this directory entry as an OS string. + #[inline] + pub fn file_name_os(&self) -> &OsStr { + &self.file_name + } + + /// Returns true if this entry is the `.` or `..` pseudo-entry. + #[inline] + pub fn is_dots(&self) -> bool { + matches!(self.file_name_u16.as_slice(), [0x2E] | [0x2E, 0x2E]) + } + + /// Return the file name in this directory entry in its original form as + /// a sequence of 16-bit code units. + /// + /// The sequence returned is not guaranteed to be valid UTF-16. + #[inline] + pub fn file_name_u16(&self) -> &[u16] { + &self.file_name_u16 + } + + /// Consume this directory entry and return its file name as an OS string. + #[inline] + pub fn into_file_name_os(self) -> OsString { + self.file_name + } + + /// Consume this directory entry and return its file name in its original + /// form as a sequence of 16-bit code units. + /// + /// The sequence returned is not guaranteed to be valid UTF-16. + #[inline] + pub fn into_file_name_u16(self) -> Vec { + self.file_name_u16 + } +} + +/// A handle to a directory opened for enumeration. +/// +/// The handle is automatically closed when it's dropped. +pub struct Dir { + handle: OwnedHandle, + buf: Box>, + /// The byte offset of the next record in `buf`, or [`None`] when the buffer is + /// exhausted and a fresh enumeration call is needed. + cursor: Option, + /// True until the first enumeration call, which must use the restart class. + restart: bool, +} + +impl fmt::Debug for Dir { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Dir").field("handle", &self.handle).finish() + } +} + +impl AsRawHandle for Dir { + fn as_raw_handle(&self) -> RawHandle { + self.handle.as_raw_handle() + } +} + +impl Dir { + fn from_handle(handle: OwnedHandle) -> Dir { + Dir { + handle, + buf: Box::new(Align8([0u8; BUF_LEN])), + cursor: None, + restart: true, + } + } + + /// Open a directory for enumeration, following a trailing symlink. + pub fn open>(dir_path: P) -> io::Result { + Dir::open_path_follow(dir_path.as_ref(), true) + } + + /// Open `path` for enumeration with the requested follow mode. + /// + /// This goes through [`OpenOptions`], which prepends the `\\?\` long path + /// prefix as needed, so it is robust to paths longer than `MAX_PATH`. + pub fn open_path_follow(path: &Path, follow: bool) -> io::Result { + let mut flags = FILE_FLAG_BACKUP_SEMANTICS; + if !follow { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + let file = OpenOptions::new() + .access_mode(FILE_LIST_DIRECTORY) + .custom_flags(flags) + .open(path)?; + + // Opening with FILE_LIST_DIRECTORY succeeds even on a plain file, so + // the type has to be verified before enumerating. + // SAFETY: file is a valid open handle and info is filled on success. + let attr = unsafe { + let mut info = mem::zeroed::(); + let res = GetFileInformationByHandle( + file.as_raw_handle() as HANDLE, + &mut info, + ); + if res == 0 { + return Err(io::Error::last_os_error()); + } + info.dwFileAttributes + }; + if attr & FILE_ATTRIBUTE_DIRECTORY == 0 { + return Err(io::Error::from_raw_os_error(ERROR_DIRECTORY as i32)); + } + Ok(Dir::from_handle(OwnedHandle::from(file))) + } + + /// Open a directory handle for the given directory name, where the given + /// handle (`parent`) corresponds to the parent directory of the given name. + /// + /// Since Win32 has no `openat`, the child is opened relative to the parent + /// handle with `NtOpenFile`. When `follow` is false it is opened with + /// [`FILE_OPEN_REPARSE_POINT`], which is the Windows equivalent of `O_NOFOLLOW`, + /// so that if the name was swapped for a reparse point it is opened as the + /// link itself rather than being followed. + /// + /// `name` must be NUL terminated, and the trailing NUL is load-bearing, + /// since per rustc#143078 some Windows builds read one `u16` past the name + /// and the buffer is not backed by the padded enumeration buffer. An error + /// is returned if the directory could not be opened, or if `name` contains + /// an interior `NUL` code unit. + pub fn openat_follow( + parent: RawHandle, + name: &[u16], + follow: bool, + ) -> io::Result { + if name.last() != Some(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "relative names must be NUL terminated", + )); + } + if name[..name.len() - 1].contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "file names on Windows cannot contain NUL code units", + )); + } + let (Some(length), Some(maximum_length)) = + ((name.len() - 1).checked_mul(2), name.len().checked_mul(2)) + else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "relative name is too long", + )); + }; + if length > u16::MAX as usize || maximum_length > u16::MAX as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "relative name is too long", + )); + } + + let name = UNICODE_STRING { + Length: length as u16, + MaximumLength: maximum_length as u16, + Buffer: name.as_ptr().cast_mut(), + }; + let object = OBJECT_ATTRIBUTES { + Length: mem::size_of::() as u32, + RootDirectory: parent as HANDLE, + ObjectName: &name, + Attributes: 0, + SecurityDescriptor: ptr::null(), + SecurityQualityOfService: ptr::null(), + }; + let access = SYNCHRONIZE | FILE_LIST_DIRECTORY; + let share = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + let mut options = FILE_SYNCHRONOUS_IO_NONALERT | FILE_DIRECTORY_FILE; + if !follow { + options |= FILE_OPEN_REPARSE_POINT; + } + + let mut handle: HANDLE = ptr::null_mut(); + let mut io_status = IO_STATUS_BLOCK { + Anonymous: IO_STATUS_BLOCK_0 { Status: STATUS_PENDING }, + Information: 0, + }; + // SAFETY: object and io_status are valid pointers that live across the + // call, and object borrows name, whose buffer is the caller's NUL + // terminated slice and so stays valid for the duration of the call. + let status = unsafe { + NtOpenFile( + &mut handle, + access, + &object, + &mut io_status, + share, + options, + ) + }; + if nt_success(status) { + // SAFETY: NtOpenFile succeeded, so handle owns a fresh handle. + let owned = unsafe { OwnedHandle::from_raw_handle(handle as _) }; + Ok(Dir::from_handle(owned)) + } else { + Err(io::Error::from_raw_os_error(nt_error(status) as i32)) + } + } + + /// Read the next directory entry from this handle. + /// + /// This returns `None` when no more directory entries could be read. + /// + /// Note that no filtering of entries (such as `.` and `..`) is performed. + pub fn read(&mut self) -> Option> { + let mut ent = DirEntry::empty(); + match self.read_into(&mut ent) { + Ok(true) => Some(Ok(ent)), + Ok(false) => None, + Err(err) => Some(Err(err)), + } + } + + /// Read the next directory entry from this handle into the given space. + /// + /// This returns false when no more directory entries could be read. + /// + /// If there was a problem reading the next directory entry, then an error + /// is returned. When an error occurs, callers can still continue to read + /// subsequent directory entries. + /// + /// The contents of `ent` when the end of the stream has been reached or + /// when an error occurs are unspecified. + /// + /// Note that no filtering of entries (such as `.` and `..`) is performed. + pub fn read_into(&mut self, ent: &mut DirEntry) -> io::Result { + let off = match self.cursor { + Some(off) => off, + None => { + if !self.fill()? { + return Ok(false); + } + 0 + } + }; + + // Drivers are not trusted here. Real filesystems return misaligned + // records (rust#104530) and a buggy or hostile one could return + // offsets past the buffer, so every field is read with read_unaligned, + // no reference is ever formed into the buffer and all offsets are + // bounds checked before any pointer arithmetic. + match off.checked_add(mem::size_of::()) { + Some(end) if end <= BUF_LEN => {} + _ => { + self.cursor = None; + return Err(malformed_record()); + } + } + // SAFETY: the check above guarantees off leaves room for a full record + // header, so every read below stays in bounds, and each one uses + // unaligned access since the records are not guaranteed to be aligned. + unsafe { + let rec = + self.buf.0.as_ptr().add(off) as *const FILE_ID_BOTH_DIR_INFO; + let next_entry = + ptr::read_unaligned(ptr::addr_of!((*rec).NextEntryOffset)); + let attr = + ptr::read_unaligned(ptr::addr_of!((*rec).FileAttributes)); + let name_len = + ptr::read_unaligned(ptr::addr_of!((*rec).FileNameLength)) + as usize; + let creation = + ptr::read_unaligned(ptr::addr_of!((*rec).CreationTime)); + let last_access = + ptr::read_unaligned(ptr::addr_of!((*rec).LastAccessTime)); + let last_write = + ptr::read_unaligned(ptr::addr_of!((*rec).LastWriteTime)); + let end_of_file = + ptr::read_unaligned(ptr::addr_of!((*rec).EndOfFile)); + // For reparse points, EaSize aliases the reparse tag in the + // FILE_ID_BOTH_DIR_INFORMATION layout that kernel32 forwards. + let reparse_tag = if attr & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + ptr::read_unaligned(ptr::addr_of!((*rec).EaSize)) + } else { + 0 + }; + let name_ptr = ptr::addr_of!((*rec).FileName) as *const u16; + let name_off = name_ptr as usize - self.buf.0.as_ptr() as usize; + match name_off.checked_add(name_len) { + Some(end) if end <= BUF_LEN => {} + _ => { + self.cursor = None; + return Err(malformed_record()); + } + } + + ent.attr = attr; + ent.reparse_tag = reparse_tag; + ent.creation_time = creation as u64; + ent.last_access_time = last_access as u64; + ent.last_write_time = last_write as u64; + ent.file_size = end_of_file as u64; + ent.file_type = FileType::from_attr(attr, reparse_tag); + + let name_units = name_len / 2; + ent.file_name_u16.clear(); + ent.file_name_u16.reserve(name_units); + for i in 0..name_units { + ent.file_name_u16.push(ptr::read_unaligned(name_ptr.add(i))); + } + + self.cursor = if next_entry == 0 { + None + } else { + match off.checked_add(next_entry as usize) { + Some(next) => Some(next), + None => { + self.cursor = None; + return Err(malformed_record()); + } + } + }; + } + + ent.file_name.clear(); + decode_utf16_into(&ent.file_name_u16, &mut ent.file_name); + Ok(true) + } + + /// Fill the buffer with the next batch of entries. + /// + /// Returns false when the directory has been fully enumerated. + fn fill(&mut self) -> io::Result { + let class = if self.restart { + FileIdBothDirectoryRestartInfo + } else { + FileIdBothDirectoryInfo + }; + // SAFETY: buf is valid for BUF_LEN bytes and outlives the call. + let res = unsafe { + GetFileInformationByHandleEx( + self.handle.as_raw_handle() as HANDLE, + class, + self.buf.0.as_mut_ptr().cast(), + BUF_LEN as u32, + ) + }; + if res == 0 { + let err = io::Error::last_os_error(); + if err.raw_os_error() == Some(ERROR_NO_MORE_FILES as i32) { + return Ok(false); + } + // Keep restart set so a retry after a transient error starts the + // enumeration over instead of continuing one that never began. + return Err(err); + } + self.restart = false; + self.cursor = Some(0); + Ok(true) + } +} + +fn malformed_record() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + "directory entry record has out of bounds offsets", + ) +} + +fn nt_success(status: NTSTATUS) -> bool { + status >= 0 +} + +fn nt_error(status: NTSTATUS) -> u32 { + if status == STATUS_DELETE_PENDING { + ERROR_DELETE_PENDING + } else { + // SAFETY: RtlNtStatusToDosError has no preconditions. + unsafe { RtlNtStatusToDosError(status) } + } +} + +/// Decode UTF-16 code units into `dst`, reusing its allocation where possible. +/// +/// `dst` must be empty on entry. On invalid UTF-16 it falls back to a fresh +/// [`OsString`] that preserves the unpaired surrogates. +fn decode_utf16_into(units: &[u16], dst: &mut OsString) { + for result in char::decode_utf16(units.iter().copied()) { + match result { + Ok(c) => { + dst.push(c.encode_utf8(&mut [0; 4])); + } + Err(_) => { + *dst = OsString::from_wide(units); + return; + } + } + } +} + +pub(crate) fn time_as_u64( + time: &windows_sys::Win32::Foundation::FILETIME, +) -> u64 { + (time.dwHighDateTime as u64) << 32 | time.dwLowDateTime as u64 +} + +pub(crate) fn intervals_to_system_time(intervals: u64) -> SystemTime { + const NANOS_IN_SECOND: u64 = 1_000_000_000; + const NANOS_PER_INTERVAL: u64 = 100; + const SECONDS_TO_UNIX: u64 = 11_644_473_600; + + let seconds_from_unix = + (intervals / (NANOS_IN_SECOND / NANOS_PER_INTERVAL)) - SECONDS_TO_UNIX; + let dur_from_unix = time::Duration::from_secs(seconds_from_unix); + SystemTime::UNIX_EPOCH + dur_from_unix +} + +pub(crate) fn escaped_u16s(slice: &[u16]) -> String { + use std::char; + + let mut buf = String::with_capacity(slice.len()); + for result in char::decode_utf16(slice.iter().cloned()) { + match result { + Ok(ch) => buf.push(ch), + Err(err) => { + let bad = err.unpaired_surrogate(); + buf.push_str(&format!(r"\u{{{:X}}}", bad)); + } + } + } + buf +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn openat_rejects_invalid_names() { + let parent = std::ptr::null_mut(); + for name in [&[][..], &[0x61][..], &[0x61, 0, 0x62, 0][..]] { + let err = Dir::openat_follow(parent, name, false).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + } + + let mut name = vec![0x61; (u16::MAX as usize / 2) + 1]; + name.push(0); + let err = Dir::openat_follow(parent, &name, false).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + } + + #[test] + fn escaping1() { + let x: Vec = "foo☃bar".encode_utf16().collect(); + let escaped = escaped_u16s(&x); + assert_eq!("foo☃bar", escaped); + } + + #[test] + fn escaping2() { + let mut x = vec![]; + x.push(0xD800); + x.extend("a".encode_utf16()); + x.push(0xDA02); + x.extend("b".encode_utf16()); + x.push(0xDFFF); + x.extend("c".encode_utf16()); + + let escaped = escaped_u16s(&x); + assert_eq!(r"\u{D800}a\u{DA02}b\u{DFFF}c", escaped); + } +} diff --git a/src/os/windows/stat.rs b/src/os/windows/stat.rs new file mode 100644 index 0000000..0a67f87 --- /dev/null +++ b/src/os/windows/stat.rs @@ -0,0 +1,299 @@ +use std::fmt; +use std::fs::OpenOptions; +use std::io; +use std::mem; +use std::os::windows::fs::OpenOptionsExt; +use std::os::windows::io::{AsRawHandle, RawHandle}; +use std::path::Path; +use std::time::SystemTime; + +use windows_sys::Win32::Foundation::HANDLE; +use windows_sys::Win32::Storage::FileSystem::{ + FileAttributeTagInfo, GetFileInformationByHandle, + GetFileInformationByHandleEx, BY_HANDLE_FILE_INFORMATION, + FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_HIDDEN, + FILE_ATTRIBUTE_REPARSE_POINT, FILE_ATTRIBUTE_TAG_INFO, + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, +}; + +use crate::os::windows::{intervals_to_system_time, time_as_u64}; + +/// The name-surrogate bit set on reparse tags for symlinks and junctions. +const IO_REPARSE_TAG_NAME_SURROGATE_BIT: u32 = 0x2000_0000; + +/// Metadata for a file, queried from an open handle. +#[derive(Clone)] +pub struct Metadata { + info: BY_HANDLE_FILE_INFORMATION, + reparse_tag: u32, +} + +impl Metadata { + /// The raw file attributes, as in `dwFileAttributes`. + pub fn file_attributes(&self) -> u32 { + self.info.dwFileAttributes + } + + /// The file type. + pub fn file_type(&self) -> FileType { + FileType::from_attr(self.file_attributes(), self.reparse_tag) + } + + /// Returns true if this file is marked hidden. + pub fn is_hidden(&self) -> bool { + self.file_attributes() & FILE_ATTRIBUTE_HIDDEN != 0 + } + + /// The creation time, when available. + pub fn created(&self) -> io::Result { + let intervals = time_as_u64(&self.info.ftCreationTime); + if intervals == 0 { + Err(io::Error::new( + io::ErrorKind::Other, + "creation time is not available on this platform currently", + )) + } else { + Ok(intervals_to_system_time(intervals)) + } + } + + /// The last access time, when available. + pub fn accessed(&self) -> io::Result { + let intervals = time_as_u64(&self.info.ftLastAccessTime); + if intervals == 0 { + Err(io::Error::new( + io::ErrorKind::Other, + "last access time is not available on this platform currently", + )) + } else { + Ok(intervals_to_system_time(intervals)) + } + } + + /// The last modification time, when available. + pub fn modified(&self) -> io::Result { + let intervals = time_as_u64(&self.info.ftLastWriteTime); + if intervals == 0 { + Err(io::Error::new( + io::ErrorKind::Other, + "last write time is not available on this platform currently", + )) + } else { + Ok(intervals_to_system_time(intervals)) + } + } + + /// The file size in bytes. + pub fn len(&self) -> u64 { + ((self.info.nFileSizeHigh as u64) << 32) + | (self.info.nFileSizeLow as u64) + } + + /// The number of hard links to the file. + pub fn number_of_links(&self) -> u64 { + self.info.nNumberOfLinks as u64 + } + + /// The serial number of the volume the file resides on. + pub fn volume_serial_number(&self) -> u64 { + self.info.dwVolumeSerialNumber as u64 + } + + /// The 64-bit file index identifying the file within its volume. + pub fn file_index(&self) -> u64 { + ((self.info.nFileIndexHigh as u64) << 32) + | (self.info.nFileIndexLow as u64) + } +} + +/// File type information discoverable from a Windows directory entry or handle. +/// +/// Note that this does not include all possible file types on Windows. +/// Instead, this only differentiates between directories, regular files and +/// symlinks. Additional file type information (such as whether a file handle +/// is a socket) can only be retrieved via the `GetFileType` winapi routines. +/// A safe wrapper for it is +/// [available in the `winapi-util` crate](https://docs.rs/winapi-util/*/x86_64-pc-windows-msvc/winapi_util/file/fn.typ.html). +#[derive(Clone, Copy)] +pub struct FileType { + attr: u32, + reparse_tag: u32, +} + +// Compare only the classification (directory/file/symlink and its target kind) +// rather than the raw attributes, since two plain files that differ only in +// HIDDEN or READONLY are the same type. +impl PartialEq for FileType { + fn eq(&self, other: &FileType) -> bool { + self.discriminant() == other.discriminant() + } +} + +impl Eq for FileType {} + +impl std::hash::Hash for FileType { + fn hash(&self, state: &mut H) { + self.discriminant().hash(state); + } +} + +impl fmt::Debug for FileType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let human = if self.is_file() { + "File" + } else if self.is_dir() { + "Directory" + } else if self.is_symlink_file() { + "Symbolic Link (File)" + } else if self.is_symlink_dir() { + "Symbolic Link (Directory)" + } else { + "Unknown" + }; + write!(f, "FileType({human})") + } +} + +impl FileType { + /// Create a file type from its raw winapi components. + /// + /// `attr` should be a file attribute bitset, corresponding to the + /// `dwFileAttributes` member of file information structs. + /// + /// `reparse_tag` should be a valid reparse tag value when the + /// `FILE_ATTRIBUTE_REPARSE_POINT` bit is set in `attr`. If the bit isn't + /// set or if the tag is not available, then the tag can be any value. + pub fn from_attr(attr: u32, reparse_tag: u32) -> FileType { + FileType { attr, reparse_tag } + } + + /// Convert this file type to the platform independent file type. + pub fn into_api(self) -> crate::FileType { + crate::FileType::from(self) + } + + /// Returns true if this file type is a regular file. + /// + /// This corresponds to any file that is neither a symlink nor a directory. + pub fn is_file(&self) -> bool { + !self.is_dir() && !self.is_symlink() + } + + /// Returns true if this file type is a directory. + /// + /// This corresponds to any file that has the `FILE_ATTRIBUTE_DIRECTORY` + /// attribute and is not a symlink. + pub fn is_dir(&self) -> bool { + self.attr & FILE_ATTRIBUTE_DIRECTORY != 0 && !self.is_symlink() + } + + /// Returns true if this file type is a symlink. This could be a symlink + /// to a directory or to a file. To distinguish between them, use + /// `is_symlink_file` and `is_symlink_dir`. + /// + /// This corresponds to any file that has a surrogate reparse point. + pub fn is_symlink(&self) -> bool { + self.reparse_tag() + .is_some_and(|tag| tag & IO_REPARSE_TAG_NAME_SURROGATE_BIT != 0) + } + + /// Returns true if this file type is a symlink to a file. + /// + /// This corresponds to any file that has a surrogate reparse point and + /// is not a symlink to a directory. + pub fn is_symlink_file(&self) -> bool { + !self.is_symlink_dir() && self.is_symlink() + } + + /// Returns true if this file type is a symlink to a directory. + /// + /// This corresponds to any file that has a surrogate reparse point and has + /// the `FILE_ATTRIBUTE_DIRECTORY` attribute. + pub fn is_symlink_dir(&self) -> bool { + self.attr & FILE_ATTRIBUTE_DIRECTORY != 0 && self.is_symlink() + } + + /// A small classification code so equal-classified types compare equal. + fn discriminant(&self) -> u8 { + if self.is_symlink() { + if self.is_symlink_dir() { + 3 + } else { + 2 + } + } else if self.is_dir() { + 1 + } else { + 0 + } + } + + fn reparse_tag(&self) -> Option { + if self.attr & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + Some(self.reparse_tag) + } else { + None + } + } +} + +/// Open a handle and query metadata, following a trailing symlink. +pub fn stat>(path: P) -> io::Result { + let file = OpenOptions::new() + // Neither read nor write permissions are needed. + .access_mode(0) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path)?; + statat(file.as_raw_handle()) +} + +/// Like stat but does not follow a trailing symlink. +pub fn lstat>(path: P) -> io::Result { + let file = OpenOptions::new() + // Neither read nor write permissions are needed. + .access_mode(0) + .custom_flags( + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + ) + .open(path)?; + statat(file.as_raw_handle()) +} + +fn statat(handle: RawHandle) -> io::Result { + // SAFETY: handle is a valid open handle for the duration of this call and + // info is fully written by GetFileInformationByHandle on success. + let info: BY_HANDLE_FILE_INFORMATION = unsafe { + let mut info = mem::zeroed(); + let res = GetFileInformationByHandle(handle as HANDLE, &mut info); + if res == 0 { + return Err(io::Error::last_os_error()); + } + info + }; + let reparse_tag = + if info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + reparse_tag(handle)? + } else { + 0 + }; + Ok(Metadata { info, reparse_tag }) +} + +/// Read the reparse tag via `FileAttributeTagInfo`. +fn reparse_tag(handle: RawHandle) -> io::Result { + // SAFETY: handle is valid and the buffer size matches the info class. + let info: FILE_ATTRIBUTE_TAG_INFO = unsafe { + let mut info: FILE_ATTRIBUTE_TAG_INFO = mem::zeroed(); + let res = GetFileInformationByHandleEx( + handle as HANDLE, + FileAttributeTagInfo, + (&mut info as *mut FILE_ATTRIBUTE_TAG_INFO).cast(), + mem::size_of::() as u32, + ); + if res == 0 { + return Err(io::Error::last_os_error()); + } + info + }; + Ok(info.ReparseTag) +} diff --git a/src/tests/linux.rs b/src/tests/linux.rs new file mode 100644 index 0000000..b1d6fe1 --- /dev/null +++ b/src/tests/linux.rs @@ -0,0 +1,199 @@ +use std::ffi::OsString; +use std::io::{self, Seek}; +use std::os::unix::io::AsRawFd; + +use crate::os::unix; +use crate::tests::util::Dir; + +#[test] +fn empty() { + let dir = Dir::tmp(); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(2, ents.len()); + assert_eq!(".", ents[0].file_name_os()); + assert_eq!("..", ents[1].file_name_os()); + assert!(ents[0].file_type().unwrap().is_dir()); + assert!(ents[1].file_type().unwrap().is_dir()); +} + +#[test] +fn one_dir() { + let dir = Dir::tmp(); + dir.mkdirp("a"); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(3, ents.len()); + assert_eq!("a", ents[2].file_name_os()); + assert_ne!(0, ents[2].ino()); + assert!(ents[2].file_type().unwrap().is_dir()); +} + +#[test] +fn one_file() { + let dir = Dir::tmp(); + dir.touch("a"); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(3, ents.len()); + assert_eq!("a", ents[2].file_name_os()); + assert_ne!(0, ents[2].ino()); + assert!(ents[2].file_type().unwrap().is_file()); +} + +#[test] +fn one_dir_file() { + let dir = Dir::tmp(); + dir.mkdirp("foo"); + dir.touch("foo/a"); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("foo")]; + assert_eq!(expected, r.sorted_file_names()); + + let mut dirfd = unix::DirFd::open(dir.path().join("foo")).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("a")]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn many_files() { + let dir = Dir::tmp(); + dir.touch_all(&["a", "b", "c", "d"]); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn many_mixed() { + let dir = Dir::tmp(); + dir.mkdirp("b"); + dir.mkdirp("d"); + dir.touch_all(&["a", "c"]); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let ents = r.sorted_ents(); + assert!(ents[2].file_type().unwrap().is_file()); + assert!(ents[3].file_type().unwrap().is_dir()); + assert!(ents[4].file_type().unwrap().is_file()); + assert!(ents[5].file_type().unwrap().is_dir()); +} + +#[test] +fn symlink() { + let dir = Dir::tmp(); + dir.touch("a"); + dir.symlink_file("a", "a-link"); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("a-link"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let ents = r.sorted_ents(); + assert!(ents[2].file_type().unwrap().is_file()); + assert!(ents[3].file_type().unwrap().is_symlink()); +} + +#[test] +fn openat() { + let dir = Dir::tmp(); + dir.mkdirp("foo"); + dir.touch("foo/a"); + + let root = unix::DirFd::open(dir.path()).unwrap(); + let mut foo = unix::DirFd::openat(root.as_raw_fd(), "foo").unwrap(); + let r = dir.run_linux(&mut foo); + r.assert_no_errors(); + + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("a")]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn rewind() { + let dir = Dir::tmp(); + dir.touch_all(&["a", "b", "c", "d"]); + + let mut dirfd = unix::DirFd::open(dir.path()).unwrap(); + + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + assert_eq!(0, r.ents().len()); + + dirfd.seek(io::SeekFrom::Start(0)).unwrap(); + let r = dir.run_linux(&mut dirfd); + r.assert_no_errors(); + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); +} diff --git a/src/tests/mod.rs b/src/tests/mod.rs index ebf952d..b5017cf 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,4 +1,12 @@ #[macro_use] mod util; +#[cfg(target_os = "linux")] +mod linux; +#[cfg(walkdir_unix)] +mod openat; mod recursive; +#[cfg(walkdir_unix)] +mod unix; +#[cfg(windows)] +mod windows; diff --git a/src/tests/openat.rs b/src/tests/openat.rs new file mode 100644 index 0000000..4bcfa05 --- /dev/null +++ b/src/tests/openat.rs @@ -0,0 +1,219 @@ +use std::ffi::CString; + +use crate::dir::DirList; +use crate::tests::util::Dir; + +fn cstr(p: &std::path::Path) -> CString { + use std::os::unix::ffi::OsStrExt; + CString::new(p.as_os_str().as_bytes()).unwrap() +} + +#[test] +fn dirlist_reads_all_entries() { + let dir = Dir::tmp(); + dir.touch_all(&["a", "b", "c"]); + + let mut list = DirList::open_path(0, &cstr(dir.path()), true); + let mut names = vec![]; + while let Some(res) = list.next() { + res.unwrap(); + let name = list.entry().file_name_bytes().to_vec(); + if name == b"." || name == b".." { + continue; + } + names.push(String::from_utf8(name).unwrap()); + } + names.sort(); + assert_eq!(names, vec!["a", "b", "c"]); +} + +#[test] +fn openat_fifo_fails_fast() { + use std::os::unix::io::AsRawFd; + + let dir = Dir::tmp(); + let fifo_path = cstr(&dir.join("fifo")); + // SAFETY: fifo_path is NUL terminated. + let rc = unsafe { libc::mkfifo(fifo_path.as_ptr(), 0o644) }; + assert!(rc == 0, "mkfifo failed: {}", std::io::Error::last_os_error()); + + // A dir swapped for a FIFO must fail to open, not block waiting for a writer. + let dfd = crate::os::unix::DirFd::open_c(&cstr(dir.path())).unwrap(); + let name = CString::new("fifo").unwrap(); + let mut list = DirList::openat(1, dfd.as_raw_fd(), &name, false); + assert!(list.is_failed()); + assert!(list.next().unwrap().is_err()); +} + +#[test] +fn file_type_eq_ignores_permissions() { + use std::collections::HashSet; + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let dir = Dir::tmp(); + dir.mkdirp("d1"); + dir.mkdirp("d2"); + dir.touch_all(&["f"]); + fs::set_permissions(dir.join("d2"), fs::Permissions::from_mode(0o700)) + .unwrap(); + + let ft = |name: &str| -> crate::FileType { + crate::os::unix::lstat(dir.join(name)).unwrap().file_type().into() + }; + assert_eq!(ft("d1"), ft("d2")); + assert_ne!(ft("d1"), ft("f")); + let mut set = HashSet::new(); + set.insert(ft("d1")); + assert!(set.contains(&ft("d2"))); +} + +#[test] +fn walk_yields_root_and_children() { + let dir = Dir::tmp(); + dir.mkdirp("a"); + dir.touch_all(&["a/x", "b"]); + + let mut paths = crate::WalkDir::new(dir.path()) + .into_iter() + .filter_map(|e| e.ok()) + .map(|e| e.path().to_path_buf()) + .collect::>(); + paths.sort(); + + let mut expected = vec![ + dir.path().to_path_buf(), + dir.join("a"), + dir.join("a").join("x"), + dir.join("b"), + ]; + expected.sort(); + assert_eq!(paths, expected); +} + +#[test] +fn deep_tree_small_max_open() { + let dir = Dir::tmp(); + // Build a chain deeper than max_open to force spill + fallback. + let mut p = String::new(); + for i in 0..20 { + p.push_str(&format!("d{i}/")); + } + dir.mkdirp(&p); + + let count = crate::WalkDir::new(dir.path()) + .max_open(2) + .into_iter() + .filter_map(|e| e.ok()) + .count(); + // root + 20 directories + assert_eq!(count, 21); +} + +#[test] +fn nofollow_symlink_dir_not_descended() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); + dir.mkdirp("real"); + dir.touch_all(&["real/inside"]); + dir.symlink_dir("real", "link"); + + let mut names = crate::WalkDir::new(dir.path()) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.path() != dir.path()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect::>(); + names.sort(); + // `link` is yielded (as a symlink) but not descended into. + assert!(names.contains(&"link".to_string())); + // `inside` appears once (under real), never under link. + assert_eq!(names.iter().filter(|n| *n == "inside").count(), 1); +} + +// Build a deep tree with openat so no syscall sees the full path. +fn mkdir_chain_deep(root: &std::path::Path, comp: &str, depth: usize) { + use crate::os::unix::DirFd; + use std::os::unix::io::AsRawFd; + + let comp_c = CString::new(comp).unwrap(); + let mut fd = DirFd::open_c(&cstr(root)).unwrap(); + for _ in 0..depth { + // SAFETY: comp_c is NUL-terminated and fd is a live directory fd. + let rc = + unsafe { libc::mkdirat(fd.as_raw_fd(), comp_c.as_ptr(), 0o755) }; + assert!( + rc == 0, + "mkdirat failed: {}", + std::io::Error::last_os_error() + ); + fd = DirFd::openat_c(fd.as_raw_fd(), &comp_c).unwrap(); + } +} + +#[test] +fn long_total_path_traversal() { + let dir = Dir::tmp(); + // Many nested dirs whose joined path exceeds PATH_MAX, each component short. + let comp = "abcdefghijklmnopqrstuvwxyz0123"; // 30 chars + mkdir_chain_deep(dir.path(), comp, 200); // assembled path > 4096 bytes + + let count = crate::WalkDir::new(dir.path()) + .into_iter() + .filter_map(|e| e.ok()) + .count(); + assert_eq!(count, 201); // root + 200 dirs, no ENAMETOOLONG + + // max_open(2) keeps the immediate parent open during linear descent. + let count_max_open_2 = crate::WalkDir::new(dir.path()) + .max_open(2) + .into_iter() + .filter_map(|e| e.ok()) + .count(); + assert_eq!(count_max_open_2, 201); +} + +#[test] +fn iterators_are_fused() { + fn assert_fused() {} + assert_fused::(); + assert_fused::< + crate::FilterEntry bool>, + >(); +} + +#[test] +fn open_failure_preserves_path() { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let dir = Dir::tmp(); + dir.mkdirp("sub"); + dir.touch_all(&["sub/inside"]); + let sub = dir.join("sub"); + fs::set_permissions(&sub, fs::Permissions::from_mode(0o000)).unwrap(); + + // Check streaming and sort_by because sort_by builds eagerly. + let streamed = crate::WalkDir::new(dir.path()) + .into_iter() + .filter_map(|e| e.err()) + .collect::>(); + let sorted = crate::WalkDir::new(dir.path()) + .sort_by(|a, b| a.file_name().cmp(b.file_name())) + .into_iter() + .filter_map(|e| e.err()) + .collect::>(); + + let _ = fs::set_permissions(&sub, fs::Permissions::from_mode(0o755)); + + // Root may bypass permissions, but any open error must keep the path. + for errs in [&streamed, &sorted] { + let had_error = !errs.is_empty(); + let has_path = errs.iter().any(|e| e.path() == Some(sub.as_path())); + assert!( + !had_error || has_path, + "dir-open error lost its path: {:?}", + errs + ); + } +} diff --git a/src/tests/recursive.rs b/src/tests/recursive.rs index e415b91..d428086 100644 --- a/src/tests/recursive.rs +++ b/src/tests/recursive.rs @@ -323,6 +323,8 @@ fn siblings() { #[test] fn sym_root_file_nofollow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.touch("a"); dir.symlink_file("a", "a-link"); @@ -354,6 +356,8 @@ fn sym_root_file_nofollow() { #[test] fn sym_root_file_follow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.touch("a"); dir.symlink_file("a", "a-link"); @@ -384,6 +388,8 @@ fn sym_root_file_follow() { #[test] fn broken_sym_root_dir_nofollow_and_root_nofollow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.symlink_dir("broken", "a-link"); @@ -400,6 +406,8 @@ fn broken_sym_root_dir_nofollow_and_root_nofollow() { #[test] fn broken_sym_root_dir_follow_and_root_nofollow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.symlink_dir("broken", "a-link"); @@ -417,6 +425,8 @@ fn broken_sym_root_dir_follow_and_root_nofollow() { #[test] fn broken_sym_root_dir_root_is_always_followed() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.symlink_dir("broken", "a-link"); @@ -435,6 +445,8 @@ fn broken_sym_root_dir_root_is_always_followed() { #[test] fn sym_root_dir_nofollow_root_nofollow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.mkdirp("a"); dir.symlink_dir("a", "a-link"); @@ -453,6 +465,8 @@ fn sym_root_dir_nofollow_root_nofollow() { #[test] fn sym_root_dir_nofollow_root_follow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.mkdirp("a"); dir.symlink_dir("a", "a-link"); @@ -489,6 +503,8 @@ fn sym_root_dir_nofollow_root_follow() { #[test] fn sym_root_dir_follow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.mkdirp("a"); dir.symlink_dir("a", "a-link"); @@ -525,6 +541,8 @@ fn sym_root_dir_follow() { #[test] fn sym_file_nofollow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.touch("a"); dir.symlink_file("a", "a-link"); @@ -561,6 +579,8 @@ fn sym_file_nofollow() { #[test] fn sym_file_follow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.touch("a"); dir.symlink_file("a", "a-link"); @@ -597,6 +617,8 @@ fn sym_file_follow() { #[test] fn sym_dir_nofollow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.mkdirp("a"); dir.symlink_dir("a", "a-link"); @@ -634,6 +656,8 @@ fn sym_dir_nofollow() { #[test] fn sym_dir_follow() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.mkdirp("a"); dir.symlink_dir("a", "a-link"); @@ -677,6 +701,8 @@ fn sym_dir_follow() { #[test] fn sym_noloop() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.mkdirp("a/b/c"); dir.symlink_dir("a", "a/b/c/a-link"); @@ -691,6 +717,8 @@ fn sym_noloop() { #[test] fn sym_loop_detect() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.mkdirp("a/b/c"); dir.symlink_dir("a", "a/b/c/a-link"); @@ -716,6 +744,8 @@ fn sym_loop_detect() { #[test] fn sym_self_loop_no_error() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.symlink_file("a", "a"); @@ -741,6 +771,8 @@ fn sym_self_loop_no_error() { #[test] fn sym_file_self_loop_io_error() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.symlink_file("a", "a"); @@ -762,6 +794,8 @@ fn sym_file_self_loop_io_error() { #[test] fn sym_dir_self_loop_io_error() { + skip_if_no_symlinks!(); + let dir = Dir::tmp(); dir.symlink_dir("a", "a"); diff --git a/src/tests/unix.rs b/src/tests/unix.rs new file mode 100644 index 0000000..758407b --- /dev/null +++ b/src/tests/unix.rs @@ -0,0 +1,214 @@ +use std::ffi::OsString; +use std::os::unix::io::AsRawFd; + +use crate::os::unix; +use crate::tests::util::Dir; + +/// Assert the type only when the platform reports `d_type`. +fn assert_file_type(ft: Option, kind: &str) { + let ft = match ft { + Some(ft) => ft, + None => return, + }; + match kind { + "dir" => assert!(ft.is_dir(), "expected dir, got {:?}", ft), + "file" => assert!(ft.is_file(), "expected file, got {:?}", ft), + "symlink" => { + assert!(ft.is_symlink(), "expected symlink, got {:?}", ft) + } + _ => unreachable!("unknown kind: {}", kind), + } +} + +#[test] +fn empty() { + let dir = Dir::tmp(); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(2, ents.len()); + assert_eq!(".", ents[0].file_name_os()); + assert_eq!("..", ents[1].file_name_os()); + assert_file_type(ents[0].file_type(), "dir"); + assert_file_type(ents[1].file_type(), "dir"); +} + +#[test] +fn one_dir() { + let dir = Dir::tmp(); + dir.mkdirp("a"); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(3, ents.len()); + assert_eq!("a", ents[2].file_name_os()); + assert_ne!(0, ents[2].ino()); + assert_file_type(ents[2].file_type(), "dir"); +} + +#[test] +fn one_file() { + let dir = Dir::tmp(); + dir.touch("a"); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(3, ents.len()); + assert_eq!("a", ents[2].file_name_os()); + assert_ne!(0, ents[2].ino()); + assert_file_type(ents[2].file_type(), "file"); +} + +#[test] +fn one_dir_file() { + let dir = Dir::tmp(); + dir.mkdirp("foo"); + dir.touch("foo/a"); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("foo")]; + assert_eq!(expected, r.sorted_file_names()); + + let mut udir = unix::Dir::open(dir.path().join("foo")).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("a")]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn many_files() { + let dir = Dir::tmp(); + dir.touch_all(&["a", "b", "c", "d"]); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn many_mixed() { + let dir = Dir::tmp(); + dir.mkdirp("b"); + dir.mkdirp("d"); + dir.touch_all(&["a", "c"]); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let ents = r.sorted_ents(); + assert_file_type(ents[2].file_type(), "file"); + assert_file_type(ents[3].file_type(), "dir"); + assert_file_type(ents[4].file_type(), "file"); + assert_file_type(ents[5].file_type(), "dir"); +} + +#[test] +fn symlink() { + let dir = Dir::tmp(); + dir.touch("a"); + dir.symlink_file("a", "a-link"); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("a-link"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let ents = r.sorted_ents(); + assert_file_type(ents[2].file_type(), "file"); + assert_file_type(ents[3].file_type(), "symlink"); +} + +#[test] +fn openat() { + let dir = Dir::tmp(); + dir.mkdirp("foo"); + dir.touch("foo/a"); + + let root = unix::Dir::open(dir.path()).unwrap(); + let mut foo = unix::Dir::openat(root.as_raw_fd(), "foo").unwrap(); + let r = dir.run_unix(&mut foo); + r.assert_no_errors(); + + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("a")]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn rewind() { + let dir = Dir::tmp(); + dir.touch_all(&["a", "b", "c", "d"]); + + let mut udir = unix::Dir::open(dir.path()).unwrap(); + + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + assert_eq!(0, r.ents().len()); + + udir.rewind(); + let r = dir.run_unix(&mut udir); + r.assert_no_errors(); + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); +} diff --git a/src/tests/util.rs b/src/tests/util.rs index fdf06f5..28d641b 100644 --- a/src/tests/util.rs +++ b/src/tests/util.rs @@ -1,12 +1,29 @@ use std::env; use std::error; +#[cfg(any(unix, windows))] +use std::ffi::OsString; use std::fs::{self, File}; use std::io; use std::path::{Path, PathBuf}; use std::result; +#[cfg(unix)] +use crate::os::unix; +#[cfg(windows)] +use crate::os::windows; use crate::{DirEntry, Error}; +/// Skip the current test if the current environment doesn't support symlinks. +#[macro_export] +macro_rules! skip_if_no_symlinks { + () => { + if !$crate::tests::util::symlink_file_works() { + eprintln!("skipping test because symlinks don't work"); + return; + } + }; +} + /// Create an error from a format!-like syntax. #[macro_export] macro_rules! err { @@ -70,6 +87,84 @@ impl RecursiveResults { } } +/// The result of running a Unix directory iterator on a single directory. +#[cfg(unix)] +#[derive(Debug)] +pub struct UnixResults { + ents: Vec, + errs: Vec, +} + +#[cfg(unix)] +impl UnixResults { + /// Assert that no errors have occurred. + pub fn assert_no_errors(&self) { + assert!( + self.errs.is_empty(), + "expected to find no errors, but found: {:?}", + self.errs + ); + } + + /// Return all the successfully retrieved directory entries in the order + /// in which they were retrieved. + pub fn ents(&self) -> &[unix::DirEntry] { + &self.ents + } + + /// Return all the successfully retrieved directory entries, sorted + /// lexicographically by their file name. + pub fn sorted_ents(&self) -> Vec { + let mut ents = self.ents.clone(); + ents.sort_by(|e1, e2| e1.file_name_bytes().cmp(e2.file_name_bytes())); + ents + } + + /// Return all file names from all successfully retrieved directory + /// entries, sorted lexicographically. + /// + /// This does not include file names that correspond to an error. + pub fn sorted_file_names(&self) -> Vec { + self.sorted_ents().into_iter().map(|d| d.into_file_name_os()).collect() + } +} + +/// The result of running a Windows directory iterator on a single directory. +#[cfg(windows)] +#[derive(Debug)] +pub struct WindowsResults { + ents: Vec, + errs: Vec, +} + +#[cfg(windows)] +impl WindowsResults { + /// Assert that no errors have occurred. + pub fn assert_no_errors(&self) { + assert!( + self.errs.is_empty(), + "expected to find no errors, but found: {:?}", + self.errs + ); + } + + /// Return all the successfully retrieved directory entries, sorted + /// lexicographically by their file name. + pub fn sorted_ents(&self) -> Vec { + let mut ents = self.ents.clone(); + ents.sort_by(|e1, e2| e1.file_name_u16().cmp(e2.file_name_u16())); + ents + } + + /// Return all file names from all successfully retrieved directory + /// entries, sorted lexicographically. + /// + /// This does not include file names that correspond to an error. + pub fn sorted_file_names(&self) -> Vec { + self.sorted_ents().into_iter().map(|d| d.into_file_name_os()).collect() + } +} + /// A helper for managing a directory in which to run tests. /// /// When manipulating paths within this directory, paths are interpreted @@ -112,6 +207,56 @@ impl Dir { results } + #[cfg(unix)] + pub fn run_unix(&self, udir: &mut unix::Dir) -> UnixResults { + let mut results = UnixResults { ents: vec![], errs: vec![] }; + while let Some(result) = udir.read() { + match result { + Ok(ent) => results.ents.push(ent), + Err(err) => results.errs.push(err), + } + } + results + } + + #[cfg(target_os = "linux")] + pub fn run_linux(&self, dirfd: &mut unix::DirFd) -> UnixResults { + use crate::os::linux::{getdents, DirEntryCursor}; + use std::os::unix::io::AsRawFd; + + let mut results = UnixResults { ents: vec![], errs: vec![] }; + let mut cursor = DirEntryCursor::new(); + loop { + match getdents(dirfd.as_raw_fd(), &mut cursor) { + Err(err) => { + results.errs.push(err); + break; + } + Ok(false) => { + break; + } + Ok(true) => { + while let Some(ent) = cursor.read_unix() { + results.ents.push(ent); + } + } + } + } + results + } + + #[cfg(windows)] + pub fn run_windows(&self, h: &mut windows::Dir) -> WindowsResults { + let mut results = WindowsResults { ents: vec![], errs: vec![] }; + while let Some(result) = h.read() { + match result { + Ok(ent) => results.ents.push(ent), + Err(err) => results.errs.push(err), + } + } + results + } + /// Create a directory at the given path, while creating all intermediate /// directories as needed. pub fn mkdirp>(&self, path: P) { @@ -148,29 +293,7 @@ impl Dir { src: P1, link_name: P2, ) { - #[cfg(windows)] - fn imp(src: &Path, link_name: &Path) -> io::Result<()> { - use std::os::windows::fs::symlink_file; - symlink_file(src, link_name) - } - - #[cfg(unix)] - fn imp(src: &Path, link_name: &Path) -> io::Result<()> { - use std::os::unix::fs::symlink; - symlink(src, link_name) - } - - let (src, link_name) = (self.join(src), self.join(link_name)); - imp(&src, &link_name) - .map_err(|e| { - err!( - "failed to symlink file {} with target {}: {}", - src.display(), - link_name.display(), - e - ) - }) - .unwrap() + symlink_file(self.join(src), self.join(link_name)).unwrap() } /// Create a directory symlink to the given src with the given link name. @@ -179,29 +302,28 @@ impl Dir { src: P1, link_name: P2, ) { - #[cfg(windows)] - fn imp(src: &Path, link_name: &Path) -> io::Result<()> { - use std::os::windows::fs::symlink_dir; - symlink_dir(src, link_name) - } + symlink_dir(self.join(src), self.join(link_name)).unwrap() + } - #[cfg(unix)] - fn imp(src: &Path, link_name: &Path) -> io::Result<()> { - use std::os::unix::fs::symlink; - symlink(src, link_name) - } + /// Create a directory junction pointing at src. Junctions need no + /// privilege, unlike symlinks. + #[cfg(windows)] + pub fn junction, P2: AsRef>( + &self, + src: P1, + link_name: P2, + ) { + use std::process::Command; - let (src, link_name) = (self.join(src), self.join(link_name)); - imp(&src, &link_name) - .map_err(|e| { - err!( - "failed to symlink directory {} with target {}: {}", - src.display(), - link_name.display(), - e - ) - }) - .unwrap() + let status = Command::new("cmd") + .arg("/c") + .arg("mklink") + .arg("/J") + .arg(self.join(link_name)) + .arg(self.join(src)) + .status() + .unwrap(); + assert!(status.success(), "failed to create junction"); } } @@ -250,3 +372,93 @@ impl TempDir { &self.0 } } + +/// Test whether file symlinks are believed to work on in this environment. +/// +/// If they work, then return true, otherwise return false. +pub fn symlink_file_works() -> bool { + use std::sync::atomic::{AtomicUsize, Ordering}; + + // 0 = untried + // 1 = works + // 2 = does not work + static WORKS: AtomicUsize = AtomicUsize::new(0); + + let status = WORKS.load(Ordering::SeqCst); + if status != 0 { + return status == 1; + } + + let tmp = TempDir::new().unwrap(); + let foo = tmp.path().join("foo"); + let foolink = tmp.path().join("foo-link"); + File::create(&foo) + .map_err(|e| { + err!("error creating file {} for link test: {}", foo.display(), e) + }) + .unwrap(); + if symlink_file(&foo, &foolink).is_err() { + WORKS.store(2, Ordering::SeqCst); + return false; + } + if fs::read(&foolink).is_err() { + WORKS.store(2, Ordering::SeqCst); + return false; + } + WORKS.store(1, Ordering::SeqCst); + true +} + +/// Create a file symlink to the given src with the given link name. +fn symlink_file, P2: AsRef>( + src: P1, + link_name: P2, +) -> Result<()> { + #[cfg(windows)] + fn imp(src: &Path, link_name: &Path) -> io::Result<()> { + use std::os::windows::fs::symlink_file; + symlink_file(src, link_name) + } + + #[cfg(unix)] + fn imp(src: &Path, link_name: &Path) -> io::Result<()> { + use std::os::unix::fs::symlink; + symlink(src, link_name) + } + + imp(src.as_ref(), link_name.as_ref()).map_err(|e| { + err!( + "failed to symlink file {} with target {}: {}", + src.as_ref().display(), + link_name.as_ref().display(), + e + ) + }) +} + +/// Create a directory symlink to the given src with the given link name. +fn symlink_dir, P2: AsRef>( + src: P1, + link_name: P2, +) -> Result<()> { + #[cfg(windows)] + fn imp(src: &Path, link_name: &Path) -> io::Result<()> { + use std::os::windows::fs::symlink_dir; + symlink_dir(src, link_name) + } + + #[cfg(unix)] + fn imp(src: &Path, link_name: &Path) -> io::Result<()> { + use std::os::unix::fs::symlink; + symlink(src, link_name) + } + + imp(src.as_ref(), link_name.as_ref()).map_err(|e| { + err!( + "failed to symlink directory {} with target {}: {}", + src.as_ref().display(), + link_name.as_ref().display(), + e + ) + }) +} diff --git a/src/tests/windows.rs b/src/tests/windows.rs new file mode 100644 index 0000000..e1bb4bf --- /dev/null +++ b/src/tests/windows.rs @@ -0,0 +1,257 @@ +use std::ffi::OsString; +use std::os::windows::ffi::OsStringExt; +use std::path::PathBuf; + +use crate::os::windows::Dir; +use crate::tests::util::Dir as TestDir; +use crate::WalkDir; + +#[test] +fn empty() { + let dir = TestDir::tmp(); + + let mut handle = Dir::open(dir.path()).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(2, ents.len()); + assert_eq!(".", ents[0].file_name_os()); + assert_eq!("..", ents[1].file_name_os()); + assert!(ents[0].file_type().is_dir()); + assert!(ents[1].file_type().is_dir()); +} + +#[test] +fn one_dir() { + let dir = TestDir::tmp(); + dir.mkdirp("a"); + + let mut handle = Dir::open(dir.path()).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(3, ents.len()); + assert_eq!("a", ents[2].file_name_os()); + assert!(ents[2].file_type().is_dir()); +} + +#[test] +fn one_file() { + let dir = TestDir::tmp(); + dir.touch("a"); + + let mut handle = Dir::open(dir.path()).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + + let ents = r.sorted_ents(); + assert_eq!(3, ents.len()); + assert_eq!("a", ents[2].file_name_os()); + assert!(ents[2].file_type().is_file()); +} + +#[test] +fn one_dir_file() { + let dir = TestDir::tmp(); + dir.mkdirp("foo"); + dir.touch("foo/a"); + + let mut handle = Dir::open(dir.path()).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("foo")]; + assert_eq!(expected, r.sorted_file_names()); + + let mut handle = Dir::open(dir.path().join("foo")).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + let expected = + vec![OsString::from("."), OsString::from(".."), OsString::from("a")]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn many_files() { + let dir = TestDir::tmp(); + dir.touch_all(&["a", "b", "c", "d"]); + + let mut handle = Dir::open(dir.path()).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); +} + +#[test] +fn many_mixed() { + let dir = TestDir::tmp(); + dir.mkdirp("b"); + dir.mkdirp("d"); + dir.touch_all(&["a", "c"]); + + let mut handle = Dir::open(dir.path()).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("b"), + OsString::from("c"), + OsString::from("d"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let ents = r.sorted_ents(); + assert!(ents[2].file_type().is_file()); + assert!(ents[3].file_type().is_dir()); + assert!(ents[4].file_type().is_file()); + assert!(ents[5].file_type().is_dir()); +} + +#[test] +fn symlink() { + skip_if_no_symlinks!(); + + let dir = TestDir::tmp(); + dir.touch("a"); + dir.symlink_file("a", "a-link"); + + let mut handle = Dir::open(dir.path()).unwrap(); + let r = dir.run_windows(&mut handle); + r.assert_no_errors(); + + let expected = vec![ + OsString::from("."), + OsString::from(".."), + OsString::from("a"), + OsString::from("a-link"), + ]; + assert_eq!(expected, r.sorted_file_names()); + + let ents = r.sorted_ents(); + assert!(ents[2].file_type().is_file()); + assert!(ents[3].file_type().is_symlink()); + assert!(ents[3].file_type().is_symlink_file()); + assert!(!ents[3].file_type().is_symlink_dir()); + assert!(!ents[3].file_type().is_file()); +} + +// A junction is classified as a symlink, so it is not descended by default +// but is followed under follow_links. +#[test] +fn junction_is_symlink_no_descent() { + let dir = TestDir::tmp(); + dir.mkdirp("target"); + dir.touch("target/child"); + dir.junction("target", "link"); + + let link = dir.join("link"); + let ty = crate::os::windows::lstat(&link).unwrap().file_type(); + assert!(ty.is_symlink()); + assert!(ty.is_symlink_dir()); + assert!(!ty.is_dir()); + + let r = dir.run_recursive(WalkDir::new(dir.path())); + r.assert_no_errors(); + let got = r.sorted_paths(); + assert!(got.contains(&link)); + assert!(!got.contains(&dir.join("link").join("child"))); +} + +#[test] +fn junction_descend_with_follow() { + let dir = TestDir::tmp(); + dir.mkdirp("target"); + dir.touch("target/child"); + dir.junction("target", "link"); + + let r = dir.run_recursive(WalkDir::new(dir.path()).follow_links(true)); + r.assert_no_errors(); + let got = r.sorted_paths(); + assert!(got.contains(&dir.join("link").join("child"))); +} + +// Relative opens off the root handle have no MAX_PATH limit, so a tree far +// deeper than 260 chars must enumerate without error. +#[test] +fn long_path() { + let dir = TestDir::tmp(); + let mut rel = PathBuf::new(); + for _ in 0..40 { + rel.push("abcdefghij"); + } + dir.mkdirp(&rel); + dir.touch(rel.join("leaf")); + + let r = dir.run_recursive(WalkDir::new(dir.path())); + r.assert_no_errors(); + assert!(r.sorted_paths().contains(&dir.join(&rel).join("leaf"))); + + // A verbatim root takes a different open path and must not be re-prefixed. + let verbatim = PathBuf::from(format!(r"\\?\{}", dir.path().display())); + let r = dir.run_recursive(WalkDir::new(&verbatim)); + r.assert_no_errors(); + assert!(r.sorted_paths().contains(&verbatim.join(&rel).join("leaf"))); +} + +// Deleting a directory out from under the walk must surface an error, not +// panic, and iteration must continue. +#[test] +fn mid_walk_delete_parent() { + let dir = TestDir::tmp(); + dir.mkdirp("a"); + dir.touch("a/f"); + dir.mkdirp("b"); + dir.touch("b/f"); + + let it = WalkDir::new(dir.path()).max_open(1).into_iter(); + let mut saw_error = false; + for res in it { + let ent = match res { + Ok(ent) => ent, + Err(_) => { + saw_error = true; + continue; + } + }; + if ent.depth() == 1 && ent.file_type().is_dir() { + let _ = std::fs::remove_dir_all(ent.path()); + } + } + // The delete may race such that enumeration still succeeds. Despite that, the + // walk must terminate without panicking. + let _ = saw_error; +} + +// A file name containing an unpaired surrogate must round-trip through the +// walk. Some filesystems reject such names, so skip if creation fails. +#[test] +fn unpaired_surrogate_name() { + let dir = TestDir::tmp(); + let name = OsString::from_wide(&[0x0061, 0xD800, 0x0062]); + let path = dir.path().join(&name); + if std::fs::File::create(&path).is_err() { + eprintln!("skipping: filesystem rejects unpaired surrogate names"); + return; + } + + let r = dir.run_recursive(WalkDir::new(dir.path()).min_depth(1)); + r.assert_no_errors(); + let names: Vec = + r.ents().iter().map(|e| e.file_name().to_os_string()).collect(); + assert_eq!(1, names.len()); + assert_eq!(name, names[0]); +} diff --git a/src/walk.rs b/src/walk.rs new file mode 100644 index 0000000..36f73f5 --- /dev/null +++ b/src/walk.rs @@ -0,0 +1,1177 @@ +use std::cmp::{self, min}; +#[cfg(walkdir_unix)] +use std::ffi::CString; +use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; +use std::result; + +use same_file::Handle; + +use crate::dent::DirEntry; +use crate::error::{Error, Result}; +use crate::util; + +/// Like try, but for iterators that return [`Option>`]. +/// +/// [`Option>`]: https://doc.rust-lang.org/stable/std/option/enum.Option.html +macro_rules! itry { + ($e:expr) => { + match $e { + Ok(v) => v, + Err(err) => return Some(Err(From::from(err))), + } + }; +} + +struct WalkDirOptions { + follow_links: bool, + follow_root_links: bool, + max_open: usize, + min_depth: usize, + max_depth: usize, + sorter: Option< + Box< + dyn FnMut(&DirEntry, &DirEntry) -> cmp::Ordering + + Send + + Sync + + 'static, + >, + >, + contents_first: bool, + same_file_system: bool, +} + +impl fmt::Debug for WalkDirOptions { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + let sorter_str = if self.sorter.is_some() { + // FnMut isn't `Debug` + "Some(...)" + } else { + "None" + }; + f.debug_struct("WalkDirOptions") + .field("follow_links", &self.follow_links) + .field("follow_root_links", &self.follow_root_links) + .field("max_open", &self.max_open) + .field("min_depth", &self.min_depth) + .field("max_depth", &self.max_depth) + .field("sorter", &sorter_str) + .field("contents_first", &self.contents_first) + .field("same_file_system", &self.same_file_system) + .finish() + } +} + +/// A builder to create an iterator for recursively walking a directory. +/// +/// Results are returned in depth first fashion, with directories yielded +/// before their contents. If [`contents_first`] is true, contents are yielded +/// before their directories. The order is unspecified but if [`sort_by`] is +/// given, directory entries are sorted according to this function. Directory +/// entries `.` and `..` are always omitted. +/// +/// If an error occurs at any point during iteration, then it is returned in +/// place of its corresponding directory entry and iteration continues as +/// normal. If an error occurs while opening a directory for reading, then it +/// is not descended into (but the error is still yielded by the iterator). +/// Iteration may be stopped at any time. When the iterator is destroyed, all +/// resources associated with it are freed. +/// +/// [`contents_first`]: struct.WalkDir.html#method.contents_first +/// [`sort_by`]: struct.WalkDir.html#method.sort_by +/// +/// # Usage +/// +/// This type implements [`IntoIterator`] so that it may be used as the subject +/// of a `for` loop. You may need to call [`into_iter`] explicitly if you want +/// to use iterator adapters such as [`filter_entry`]. +/// +/// Idiomatic use of this type should use method chaining to set desired +/// options. For example, this only shows entries with a depth of `1`, `2` or +/// `3` (relative to `foo`): +/// +/// ```no_run +/// use walkdir::WalkDir; +/// # use walkdir::Error; +/// +/// # fn try_main() -> Result<(), Error> { +/// for entry in WalkDir::new("foo").min_depth(1).max_depth(3) { +/// println!("{}", entry?.path().display()); +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// [`IntoIterator`]: https://doc.rust-lang.org/stable/std/iter/trait.IntoIterator.html +/// [`into_iter`]: https://doc.rust-lang.org/nightly/core/iter/trait.IntoIterator.html#tymethod.into_iter +/// [`filter_entry`]: struct.IntoIter.html#method.filter_entry +/// +/// Note that the iterator by default includes the top-most directory. Since +/// this is the only directory yielded with depth `0`, it is easy to ignore it +/// with the [`min_depth`] setting: +/// +/// ```no_run +/// use walkdir::WalkDir; +/// # use walkdir::Error; +/// +/// # fn try_main() -> Result<(), Error> { +/// for entry in WalkDir::new("foo").min_depth(1) { +/// println!("{}", entry?.path().display()); +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// [`min_depth`]: struct.WalkDir.html#method.min_depth +/// +/// This will only return descendents of the `foo` directory and not `foo` +/// itself. +/// +/// # Loops +/// +/// This iterator (like most/all recursive directory iterators) assumes that +/// no loops can be made with *hard* links on your file system. In particular, +/// this would require creating a hard link to a directory such that it creates +/// a loop. On most platforms, this operation is illegal. +/// +/// Note that when following symbolic/soft links, loops are detected and an +/// error is reported. +#[derive(Debug)] +pub struct WalkDir { + root: PathBuf, + opts: WalkDirOptions, +} + +impl IntoIterator for WalkDir { + type Item = Result; + type IntoIter = IntoIter; + + fn into_iter(self) -> IntoIter { + IntoIter { + opts: self.opts, + start: Some(self.root), + stack_list: vec![], + stack_path: vec![], + path_buf: PathBuf::new(), + oldest_opened: 0, + depth: 0, + deferred_dirs: vec![], + root_device: None, + } + } +} + +impl WalkDir { + /// Create a builder for a recursive directory iterator starting at the + /// file path `root`. If `root` is a directory, then it is the first item + /// yielded by the iterator. If `root` is a file, then it is the first + /// and only item yielded by the iterator. If `root` is a symlink, then it + /// is followed for the purposes of directory traversal unless + /// [`follow_root_links`] is disabled. (A root `DirEntry` still obeys its + /// documentation with respect to symlinks and the `follow_links` setting.) + /// + /// [`follow_root_links`]: struct.WalkDir.html#method.follow_root_links + pub fn new>(root: P) -> WalkDir { + WalkDir { + root: root.into(), + opts: WalkDirOptions { + follow_links: false, + follow_root_links: true, + max_open: 10, + min_depth: 0, + max_depth: usize::MAX, + sorter: None, + contents_first: false, + same_file_system: false, + }, + } + } + + /// Set the minimum depth of entries yielded by the iterator. + /// + /// The smallest depth is `0` and always corresponds to the path given + /// to the `new` function on this type. Its direct descendents have depth + /// `1`, and their descendents have depth `2`, and so on. + pub fn min_depth(mut self, depth: usize) -> WalkDir { + self.opts.min_depth = depth; + if self.opts.min_depth > self.opts.max_depth { + self.opts.min_depth = self.opts.max_depth; + } + self + } + + /// Set the maximum depth of entries yield by the iterator. + /// + /// The smallest depth is `0` and always corresponds to the path given + /// to the `new` function on this type. Its direct descendents have depth + /// `1`, and their descendents have depth `2`, and so on. + /// + /// Note that this will not simply filter the entries of the iterator, but + /// it will actually avoid descending into directories when the depth is + /// exceeded. + pub fn max_depth(mut self, depth: usize) -> WalkDir { + self.opts.max_depth = depth; + if self.opts.max_depth < self.opts.min_depth { + self.opts.max_depth = self.opts.min_depth; + } + self + } + + /// Follow symbolic links. By default, this is disabled. + /// + /// When `yes` is `true`, symbolic links are followed as if they were + /// normal directories and files. If a symbolic link is broken or is + /// involved in a loop, an error is yielded. + /// + /// When enabled, the yielded [`DirEntry`] values represent the target of + /// the link while the path corresponds to the link. See the [`DirEntry`] + /// type for more details. + /// + /// [`DirEntry`]: struct.DirEntry.html + pub fn follow_links(mut self, yes: bool) -> WalkDir { + self.opts.follow_links = yes; + self + } + + /// Follow symbolic links if these are the root of the traversal. + /// By default, this is enabled. + /// + /// When `yes` is `true`, symbolic links on root paths are followed + /// which is effective if the symbolic link points to a directory. + /// If a symbolic link is broken or is involved in a loop, an error is + /// yielded as the first entry of the traversal. + /// + /// When enabled, the yielded [`DirEntry`] values represent the target of + /// the link while the path corresponds to the link. See the [`DirEntry`] + /// type for more details, and all future entries will be contained within + /// the resolved directory behind the symbolic link of the root path. + /// + /// [`DirEntry`]: struct.DirEntry.html + pub fn follow_root_links(mut self, yes: bool) -> WalkDir { + self.opts.follow_root_links = yes; + self + } + + /// Set the maximum number of simultaneously open file descriptors used + /// by the iterator. + /// + /// `n` must be greater than or equal to `1`. If `n` is `0`, then it is set + /// to `1` automatically. If this is not set, then it defaults to some + /// reasonably low number. + /// + /// This setting has no impact on the results yielded by the iterator + /// (even when `n` is `1`). Instead, this setting represents a trade off + /// between scarce resources (file descriptors) and memory. Namely, when + /// the maximum number of file descriptors is reached and a new directory + /// needs to be opened to continue iteration, then a previous directory + /// handle is closed and has its unyielded entries stored in memory. In + /// practice, this is a satisfying trade off because it scales with respect + /// to the *depth* of your file tree. Therefore, low values (even `1`) are + /// acceptable. + /// + /// Note that this value does not impact the number of system calls made by + /// an exhausted iterator. + /// + /// # Platform behavior + /// + /// On Windows, if `follow_links` is enabled, then this limit is not + /// respected. In particular, the maximum number of file descriptors opened + /// is proportional to the depth of the directory tree traversed. + pub fn max_open(mut self, mut n: usize) -> WalkDir { + if n == 0 { + n = 1; + } + self.opts.max_open = n; + self + } + + /// Set a function for sorting directory entries. + /// + /// If a compare function is set, the resulting iterator will return all + /// paths in sorted order. The compare function will be called to compare + /// entries from the same directory. + /// + /// ```rust,no_run + /// use std::cmp; + /// use std::ffi::OsString; + /// use walkdir::WalkDir; + /// + /// WalkDir::new("foo").sort_by(|a,b| a.file_name().cmp(b.file_name())); + /// ``` + pub fn sort_by(mut self, cmp: F) -> WalkDir + where + F: FnMut(&DirEntry, &DirEntry) -> cmp::Ordering + + Send + + Sync + + 'static, + { + self.opts.sorter = Some(Box::new(cmp)); + self + } + + /// Set a function for sorting directory entries with a key extraction + /// function. + /// + /// If a compare function is set, the resulting iterator will return all + /// paths in sorted order. The compare function will be called to compare + /// entries from the same directory. + /// + /// ```rust,no_run + /// use std::cmp; + /// use std::ffi::OsString; + /// use walkdir::WalkDir; + /// + /// WalkDir::new("foo").sort_by_key(|a| a.file_name().to_owned()); + /// ``` + pub fn sort_by_key(self, mut cmp: F) -> WalkDir + where + F: FnMut(&DirEntry) -> K + Send + Sync + 'static, + K: Ord, + { + self.sort_by(move |a, b| cmp(a).cmp(&cmp(b))) + } + + /// Sort directory entries by file name, to ensure a deterministic order. + /// + /// This is a convenience function for calling [`Self::sort_by()`]. + /// + /// ```rust,no_run + /// use walkdir::WalkDir; + /// + /// WalkDir::new("foo").sort_by_file_name(); + /// ``` + pub fn sort_by_file_name(self) -> WalkDir { + self.sort_by(|a, b| a.file_name().cmp(b.file_name())) + } + + /// Yield a directory's contents before the directory itself. By default, + /// this is disabled. + /// + /// When `yes` is `false` (as is the default), the directory is yielded + /// before its contents are read. This is useful when, e.g. you want to + /// skip processing of some directories. + /// + /// When `yes` is `true`, the iterator yields the contents of a directory + /// before yielding the directory itself. This is useful when, e.g. you + /// want to recursively delete a directory. + /// + /// # Example + /// + /// Assume the following directory tree: + /// + /// ```text + /// foo/ + /// abc/ + /// qrs + /// tuv + /// def/ + /// ``` + /// + /// With contents_first disabled (the default), the following code visits + /// the directory tree in depth-first order: + /// + /// ```no_run + /// use walkdir::WalkDir; + /// + /// for entry in WalkDir::new("foo") { + /// let entry = entry.unwrap(); + /// println!("{}", entry.path().display()); + /// } + /// + /// // foo + /// // foo/abc + /// // foo/abc/qrs + /// // foo/abc/tuv + /// // foo/def + /// ``` + /// + /// With contents_first enabled: + /// + /// ```no_run + /// use walkdir::WalkDir; + /// + /// for entry in WalkDir::new("foo").contents_first(true) { + /// let entry = entry.unwrap(); + /// println!("{}", entry.path().display()); + /// } + /// + /// // foo/abc/qrs + /// // foo/abc/tuv + /// // foo/abc + /// // foo/def + /// // foo + /// ``` + pub fn contents_first(mut self, yes: bool) -> WalkDir { + self.opts.contents_first = yes; + self + } + + /// Do not cross file system boundaries. + /// + /// When this option is enabled, directory traversal will not descend into + /// directories that are on a different file system from the root path. + /// + /// Currently, this option is only supported on Unix and Windows. If this + /// option is used on an unsupported platform, then directory traversal + /// will immediately return an error and will not yield any entries. + pub fn same_file_system(mut self, yes: bool) -> WalkDir { + self.opts.same_file_system = yes; + self + } +} + +/// An iterator for recursively descending into a directory. +/// +/// A value with this type must be constructed with the [`WalkDir`] type, which +/// uses a builder pattern to set options such as min/max depth, max open file +/// descriptors and whether the iterator should follow symbolic links. After +/// constructing a `WalkDir`, call [`.into_iter()`] at the end of the chain. +/// +/// The order of elements yielded by this iterator is unspecified. +/// +/// [`WalkDir`]: struct.WalkDir.html +/// [`.into_iter()`]: struct.WalkDir.html#into_iter.v +#[derive(Debug)] +pub struct IntoIter { + opts: WalkDirOptions, + /// The start path. + /// + /// This is only [`Some(...)`](`Some`) at the beginning. After the first iteration, + /// this is always [`None`]. + start: Option, + /// A stack of open (up to max fd) or spilled handles to directories. + stack_list: Vec, + /// A stack of file paths. + /// + /// This is *only* used when `follow_links` is enabled. In all other cases + /// this stack is empty. + stack_path: Vec, + /// The path of the directory currently on top of `stack_list`. + path_buf: PathBuf, + /// An index into `stack_list` that points to the oldest open directory + /// handle. If the maximum fd limit is reached and a new directory needs to + /// be read, the handle at this index is spilled before the new directory is + /// opened. + oldest_opened: usize, + /// The current depth of iteration. + depth: usize, + /// A list of [`DirEntry`]s corresponding to directories, that are yielded + /// after their contents has been fully yielded. This is only used when + /// `contents_first` is enabled. + deferred_dirs: Vec, + /// The device of the root file path when the first call to `next` was made. + /// + /// If the `same_file_system` option isn't enabled, then this is always + /// [`None`]. Conversely, if it is enabled, this is always [`Some(...)`](`Some`) + /// after handling the root path. + root_device: Option, +} + +/// An ancestor is an item in the directory tree traversed by walkdir, and is +/// used to check for loops in the tree when traversing symlinks. +#[derive(Debug)] +struct Ancestor { + /// The path of this ancestor. + path: PathBuf, + /// An open file to this ancesor. This is only used on Windows where + /// opening a file handle appears to be quite expensive, so we choose to + /// cache it. This comes at the cost of not respecting the file descriptor + /// limit set by the user. + #[cfg(windows)] + handle: Handle, +} + +impl Ancestor { + /// Create a new ancestor from the given directory path. + #[cfg(windows)] + fn new(dent: &DirEntry) -> std::io::Result { + let handle = Handle::from_path(dent.path())?; + Ok(Ancestor { path: dent.path().to_path_buf(), handle }) + } + + /// Create a new ancestor from the given directory path. + #[cfg(not(windows))] + fn new(dent: &DirEntry) -> std::io::Result { + Ok(Ancestor { path: dent.path().to_path_buf() }) + } + + /// Returns true if and only if the given open file handle corresponds to + /// the same directory as this ancestor. + #[cfg(windows)] + fn is_same(&self, child: &Handle) -> std::io::Result { + Ok(child == &self.handle) + } + + /// Returns true if and only if the given open file handle corresponds to + /// the same directory as this ancestor. + #[cfg(not(windows))] + fn is_same(&self, child: &Handle) -> std::io::Result { + Ok(child == &Handle::from_path(&self.path)?) + } +} + +impl Iterator for IntoIter { + type Item = Result; + /// Advances the iterator and returns the next value. + /// + /// # Errors + /// + /// If the iterator fails to retrieve the next value, this method returns + /// an error value. The error will be wrapped in an [`Option::Some`]. + fn next(&mut self) -> Option> { + if let Some(start) = self.start.take() { + if self.opts.same_file_system { + let result = util::device_num(&start) + .map_err(|e| Error::from_path(0, start.clone(), e)); + self.root_device = Some(itry!(result)); + } + let dent = itry!(DirEntry::from_path(0, start, false)); + if let Some(result) = self.handle_entry(dent) { + return Some(result); + } + } + while !self.stack_list.is_empty() { + self.depth = self.stack_list.len(); + if let Some(dentry) = self.get_deferred_dir() { + return Some(Ok(dentry)); + } + if self.depth > self.opts.max_depth { + // If we've exceeded the max depth, pop the current dir + // so that we don't descend. + self.pop(); + continue; + } + match self.next_entry() { + None => self.pop(), + Some(Err(err)) => return Some(Err(err)), + Some(Ok(dent)) => { + if let Some(result) = self.handle_entry(dent) { + return Some(result); + } + } + } + } + if self.opts.contents_first { + self.depth = self.stack_list.len(); + if let Some(dentry) = self.get_deferred_dir() { + return Some(Ok(dentry)); + } + } + None + } +} + +impl IntoIter { + /// Skips the current directory. + /// + /// This causes the iterator to stop traversing the contents of the least + /// recently yielded directory. This means any remaining entries in that + /// directory will be skipped (including sub-directories). + /// + /// Note that the ergonomics of this method are questionable since it + /// borrows the iterator mutably. Namely, you must write out the looping + /// condition manually. For example, to skip hidden entries efficiently on + /// unix systems: + /// + /// ```no_run + /// use walkdir::{DirEntry, WalkDir}; + /// + /// fn is_hidden(entry: &DirEntry) -> bool { + /// entry.file_name() + /// .to_str() + /// .map(|s| s.starts_with(".")) + /// .unwrap_or(false) + /// } + /// + /// let mut it = WalkDir::new("foo").into_iter(); + /// loop { + /// let entry = match it.next() { + /// None => break, + /// Some(Err(err)) => panic!("ERROR: {}", err), + /// Some(Ok(entry)) => entry, + /// }; + /// if is_hidden(&entry) { + /// if entry.file_type().is_dir() { + /// it.skip_current_dir(); + /// } + /// continue; + /// } + /// println!("{}", entry.path().display()); + /// } + /// ``` + /// + /// You may find it more convenient to use the [`filter_entry`] iterator + /// adapter. (See its documentation for the same example functionality as + /// above.) + /// + /// [`filter_entry`]: #method.filter_entry + pub fn skip_current_dir(&mut self) { + if !self.stack_list.is_empty() { + self.pop(); + } + } + + /// Yields only entries which satisfy the given predicate and skips + /// descending into directories that do not satisfy the given predicate. + /// + /// The predicate is applied to all entries. If the predicate is + /// true, iteration carries on as normal. If the predicate is false, the + /// entry is ignored and if it is a directory, it is not descended into. + /// + /// This is often more convenient to use than [`skip_current_dir`]. For + /// example, to skip hidden files and directories efficiently on unix + /// systems: + /// + /// ```no_run + /// use walkdir::{DirEntry, WalkDir}; + /// # use walkdir::Error; + /// + /// fn is_hidden(entry: &DirEntry) -> bool { + /// entry.file_name() + /// .to_str() + /// .map(|s| s.starts_with(".")) + /// .unwrap_or(false) + /// } + /// + /// # fn try_main() -> Result<(), Error> { + /// for entry in WalkDir::new("foo") + /// .into_iter() + /// .filter_entry(|e| !is_hidden(e)) { + /// println!("{}", entry?.path().display()); + /// } + /// # Ok(()) + /// # } + /// ``` + /// + /// Note that the iterator will still yield errors for reading entries that + /// may not satisfy the predicate. + /// + /// Note that entries skipped with [`min_depth`] and [`max_depth`] are not + /// passed to this predicate. + /// + /// Note that if the iterator has `contents_first` enabled, then this + /// method is no different than calling the standard `Iterator::filter` + /// method (because directory entries are yielded after they've been + /// descended into). + /// + /// [`skip_current_dir`]: #method.skip_current_dir + /// [`min_depth`]: struct.WalkDir.html#method.min_depth + /// [`max_depth`]: struct.WalkDir.html#method.max_depth + pub fn filter_entry

(self, predicate: P) -> FilterEntry + where + P: FnMut(&DirEntry) -> bool, + { + FilterEntry { it: self, predicate } + } + + fn handle_entry( + &mut self, + mut dent: DirEntry, + ) -> Option> { + if self.opts.follow_links && dent.file_type().is_symlink() { + dent = itry!(self.follow(dent)); + } + let is_normal_dir = !dent.file_type().is_symlink() && dent.is_dir(); + if is_normal_dir { + if self.opts.same_file_system && dent.depth() > 0 { + if itry!(self.is_same_file_system(&dent)) { + itry!(self.push(&dent)); + } + } else { + itry!(self.push(&dent)); + } + } else if dent.depth() == 0 + && dent.file_type().is_symlink() + && self.opts.follow_root_links + { + // As a special case, if we are processing a root entry, then we + // always follow it even if it's a symlink and follow_links is + // false. We are careful to not let this change the semantics of + // the DirEntry however. Namely, the DirEntry should still respect + // the follow_links setting. When it's disabled, it should report + // itself as a symlink. When it's enabled, it should always report + // itself as the target. + let md = itry!(fs::metadata(dent.path()).map_err(|err| { + Error::from_path(dent.depth(), dent.path().to_path_buf(), err) + })); + if md.file_type().is_dir() { + itry!(self.push(&dent)); + } + } + if is_normal_dir && self.opts.contents_first { + self.deferred_dirs.push(dent); + None + } else if self.skippable() { + None + } else { + Some(Ok(dent)) + } + } + + fn get_deferred_dir(&mut self) -> Option { + if self.opts.contents_first && self.depth < self.deferred_dirs.len() { + // Unwrap is safe here because we've guaranteed that + // `self.deferred_dirs.len()` can never be less than 1 + let deferred: DirEntry = self + .deferred_dirs + .pop() + .expect("BUG: deferred_dirs should be non-empty"); + if !self.skippable() { + return Some(deferred); + } + } + None + } + + fn push(&mut self, dent: &DirEntry) -> Result<()> { + // Spill the shallowest handle so the immediate parent stays open. + let free = + self.stack_list.len().checked_sub(self.oldest_opened).unwrap(); + if free == self.opts.max_open { + self.stack_list[self.oldest_opened].spill(); + self.oldest_opened = self.oldest_opened.checked_add(1).unwrap(); + } + + if self.opts.follow_links { + let ancestor = Ancestor::new(dent) + .map_err(|err| Error::from_io(self.depth, err))?; + self.stack_path.push(ancestor); + } + if dent.depth() == 0 { + self.path_buf.clear(); + self.path_buf.push(dent.path()); + } else { + self.path_buf.push(dent.file_name()); + } + + #[cfg(walkdir_unix)] + let mut list = { + let follow = self.opts.follow_links + || (dent.depth() == 0 && self.opts.follow_root_links); + match self.stack_list.last().and_then(|list| list.parent_handle()) + { + Some(parent) => crate::dir::DirList::openat( + self.depth, + parent, + &cstr_of_file_name(dent.path()), + follow, + ), + None => crate::dir::DirList::open_path( + self.depth, + &cstr_of_path(dent.path()), + follow, + ), + } + }; + #[cfg(windows)] + let mut list = { + let follow = self.opts.follow_links + || (dent.depth() == 0 && self.opts.follow_root_links); + match self.stack_list.last().and_then(|list| list.parent_handle()) + { + Some(parent) => crate::dir::DirList::openat( + self.depth, + parent, + &wide_of_file_name(dent.path()), + follow, + ), + None => crate::dir::DirList::open_path( + self.depth, + dent.path(), + follow, + ), + } + }; + #[cfg(not(any(walkdir_unix, windows)))] + let mut list = crate::dir::DirList::open_path(self.depth, dent.path()); + + // Sorters need a materialized directory. + if self.opts.sorter.is_some() { + // Failed opens keep the directory path. + let open_failure = list.is_failed(); + let mut ents: Vec> = Vec::new(); + while let Some(res) = list.next() { + match res { + Ok(()) => { + if let Some(entry) = entry_from_list( + self.depth + 1, + &self.path_buf, + &list, + ) { + ents.push(entry); + } + } + Err(err) => ents.push(Err(dir_read_error( + open_failure, + self.depth, + &self.path_buf, + self.depth + 1, + err, + ))), + } + } + if let Some(ref mut cmp) = self.opts.sorter { + ents.sort_by(|a, b| match (a, b) { + (Ok(a), Ok(b)) => cmp(a, b), + (&Err(_), &Err(_)) => cmp::Ordering::Equal, + (&Ok(_), &Err(_)) => cmp::Ordering::Greater, + (&Err(_), &Ok(_)) => cmp::Ordering::Less, + }); + } + list = list.into_built(ents); + } + + self.stack_list.push(list); + Ok(()) + } + + fn pop(&mut self) { + self.stack_list.pop().expect("BUG: cannot pop from empty stack"); + if self.opts.follow_links { + self.stack_path.pop().expect("BUG: list/path stacks out of sync"); + } + if self.stack_list.is_empty() { + self.path_buf.clear(); + } else { + self.path_buf.pop(); + } + // If everything in the stack is already closed, then there is + // room for at least one more open descriptor and it will + // always be at the top of the stack. + self.oldest_opened = min(self.oldest_opened, self.stack_list.len()); + } + + /// Pull the next entry from the active directory. + fn next_entry(&mut self) -> Option> { + let depth = self.depth; + let path = &self.path_buf; + let list = self.stack_list.last_mut()?; + loop { + if list.is_built() { + return list.next_built(); + } + // Failed opens keep the directory path and depth. + let open_failure = list.is_failed(); + let list_depth = list.depth(); + match list.next() { + None => return None, + Some(Err(err)) => { + return Some(Err(dir_read_error( + open_failure, + list_depth, + path, + depth, + err, + ))) + } + Some(Ok(())) => { + if let Some(entry) = entry_from_list(depth, path, list) { + return Some(entry); + } + } + } + } + } + + fn follow(&self, mut dent: DirEntry) -> Result { + dent = + DirEntry::from_path(self.depth, dent.path().to_path_buf(), true)?; + // The only way a symlink can cause a loop is if it points + // to a directory. Otherwise, it always points to a leaf + // and we can omit any loop checks. + if dent.is_dir() { + self.check_loop(dent.path())?; + } + Ok(dent) + } + + fn check_loop>(&self, child: P) -> Result<()> { + let hchild = Handle::from_path(&child) + .map_err(|err| Error::from_io(self.depth, err))?; + for ancestor in self.stack_path.iter().rev() { + let is_same = ancestor + .is_same(&hchild) + .map_err(|err| Error::from_io(self.depth, err))?; + if is_same { + return Err(Error::from_loop( + self.depth, + &ancestor.path, + child.as_ref(), + )); + } + } + Ok(()) + } + + #[cfg(walkdir_unix)] + fn is_same_file_system(&mut self, dent: &DirEntry) -> Result { + let parent = + self.stack_list.last().and_then(|list| list.parent_handle()); + let dent_device = match parent { + Some(fd) => { + crate::os::unix::statat_c(fd, &cstr_of_file_name(dent.path())) + .map(|metadata| metadata.dev()) + } + None => util::device_num(dent.path()), + } + .map_err(|err| Error::from_entry(dent, err))?; + Ok(self + .root_device + .map(|device| device == dent_device) + .expect("BUG: called is_same_file_system without root device")) + } + + #[cfg(not(walkdir_unix))] + fn is_same_file_system(&mut self, dent: &DirEntry) -> Result { + let dent_device = util::device_num(dent.path()) + .map_err(|err| Error::from_entry(dent, err))?; + Ok(self + .root_device + .map(|d| d == dent_device) + .expect("BUG: called is_same_file_system without root device")) + } + + fn skippable(&self) -> bool { + self.depth < self.opts.min_depth || self.depth > self.opts.max_depth + } +} + +impl std::iter::FusedIterator for IntoIter {} + +#[cfg(walkdir_unix)] +fn entry_from_list( + depth: usize, + path: &Path, + list: &crate::dir::DirList, +) -> Option> { + let entry = list.entry(); + if entry.file_name_bytes() == b"." || entry.file_name_bytes() == b".." { + return None; + } + Some(DirEntry::from_os_entry(depth, path, list.parent_handle(), entry)) +} + +#[cfg(windows)] +fn entry_from_list( + depth: usize, + path: &Path, + list: &crate::dir::DirList, +) -> Option> { + let entry = list.entry(); + if entry.is_dots() { + return None; + } + Some(DirEntry::from_os_entry(depth, path, entry)) +} + +#[cfg(not(any(walkdir_unix, windows)))] +fn entry_from_list( + depth: usize, + _path: &Path, + list: &crate::dir::DirList, +) -> Option> { + Some(DirEntry::from_entry(depth, list.entry())) +} + +/// Failed opens keep the directory's own path and depth, other errors use +/// the entry depth. +fn dir_read_error( + open_failure: bool, + dir_depth: usize, + dir_path: &Path, + entry_depth: usize, + err: std::io::Error, +) -> Error { + if open_failure { + Error::from_path(dir_depth, dir_path.to_path_buf(), err) + } else { + Error::from_io(entry_depth, err) + } +} + +#[cfg(walkdir_unix)] +fn cstr_of_path(path: &Path) -> CString { + use std::os::unix::ffi::OsStrExt; + + CString::new(path.as_os_str().as_bytes()) + .expect("path has no interior NUL") +} + +#[cfg(walkdir_unix)] +fn cstr_of_file_name(path: &Path) -> CString { + use std::os::unix::ffi::OsStrExt; + + let name = path.file_name().unwrap_or(path.as_os_str()); + CString::new(name.as_bytes()).expect("name has no interior NUL") +} + +#[cfg(windows)] +fn wide_of_file_name(path: &Path) -> Vec { + use std::os::windows::ffi::OsStrExt; + + let name = path.file_name().unwrap_or(path.as_os_str()); + name.encode_wide().chain(std::iter::once(0)).collect() +} + +/// A recursive directory iterator that skips entries. +/// +/// Values of this type are created by calling [`.filter_entry()`] on an +/// [`IntoIter`], which is formed by calling [`.into_iter()`] on a [`WalkDir`]. +/// +/// Directories that fail the predicate `P` are skipped. Namely, they are +/// never yielded and never descended into. +/// +/// Entries that are skipped with the [`min_depth`] and [`max_depth`] options +/// are not passed through this filter. +/// +/// If opening a handle to a directory resulted in an error, then it is yielded +/// and no corresponding call to the predicate is made. +/// +/// Type parameter `I` refers to the underlying iterator and `P` refers to the +/// predicate, which is usually `FnMut(&DirEntry) -> bool`. +/// +/// [`.filter_entry()`]: struct.IntoIter.html#method.filter_entry +/// [`.into_iter()`]: struct.WalkDir.html#into_iter.v +/// [`min_depth`]: struct.WalkDir.html#method.min_depth +/// [`max_depth`]: struct.WalkDir.html#method.max_depth +#[derive(Debug)] +pub struct FilterEntry { + it: I, + predicate: P, +} + +impl

std::iter::FusedIterator for FilterEntry where + P: FnMut(&DirEntry) -> bool +{ +} + +impl

Iterator for FilterEntry +where + P: FnMut(&DirEntry) -> bool, +{ + type Item = Result; + + /// Advances the iterator and returns the next value. + /// + /// # Errors + /// + /// If the iterator fails to retrieve the next value, this method returns + /// an error value. The error will be wrapped in an [`Option::Some`]. + fn next(&mut self) -> Option> { + loop { + let dent = itry!(self.it.next()?); + + if !(self.predicate)(&dent) { + if dent.is_dir() { + self.it.skip_current_dir(); + } + continue; + } + + return Some(Ok(dent)); + } + } +} + +impl

FilterEntry +where + P: FnMut(&DirEntry) -> bool, +{ + /// Yields only entries which satisfy the given predicate and skips + /// descending into directories that do not satisfy the given predicate. + /// + /// The predicate is applied to all entries. If the predicate is + /// true, iteration carries on as normal. If the predicate is false, the + /// entry is ignored and if it is a directory, it is not descended into. + /// + /// This is often more convenient to use than [`skip_current_dir`]. For + /// example, to skip hidden files and directories efficiently on unix + /// systems: + /// + /// ```no_run + /// use walkdir::{DirEntry, WalkDir}; + /// # use walkdir::Error; + /// + /// fn is_hidden(entry: &DirEntry) -> bool { + /// entry.file_name() + /// .to_str() + /// .map(|s| s.starts_with(".")) + /// .unwrap_or(false) + /// } + /// + /// # fn try_main() -> Result<(), Error> { + /// for entry in WalkDir::new("foo") + /// .into_iter() + /// .filter_entry(|e| !is_hidden(e)) { + /// println!("{}", entry?.path().display()); + /// } + /// # Ok(()) + /// # } + /// ``` + /// + /// Note that the iterator will still yield errors for reading entries that + /// may not satisfy the predicate. + /// + /// Note that entries skipped with [`min_depth`] and [`max_depth`] are not + /// passed to this predicate. + /// + /// Note that if the iterator has `contents_first` enabled, then this + /// method is no different than calling the standard [`Iterator::filter`] + /// method (because directory entries are yielded after they've been + /// descended into). + /// + /// [`skip_current_dir`]: #method.skip_current_dir + /// [`min_depth`]: struct.WalkDir.html#method.min_depth + /// [`max_depth`]: struct.WalkDir.html#method.max_depth + pub fn filter_entry(self, predicate: P) -> FilterEntry { + FilterEntry { it: self, predicate } + } + + /// Skips the current directory. + /// + /// This causes the iterator to stop traversing the contents of the least + /// recently yielded directory. This means any remaining entries in that + /// directory will be skipped (including sub-directories). + /// + /// Note that the ergonomics of this method are questionable since it + /// borrows the iterator mutably. Namely, you must write out the looping + /// condition manually. For example, to skip hidden entries efficiently on + /// unix systems: + /// + /// ```no_run + /// use walkdir::{DirEntry, WalkDir}; + /// + /// fn is_hidden(entry: &DirEntry) -> bool { + /// entry.file_name() + /// .to_str() + /// .map(|s| s.starts_with(".")) + /// .unwrap_or(false) + /// } + /// + /// let mut it = WalkDir::new("foo").into_iter(); + /// loop { + /// let entry = match it.next() { + /// None => break, + /// Some(Err(err)) => panic!("ERROR: {}", err), + /// Some(Ok(entry)) => entry, + /// }; + /// if is_hidden(&entry) { + /// if entry.file_type().is_dir() { + /// it.skip_current_dir(); + /// } + /// continue; + /// } + /// println!("{}", entry.path().display()); + /// } + /// ``` + /// + /// You may find it more convenient to use the [`filter_entry`] iterator + /// adapter. (See its documentation for the same example functionality as + /// above.) + /// + /// [`filter_entry`]: #method.filter_entry + pub fn skip_current_dir(&mut self) { + self.it.skip_current_dir(); + } +} diff --git a/walkdir-list/main.rs b/walkdir-list/main.rs index 5b7f7fb..ec7c79d 100644 --- a/walkdir-list/main.rs +++ b/walkdir-list/main.rs @@ -13,6 +13,7 @@ use std::error::Error; use std::ffi::OsStr; +use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process; @@ -63,11 +64,20 @@ where W1: io::Write, W2: io::Write, { - let mut count: u64 = 0; - for dir in &args.dirs { + fn count_walkdir( + args: &Args, + mut stderr: W, + dir: &Path, + ) -> Result { + let mut res = args.empty_count_result(); for result in args.walkdir(dir) { match result { - Ok(_) => count += 1, + Ok(dent) => { + res.count += 1; + if let Some(ref mut size) = res.size { + *size = dent.metadata()?.len(); + } + } Err(err) => { if !args.ignore_errors { writeln!(stderr, "ERROR: {}", err)?; @@ -75,79 +85,396 @@ where } } } + Ok(res) } - writeln!(stdout, "{}", count)?; - Ok(()) -} -fn print_paths( - args: &Args, - mut stdout: W1, - mut stderr: W2, -) -> Result<()> -where - W1: io::Write, - W2: io::Write, -{ + fn count_std( + args: &Args, + mut stderr: W, + dir: &Path, + ) -> Result { + let mut res = args.empty_count_result(); + for result in fs::read_dir(dir)? { + match result { + Ok(dent) => { + res.count += 1; + if let Some(ref mut size) = res.size { + *size = dent.metadata()?.len(); + } + } + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } + } + } + } + Ok(res) + } + + #[cfg(windows)] + fn count_windows( + args: &Args, + mut stderr: W, + dir: &Path, + ) -> Result { + use walkdir::os::windows; + + let mut res = args.empty_count_result(); + let mut handle = windows::Dir::open(dir)?; + let mut dent = windows::DirEntry::empty(); + loop { + match handle.read_into(&mut dent) { + Ok(true) => { + res.count += 1; + if let Some(ref mut size) = res.size { + let md = dir.join(dent.file_name_os()).metadata()?; + *size = md.len(); + } + } + Ok(false) => break, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } + } + } + } + Ok(res) + } + + #[cfg(not(windows))] + fn count_windows( + _args: &Args, + _stderr: W, + _dir: &Path, + ) -> Result { + err!("cannot use --flat-windows on non-Windows platform") + } + + #[cfg(walkdir_unix)] + fn count_unix( + args: &Args, + mut stderr: W, + dir: &Path, + ) -> Result { + use walkdir::os::unix; + + let mut res = args.empty_count_result(); + let mut udir = unix::Dir::open(dir)?; + let mut dent = unix::DirEntry::empty(); + loop { + match udir.read_into(&mut dent) { + Ok(true) => { + res.count += 1; + if let Some(ref mut size) = res.size { + let md = dir.join(dent.file_name_os()).metadata()?; + *size = md.len(); + } + } + Ok(false) => break, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } + } + } + } + Ok(res) + } + + #[cfg(not(walkdir_unix))] + fn count_unix( + _args: &Args, + _stderr: W, + _dir: &Path, + ) -> Result { + err!("cannot use --flat-unix on non-Unix platform") + } + + #[cfg(target_os = "linux")] + fn count_linux( + args: &Args, + mut stderr: W, + dir: &Path, + ) -> Result { + use std::os::unix::io::AsRawFd; + use walkdir::os::{linux, unix}; + + let mut res = args.empty_count_result(); + let udir = unix::Dir::open(dir)?; + let mut cursor = linux::DirEntryCursor::new(); + loop { + match linux::getdents(udir.as_raw_fd(), &mut cursor) { + Ok(true) => { + while let Some(dent) = cursor.read() { + res.count += 1; + if let Some(ref mut size) = res.size { + let md = + dir.join(dent.file_name_os()).metadata()?; + *size = md.len(); + } + } + } + Ok(false) => break, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } + } + } + } + Ok(res) + } + + #[cfg(not(target_os = "linux"))] + fn count_linux( + _args: &Args, + _stderr: W, + _dir: &Path, + ) -> Result { + err!("cannot use --flat-linux on non-Linux platform") + } + + let mut res = args.empty_count_result(); for dir in &args.dirs { - if args.tree { - print_paths_tree(&args, &mut stdout, &mut stderr, dir)?; + if args.flat_std { + res = res.add(count_std(args, &mut stderr, &dir)?); + } else if args.flat_windows { + res = res.add(count_windows(args, &mut stderr, &dir)?); + } else if args.flat_unix { + res = res.add(count_unix(args, &mut stderr, &dir)?); + } else if args.flat_linux { + res = res.add(count_linux(args, &mut stderr, &dir)?); } else { - print_paths_flat(&args, &mut stdout, &mut stderr, dir)?; + res = res.add(count_walkdir(args, &mut stderr, &dir)?); + } + } + match res.size { + Some(size) => { + writeln!(stdout, "{} (file size: {})", res.count, size)?; + } + None => { + writeln!(stdout, "{}", res.count)?; } } Ok(()) } -fn print_paths_flat( +fn print_paths( args: &Args, mut stdout: W1, mut stderr: W2, - dir: &Path, ) -> Result<()> where W1: io::Write, W2: io::Write, { - for result in args.walkdir(dir) { - let dent = match result { - Ok(dent) => dent, - Err(err) => { - if !args.ignore_errors { - writeln!(stderr, "ERROR: {}", err)?; + fn print_walkdir( + args: &Args, + mut stdout: W1, + mut stderr: W2, + dir: &Path, + ) -> Result<()> + where + W1: io::Write, + W2: io::Write, + { + for result in args.walkdir(dir) { + let dent = match result { + Ok(dent) => dent, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } + continue; + } + }; + write_path(&mut stdout, dent.path())?; + stdout.write_all(b"\n")?; + } + Ok(()) + } + + fn print_std( + args: &Args, + mut stdout: W1, + mut stderr: W2, + dir: &Path, + ) -> Result<()> + where + W1: io::Write, + W2: io::Write, + { + for result in fs::read_dir(dir)? { + let dent = match result { + Ok(dent) => dent, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } + continue; + } + }; + write_os_str(&mut stdout, &dent.file_name())?; + stdout.write_all(b"\n")?; + } + Ok(()) + } + + #[cfg(windows)] + fn print_windows( + args: &Args, + mut stdout: W1, + mut stderr: W2, + dir: &Path, + ) -> Result<()> + where + W1: io::Write, + W2: io::Write, + { + use walkdir::os::windows; + + let mut handle = windows::Dir::open(dir)?; + let mut dent = windows::DirEntry::empty(); + loop { + match handle.read_into(&mut dent) { + Ok(true) => { + write_os_str(&mut stdout, dent.file_name_os())?; + stdout.write_all(b"\n")?; + } + Ok(false) => break, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } } - continue; } - }; - write_path(&mut stdout, dent.path())?; - stdout.write_all(b"\n")?; + } + Ok(()) } - Ok(()) -} -fn print_paths_tree( - args: &Args, - mut stdout: W1, - mut stderr: W2, - dir: &Path, -) -> Result<()> -where - W1: io::Write, - W2: io::Write, -{ - for result in args.walkdir(dir) { - let dent = match result { - Ok(dent) => dent, - Err(err) => { - if !args.ignore_errors { - writeln!(stderr, "ERROR: {}", err)?; + #[cfg(not(windows))] + fn print_windows( + _args: &Args, + _stdout: W1, + _stderr: W2, + _dir: &Path, + ) -> Result + where + W1: io::Write, + W2: io::Write, + { + err!("cannot use --flat-windows on non-Windows platform") + } + + #[cfg(walkdir_unix)] + fn print_unix( + args: &Args, + mut stdout: W1, + mut stderr: W2, + dir: &Path, + ) -> Result<()> + where + W1: io::Write, + W2: io::Write, + { + use walkdir::os::unix; + + let mut dir = unix::Dir::open(dir)?; + let mut dent = unix::DirEntry::empty(); + loop { + match dir.read_into(&mut dent) { + Ok(true) => { + write_os_str(&mut stdout, dent.file_name_os())?; + stdout.write_all(b"\n")?; + } + Ok(false) => break, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } } - continue; } - }; - stdout.write_all(" ".repeat(dent.depth()).as_bytes())?; - write_os_str(&mut stdout, dent.file_name())?; - stdout.write_all(b"\n")?; + } + Ok(()) + } + + #[cfg(not(walkdir_unix))] + fn print_unix( + _args: &Args, + _stdout: W1, + _stderr: W2, + _dir: &Path, + ) -> Result<()> + where + W1: io::Write, + W2: io::Write, + { + err!("cannot use --flat-unix on non-Unix platform") + } + + #[cfg(target_os = "linux")] + fn print_linux( + args: &Args, + mut stdout: W1, + mut stderr: W2, + dir: &Path, + ) -> Result<()> + where + W1: io::Write, + W2: io::Write, + { + use std::os::unix::io::AsRawFd; + use walkdir::os::{linux, unix}; + + let dir = unix::Dir::open(dir)?; + let mut cursor = linux::DirEntryCursor::new(); + loop { + match linux::getdents(dir.as_raw_fd(), &mut cursor) { + Ok(true) => { + while let Some(dent) = cursor.read() { + write_os_str(&mut stdout, dent.file_name_os())?; + stdout.write_all(b"\n")?; + } + } + Ok(false) => break, + Err(err) => { + if !args.ignore_errors { + writeln!(stderr, "ERROR: {}", err)?; + } + } + } + } + Ok(()) + } + + #[cfg(not(target_os = "linux"))] + fn print_linux( + _args: &Args, + _stdout: W1, + _stderr: W2, + _dir: &Path, + ) -> Result<()> + where + W1: io::Write, + W2: io::Write, + { + err!("cannot use --flat-linux on non-Linux platform") + } + + for dir in &args.dirs { + if args.flat_std { + print_std(&args, &mut stdout, &mut stderr, dir)?; + } else if args.flat_windows { + print_windows(&args, &mut stdout, &mut stderr, dir)?; + } else if args.flat_unix { + print_unix(&args, &mut stdout, &mut stderr, dir)?; + } else if args.flat_linux { + print_linux(&args, &mut stdout, &mut stderr, dir)?; + } else { + print_walkdir(&args, &mut stdout, &mut stderr, dir)?; + } } Ok(()) } @@ -159,20 +486,24 @@ struct Args { min_depth: Option, max_depth: Option, max_open: Option, - tree: bool, ignore_errors: bool, sort: bool, depth_first: bool, same_file_system: bool, timeit: bool, count: bool, + file_size: bool, + flat_std: bool, + flat_windows: bool, + flat_unix: bool, + flat_linux: bool, } impl Args { fn parse() -> Result { use clap::{crate_authors, crate_version, App, Arg}; - let parsed = App::new("List files using walkdir") + let mut app = App::new("List files using walkdir") .author(crate_authors!()) .version(crate_version!()) .max_term_width(100) @@ -213,11 +544,6 @@ impl Args { .short("q") .help("Don't print error messages."), ) - .arg( - Arg::with_name("sort") - .long("sort") - .help("Sort file paths lexicographically."), - ) .arg( Arg::with_name("depth-first").long("depth-first").help( "Show directory contents before the directory path.", @@ -243,7 +569,59 @@ impl Args { .short("c") .help("Print only a total count of all file paths."), ) - .get_matches(); + .arg(Arg::with_name("file-size").long("file-size").help( + "Print the total file size of all files. This \ + implies --count.", + )) + .arg( + Arg::with_name("flat-std") + .long("flat-std") + .conflicts_with("flat-unix") + .conflicts_with("flat-linux") + .conflicts_with("flat-windows") + .help( + "Use std::fs::read_dir to list contents of a single \ + directory. This is NOT recursive.", + ), + ); + if cfg!(walkdir_unix) { + app = app.arg( + Arg::with_name("flat-unix") + .long("flat-unix") + .conflicts_with("flat-std") + .conflicts_with("flat-linux") + .conflicts_with("flat-windows") + .help( + "Use Unix-specific APIs to list contents of a single \ + directory. This is NOT recursive.", + ), + ); + } + if cfg!(target_os = "linux") { + app = app.arg( + Arg::with_name("flat-linux") + .long("flat-linux") + .conflicts_with("flat-std") + .conflicts_with("flat-unix") + .conflicts_with("flat-windows") + .help( + "Use Linux-specific syscalls (getdents64) to list \ + contents of a single directory. This is NOT \ + recursive.", + ), + ); + } + if cfg!(windows) { + app = app + .arg(Arg::with_name("flat-windows") + .long("flat-windows") + .conflicts_with("flat-std") + .conflicts_with("flat-unix") + .conflicts_with("flat-linux") + .help("Use Windows-specific APIs to list contents of a single \ + directory. This is NOT recursive.")); + } + let parsed = app.get_matches(); let dirs = match parsed.values_of_os("dirs") { None => vec![PathBuf::from("./")], @@ -255,13 +633,17 @@ impl Args { min_depth: parse_usize(&parsed, "min-depth")?, max_depth: parse_usize(&parsed, "max-depth")?, max_open: parse_usize(&parsed, "max-open")?, - tree: parsed.is_present("tree"), ignore_errors: parsed.is_present("ignore-errors"), sort: parsed.is_present("sort"), depth_first: parsed.is_present("depth-first"), same_file_system: parsed.is_present("same-file-system"), timeit: parsed.is_present("timeit"), count: parsed.is_present("count"), + file_size: parsed.is_present("file-size"), + flat_std: parsed.is_present("flat-std"), + flat_windows: parsed.is_present("flat-windows"), + flat_unix: parsed.is_present("flat-unix"), + flat_linux: parsed.is_present("flat-linux"), }) } @@ -284,6 +666,28 @@ impl Args { } walkdir } + + fn empty_count_result(&self) -> CountResult { + CountResult { + count: 0, + size: if self.file_size { Some(0) } else { None }, + } + } +} + +#[derive(Clone, Copy, Debug)] +struct CountResult { + count: u64, + size: Option, +} + +impl CountResult { + fn add(self, other: CountResult) -> CountResult { + CountResult { + count: self.count + other.count, + size: self.size.and_then(|s1| other.size.map(|s2| s1 + s2)), + } + } } fn parse_usize(