Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 237 additions & 13 deletions rust_port/crates/flow_common/src/files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -186,29 +186,80 @@ impl Default for FileOptions {
pub static node_modules_containers: LazyLock<RwLock<BTreeMap<FlowSmolStr, BTreeSet<FlowSmolStr>>>> =
LazyLock::new(|| RwLock::new(BTreeMap::new()));

/// Logical-to-real path mappings recorded by the initial crawl.
#[derive(Debug, Default)]
struct SymlinkPaths {
/// Files reached through a symlink, keyed by the path the crawl reached them by.
files: HashMap<PathBuf, PathBuf>,
/// Directory symlinks the crawl didn't follow, mapped to a directory it read instead.
dirs: HashMap<PathBuf, PathBuf>,
}

impl SymlinkPaths {
fn resolve<'a>(&self, path: &'a Path) -> Cow<'a, Path> {
let mut path = Cow::Borrowed(path);
loop {
if let Some(real_path) = self.files.get(path.as_ref()) {
return Cow::Owned(real_path.clone());
}
if self.dirs.is_empty() {
return path;
}
// Each target is a directory the crawl read, so no skipped symlink is at or above it,
// and each step resolves part of the rest of the path.
let Some((link, target)) = path
.ancestors()
.find_map(|dir| self.dirs.get_key_value(dir))
else {
return path;
};
let rest = path
.strip_prefix(link)
.expect("a path should start with its ancestor");
path = Cow::Owned(if rest.as_os_str().is_empty() {
target.clone()
} else {
target.join(rest)
});
}
}
}

#[derive(Debug)]
pub struct SymlinkMap {
building_paths: Mutex<Option<HashMap<PathBuf, PathBuf>>>,
complete_paths: OnceLock<HashMap<PathBuf, PathBuf>>,
building_paths: Mutex<Option<SymlinkPaths>>,
complete_paths: OnceLock<SymlinkPaths>,
}

impl Default for SymlinkMap {
fn default() -> Self {
Self {
building_paths: Mutex::new(Some(HashMap::new())),
building_paths: Mutex::new(Some(SymlinkPaths::default())),
complete_paths: OnceLock::new(),
}
}
}

impl SymlinkMap {
fn insert(&self, path: PathBuf, real_path: PathBuf) {
self.building_paths
fn with_building_paths(&self, f: impl FnOnce(&mut SymlinkPaths)) {
f(self
.building_paths
.lock()
.expect("symlink map lock should not be poisoned")
.as_mut()
.expect("paths cannot be inserted after the map is complete")
.insert(path, real_path);
.expect("paths cannot be inserted after the map is complete"))
}

fn insert(&self, path: PathBuf, real_path: PathBuf) {
self.with_building_paths(|paths| {
paths.files.insert(path, real_path);
});
}

fn insert_dir(&self, link: PathBuf, target: PathBuf) {
self.with_building_paths(|paths| {
paths.dirs.insert(link, target);
});
}

pub fn mark_complete(&self) {
Expand All @@ -230,11 +281,8 @@ impl SymlinkMap {
/// Returns `None` when the map is not complete (e.g. not a full init run).
pub fn try_resolve<'a>(&self, normalized_path: &'a str) -> Option<Cow<'a, str>> {
let paths = self.complete_paths.get()?;
let normalized_path_buf = Path::new(normalized_path);
let resolved = paths
.get(normalized_path_buf)
.map_or(normalized_path_buf, PathBuf::as_path);
Some(resolved_path(normalized_path, resolved))
let resolved = paths.resolve(Path::new(normalized_path));
Some(resolved_path(normalized_path, &resolved))
}
}

Expand Down Expand Up @@ -1022,6 +1070,39 @@ fn crawl_path_filter(
&& realpath_filter(options, path, all, include_libdef)
}

/// Whether the crawl skips the contents of the directory at `path`.
fn crawl_prunes_dir(
options: &FileOptions,
can_prune: bool,
path: &Path,
all: bool,
include_libdef: bool,
) -> bool {
can_prune && !wanted_filter(options, path, all, include_libdef)
}

/// Whether the crawl reads the directory at the real path `path` and reports each file under it
/// that it reports through a symlink to it: the directory is under the implicitly included root
/// (and `subdir`), and isn't pruned. `[include]` and libdef paths don't count, as a pattern that
/// matches a directory needn't match the files under it.
fn crawl_reaches_dir(
options: &FileOptions,
root: &Path,
subdir: Option<&Path>,
path: &Path,
can_prune: bool,
all: bool,
include_libdef: bool,
) -> bool {
options.implicitly_include_root
&& path.starts_with(root)
&& subdir.is_none_or(|subdir| path.starts_with(subdir))
&& !path
.ancestors()
.take_while(|dir| dir.starts_with(root))
.any(|dir| crawl_prunes_dir(options, can_prune, dir, all, include_libdef))
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This still feels like reimplementing things and why does it only work for options.implicitly_include_root


/// Whether the initial full crawl covers a path, making an absent alias mapping conclusive.
pub fn initial_crawl_covers_path(options: &FileOptions, root: &Path, path: &Path) -> bool {
path_is_in_crawl_scope(options, root, None, path, true)
Expand Down Expand Up @@ -1077,20 +1158,50 @@ pub fn make_next_files(
};

let can_prune = can_prune_dir(&options);
let skipped_dir_symlinks = symlink_map.map(|_| Arc::new(Mutex::new(Vec::new())));
let mut chunk = Vec::new();
let mut symlink_ancestor_cache = HashMap::new();
let mut reported_real_paths = HashSet::new();
for starting_point_path in starting_point_paths {
let duped_options = options.dupe();
let root_buf = root.to_path_buf();
let subdir_buf = subdir.map(Path::to_path_buf);
let skipped_dir_symlinks = skipped_dir_symlinks.clone();
for entry in jwalk::WalkDir::new(starting_point_path)
.skip_hidden(false)
.sort(sort)
.follow_links(true)
.parallelism(jwalk::Parallelism::RayonNewPool(2))
.process_read_dir(move |_depth, path, _read_dir_state, children| {
if can_prune && !wanted_filter(&duped_options, path, all, include_libdef) {
if crawl_prunes_dir(&duped_options, can_prune, path, all, include_libdef) {
children.clear();
}
// Files reached through a symlink are reported by their real path, so a symlink to a
// directory the crawl reads anyway adds nothing. Following it would also never end
// for a cycle, eg between packages linked into each other's node_modules.
for entry in children.iter_mut().flatten() {
if entry.read_children.is_some()
&& entry.path_is_symlink()
&& let Ok(target) = cached_canonicalize(&entry.path())
&& crawl_reaches_dir(
&duped_options,
&root_buf,
subdir_buf.as_deref(),
&target,
can_prune,
all,
include_libdef,
)
{
entry.read_children = None;
if let Some(skipped_dir_symlinks) = &skipped_dir_symlinks {
skipped_dir_symlinks
.lock()
.expect("skipped symlinks lock should not be poisoned")
.push((entry.path(), target));
}
}
}
})
{
let Ok(entry) = entry else {
Expand Down Expand Up @@ -1165,6 +1276,18 @@ pub fn make_next_files(
}
}
}
if let Some(symlink_map) = symlink_map
&& let Some(skipped_dir_symlinks) = skipped_dir_symlinks
{
let skipped_dir_symlinks = std::mem::take(
&mut *skipped_dir_symlinks
.lock()
.expect("skipped symlinks lock should not be poisoned"),
);
for (link, target) in skipped_dir_symlinks {
symlink_map.insert_dir(link, target);
}
}
if !chunk.is_empty() {
chunk.reverse();
send_chunked(chunk);
Expand Down Expand Up @@ -1791,6 +1914,107 @@ mod tests {
);
}

#[test]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are these tests actually useful or just repeating what the integration tests are doing

fn symlink_paths_resolve_through_chained_dir_aliases() {
let mut paths = super::SymlinkPaths::default();
paths.files.insert(
"/project/link/file.js".into(),
"/outside/target/file.js".into(),
);
paths
.dirs
.insert("/project/other_link".into(), "/project/link".into());
paths.dirs.insert(
"/project/pkg/node_modules/dep".into(),
"/project/dep".into(),
);

let resolve = |path: &str| paths.resolve(Path::new(path)).into_owned();
assert_eq!(
resolve("/project/other_link/file.js"),
Path::new("/outside/target/file.js")
);
assert_eq!(
resolve("/project/pkg/node_modules/dep/src/index.js"),
Path::new("/project/dep/src/index.js")
);
assert_eq!(
resolve("/project/pkg/node_modules/dep"),
Path::new("/project/dep")
);
assert_eq!(
resolve("/project/unrelated.js"),
Path::new("/project/unrelated.js")
);
}

#[cfg(unix)]
#[test]
fn make_next_files_does_not_follow_symlinks_into_crawled_directories() {
use std::os::unix::fs::symlink;
use std::sync::Arc;

let tmp = std::env::temp_dir().join(format!(
"flow_make_next_files_crawled_symlinks_test_{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&tmp);
// Packages linked into each other's node_modules, as pnpm links workspace packages, form a
// cycle between `a` and `b`.
let packages = tmp.join("packages");
for (package, links) in [("a", &["b"][..]), ("b", &["a", "c"]), ("c", &[])] {
let dir = packages.join(package);
std::fs::create_dir_all(dir.join("node_modules")).unwrap();
std::fs::write(dir.join("index.js"), "").unwrap();
for link in links {
symlink(
Path::new("../..").join(link),
dir.join("node_modules").join(link),
)
.unwrap();
}
}
let root = super::cached_canonicalize(&tmp).unwrap();
let packages = root.join("packages");

let mut options = FileOptions {
module_file_exts: vec![".js".into()],
..FileOptions::default()
};
options.includes.add_path(&root);
let symlink_map = super::SymlinkMap::default();
let mut files = Vec::new();
super::make_next_files(
&root,
None,
false,
true,
Arc::new(options),
false,
Some(&symlink_map),
|chunk| files.extend(chunk),
);
symlink_map.mark_complete();

files.sort();
let expected: Vec<_> = ["a", "b", "c"]
.iter()
.map(|package| packages.join(package).join("index.js"))
.collect();
assert_eq!(files, expected);

let paths = symlink_map.complete_paths.get().unwrap();
let through_link = packages.join("a/node_modules/b");
assert!(paths.dirs.contains_key(&through_link));
assert!(!paths.files.contains_key(&through_link.join("index.js")));
assert_eq!(
paths.resolve(&through_link.join("node_modules/c/index.js")),
packages.join("c/index.js")
);

let _ = std::fs::remove_dir_all(&tmp);
}

#[cfg(windows)]
#[test]
fn normalize_path_accepts_msys_drive_paths() {
Expand Down
3 changes: 3 additions & 0 deletions tests/node_modules_symlink_cycles/.testconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
cwd: root
shell: test.sh
auto_start: false
31 changes: 31 additions & 0 deletions tests/node_modules_symlink_cycles/node_modules_symlink_cycles.exp
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
flow ls lists each file once:
.flowconfig
node_modules/.pnpm/dep@1.0.0/node_modules/dep/index.js
node_modules/.pnpm/dep@1.0.0/node_modules/dep/package.json
node_modules/.pnpm/peer@1.0.0/node_modules/peer/index.js
node_modules/.pnpm/peer@1.0.0/node_modules/peer/package.json
packages/a/index.js
packages/a/package.json
packages/b/index.js
packages/b/package.json

flow check resolves modules through the links:
Error ----------------------------------------------------------------------------------------- packages/a/index.js:3:23

Cannot assign `require(...)` to `wrong` because `number` [1] is incompatible with `string` [2]. [incompatible-type]

packages/a/index.js:3:23
3| const wrong: string = require('b'); // error: number is incompatible with string
^^^^^^^^^^^^

References:
packages/b/index.js:2:18
2| module.exports = 2;
^ [1]
packages/a/index.js:3:14
3| const wrong: string = require('b'); // error: number is incompatible with string
^^^^^^ [2]



Found 1 error
3 changes: 3 additions & 0 deletions tests/node_modules_symlink_cycles/root/.flowconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[options]
all=true
builtin_lib=core-only

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions tests/node_modules_symlink_cycles/root/packages/a/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
const b: number = require('b');
const dep: string = require('dep');
const wrong: string = require('b'); // error: number is incompatible with string
module.exports = 1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"name": "a", "main": "index.js"}
2 changes: 2 additions & 0 deletions tests/node_modules_symlink_cycles/root/packages/b/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
const a: number = require('a');
module.exports = 2;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"name": "b", "main": "index.js"}
Loading
Loading