Repository navigation
[flow] Fix file crawl hanging on pnpm workspace links #9534
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
1c9db38
4f23b50
5f6885e
3f63cca
1e46e2e
77297c3
e4b493f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -186,29 +186,80 @@ impl Default for FileOptions { | |
| pub static node_modules_containers: LazyLock<RwLock<BTreeMap<FlowSmolStr, BTreeSet<FlowSmolStr>>>> = | ||
| LazyLock::new(|| RwLock::new(BTreeMap::new())); | ||
|
|
||
| /// Logical-to-real path mappings recorded by the initial crawl. | ||
| #[derive(Debug, Default)] | ||
| struct SymlinkPaths { | ||
| /// Files reached through a symlink, keyed by the path the crawl reached them by. | ||
| files: HashMap<PathBuf, PathBuf>, | ||
| /// Directory symlinks the crawl didn't follow, mapped to a directory it read instead. | ||
| dirs: HashMap<PathBuf, PathBuf>, | ||
| } | ||
|
|
||
| impl SymlinkPaths { | ||
| fn resolve<'a>(&self, path: &'a Path) -> Cow<'a, Path> { | ||
| let mut path = Cow::Borrowed(path); | ||
| loop { | ||
| if let Some(real_path) = self.files.get(path.as_ref()) { | ||
| return Cow::Owned(real_path.clone()); | ||
| } | ||
| if self.dirs.is_empty() { | ||
| return path; | ||
| } | ||
| // Each target is a directory the crawl read, so no skipped symlink is at or above it, | ||
| // and each step resolves part of the rest of the path. | ||
| let Some((link, target)) = path | ||
| .ancestors() | ||
| .find_map(|dir| self.dirs.get_key_value(dir)) | ||
| else { | ||
| return path; | ||
| }; | ||
| let rest = path | ||
| .strip_prefix(link) | ||
| .expect("a path should start with its ancestor"); | ||
| path = Cow::Owned(if rest.as_os_str().is_empty() { | ||
| target.clone() | ||
| } else { | ||
| target.join(rest) | ||
| }); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[derive(Debug)] | ||
| pub struct SymlinkMap { | ||
| building_paths: Mutex<Option<HashMap<PathBuf, PathBuf>>>, | ||
| complete_paths: OnceLock<HashMap<PathBuf, PathBuf>>, | ||
| building_paths: Mutex<Option<SymlinkPaths>>, | ||
| complete_paths: OnceLock<SymlinkPaths>, | ||
| } | ||
|
|
||
| impl Default for SymlinkMap { | ||
| fn default() -> Self { | ||
| Self { | ||
| building_paths: Mutex::new(Some(HashMap::new())), | ||
| building_paths: Mutex::new(Some(SymlinkPaths::default())), | ||
| complete_paths: OnceLock::new(), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl SymlinkMap { | ||
| fn insert(&self, path: PathBuf, real_path: PathBuf) { | ||
| self.building_paths | ||
| fn with_building_paths(&self, f: impl FnOnce(&mut SymlinkPaths)) { | ||
| f(self | ||
| .building_paths | ||
| .lock() | ||
| .expect("symlink map lock should not be poisoned") | ||
| .as_mut() | ||
| .expect("paths cannot be inserted after the map is complete") | ||
| .insert(path, real_path); | ||
| .expect("paths cannot be inserted after the map is complete")) | ||
| } | ||
|
|
||
| fn insert(&self, path: PathBuf, real_path: PathBuf) { | ||
| self.with_building_paths(|paths| { | ||
| paths.files.insert(path, real_path); | ||
| }); | ||
| } | ||
|
|
||
| fn insert_dir(&self, link: PathBuf, target: PathBuf) { | ||
| self.with_building_paths(|paths| { | ||
| paths.dirs.insert(link, target); | ||
| }); | ||
| } | ||
|
|
||
| pub fn mark_complete(&self) { | ||
|
|
@@ -230,11 +281,8 @@ impl SymlinkMap { | |
| /// Returns `None` when the map is not complete (e.g. not a full init run). | ||
| pub fn try_resolve<'a>(&self, normalized_path: &'a str) -> Option<Cow<'a, str>> { | ||
| let paths = self.complete_paths.get()?; | ||
| let normalized_path_buf = Path::new(normalized_path); | ||
| let resolved = paths | ||
| .get(normalized_path_buf) | ||
| .map_or(normalized_path_buf, PathBuf::as_path); | ||
| Some(resolved_path(normalized_path, resolved)) | ||
| let resolved = paths.resolve(Path::new(normalized_path)); | ||
| Some(resolved_path(normalized_path, &resolved)) | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -1022,6 +1070,39 @@ fn crawl_path_filter( | |
| && realpath_filter(options, path, all, include_libdef) | ||
| } | ||
|
|
||
| /// Whether the crawl skips the contents of the directory at `path`. | ||
| fn crawl_prunes_dir( | ||
| options: &FileOptions, | ||
| can_prune: bool, | ||
| path: &Path, | ||
| all: bool, | ||
| include_libdef: bool, | ||
| ) -> bool { | ||
| can_prune && !wanted_filter(options, path, all, include_libdef) | ||
| } | ||
|
|
||
| /// Whether the crawl reads the directory at the real path `path` and reports each file under it | ||
| /// that it reports through a symlink to it: the directory is under the implicitly included root | ||
| /// (and `subdir`), and isn't pruned. `[include]` and libdef paths don't count, as a pattern that | ||
| /// matches a directory needn't match the files under it. | ||
| fn crawl_reaches_dir( | ||
| options: &FileOptions, | ||
| root: &Path, | ||
| subdir: Option<&Path>, | ||
| path: &Path, | ||
| can_prune: bool, | ||
| all: bool, | ||
| include_libdef: bool, | ||
| ) -> bool { | ||
| options.implicitly_include_root | ||
| && path.starts_with(root) | ||
| && subdir.is_none_or(|subdir| path.starts_with(subdir)) | ||
| && !path | ||
| .ancestors() | ||
| .take_while(|dir| dir.starts_with(root)) | ||
| .any(|dir| crawl_prunes_dir(options, can_prune, dir, all, include_libdef)) | ||
| } | ||
|
|
||
| /// Whether the initial full crawl covers a path, making an absent alias mapping conclusive. | ||
| pub fn initial_crawl_covers_path(options: &FileOptions, root: &Path, path: &Path) -> bool { | ||
| path_is_in_crawl_scope(options, root, None, path, true) | ||
|
|
@@ -1077,20 +1158,50 @@ pub fn make_next_files( | |
| }; | ||
|
|
||
| let can_prune = can_prune_dir(&options); | ||
| let skipped_dir_symlinks = symlink_map.map(|_| Arc::new(Mutex::new(Vec::new()))); | ||
| let mut chunk = Vec::new(); | ||
| let mut symlink_ancestor_cache = HashMap::new(); | ||
| let mut reported_real_paths = HashSet::new(); | ||
| for starting_point_path in starting_point_paths { | ||
| let duped_options = options.dupe(); | ||
| let root_buf = root.to_path_buf(); | ||
| let subdir_buf = subdir.map(Path::to_path_buf); | ||
| let skipped_dir_symlinks = skipped_dir_symlinks.clone(); | ||
| for entry in jwalk::WalkDir::new(starting_point_path) | ||
| .skip_hidden(false) | ||
| .sort(sort) | ||
| .follow_links(true) | ||
| .parallelism(jwalk::Parallelism::RayonNewPool(2)) | ||
| .process_read_dir(move |_depth, path, _read_dir_state, children| { | ||
| if can_prune && !wanted_filter(&duped_options, path, all, include_libdef) { | ||
| if crawl_prunes_dir(&duped_options, can_prune, path, all, include_libdef) { | ||
| children.clear(); | ||
| } | ||
| // Files reached through a symlink are reported by their real path, so a symlink to a | ||
| // directory the crawl reads anyway adds nothing. Following it would also never end | ||
| // for a cycle, eg between packages linked into each other's node_modules. | ||
| for entry in children.iter_mut().flatten() { | ||
| if entry.read_children.is_some() | ||
| && entry.path_is_symlink() | ||
| && let Ok(target) = cached_canonicalize(&entry.path()) | ||
| && crawl_reaches_dir( | ||
| &duped_options, | ||
| &root_buf, | ||
| subdir_buf.as_deref(), | ||
| &target, | ||
| can_prune, | ||
| all, | ||
| include_libdef, | ||
| ) | ||
| { | ||
| entry.read_children = None; | ||
| if let Some(skipped_dir_symlinks) = &skipped_dir_symlinks { | ||
| skipped_dir_symlinks | ||
| .lock() | ||
| .expect("skipped symlinks lock should not be poisoned") | ||
| .push((entry.path(), target)); | ||
| } | ||
| } | ||
| } | ||
| }) | ||
| { | ||
| let Ok(entry) = entry else { | ||
|
|
@@ -1165,6 +1276,18 @@ pub fn make_next_files( | |
| } | ||
| } | ||
| } | ||
| if let Some(symlink_map) = symlink_map | ||
| && let Some(skipped_dir_symlinks) = skipped_dir_symlinks | ||
| { | ||
| let skipped_dir_symlinks = std::mem::take( | ||
| &mut *skipped_dir_symlinks | ||
| .lock() | ||
| .expect("skipped symlinks lock should not be poisoned"), | ||
| ); | ||
| for (link, target) in skipped_dir_symlinks { | ||
| symlink_map.insert_dir(link, target); | ||
| } | ||
| } | ||
| if !chunk.is_empty() { | ||
| chunk.reverse(); | ||
| send_chunked(chunk); | ||
|
|
@@ -1791,6 +1914,107 @@ mod tests { | |
| ); | ||
| } | ||
|
|
||
| #[test] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. are these tests actually useful or just repeating what the integration tests are doing |
||
| fn symlink_paths_resolve_through_chained_dir_aliases() { | ||
| let mut paths = super::SymlinkPaths::default(); | ||
| paths.files.insert( | ||
| "/project/link/file.js".into(), | ||
| "/outside/target/file.js".into(), | ||
| ); | ||
| paths | ||
| .dirs | ||
| .insert("/project/other_link".into(), "/project/link".into()); | ||
| paths.dirs.insert( | ||
| "/project/pkg/node_modules/dep".into(), | ||
| "/project/dep".into(), | ||
| ); | ||
|
|
||
| let resolve = |path: &str| paths.resolve(Path::new(path)).into_owned(); | ||
| assert_eq!( | ||
| resolve("/project/other_link/file.js"), | ||
| Path::new("/outside/target/file.js") | ||
| ); | ||
| assert_eq!( | ||
| resolve("/project/pkg/node_modules/dep/src/index.js"), | ||
| Path::new("/project/dep/src/index.js") | ||
| ); | ||
| assert_eq!( | ||
| resolve("/project/pkg/node_modules/dep"), | ||
| Path::new("/project/dep") | ||
| ); | ||
| assert_eq!( | ||
| resolve("/project/unrelated.js"), | ||
| Path::new("/project/unrelated.js") | ||
| ); | ||
| } | ||
|
|
||
| #[cfg(unix)] | ||
| #[test] | ||
| fn make_next_files_does_not_follow_symlinks_into_crawled_directories() { | ||
| use std::os::unix::fs::symlink; | ||
| use std::sync::Arc; | ||
|
|
||
| let tmp = std::env::temp_dir().join(format!( | ||
| "flow_make_next_files_crawled_symlinks_test_{}", | ||
| std::process::id() | ||
| )); | ||
| let _ = std::fs::remove_dir_all(&tmp); | ||
| // Packages linked into each other's node_modules, as pnpm links workspace packages, form a | ||
| // cycle between `a` and `b`. | ||
| let packages = tmp.join("packages"); | ||
| for (package, links) in [("a", &["b"][..]), ("b", &["a", "c"]), ("c", &[])] { | ||
| let dir = packages.join(package); | ||
| std::fs::create_dir_all(dir.join("node_modules")).unwrap(); | ||
| std::fs::write(dir.join("index.js"), "").unwrap(); | ||
| for link in links { | ||
| symlink( | ||
| Path::new("../..").join(link), | ||
| dir.join("node_modules").join(link), | ||
| ) | ||
| .unwrap(); | ||
| } | ||
| } | ||
| let root = super::cached_canonicalize(&tmp).unwrap(); | ||
| let packages = root.join("packages"); | ||
|
|
||
| let mut options = FileOptions { | ||
| module_file_exts: vec![".js".into()], | ||
| ..FileOptions::default() | ||
| }; | ||
| options.includes.add_path(&root); | ||
| let symlink_map = super::SymlinkMap::default(); | ||
| let mut files = Vec::new(); | ||
| super::make_next_files( | ||
| &root, | ||
| None, | ||
| false, | ||
| true, | ||
| Arc::new(options), | ||
| false, | ||
| Some(&symlink_map), | ||
| |chunk| files.extend(chunk), | ||
| ); | ||
| symlink_map.mark_complete(); | ||
|
|
||
| files.sort(); | ||
| let expected: Vec<_> = ["a", "b", "c"] | ||
| .iter() | ||
| .map(|package| packages.join(package).join("index.js")) | ||
| .collect(); | ||
| assert_eq!(files, expected); | ||
|
|
||
| let paths = symlink_map.complete_paths.get().unwrap(); | ||
| let through_link = packages.join("a/node_modules/b"); | ||
| assert!(paths.dirs.contains_key(&through_link)); | ||
| assert!(!paths.files.contains_key(&through_link.join("index.js"))); | ||
| assert_eq!( | ||
| paths.resolve(&through_link.join("node_modules/c/index.js")), | ||
| packages.join("c/index.js") | ||
| ); | ||
|
|
||
| let _ = std::fs::remove_dir_all(&tmp); | ||
| } | ||
|
|
||
| #[cfg(windows)] | ||
| #[test] | ||
| fn normalize_path_accepts_msys_drive_paths() { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| cwd: root | ||
| shell: test.sh | ||
| auto_start: false |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| flow ls lists each file once: | ||
| .flowconfig | ||
| node_modules/.pnpm/dep@1.0.0/node_modules/dep/index.js | ||
| node_modules/.pnpm/dep@1.0.0/node_modules/dep/package.json | ||
| node_modules/.pnpm/peer@1.0.0/node_modules/peer/index.js | ||
| node_modules/.pnpm/peer@1.0.0/node_modules/peer/package.json | ||
| packages/a/index.js | ||
| packages/a/package.json | ||
| packages/b/index.js | ||
| packages/b/package.json | ||
|
|
||
| flow check resolves modules through the links: | ||
| Error ----------------------------------------------------------------------------------------- packages/a/index.js:3:23 | ||
|
|
||
| Cannot assign `require(...)` to `wrong` because `number` [1] is incompatible with `string` [2]. [incompatible-type] | ||
|
|
||
| packages/a/index.js:3:23 | ||
| 3| const wrong: string = require('b'); // error: number is incompatible with string | ||
| ^^^^^^^^^^^^ | ||
|
|
||
| References: | ||
| packages/b/index.js:2:18 | ||
| 2| module.exports = 2; | ||
| ^ [1] | ||
| packages/a/index.js:3:14 | ||
| 3| const wrong: string = require('b'); // error: number is incompatible with string | ||
| ^^^^^^ [2] | ||
|
|
||
|
|
||
|
|
||
| Found 1 error |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| [options] | ||
| all=true | ||
| builtin_lib=core-only |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| const b: number = require('b'); | ||
| const dep: string = require('dep'); | ||
| const wrong: string = require('b'); // error: number is incompatible with string | ||
| module.exports = 1; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {"name": "a", "main": "index.js"} |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| const a: number = require('a'); | ||
| module.exports = 2; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {"name": "b", "main": "index.js"} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This still feels like reimplementing things and why does it only work for
options.implicitly_include_root