Repository navigation
[flow] Fix file crawl hanging on pnpm workspace links #9534
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
1c9db38
[flow] Fix file crawl hanging on pnpm workspace links
robhogan 4f23b50
[flow] Reuse the crawl scope check for symlink targets
robhogan 5f6885e
[flow] Check symlink targets with the crawl's own scope and prune rules
robhogan 3f63cca
[flow] Simplify symlink skipping and fold its test into node_modules_…
robhogan 1e46e2e
[flow] Only skip symlinks under the root, and only treat include glob…
robhogan 77297c3
[flow] Use one symlink map, skip only links between directories under…
robhogan e4b493f
[flow] Check include globs at the call site, bound alias resolution, …
robhogan File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -230,11 +230,22 @@ impl SymlinkMap { | |
| /// Returns `None` when the map is not complete (e.g. not a full init run). | ||
| pub fn try_resolve<'a>(&self, normalized_path: &'a str) -> Option<Cow<'a, str>> { | ||
| let paths = self.complete_paths.get()?; | ||
| let normalized_path_buf = Path::new(normalized_path); | ||
| let resolved = paths | ||
| .get(normalized_path_buf) | ||
| .map_or(normalized_path_buf, PathBuf::as_path); | ||
| Some(resolved_path(normalized_path, resolved)) | ||
| let mut path = Cow::Borrowed(Path::new(normalized_path)); | ||
| // A path can be under a directory symlink the crawl didn't follow. Each real path has no | ||
| // symlink at or above it, so each step resolves part of the rest of the path. | ||
| while let Some((link, real_path)) = | ||
| path.ancestors().find_map(|dir| paths.get_key_value(dir)) | ||
| { | ||
| let rest = path | ||
| .strip_prefix(link) | ||
| .expect("a path should start with its ancestor"); | ||
| path = Cow::Owned(if rest.as_os_str().is_empty() { | ||
| real_path.clone() | ||
| } else { | ||
| real_path.join(rest) | ||
| }); | ||
| } | ||
| Some(resolved_path(normalized_path, &path)) | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -991,21 +1002,28 @@ fn realpath_filter(options: &FileOptions, path: &Path, all: bool, include_libdef | |
| } | ||
|
|
||
| /// Whether `path` is somewhere the crawl looks: under `subdir` when the crawl is limited to one, | ||
| /// and under the root (when it is implicitly included), an `[include]` path or a libdef. | ||
| /// and under the root (when it is implicitly included), an `[include]` path or a libdef. With | ||
| /// `subtree`, `path` is a directory, an `[include]` glob must cover everything under it, and | ||
| /// libdefs don't count. | ||
| fn path_is_in_crawl_scope( | ||
| options: &FileOptions, | ||
| root: &Path, | ||
| subdir: Option<&Path>, | ||
| path: &Path, | ||
| include_libdef: bool, | ||
| subtree: bool, | ||
| ) -> bool { | ||
| subdir.is_none_or(|subdir| path.starts_with(subdir)) | ||
| && ((options.implicitly_include_root && path.starts_with(root)) || { | ||
| let path_str = path.to_string_lossy(); | ||
| is_included(options, &path_str) | ||
| || (include_libdef | ||
| && is_dynamically_discovered_builtin_ts_libdef(options, &path_str)) | ||
| || (include_libdef && is_configured_lib_file(options, &path_str)) | ||
| if subtree { | ||
| options.includes.matches_subtree(&path_str) | ||
| } else { | ||
| is_included(options, &path_str) | ||
| || (include_libdef | ||
| && is_dynamically_discovered_builtin_ts_libdef(options, &path_str)) | ||
| || (include_libdef && is_configured_lib_file(options, &path_str)) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
|
|
@@ -1018,13 +1036,13 @@ fn crawl_path_filter( | |
| all: bool, | ||
| include_libdef: bool, | ||
| ) -> bool { | ||
| path_is_in_crawl_scope(options, root, subdir, path, include_libdef) | ||
| path_is_in_crawl_scope(options, root, subdir, path, include_libdef, false) | ||
| && realpath_filter(options, path, all, include_libdef) | ||
| } | ||
|
|
||
| /// Whether the initial full crawl covers a path, making an absent alias mapping conclusive. | ||
| pub fn initial_crawl_covers_path(options: &FileOptions, root: &Path, path: &Path) -> bool { | ||
| path_is_in_crawl_scope(options, root, None, path, true) | ||
| path_is_in_crawl_scope(options, root, None, path, true, false) | ||
| && (wanted_filter(options, path, false, true) | ||
| || has_symlink_ancestor_under_root(root, path, &mut HashMap::new())) | ||
| } | ||
|
|
@@ -1082,15 +1100,44 @@ pub fn make_next_files( | |
| let mut reported_real_paths = HashSet::new(); | ||
| for starting_point_path in starting_point_paths { | ||
| let duped_options = options.dupe(); | ||
| let root_buf = root.to_path_buf(); | ||
| let subdir_buf = subdir.map(Path::to_path_buf); | ||
| for entry in jwalk::WalkDir::new(starting_point_path) | ||
| .skip_hidden(false) | ||
| .sort(sort) | ||
| .follow_links(true) | ||
| .parallelism(jwalk::Parallelism::RayonNewPool(2)) | ||
| .process_read_dir(move |_depth, path, _read_dir_state, children| { | ||
| if can_prune && !wanted_filter(&duped_options, path, all, include_libdef) { | ||
| let prunes = |dir: &Path| { | ||
| can_prune && !wanted_filter(&duped_options, dir, all, include_libdef) | ||
| }; | ||
| if prunes(path) { | ||
| children.clear(); | ||
| } | ||
| // Files reached through a symlink under the root are reported by their real path, | ||
| // so following one to a directory under the root that the crawl reads anyway | ||
| // reports nothing new. It would also never end for a cycle, e.g. between packages | ||
| // linked into each other's node_modules. A pruned target is still followed, so that | ||
| // the symlinks inside it can be resolved. | ||
| for entry in children.iter_mut().flatten() { | ||
| if entry.read_children.is_some() | ||
| && entry.path_is_symlink() | ||
| && entry.parent_path().starts_with(&root_buf) | ||
| && let Ok(target) = cached_canonicalize(&entry.path()) | ||
| && target.starts_with(&root_buf) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. still unclear to me why there are so many conditions needed here |
||
| && path_is_in_crawl_scope( | ||
| &duped_options, | ||
| &root_buf, | ||
| subdir_buf.as_deref(), | ||
| &target, | ||
| include_libdef, | ||
| true, | ||
| ) | ||
| && !prunes(&target) | ||
| { | ||
| entry.read_children = None; | ||
| } | ||
| } | ||
| }) | ||
| { | ||
| let Ok(entry) = entry else { | ||
|
|
@@ -1114,6 +1161,15 @@ pub fn make_next_files( | |
| .or_default() | ||
| .insert(FlowSmolStr::new(&basename)); | ||
| } | ||
| // A directory symlink that `process_read_dir` didn't follow. | ||
| if let Some(symlink_map) = symlink_map | ||
| && entry.path_is_symlink() | ||
| && entry.read_children.is_none() | ||
| && let Ok(real_path) = cached_canonicalize(&path) | ||
| && real_path != path | ||
| { | ||
| symlink_map.insert(path, real_path); | ||
| } | ||
| continue; | ||
| } | ||
| if !entry.file_type().is_file() { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| require('dep'); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,7 @@ | ||
| [ignore] | ||
| ignored_alias/** | ||
| ign/** | ||
| ignored_link/** | ||
|
|
||
| [options] | ||
| all=true | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| module.exports = 1; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| cwd: root | ||
| shell: test.sh | ||
| auto_start: false |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| // @flow |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| [include] | ||
| ../shared/* | ||
| ../other/** | ||
|
|
||
| [options] | ||
| all=false |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| // @flow |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| #!/bin/bash | ||
| # Copyright (c) Meta Platforms, Inc. and affiliates. | ||
| # | ||
| # This source code is licensed under the MIT license found in the | ||
| # LICENSE file in the root directory of this source tree. | ||
|
|
||
| # The crawl must still follow these symlinks, as it doesn't read their targets, or doesn't report | ||
| # the files under them by their real paths. | ||
|
|
||
| export LC_ALL=C | ||
|
|
||
| link() { | ||
| MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s "$1" "$2" | ||
| } | ||
|
|
||
| # `../shared/*` doesn't cover `../shared/sub/deep.js`. | ||
| link ../shared/sub sublink | ||
| # `../root-lib` is outside the root, though its path starts with the root's. | ||
| link ../root-lib root-lib | ||
| # `../other/sub` is outside the root, so the symlink inside it isn't reported by its real path. | ||
| mkdir -p ../other/sub | ||
| link ../other/sub other_sub | ||
| link ../../third ../other/sub/nested | ||
| # A symlink outside the root, to a directory under it. | ||
| link ../root/pkg ../other/to_root | ||
|
|
||
| assert_ok "$FLOW" ls --strip-root | sort |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| // @flow |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| ../other/sub/nested/y.js | ||
| ../other/to_root/z.js | ||
| ../root-lib/index.js | ||
| ../shared/sub/deep.js | ||
| ../third/y.js | ||
| .flowconfig | ||
| pkg/z.js |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| // @flow |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is "is this specific path included?" vs "is everything under this directory included?".
Part of an effort to reuse this code where we previously had a second method you correctly called out as a reimplementation. I'll see if we can refine it a bit further.