Skip to content
Closed
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 69 additions & 13 deletions rust_port/crates/flow_common/src/files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -230,11 +230,22 @@ impl SymlinkMap {
/// Returns `None` when the map is not complete (e.g. not a full init run).
pub fn try_resolve<'a>(&self, normalized_path: &'a str) -> Option<Cow<'a, str>> {
let paths = self.complete_paths.get()?;
let normalized_path_buf = Path::new(normalized_path);
let resolved = paths
.get(normalized_path_buf)
.map_or(normalized_path_buf, PathBuf::as_path);
Some(resolved_path(normalized_path, resolved))
let mut path = Cow::Borrowed(Path::new(normalized_path));
// A path can be under a directory symlink the crawl didn't follow. Each real path has no
// symlink at or above it, so each step resolves part of the rest of the path.
while let Some((link, real_path)) =
path.ancestors().find_map(|dir| paths.get_key_value(dir))
{
let rest = path
.strip_prefix(link)
.expect("a path should start with its ancestor");
path = Cow::Owned(if rest.as_os_str().is_empty() {
real_path.clone()
} else {
real_path.join(rest)
});
}
Some(resolved_path(normalized_path, &path))
}
}

Expand Down Expand Up @@ -991,21 +1002,28 @@ fn realpath_filter(options: &FileOptions, path: &Path, all: bool, include_libdef
}

/// Whether `path` is somewhere the crawl looks: under `subdir` when the crawl is limited to one,
/// and under the root (when it is implicitly included), an `[include]` path or a libdef.
/// and under the root (when it is implicitly included), an `[include]` path or a libdef. With
/// `subtree`, `path` is a directory, an `[include]` glob must cover everything under it, and
/// libdefs don't count.
fn path_is_in_crawl_scope(
options: &FileOptions,
root: &Path,
subdir: Option<&Path>,
path: &Path,
include_libdef: bool,
subtree: bool,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is "is this specific path included?" vs "is everything under this directory included?".

Part of an effort to reuse this code where we previously had a second method you correctly called out as a reimplementation. I'll see if we can refine it a bit further.

) -> bool {
subdir.is_none_or(|subdir| path.starts_with(subdir))
&& ((options.implicitly_include_root && path.starts_with(root)) || {
let path_str = path.to_string_lossy();
is_included(options, &path_str)
|| (include_libdef
&& is_dynamically_discovered_builtin_ts_libdef(options, &path_str))
|| (include_libdef && is_configured_lib_file(options, &path_str))
if subtree {
options.includes.matches_subtree(&path_str)
} else {
is_included(options, &path_str)
|| (include_libdef
&& is_dynamically_discovered_builtin_ts_libdef(options, &path_str))
|| (include_libdef && is_configured_lib_file(options, &path_str))
}
})
}

Expand All @@ -1018,13 +1036,13 @@ fn crawl_path_filter(
all: bool,
include_libdef: bool,
) -> bool {
path_is_in_crawl_scope(options, root, subdir, path, include_libdef)
path_is_in_crawl_scope(options, root, subdir, path, include_libdef, false)
&& realpath_filter(options, path, all, include_libdef)
}

/// Whether the initial full crawl covers a path, making an absent alias mapping conclusive.
pub fn initial_crawl_covers_path(options: &FileOptions, root: &Path, path: &Path) -> bool {
path_is_in_crawl_scope(options, root, None, path, true)
path_is_in_crawl_scope(options, root, None, path, true, false)
&& (wanted_filter(options, path, false, true)
|| has_symlink_ancestor_under_root(root, path, &mut HashMap::new()))
}
Expand Down Expand Up @@ -1082,15 +1100,44 @@ pub fn make_next_files(
let mut reported_real_paths = HashSet::new();
for starting_point_path in starting_point_paths {
let duped_options = options.dupe();
let root_buf = root.to_path_buf();
let subdir_buf = subdir.map(Path::to_path_buf);
for entry in jwalk::WalkDir::new(starting_point_path)
.skip_hidden(false)
.sort(sort)
.follow_links(true)
.parallelism(jwalk::Parallelism::RayonNewPool(2))
.process_read_dir(move |_depth, path, _read_dir_state, children| {
if can_prune && !wanted_filter(&duped_options, path, all, include_libdef) {
let prunes = |dir: &Path| {
can_prune && !wanted_filter(&duped_options, dir, all, include_libdef)
};
if prunes(path) {
children.clear();
}
// Files reached through a symlink under the root are reported by their real path,
// so following one to a directory under the root that the crawl reads anyway
// reports nothing new. It would also never end for a cycle, e.g. between packages
// linked into each other's node_modules. A pruned target is still followed, so that
// the symlinks inside it can be resolved.
for entry in children.iter_mut().flatten() {
if entry.read_children.is_some()
&& entry.path_is_symlink()
&& entry.parent_path().starts_with(&root_buf)
&& let Ok(target) = cached_canonicalize(&entry.path())
&& target.starts_with(&root_buf)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

still unclear to me why there are so many conditions needed here

&& path_is_in_crawl_scope(
&duped_options,
&root_buf,
subdir_buf.as_deref(),
&target,
include_libdef,
true,
)
&& !prunes(&target)
{
entry.read_children = None;
}
}
})
{
let Ok(entry) = entry else {
Expand All @@ -1114,6 +1161,15 @@ pub fn make_next_files(
.or_default()
.insert(FlowSmolStr::new(&basename));
}
// A directory symlink that `process_read_dir` didn't follow.
if let Some(symlink_map) = symlink_map
&& entry.path_is_symlink()
&& entry.read_children.is_none()
&& let Ok(real_path) = cached_canonicalize(&path)
&& real_path != path
{
symlink_map.insert(path, real_path);
}
continue;
}
if !entry.file_type().is_file() {
Expand Down
8 changes: 8 additions & 0 deletions rust_port/crates/flow_common/src/path_matcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,14 @@ impl PathMatcher {
}
}) || self.glob_patterns.iter().any(|glob| glob.is_match(f))
}

/// Check if a glob matches the directory `dir` and everything under it. Paths are left out, as
/// they match by string prefix, e.g. the root `/x/root` matches `/x/root-lib`.
pub fn matches_subtree(&self, dir: &str) -> bool {
self.glob_patterns
.iter()
.any(|glob| glob.supports_directory_pruning() && glob.is_match(dir))
}
}

#[cfg(test)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ node_modules/.pnpm/dep@1.0.0/node_modules/dep/index.js
node_modules/.pnpm/dep@1.0.0/node_modules/dep/package.json
packages/a/index.js
packages/b/index.js
packages/c/index.js
1 change: 1 addition & 0 deletions tests/node_modules_symlink_duplicates/packages/c/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
require('dep');
12 changes: 10 additions & 2 deletions tests/node_modules_symlink_duplicates/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@
# LICENSE file in the root directory of this source tree.

# pnpm links each package from its store into the node_modules of every package that depends on
# it. The crawl reaches the store's files through each link, and reports each one once, by its
# real path.
# it, and links workspace packages into each other's node_modules. The crawl reports each file
# once, by its real path, including one also reached through a file symlink, and doesn't walk every
# path through the cycles between the packages.

export LC_ALL=C

Expand All @@ -20,5 +21,12 @@ link_dir() {
link_dir .pnpm/dep@1.0.0/node_modules/dep node_modules/dep
link_dir ../../../node_modules/.pnpm/dep@1.0.0/node_modules/dep packages/a/node_modules/dep
link_dir ../../../node_modules/.pnpm/dep@1.0.0/node_modules/dep packages/b/node_modules/dep
link_dir ../../b packages/a/node_modules/b
link_dir ../../c packages/a/node_modules/c
link_dir ../../a packages/b/node_modules/a
link_dir ../../c packages/b/node_modules/c
link_dir ../../a packages/c/node_modules/a
link_dir ../../b packages/c/node_modules/b
MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s ../../node_modules/dep/index.js packages/a/dep.js

assert_ok "$FLOW" ls --strip-root | sort
2 changes: 2 additions & 0 deletions tests/node_modules_with_symlinks/root/.flowconfig
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
[ignore]
ignored_alias/**
ign/**
ignored_link/**

[options]
all=true
Expand Down
3 changes: 3 additions & 0 deletions tests/node_modules_with_symlinks/root/foo.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,6 @@ console.log(symlinked_file);
console.log(nested_symlink);
console.log(ignored_symlink);
console.log(extensionless_symlink);
var through_two_links = require('./lib2_link/n/placeholder');
var through_pruned_link = require('./ign_link/n/placeholder');
var under_ignored_link = require('./ignored_link/sub/x');
1 change: 1 addition & 0 deletions tests/node_modules_with_symlinks/root/lib2/sub/x.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
module.exports = 1;
9 changes: 9 additions & 0 deletions tests/node_modules_with_symlinks/root/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@ if command -v cmd.exe > /dev/null 2>&1; then
cmd.exe /D /S /C 'mklink /D ..\unwatched\dir_link ..\root\dir' > /dev/null
fi

# Symlinks the crawl doesn't follow, as it reads their targets anyway: one to a directory holding
# another, one inside a pruned directory, and one whose own path is ignored.
mkdir -p ign/sub
MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s ../dir lib2/n
MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s lib2 lib2_link
MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s ../../dir ign/sub/n
MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s ign/sub ign_link
MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s lib2 ignored_link

printf "Scratch init:\n"
assert_errors "$FLOW" status .

Expand Down
3 changes: 3 additions & 0 deletions tests/symlinks_out_of_root/.testconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
cwd: root
shell: test.sh
auto_start: false
1 change: 1 addition & 0 deletions tests/symlinks_out_of_root/root-lib/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// @flow
6 changes: 6 additions & 0 deletions tests/symlinks_out_of_root/root/.flowconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[include]
../shared/*
../other/**

[options]
all=false
1 change: 1 addition & 0 deletions tests/symlinks_out_of_root/root/pkg/z.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// @flow
27 changes: 27 additions & 0 deletions tests/symlinks_out_of_root/root/test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# The crawl must still follow these symlinks, as it doesn't read their targets, or doesn't report
# the files under them by their real paths.

export LC_ALL=C

link() {
MSYS=winsymlinks:nativestrict CYGWIN=winsymlinks:nativestrict ln -s "$1" "$2"
}

# `../shared/*` doesn't cover `../shared/sub/deep.js`.
link ../shared/sub sublink
# `../root-lib` is outside the root, though its path starts with the root's.
link ../root-lib root-lib
# `../other/sub` is outside the root, so the symlink inside it isn't reported by its real path.
mkdir -p ../other/sub
link ../other/sub other_sub
link ../../third ../other/sub/nested
# A symlink outside the root, to a directory under it.
link ../root/pkg ../other/to_root

assert_ok "$FLOW" ls --strip-root | sort
1 change: 1 addition & 0 deletions tests/symlinks_out_of_root/shared/sub/deep.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// @flow
7 changes: 7 additions & 0 deletions tests/symlinks_out_of_root/symlinks_out_of_root.exp
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
../other/sub/nested/y.js
../other/to_root/z.js
../root-lib/index.js
../shared/sub/deep.js
../third/y.js
.flowconfig
pkg/z.js
1 change: 1 addition & 0 deletions tests/symlinks_out_of_root/third/y.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// @flow
Loading